1 /* 2 * CDDL HEADER START 3 * 4 * The contents of this file are subject to the terms of the 5 * Common Development and Distribution License, Version 1.0 only 6 * (the "License"). You may not use this file except in compliance 7 * with the License. 8 * 9 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE 10 * or http://www.opensolaris.org/os/licensing. 11 * See the License for the specific language governing permissions 12 * and limitations under the License. 13 * 14 * When distributing Covered Code, include this CDDL HEADER in each 15 * file and include the License file at usr/src/OPENSOLARIS.LICENSE. 16 * If applicable, add the following below this CDDL HEADER, with the 17 * fields enclosed by brackets "[]" replaced with your own identifying 18 * information: Portions Copyright [yyyy] [name of copyright owner] 19 * 20 * CDDL HEADER END 21 */ 22 /* 23 * Copyright 2005 Sun Microsystems, Inc. All rights reserved. 24 * Use is subject to license terms. 25 */ 26 27 #ifndef _ASM_ATOMIC_H 28 #define _ASM_ATOMIC_H 29 30 #include <sys/ccompile.h> 31 #include <sys/types.h> 32 33 #ifdef __cplusplus 34 extern "C" { 35 #endif 36 37 #if !defined(__lint) && defined(__GNUC__) 38 39 /* 40 * This file contains a number of static inline functions implementing 41 * various atomic variable functions. Note that these are *not* all of the 42 * atomic_* functions as defined in usr/src/uts/common/sys/atomic.h. All 43 * possible atomic_* functions are implemented in usr/src/common/atomic in 44 * pure assembly. In the absence of an identically named function in this 45 * header file, any use of the function will result in the compiler emitting 46 * a function call as usual. On the other hand, if an identically named 47 * function exists in this header as a static inline, the compiler will 48 * inline its contents and the linker never sees the symbol reference. We 49 * use this to avoid implementing some of the more complex and less used 50 * functions and instead falling back to function calls. Note that in some 51 * cases (e.g., atomic_inc_64) we implement a static inline only on AMD64 52 * but not i386. 53 */ 54 55 /* 56 * Instruction suffixes for various operand sizes (assuming AMD64) 57 */ 58 #define SUF_8 "b" 59 #define SUF_16 "w" 60 #define SUF_32 "l" 61 #define SUF_64 "q" 62 63 #if defined(__amd64) 64 #define SUF_LONG SUF_64 65 #define SUF_PTR SUF_64 66 #define __ATOMIC_OP64(...) __ATOMIC_OPXX(__VA_ARGS__) 67 #elif defined(__i386) 68 #define SUF_LONG SUF_32 69 #define SUF_PTR SUF_32 70 #define __ATOMIC_OP64(...) 71 #else 72 #error "port me" 73 #endif 74 75 #if defined(__amd64) || defined(__i386) 76 77 #define __ATOMIC_OPXX(fxn, type, op) \ 78 extern __GNU_INLINE void \ 79 fxn(volatile type *target) \ 80 { \ 81 __asm__ __volatile__( \ 82 "lock; " op " %0" \ 83 : "+m" (*target)); \ 84 } 85 86 __ATOMIC_OPXX(atomic_inc_8, uint8_t, "inc" SUF_8) 87 __ATOMIC_OPXX(atomic_inc_16, uint16_t, "inc" SUF_16) 88 __ATOMIC_OPXX(atomic_inc_32, uint32_t, "inc" SUF_32) 89 __ATOMIC_OP64(atomic_inc_64, uint64_t, "inc" SUF_64) 90 __ATOMIC_OPXX(atomic_inc_uchar, uchar_t, "inc" SUF_8) 91 __ATOMIC_OPXX(atomic_inc_ushort, ushort_t, "inc" SUF_16) 92 __ATOMIC_OPXX(atomic_inc_uint, uint_t, "inc" SUF_32) 93 __ATOMIC_OPXX(atomic_inc_ulong, ulong_t, "inc" SUF_LONG) 94 95 __ATOMIC_OPXX(atomic_dec_8, uint8_t, "dec" SUF_8) 96 __ATOMIC_OPXX(atomic_dec_16, uint16_t, "dec" SUF_16) 97 __ATOMIC_OPXX(atomic_dec_32, uint32_t, "dec" SUF_32) 98 __ATOMIC_OP64(atomic_dec_64, uint64_t, "dec" SUF_64) 99 __ATOMIC_OPXX(atomic_dec_uchar, uchar_t, "dec" SUF_8) 100 __ATOMIC_OPXX(atomic_dec_ushort, ushort_t, "dec" SUF_16) 101 __ATOMIC_OPXX(atomic_dec_uint, uint_t, "dec" SUF_32) 102 __ATOMIC_OPXX(atomic_dec_ulong, ulong_t, "dec" SUF_LONG) 103 104 #undef __ATOMIC_OPXX 105 106 #define __ATOMIC_OPXX(fxn, type1, type2, op) \ 107 extern __GNU_INLINE void \ 108 fxn(volatile type1 *target, type2 delta) \ 109 { \ 110 __asm__ __volatile__( \ 111 "lock; " op " %1,%0" \ 112 : "+m" (*target) \ 113 : "ir" (delta)); \ 114 } 115 116 __ATOMIC_OPXX(atomic_add_8, uint8_t, int8_t, "add" SUF_8) 117 __ATOMIC_OPXX(atomic_add_16, uint16_t, int16_t, "add" SUF_16) 118 __ATOMIC_OPXX(atomic_add_32, uint32_t, int32_t, "add" SUF_32) 119 __ATOMIC_OP64(atomic_add_64, uint64_t, int64_t, "add" SUF_64) 120 __ATOMIC_OPXX(atomic_add_char, uchar_t, signed char, "add" SUF_8) 121 __ATOMIC_OPXX(atomic_add_short, ushort_t, short, "add" SUF_16) 122 __ATOMIC_OPXX(atomic_add_int, uint_t, int, "add" SUF_32) 123 __ATOMIC_OPXX(atomic_add_long, ulong_t, long, "add" SUF_LONG) 124 125 /* 126 * We don't use the above macro here because atomic_add_ptr has an 127 * inconsistent type. The first argument should really be a 'volatile void 128 * **'. 129 */ 130 extern __GNU_INLINE void 131 atomic_add_ptr(volatile void *target, ssize_t delta) 132 { 133 volatile void **tmp = (volatile void **)target; 134 135 __asm__ __volatile__( 136 "lock; add" SUF_PTR " %1,%0" 137 : "+m" (*tmp) 138 : "ir" (delta)); 139 } 140 141 __ATOMIC_OPXX(atomic_or_8, uint8_t, uint8_t, "or" SUF_8) 142 __ATOMIC_OPXX(atomic_or_16, uint16_t, uint16_t, "or" SUF_16) 143 __ATOMIC_OPXX(atomic_or_32, uint32_t, uint32_t, "or" SUF_32) 144 __ATOMIC_OP64(atomic_or_64, uint64_t, uint64_t, "or" SUF_64) 145 __ATOMIC_OPXX(atomic_or_uchar, uchar_t, uchar_t, "or" SUF_8) 146 __ATOMIC_OPXX(atomic_or_ushort, ushort_t, ushort_t, "or" SUF_16) 147 __ATOMIC_OPXX(atomic_or_uint, uint_t, uint_t, "or" SUF_32) 148 __ATOMIC_OPXX(atomic_or_ulong, ulong_t, ulong_t, "or" SUF_LONG) 149 150 __ATOMIC_OPXX(atomic_and_8, uint8_t, uint8_t, "and" SUF_8) 151 __ATOMIC_OPXX(atomic_and_16, uint16_t, uint16_t, "and" SUF_16) 152 __ATOMIC_OPXX(atomic_and_32, uint32_t, uint32_t, "and" SUF_32) 153 __ATOMIC_OP64(atomic_and_64, uint64_t, uint64_t, "and" SUF_64) 154 __ATOMIC_OPXX(atomic_and_uchar, uchar_t, uchar_t, "and" SUF_8) 155 __ATOMIC_OPXX(atomic_and_ushort, ushort_t, ushort_t, "and" SUF_16) 156 __ATOMIC_OPXX(atomic_and_uint, uint_t, uint_t, "and" SUF_32) 157 __ATOMIC_OPXX(atomic_and_ulong, ulong_t, ulong_t, "and" SUF_LONG) 158 159 #undef __ATOMIC_OPXX 160 161 #define __ATOMIC_OPXX(fxn, type, op, reg) \ 162 extern __GNU_INLINE type \ 163 fxn(volatile type *target, type cmp, type new) \ 164 { \ 165 type ret; \ 166 __asm__ __volatile__( \ 167 "lock; " op " %2,%0" \ 168 : "+m" (*target), "=a" (ret) \ 169 : reg (new), "1" (cmp) \ 170 : "cc"); \ 171 return (ret); \ 172 } 173 174 __ATOMIC_OPXX(atomic_cas_8, uint8_t, "cmpxchg" SUF_8, "q") 175 __ATOMIC_OPXX(atomic_cas_16, uint16_t, "cmpxchg" SUF_16, "r") 176 __ATOMIC_OPXX(atomic_cas_32, uint32_t, "cmpxchg" SUF_32, "r") 177 __ATOMIC_OP64(atomic_cas_64, uint64_t, "cmpxchg" SUF_64, "r") 178 __ATOMIC_OPXX(atomic_cas_uchar, uchar_t, "cmpxchg" SUF_8, "q") 179 __ATOMIC_OPXX(atomic_cas_ushort, ushort_t, "cmpxchg" SUF_16, "r") 180 __ATOMIC_OPXX(atomic_cas_uint, uint_t, "cmpxchg" SUF_32, "r") 181 __ATOMIC_OPXX(atomic_cas_ulong, ulong_t, "cmpxchg" SUF_LONG, "r") 182 183 #undef __ATOMIC_OPXX 184 185 /* 186 * We don't use the above macro here because atomic_cas_ptr has an 187 * inconsistent type. The first argument should really be a 'volatile void 188 * **'. 189 */ 190 extern __GNU_INLINE void * 191 atomic_cas_ptr(volatile void *target, void *cmp, void *new) 192 { 193 volatile void **tmp = (volatile void **)target; 194 void *ret; 195 196 __asm__ __volatile__( 197 "lock; cmpxchg" SUF_PTR " %2,%0" 198 : "+m" (*tmp), "=a" (ret) 199 : "r" (new), "1" (cmp) 200 : "cc"); 201 202 return (ret); 203 } 204 205 #define __ATOMIC_OPXX(fxn, type, op, reg) \ 206 extern __GNU_INLINE type \ 207 fxn(volatile type *target, type val) \ 208 { \ 209 __asm__ __volatile__( \ 210 op " %1,%0" \ 211 : "+m" (*target), "+" reg (val)); \ 212 return (val); \ 213 } 214 215 __ATOMIC_OPXX(atomic_swap_8, uint8_t, "xchg" SUF_8, "q") 216 __ATOMIC_OPXX(atomic_swap_16, uint16_t, "xchg" SUF_16, "r") 217 __ATOMIC_OPXX(atomic_swap_32, uint32_t, "xchg" SUF_32, "r") 218 __ATOMIC_OP64(atomic_swap_64, uint64_t, "xchg" SUF_64, "r") 219 __ATOMIC_OPXX(atomic_swap_uchar, uchar_t, "xchg" SUF_8, "q") 220 __ATOMIC_OPXX(atomic_swap_ushort, ushort_t, "xchg" SUF_16, "r") 221 __ATOMIC_OPXX(atomic_swap_uint, uint_t, "xchg" SUF_32, "r") 222 __ATOMIC_OPXX(atomic_swap_ulong, ulong_t, "xchg" SUF_LONG, "r") 223 224 #undef __ATOMIC_OPXX 225 226 /* 227 * We don't use the above macro here because atomic_swap_ptr has an 228 * inconsistent type. The first argument should really be a 'volatile void 229 * **'. 230 */ 231 extern __GNU_INLINE void * 232 atomic_swap_ptr(volatile void *target, void *val) 233 { 234 volatile void **tmp = (volatile void **)target; 235 236 __asm__ __volatile__( 237 "xchg" SUF_PTR " %1,%0" 238 : "+m" (*tmp), "+r" (val)); 239 240 return (val); 241 } 242 243 #else 244 #error "port me" 245 #endif 246 247 #undef SUF_8 248 #undef SUF_16 249 #undef SUF_32 250 #undef SUF_64 251 #undef SUF_LONG 252 #undef SUF_PTR 253 254 #undef __ATOMIC_OP64 255 256 #endif /* !__lint && __GNUC__ */ 257 258 #ifdef __cplusplus 259 } 260 #endif 261 262 #endif /* _ASM_ATOMIC_H */