19 *
20 * CDDL HEADER END
21 */
22 /*
23 * Copyright 2005 Sun Microsystems, Inc. All rights reserved.
24 * Use is subject to license terms.
25 */
26
27 #ifndef _ASM_ATOMIC_H
28 #define _ASM_ATOMIC_H
29
30 #include <sys/ccompile.h>
31 #include <sys/types.h>
32
33 #ifdef __cplusplus
34 extern "C" {
35 #endif
36
37 #if !defined(__lint) && defined(__GNUC__)
38
39 #if defined(__amd64)
40
41 #elif defined(__i386)
42
43 #else
44 #error "port me"
45 #endif
46
47 #endif /* !__lint && __GNUC__ */
48
49 #ifdef __cplusplus
50 }
51 #endif
52
53 #endif /* _ASM_ATOMIC_H */
|
19 *
20 * CDDL HEADER END
21 */
22 /*
23 * Copyright 2005 Sun Microsystems, Inc. All rights reserved.
24 * Use is subject to license terms.
25 */
26
27 #ifndef _ASM_ATOMIC_H
28 #define _ASM_ATOMIC_H
29
30 #include <sys/ccompile.h>
31 #include <sys/types.h>
32
33 #ifdef __cplusplus
34 extern "C" {
35 #endif
36
37 #if !defined(__lint) && defined(__GNUC__)
38
39 /*
40 * This file contains a number of static inline functions implementing
41 * various atomic variable functions. Note that these are *not* all of the
42 * atomic_* functions as defined in usr/src/uts/common/sys/atomic.h. All
43 * possible atomic_* functions are implemented in usr/src/common/atomic in
44 * pure assembly. In the absence of an identically named function in this
45 * header file, any use of the function will result in the compiler emitting
46 * a function call as usual. On the other hand, if an identically named
47 * function exists in this header as a static inline, the compiler will
48 * inline its contents and the linker never sees the symbol reference. We
49 * use this to avoid implementing some of the more complex and less used
50 * functions and instead falling back to function calls. Note that in some
51 * cases (e.g., atomic_inc_64) we implement a static inline only on AMD64
52 * but not i386.
53 */
54
55 /*
56 * Instruction suffixes for various operand sizes (assuming AMD64)
57 */
58 #define SUF_8 "b"
59 #define SUF_16 "w"
60 #define SUF_32 "l"
61 #define SUF_64 "q"
62
63 #if defined(__amd64)
64 #define SUF_LONG SUF_64
65 #define SUF_PTR SUF_64
66 #define __ATOMIC_OP64(...) __ATOMIC_OPXX(__VA_ARGS__)
67 #elif defined(__i386)
68 #define SUF_LONG SUF_32
69 #define SUF_PTR SUF_32
70 #define __ATOMIC_OP64(...)
71 #else
72 #error "port me"
73 #endif
74
75 #if defined(__amd64) || defined(__i386)
76
77 #define __ATOMIC_OPXX(fxn, type, op) \
78 extern __GNU_INLINE void \
79 fxn(volatile type *target) \
80 { \
81 __asm__ __volatile__( \
82 "lock; " op " %0" \
83 : "+m" (*target)); \
84 }
85
86 __ATOMIC_OPXX(atomic_inc_8, uint8_t, "inc" SUF_8)
87 __ATOMIC_OPXX(atomic_inc_16, uint16_t, "inc" SUF_16)
88 __ATOMIC_OPXX(atomic_inc_32, uint32_t, "inc" SUF_32)
89 __ATOMIC_OP64(atomic_inc_64, uint64_t, "inc" SUF_64)
90 __ATOMIC_OPXX(atomic_inc_uchar, uchar_t, "inc" SUF_8)
91 __ATOMIC_OPXX(atomic_inc_ushort, ushort_t, "inc" SUF_16)
92 __ATOMIC_OPXX(atomic_inc_uint, uint_t, "inc" SUF_32)
93 __ATOMIC_OPXX(atomic_inc_ulong, ulong_t, "inc" SUF_LONG)
94
95 __ATOMIC_OPXX(atomic_dec_8, uint8_t, "dec" SUF_8)
96 __ATOMIC_OPXX(atomic_dec_16, uint16_t, "dec" SUF_16)
97 __ATOMIC_OPXX(atomic_dec_32, uint32_t, "dec" SUF_32)
98 __ATOMIC_OP64(atomic_dec_64, uint64_t, "dec" SUF_64)
99 __ATOMIC_OPXX(atomic_dec_uchar, uchar_t, "dec" SUF_8)
100 __ATOMIC_OPXX(atomic_dec_ushort, ushort_t, "dec" SUF_16)
101 __ATOMIC_OPXX(atomic_dec_uint, uint_t, "dec" SUF_32)
102 __ATOMIC_OPXX(atomic_dec_ulong, ulong_t, "dec" SUF_LONG)
103
104 #undef __ATOMIC_OPXX
105
106 #define __ATOMIC_OPXX(fxn, type1, type2, op) \
107 extern __GNU_INLINE void \
108 fxn(volatile type1 *target, type2 delta) \
109 { \
110 __asm__ __volatile__( \
111 "lock; " op " %1,%0" \
112 : "+m" (*target) \
113 : "ir" (delta)); \
114 }
115
116 __ATOMIC_OPXX(atomic_add_8, uint8_t, int8_t, "add" SUF_8)
117 __ATOMIC_OPXX(atomic_add_16, uint16_t, int16_t, "add" SUF_16)
118 __ATOMIC_OPXX(atomic_add_32, uint32_t, int32_t, "add" SUF_32)
119 __ATOMIC_OP64(atomic_add_64, uint64_t, int64_t, "add" SUF_64)
120 __ATOMIC_OPXX(atomic_add_char, uchar_t, signed char, "add" SUF_8)
121 __ATOMIC_OPXX(atomic_add_short, ushort_t, short, "add" SUF_16)
122 __ATOMIC_OPXX(atomic_add_int, uint_t, int, "add" SUF_32)
123 __ATOMIC_OPXX(atomic_add_long, ulong_t, long, "add" SUF_LONG)
124
125 /*
126 * We don't use the above macro here because atomic_add_ptr has an
127 * inconsistent type. The first argument should really be a 'volatile void
128 * **'.
129 */
130 extern __GNU_INLINE void
131 atomic_add_ptr(volatile void *target, ssize_t delta)
132 {
133 volatile void **tmp = (volatile void **)target;
134
135 __asm__ __volatile__(
136 "lock; add" SUF_PTR " %1,%0"
137 : "+m" (*tmp)
138 : "ir" (delta));
139 }
140
141 __ATOMIC_OPXX(atomic_or_8, uint8_t, uint8_t, "or" SUF_8)
142 __ATOMIC_OPXX(atomic_or_16, uint16_t, uint16_t, "or" SUF_16)
143 __ATOMIC_OPXX(atomic_or_32, uint32_t, uint32_t, "or" SUF_32)
144 __ATOMIC_OP64(atomic_or_64, uint64_t, uint64_t, "or" SUF_64)
145 __ATOMIC_OPXX(atomic_or_uchar, uchar_t, uchar_t, "or" SUF_8)
146 __ATOMIC_OPXX(atomic_or_ushort, ushort_t, ushort_t, "or" SUF_16)
147 __ATOMIC_OPXX(atomic_or_uint, uint_t, uint_t, "or" SUF_32)
148 __ATOMIC_OPXX(atomic_or_ulong, ulong_t, ulong_t, "or" SUF_LONG)
149
150 __ATOMIC_OPXX(atomic_and_8, uint8_t, uint8_t, "and" SUF_8)
151 __ATOMIC_OPXX(atomic_and_16, uint16_t, uint16_t, "and" SUF_16)
152 __ATOMIC_OPXX(atomic_and_32, uint32_t, uint32_t, "and" SUF_32)
153 __ATOMIC_OP64(atomic_and_64, uint64_t, uint64_t, "and" SUF_64)
154 __ATOMIC_OPXX(atomic_and_uchar, uchar_t, uchar_t, "and" SUF_8)
155 __ATOMIC_OPXX(atomic_and_ushort, ushort_t, ushort_t, "and" SUF_16)
156 __ATOMIC_OPXX(atomic_and_uint, uint_t, uint_t, "and" SUF_32)
157 __ATOMIC_OPXX(atomic_and_ulong, ulong_t, ulong_t, "and" SUF_LONG)
158
159 #undef __ATOMIC_OPXX
160
161 #define __ATOMIC_OPXX(fxn, type, op, reg) \
162 extern __GNU_INLINE type \
163 fxn(volatile type *target, type cmp, type new) \
164 { \
165 type ret; \
166 __asm__ __volatile__( \
167 "lock; " op " %2,%0" \
168 : "+m" (*target), "=a" (ret) \
169 : reg (new), "1" (cmp) \
170 : "cc"); \
171 return (ret); \
172 }
173
174 __ATOMIC_OPXX(atomic_cas_8, uint8_t, "cmpxchg" SUF_8, "q")
175 __ATOMIC_OPXX(atomic_cas_16, uint16_t, "cmpxchg" SUF_16, "r")
176 __ATOMIC_OPXX(atomic_cas_32, uint32_t, "cmpxchg" SUF_32, "r")
177 __ATOMIC_OP64(atomic_cas_64, uint64_t, "cmpxchg" SUF_64, "r")
178 __ATOMIC_OPXX(atomic_cas_uchar, uchar_t, "cmpxchg" SUF_8, "q")
179 __ATOMIC_OPXX(atomic_cas_ushort, ushort_t, "cmpxchg" SUF_16, "r")
180 __ATOMIC_OPXX(atomic_cas_uint, uint_t, "cmpxchg" SUF_32, "r")
181 __ATOMIC_OPXX(atomic_cas_ulong, ulong_t, "cmpxchg" SUF_LONG, "r")
182
183 #undef __ATOMIC_OPXX
184
185 /*
186 * We don't use the above macro here because atomic_cas_ptr has an
187 * inconsistent type. The first argument should really be a 'volatile void
188 * **'.
189 */
190 extern __GNU_INLINE void *
191 atomic_cas_ptr(volatile void *target, void *cmp, void *new)
192 {
193 volatile void **tmp = (volatile void **)target;
194 void *ret;
195
196 __asm__ __volatile__(
197 "lock; cmpxchg" SUF_PTR " %2,%0"
198 : "+m" (*tmp), "=a" (ret)
199 : "r" (new), "1" (cmp)
200 : "cc");
201
202 return (ret);
203 }
204
205 #define __ATOMIC_OPXX(fxn, type, op, reg) \
206 extern __GNU_INLINE type \
207 fxn(volatile type *target, type val) \
208 { \
209 __asm__ __volatile__( \
210 op " %1,%0" \
211 : "+m" (*target), "+" reg (val)); \
212 return (val); \
213 }
214
215 __ATOMIC_OPXX(atomic_swap_8, uint8_t, "xchg" SUF_8, "q")
216 __ATOMIC_OPXX(atomic_swap_16, uint16_t, "xchg" SUF_16, "r")
217 __ATOMIC_OPXX(atomic_swap_32, uint32_t, "xchg" SUF_32, "r")
218 __ATOMIC_OP64(atomic_swap_64, uint64_t, "xchg" SUF_64, "r")
219 __ATOMIC_OPXX(atomic_swap_uchar, uchar_t, "xchg" SUF_8, "q")
220 __ATOMIC_OPXX(atomic_swap_ushort, ushort_t, "xchg" SUF_16, "r")
221 __ATOMIC_OPXX(atomic_swap_uint, uint_t, "xchg" SUF_32, "r")
222 __ATOMIC_OPXX(atomic_swap_ulong, ulong_t, "xchg" SUF_LONG, "r")
223
224 #undef __ATOMIC_OPXX
225
226 /*
227 * We don't use the above macro here because atomic_swap_ptr has an
228 * inconsistent type. The first argument should really be a 'volatile void
229 * **'.
230 */
231 extern __GNU_INLINE void *
232 atomic_swap_ptr(volatile void *target, void *val)
233 {
234 volatile void **tmp = (volatile void **)target;
235
236 __asm__ __volatile__(
237 "xchg" SUF_PTR " %1,%0"
238 : "+m" (*tmp), "+r" (val));
239
240 return (val);
241 }
242
243 #else
244 #error "port me"
245 #endif
246
247 #undef SUF_8
248 #undef SUF_16
249 #undef SUF_32
250 #undef SUF_64
251 #undef SUF_LONG
252 #undef SUF_PTR
253
254 #undef __ATOMIC_OP64
255
256 #endif /* !__lint && __GNUC__ */
257
258 #ifdef __cplusplus
259 }
260 #endif
261
262 #endif /* _ASM_ATOMIC_H */
|