Print this page
6137 implement static inlines for atomic_{add,inc,dec,or,and}_*_nv on intel
| Split |
Close |
| Expand all |
| Collapse all |
--- old/usr/src/uts/intel/asm/atomic.h
+++ new/usr/src/uts/intel/asm/atomic.h
1 1 /*
2 2 * CDDL HEADER START
3 3 *
4 4 * The contents of this file are subject to the terms of the
5 5 * Common Development and Distribution License, Version 1.0 only
6 6 * (the "License"). You may not use this file except in compliance
7 7 * with the License.
8 8 *
9 9 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
10 10 * or http://www.opensolaris.org/os/licensing.
11 11 * See the License for the specific language governing permissions
12 12 * and limitations under the License.
13 13 *
14 14 * When distributing Covered Code, include this CDDL HEADER in each
15 15 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
16 16 * If applicable, add the following below this CDDL HEADER, with the
17 17 * fields enclosed by brackets "[]" replaced with your own identifying
18 18 * information: Portions Copyright [yyyy] [name of copyright owner]
19 19 *
20 20 * CDDL HEADER END
21 21 */
22 22 /*
23 23 * Copyright 2005 Sun Microsystems, Inc. All rights reserved.
24 24 * Use is subject to license terms.
25 25 * Copyright 2014 Nexenta Systems, Inc. All rights reserved.
26 26 * Copyright 2015 Josef 'Jeff' Sipek <jeffpc@josefsipek.net>
27 27 */
28 28
29 29 #ifndef _ASM_ATOMIC_H
30 30 #define _ASM_ATOMIC_H
31 31
32 32 #include <sys/ccompile.h>
33 33 #include <sys/types.h>
34 34
35 35 #ifdef __cplusplus
36 36 extern "C" {
37 37 #endif
38 38
39 39 #if !defined(__lint) && defined(__GNUC__)
40 40
41 41 /* BEGIN CSTYLED */
42 42 /*
43 43 * This file contains a number of static inline functions implementing
44 44 * various atomic variable functions. Note that these are *not* all of the
45 45 * atomic_* functions as defined in usr/src/uts/common/sys/atomic.h. All
46 46 * possible atomic_* functions are implemented in usr/src/common/atomic in
47 47 * pure assembly. In the absence of an identically named function in this
48 48 * header file, any use of the function will result in the compiler emitting
49 49 * a function call as usual. On the other hand, if an identically named
50 50 * function exists in this header as a static inline, the compiler will
51 51 * inline its contents and the linker never sees the symbol reference. We
52 52 * use this to avoid implementing some of the more complex and less used
53 53 * functions and instead falling back to function calls. Note that in some
54 54 * cases (e.g., atomic_inc_64) we implement a static inline only on AMD64
55 55 * but not i386.
56 56 */
57 57
58 58 /*
59 59 * Instruction suffixes for various operand sizes (assuming AMD64)
60 60 */
61 61 #define SUF_8 "b"
62 62 #define SUF_16 "w"
63 63 #define SUF_32 "l"
64 64 #define SUF_64 "q"
65 65
66 66 #if defined(__amd64)
67 67 #define SUF_LONG SUF_64
68 68 #define SUF_PTR SUF_64
69 69 #define __ATOMIC_OP64(...) __ATOMIC_OPXX(__VA_ARGS__)
70 70 #elif defined(__i386)
71 71 #define SUF_LONG SUF_32
72 72 #define SUF_PTR SUF_32
73 73 #define __ATOMIC_OP64(...)
74 74 #else
75 75 #error "port me"
76 76 #endif
77 77
78 78 #if defined(__amd64) || defined(__i386)
79 79
80 80 #define __ATOMIC_OPXX(fxn, type, op) \
81 81 extern __GNU_INLINE void \
82 82 fxn(volatile type *target) \
83 83 { \
84 84 __asm__ __volatile__( \
85 85 "lock; " op " %0" \
86 86 : "+m" (*target) \
87 87 : /* no inputs */ \
88 88 : "cc"); \
89 89 }
90 90
91 91 __ATOMIC_OPXX(atomic_inc_8, uint8_t, "inc" SUF_8)
92 92 __ATOMIC_OPXX(atomic_inc_16, uint16_t, "inc" SUF_16)
93 93 __ATOMIC_OPXX(atomic_inc_32, uint32_t, "inc" SUF_32)
94 94 __ATOMIC_OP64(atomic_inc_64, uint64_t, "inc" SUF_64)
95 95 __ATOMIC_OPXX(atomic_inc_uchar, uchar_t, "inc" SUF_8)
96 96 __ATOMIC_OPXX(atomic_inc_ushort, ushort_t, "inc" SUF_16)
97 97 __ATOMIC_OPXX(atomic_inc_uint, uint_t, "inc" SUF_32)
98 98 __ATOMIC_OPXX(atomic_inc_ulong, ulong_t, "inc" SUF_LONG)
99 99
100 100 __ATOMIC_OPXX(atomic_dec_8, uint8_t, "dec" SUF_8)
101 101 __ATOMIC_OPXX(atomic_dec_16, uint16_t, "dec" SUF_16)
102 102 __ATOMIC_OPXX(atomic_dec_32, uint32_t, "dec" SUF_32)
103 103 __ATOMIC_OP64(atomic_dec_64, uint64_t, "dec" SUF_64)
104 104 __ATOMIC_OPXX(atomic_dec_uchar, uchar_t, "dec" SUF_8)
105 105 __ATOMIC_OPXX(atomic_dec_ushort, ushort_t, "dec" SUF_16)
106 106 __ATOMIC_OPXX(atomic_dec_uint, uint_t, "dec" SUF_32)
107 107 __ATOMIC_OPXX(atomic_dec_ulong, ulong_t, "dec" SUF_LONG)
108 108
109 109 #undef __ATOMIC_OPXX
110 110
111 111 #define __ATOMIC_OPXX(fxn, type1, type2, op, reg) \
112 112 extern __GNU_INLINE void \
113 113 fxn(volatile type1 *target, type2 delta) \
114 114 { \
115 115 __asm__ __volatile__( \
116 116 "lock; " op " %1,%0" \
117 117 : "+m" (*target) \
118 118 : "i" reg (delta) \
119 119 : "cc"); \
120 120 }
121 121
122 122 __ATOMIC_OPXX(atomic_add_8, uint8_t, int8_t, "add" SUF_8, "q")
123 123 __ATOMIC_OPXX(atomic_add_16, uint16_t, int16_t, "add" SUF_16, "r")
124 124 __ATOMIC_OPXX(atomic_add_32, uint32_t, int32_t, "add" SUF_32, "r")
125 125 __ATOMIC_OP64(atomic_add_64, uint64_t, int64_t, "add" SUF_64, "r")
126 126 __ATOMIC_OPXX(atomic_add_char, uchar_t, signed char, "add" SUF_8, "q")
127 127 __ATOMIC_OPXX(atomic_add_short, ushort_t, short, "add" SUF_16, "r")
128 128 __ATOMIC_OPXX(atomic_add_int, uint_t, int, "add" SUF_32, "r")
129 129 __ATOMIC_OPXX(atomic_add_long, ulong_t, long, "add" SUF_LONG, "r")
130 130
131 131 /*
132 132 * We don't use the above macro here because atomic_add_ptr has an
133 133 * inconsistent type. The first argument should really be a 'volatile void
134 134 * **'.
135 135 */
136 136 extern __GNU_INLINE void
137 137 atomic_add_ptr(volatile void *target, ssize_t delta)
138 138 {
139 139 volatile void **tmp = (volatile void **)target;
140 140
141 141 __asm__ __volatile__(
142 142 "lock; add" SUF_PTR " %1,%0"
143 143 : "+m" (*tmp)
144 144 : "ir" (delta)
145 145 : "cc");
146 146 }
147 147
148 148 __ATOMIC_OPXX(atomic_or_8, uint8_t, uint8_t, "or" SUF_8, "q")
149 149 __ATOMIC_OPXX(atomic_or_16, uint16_t, uint16_t, "or" SUF_16, "r")
150 150 __ATOMIC_OPXX(atomic_or_32, uint32_t, uint32_t, "or" SUF_32, "r")
151 151 __ATOMIC_OP64(atomic_or_64, uint64_t, uint64_t, "or" SUF_64, "r")
152 152 __ATOMIC_OPXX(atomic_or_uchar, uchar_t, uchar_t, "or" SUF_8, "q")
153 153 __ATOMIC_OPXX(atomic_or_ushort, ushort_t, ushort_t, "or" SUF_16, "r")
154 154 __ATOMIC_OPXX(atomic_or_uint, uint_t, uint_t, "or" SUF_32, "r")
155 155 __ATOMIC_OPXX(atomic_or_ulong, ulong_t, ulong_t, "or" SUF_LONG, "r")
156 156
157 157 __ATOMIC_OPXX(atomic_and_8, uint8_t, uint8_t, "and" SUF_8, "q")
158 158 __ATOMIC_OPXX(atomic_and_16, uint16_t, uint16_t, "and" SUF_16, "r")
159 159 __ATOMIC_OPXX(atomic_and_32, uint32_t, uint32_t, "and" SUF_32, "r")
160 160 __ATOMIC_OP64(atomic_and_64, uint64_t, uint64_t, "and" SUF_64, "r")
161 161 __ATOMIC_OPXX(atomic_and_uchar, uchar_t, uchar_t, "and" SUF_8, "q")
162 162 __ATOMIC_OPXX(atomic_and_ushort, ushort_t, ushort_t, "and" SUF_16, "r")
163 163 __ATOMIC_OPXX(atomic_and_uint, uint_t, uint_t, "and" SUF_32, "r")
164 164 __ATOMIC_OPXX(atomic_and_ulong, ulong_t, ulong_t, "and" SUF_LONG, "r")
165 165
166 166 #undef __ATOMIC_OPXX
167 167
168 168 #define __ATOMIC_OPXX(fxn, type, op, reg) \
169 169 extern __GNU_INLINE type \
170 170 fxn(volatile type *target, type cmp, type new) \
171 171 { \
172 172 type ret; \
173 173 __asm__ __volatile__( \
174 174 "lock; " op " %2,%0" \
175 175 : "+m" (*target), "=a" (ret) \
176 176 : reg (new), "1" (cmp) \
177 177 : "cc"); \
178 178 return (ret); \
179 179 }
180 180
181 181 __ATOMIC_OPXX(atomic_cas_8, uint8_t, "cmpxchg" SUF_8, "q")
182 182 __ATOMIC_OPXX(atomic_cas_16, uint16_t, "cmpxchg" SUF_16, "r")
183 183 __ATOMIC_OPXX(atomic_cas_32, uint32_t, "cmpxchg" SUF_32, "r")
184 184 __ATOMIC_OP64(atomic_cas_64, uint64_t, "cmpxchg" SUF_64, "r")
185 185 __ATOMIC_OPXX(atomic_cas_uchar, uchar_t, "cmpxchg" SUF_8, "q")
186 186 __ATOMIC_OPXX(atomic_cas_ushort, ushort_t, "cmpxchg" SUF_16, "r")
187 187 __ATOMIC_OPXX(atomic_cas_uint, uint_t, "cmpxchg" SUF_32, "r")
188 188 __ATOMIC_OPXX(atomic_cas_ulong, ulong_t, "cmpxchg" SUF_LONG, "r")
189 189
190 190 #undef __ATOMIC_OPXX
191 191
192 192 /*
193 193 * We don't use the above macro here because atomic_cas_ptr has an
194 194 * inconsistent type. The first argument should really be a 'volatile void
195 195 * **'.
196 196 */
197 197 extern __GNU_INLINE void *
198 198 atomic_cas_ptr(volatile void *target, void *cmp, void *new)
199 199 {
200 200 volatile void **tmp = (volatile void **)target;
201 201 void *ret;
202 202
203 203 __asm__ __volatile__(
204 204 "lock; cmpxchg" SUF_PTR " %2,%0"
205 205 : "+m" (*tmp), "=a" (ret)
206 206 : "r" (new), "1" (cmp)
207 207 : "cc");
208 208
209 209 return (ret);
210 210 }
211 211
212 212 #define __ATOMIC_OPXX(fxn, type, op, reg) \
213 213 extern __GNU_INLINE type \
214 214 fxn(volatile type *target, type val) \
215 215 { \
216 216 __asm__ __volatile__( \
217 217 op " %1,%0" \
218 218 : "+m" (*target), "+" reg (val)); \
219 219 return (val); \
220 220 }
221 221
222 222 __ATOMIC_OPXX(atomic_swap_8, uint8_t, "xchg" SUF_8, "q")
223 223 __ATOMIC_OPXX(atomic_swap_16, uint16_t, "xchg" SUF_16, "r")
224 224 __ATOMIC_OPXX(atomic_swap_32, uint32_t, "xchg" SUF_32, "r")
225 225 __ATOMIC_OP64(atomic_swap_64, uint64_t, "xchg" SUF_64, "r")
226 226 __ATOMIC_OPXX(atomic_swap_uchar, uchar_t, "xchg" SUF_8, "q")
227 227 __ATOMIC_OPXX(atomic_swap_ushort, ushort_t, "xchg" SUF_16, "r")
228 228 __ATOMIC_OPXX(atomic_swap_uint, uint_t, "xchg" SUF_32, "r")
229 229 __ATOMIC_OPXX(atomic_swap_ulong, ulong_t, "xchg" SUF_LONG, "r")
230 230
231 231 #undef __ATOMIC_OPXX
232 232
233 233 /*
234 234 * We don't use the above macro here because atomic_swap_ptr has an
235 235 * inconsistent type. The first argument should really be a 'volatile void
236 236 * **'.
237 237 */
238 238 extern __GNU_INLINE void *
239 239 atomic_swap_ptr(volatile void *target, void *val)
|
↓ open down ↓ |
239 lines elided |
↑ open up ↑ |
240 240 {
241 241 volatile void **tmp = (volatile void **)target;
242 242
243 243 __asm__ __volatile__(
244 244 "xchg" SUF_PTR " %1,%0"
245 245 : "+m" (*tmp), "+r" (val));
246 246
247 247 return (val);
248 248 }
249 249
250 +#define __ATOMIC_OPXX(fxn, type1, type2, suf, reg) \
251 +extern __GNU_INLINE type1 \
252 +fxn(volatile type1 *target, type2 delta) \
253 +{ \
254 + type1 orig; \
255 + __asm__ __volatile__( \
256 + "lock; xadd" suf " %1, %0" \
257 + : "+m" (*target), "=" reg (orig) \
258 + : "1" (delta) \
259 + : "cc"); \
260 + return (orig + delta); \
261 +}
262 +
263 +__ATOMIC_OPXX(atomic_add_8_nv, uint8_t, int8_t, SUF_8, "q")
264 +__ATOMIC_OPXX(atomic_add_16_nv, uint16_t, int16_t, SUF_16, "r")
265 +__ATOMIC_OPXX(atomic_add_32_nv, uint32_t, int32_t, SUF_32, "r")
266 +__ATOMIC_OP64(atomic_add_64_nv, uint64_t, int64_t, SUF_64, "r")
267 +__ATOMIC_OPXX(atomic_add_char_nv, unsigned char, signed char, SUF_8, "q")
268 +__ATOMIC_OPXX(atomic_add_short_nv, ushort_t, short, SUF_16, "r")
269 +__ATOMIC_OPXX(atomic_add_int_nv, uint_t, int, SUF_32, "r")
270 +__ATOMIC_OPXX(atomic_add_long_nv, ulong_t, long, SUF_LONG, "r")
271 +
272 +#undef __ATOMIC_OPXX
273 +
274 +/*
275 + * We don't use the above macro here because atomic_add_ptr_nv has an
276 + * inconsistent type. The first argument should really be a 'volatile void
277 + * **'.
278 + */
279 +extern __GNU_INLINE void *
280 +atomic_add_ptr_nv(volatile void *target, ssize_t delta)
281 +{
282 + return ((void *)atomic_add_long_nv((volatile ulong_t *)target, delta));
283 +}
284 +
285 +#define __ATOMIC_OPXX(fxn, implfxn, type, c) \
286 +extern __GNU_INLINE type \
287 +fxn(volatile type *target) \
288 +{ \
289 + return (implfxn(target, c)); \
290 +}
291 +
292 +__ATOMIC_OPXX(atomic_inc_8_nv, atomic_add_8_nv, uint8_t, 1)
293 +__ATOMIC_OPXX(atomic_inc_16_nv, atomic_add_16_nv, uint16_t, 1)
294 +__ATOMIC_OPXX(atomic_inc_32_nv, atomic_add_32_nv, uint32_t, 1)
295 +__ATOMIC_OP64(atomic_inc_64_nv, atomic_add_64_nv, uint64_t, 1)
296 +__ATOMIC_OPXX(atomic_inc_uchar_nv, atomic_add_char_nv, uchar_t, 1)
297 +__ATOMIC_OPXX(atomic_inc_ushort_nv, atomic_add_short_nv, ushort_t, 1)
298 +__ATOMIC_OPXX(atomic_inc_uint_nv, atomic_add_int_nv, uint_t, 1)
299 +__ATOMIC_OPXX(atomic_inc_ulong_nv, atomic_add_long_nv, ulong_t, 1)
300 +
301 +__ATOMIC_OPXX(atomic_dec_8_nv, atomic_add_8_nv, uint8_t, -1)
302 +__ATOMIC_OPXX(atomic_dec_16_nv, atomic_add_16_nv, uint16_t, -1)
303 +__ATOMIC_OPXX(atomic_dec_32_nv, atomic_add_32_nv, uint32_t, -1)
304 +__ATOMIC_OP64(atomic_dec_64_nv, atomic_add_64_nv, uint64_t, -1)
305 +__ATOMIC_OPXX(atomic_dec_uchar_nv, atomic_add_char_nv, uchar_t, -1)
306 +__ATOMIC_OPXX(atomic_dec_ushort_nv, atomic_add_short_nv, ushort_t, -1)
307 +__ATOMIC_OPXX(atomic_dec_uint_nv, atomic_add_int_nv, uint_t, -1)
308 +__ATOMIC_OPXX(atomic_dec_ulong_nv, atomic_add_long_nv, ulong_t, -1)
309 +
310 +#undef __ATOMIC_OPXX
311 +
312 +#define __ATOMIC_OPXX(fxn, cas, op, type) \
313 +extern __GNU_INLINE type \
314 +fxn(volatile type *target, type delta) \
315 +{ \
316 + type old; \
317 + do { \
318 + old = *target; \
319 + } while (cas(target, old, old op delta) != old); \
320 + return (old op delta); \
321 +}
322 +
323 +__ATOMIC_OPXX(atomic_or_8_nv, atomic_cas_8, |, uint8_t)
324 +__ATOMIC_OPXX(atomic_or_16_nv, atomic_cas_16, |, uint16_t)
325 +__ATOMIC_OPXX(atomic_or_32_nv, atomic_cas_32, |, uint32_t)
326 +__ATOMIC_OP64(atomic_or_64_nv, atomic_cas_64, |, uint64_t)
327 +__ATOMIC_OPXX(atomic_or_uchar_nv, atomic_cas_uchar, |, uchar_t)
328 +__ATOMIC_OPXX(atomic_or_ushort_nv, atomic_cas_ushort, |, ushort_t)
329 +__ATOMIC_OPXX(atomic_or_uint_nv, atomic_cas_uint, |, uint_t)
330 +__ATOMIC_OPXX(atomic_or_ulong_nv, atomic_cas_ulong, |, ulong_t)
331 +
332 +__ATOMIC_OPXX(atomic_and_8_nv, atomic_cas_8, &, uint8_t)
333 +__ATOMIC_OPXX(atomic_and_16_nv, atomic_cas_16, &, uint16_t)
334 +__ATOMIC_OPXX(atomic_and_32_nv, atomic_cas_32, &, uint32_t)
335 +__ATOMIC_OP64(atomic_and_64_nv, atomic_cas_64, &, uint64_t)
336 +__ATOMIC_OPXX(atomic_and_uchar_nv, atomic_cas_uchar, &, uchar_t)
337 +__ATOMIC_OPXX(atomic_and_ushort_nv, atomic_cas_ushort, &, ushort_t)
338 +__ATOMIC_OPXX(atomic_and_uint_nv, atomic_cas_uint, &, uint_t)
339 +__ATOMIC_OPXX(atomic_and_ulong_nv, atomic_cas_ulong, &, ulong_t)
340 +
341 +#undef __ATOMIC_OPXX
342 +
250 343 #else
251 344 #error "port me"
252 345 #endif
253 346
254 347 #undef SUF_8
255 348 #undef SUF_16
256 349 #undef SUF_32
257 350 #undef SUF_64
258 351 #undef SUF_LONG
259 352 #undef SUF_PTR
260 353
261 354 #undef __ATOMIC_OP64
262 355
263 356 /* END CSTYLED */
264 357
265 358 #endif /* !__lint && __GNUC__ */
266 359
267 360 #ifdef __cplusplus
268 361 }
269 362 #endif
270 363
271 364 #endif /* _ASM_ATOMIC_H */
|
↓ open down ↓ |
12 lines elided |
↑ open up ↑ |
XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX