1 /*
   2  * CDDL HEADER START
   3  *
   4  * The contents of this file are subject to the terms of the
   5  * Common Development and Distribution License (the "License").
   6  * You may not use this file except in compliance with the License.
   7  *
   8  * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
   9  * or http://www.opensolaris.org/os/licensing.
  10  * See the License for the specific language governing permissions
  11  * and limitations under the License.
  12  *
  13  * When distributing Covered Code, include this CDDL HEADER in each
  14  * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
  15  * If applicable, add the following below this CDDL HEADER, with the
  16  * fields enclosed by brackets "[]" replaced with your own identifying
  17  * information: Portions Copyright [yyyy] [name of copyright owner]
  18  *
  19  * CDDL HEADER END
  20  */
  21 
  22 /*
  23  * Copyright 2008 Sun Microsystems, Inc.  All rights reserved.
  24  * Use is subject to license terms.
  25  */
  26 
  27         .file   "atomic.s"
  28 
  29 #include <sys/asm_linkage.h>
  30 
  31 #if defined(_KERNEL)
  32         /*
  33          * Legacy kernel interfaces; they will go away the moment our closed
  34          * bins no longer require them.
  35          */
  36         ANSI_PRAGMA_WEAK2(cas8,atomic_cas_8,function)
  37         ANSI_PRAGMA_WEAK2(cas32,atomic_cas_32,function)
  38         ANSI_PRAGMA_WEAK2(cas64,atomic_cas_64,function)
  39         ANSI_PRAGMA_WEAK2(caslong,atomic_cas_ulong,function)
  40         ANSI_PRAGMA_WEAK2(casptr,atomic_cas_ptr,function)
  41         ANSI_PRAGMA_WEAK2(atomic_and_long,atomic_and_ulong,function)
  42         ANSI_PRAGMA_WEAK2(atomic_or_long,atomic_or_ulong,function)
  43         ANSI_PRAGMA_WEAK2(swapl,atomic_swap_32,function)
  44 #endif
  45 
  46         /*
  47          * NOTE: If atomic_inc_8 and atomic_inc_8_nv are ever
  48          * separated, you need to also edit the libc sparc platform
  49          * specific mapfile and remove the NODYNSORT attribute
  50          * from atomic_inc_8_nv.
  51          */
  52         ENTRY(atomic_inc_8)
  53         ALTENTRY(atomic_inc_8_nv)
  54         ALTENTRY(atomic_inc_uchar)
  55         ALTENTRY(atomic_inc_uchar_nv)
  56         ba      add_8
  57           add   %g0, 1, %o1
  58         SET_SIZE(atomic_inc_uchar_nv)
  59         SET_SIZE(atomic_inc_uchar)
  60         SET_SIZE(atomic_inc_8_nv)
  61         SET_SIZE(atomic_inc_8)
  62 
  63         /*
  64          * NOTE: If atomic_dec_8 and atomic_dec_8_nv are ever
  65          * separated, you need to also edit the libc sparc platform
  66          * specific mapfile and remove the NODYNSORT attribute
  67          * from atomic_dec_8_nv.
  68          */
  69         ENTRY(atomic_dec_8)
  70         ALTENTRY(atomic_dec_8_nv)
  71         ALTENTRY(atomic_dec_uchar)
  72         ALTENTRY(atomic_dec_uchar_nv)
  73         ba      add_8
  74           sub   %g0, 1, %o1
  75         SET_SIZE(atomic_dec_uchar_nv)
  76         SET_SIZE(atomic_dec_uchar)
  77         SET_SIZE(atomic_dec_8_nv)
  78         SET_SIZE(atomic_dec_8)
  79 
  80         /*
  81          * NOTE: If atomic_add_8 and atomic_add_8_nv are ever
  82          * separated, you need to also edit the libc sparc platform
  83          * specific mapfile and remove the NODYNSORT attribute
  84          * from atomic_add_8_nv.
  85          */
  86         ENTRY(atomic_add_8)
  87         ALTENTRY(atomic_add_8_nv)
  88         ALTENTRY(atomic_add_char)
  89         ALTENTRY(atomic_add_char_nv)
  90 add_8:
  91         and     %o0, 0x3, %o4           ! %o4 = byte offset, left-to-right
  92         xor     %o4, 0x3, %g1           ! %g1 = byte offset, right-to-left
  93         sll     %g1, 3, %g1             ! %g1 = bit offset, right-to-left
  94         set     0xff, %o3               ! %o3 = mask
  95         sll     %o3, %g1, %o3           ! %o3 = shifted to bit offset
  96         sll     %o1, %g1, %o1           ! %o1 = shifted to bit offset
  97         and     %o1, %o3, %o1           ! %o1 = single byte value
  98         andn    %o0, 0x3, %o0           ! %o0 = word address
  99         ld      [%o0], %o2              ! read old value
 100 1:
 101         add     %o2, %o1, %o5           ! add value to the old value
 102         and     %o5, %o3, %o5           ! clear other bits
 103         andn    %o2, %o3, %o4           ! clear target bits
 104         or      %o4, %o5, %o5           ! insert the new value
 105         cas     [%o0], %o2, %o5
 106         cmp     %o2, %o5
 107         bne,a,pn %icc, 1b
 108           mov   %o5, %o2                ! %o2 = old value
 109         add     %o2, %o1, %o5
 110         and     %o5, %o3, %o5
 111         retl
 112         srl     %o5, %g1, %o0           ! %o0 = new value
 113         SET_SIZE(atomic_add_char_nv)
 114         SET_SIZE(atomic_add_char)
 115         SET_SIZE(atomic_add_8_nv)
 116         SET_SIZE(atomic_add_8)
 117 
 118         /*
 119          * NOTE: If atomic_inc_16 and atomic_inc_16_nv are ever
 120          * separated, you need to also edit the libc sparc platform
 121          * specific mapfile and remove the NODYNSORT attribute
 122          * from atomic_inc_16_nv.
 123          */
 124         ENTRY(atomic_inc_16)
 125         ALTENTRY(atomic_inc_16_nv)
 126         ALTENTRY(atomic_inc_ushort)
 127         ALTENTRY(atomic_inc_ushort_nv)
 128         ba      add_16
 129           add   %g0, 1, %o1
 130         SET_SIZE(atomic_inc_ushort_nv)
 131         SET_SIZE(atomic_inc_ushort)
 132         SET_SIZE(atomic_inc_16_nv)
 133         SET_SIZE(atomic_inc_16)
 134 
 135         /*
 136          * NOTE: If atomic_dec_16 and atomic_dec_16_nv are ever
 137          * separated, you need to also edit the libc sparc platform
 138          * specific mapfile and remove the NODYNSORT attribute
 139          * from atomic_dec_16_nv.
 140          */
 141         ENTRY(atomic_dec_16)
 142         ALTENTRY(atomic_dec_16_nv)
 143         ALTENTRY(atomic_dec_ushort)
 144         ALTENTRY(atomic_dec_ushort_nv)
 145         ba      add_16
 146           sub   %g0, 1, %o1
 147         SET_SIZE(atomic_dec_ushort_nv)
 148         SET_SIZE(atomic_dec_ushort)
 149         SET_SIZE(atomic_dec_16_nv)
 150         SET_SIZE(atomic_dec_16)
 151 
 152         /*
 153          * NOTE: If atomic_add_16 and atomic_add_16_nv are ever
 154          * separated, you need to also edit the libc sparc platform
 155          * specific mapfile and remove the NODYNSORT attribute
 156          * from atomic_add_16_nv.
 157          */
 158         ENTRY(atomic_add_16)
 159         ALTENTRY(atomic_add_16_nv)
 160         ALTENTRY(atomic_add_short)
 161         ALTENTRY(atomic_add_short_nv)
 162 add_16:
 163         and     %o0, 0x2, %o4           ! %o4 = byte offset, left-to-right
 164         xor     %o4, 0x2, %g1           ! %g1 = byte offset, right-to-left
 165         sll     %o4, 3, %o4             ! %o4 = bit offset, left-to-right
 166         sll     %g1, 3, %g1             ! %g1 = bit offset, right-to-left
 167         sethi   %hi(0xffff0000), %o3    ! %o3 = mask
 168         srl     %o3, %o4, %o3           ! %o3 = shifted to bit offset
 169         sll     %o1, %g1, %o1           ! %o1 = shifted to bit offset
 170         and     %o1, %o3, %o1           ! %o1 = single short value
 171         andn    %o0, 0x2, %o0           ! %o0 = word address
 172         ! if low-order bit is 1, we will properly get an alignment fault here
 173         ld      [%o0], %o2              ! read old value
 174 1:
 175         add     %o1, %o2, %o5           ! add value to the old value
 176         and     %o5, %o3, %o5           ! clear other bits
 177         andn    %o2, %o3, %o4           ! clear target bits
 178         or      %o4, %o5, %o5           ! insert the new value
 179         cas     [%o0], %o2, %o5
 180         cmp     %o2, %o5
 181         bne,a,pn %icc, 1b
 182           mov   %o5, %o2                ! %o2 = old value
 183         add     %o1, %o2, %o5
 184         and     %o5, %o3, %o5
 185         retl
 186         srl     %o5, %g1, %o0           ! %o0 = new value
 187         SET_SIZE(atomic_add_short_nv)
 188         SET_SIZE(atomic_add_short)
 189         SET_SIZE(atomic_add_16_nv)
 190         SET_SIZE(atomic_add_16)
 191 
 192         /*
 193          * NOTE: If atomic_inc_32 and atomic_inc_32_nv are ever
 194          * separated, you need to also edit the libc sparc platform
 195          * specific mapfile and remove the NODYNSORT attribute
 196          * from atomic_inc_32_nv.
 197          */
 198         ENTRY(atomic_inc_32)
 199         ALTENTRY(atomic_inc_32_nv)
 200         ALTENTRY(atomic_inc_uint)
 201         ALTENTRY(atomic_inc_uint_nv)
 202         ALTENTRY(atomic_inc_ulong)
 203         ALTENTRY(atomic_inc_ulong_nv)
 204         ba      add_32
 205           add   %g0, 1, %o1
 206         SET_SIZE(atomic_inc_ulong_nv)
 207         SET_SIZE(atomic_inc_ulong)
 208         SET_SIZE(atomic_inc_uint_nv)
 209         SET_SIZE(atomic_inc_uint)
 210         SET_SIZE(atomic_inc_32_nv)
 211         SET_SIZE(atomic_inc_32)
 212 
 213         /*
 214          * NOTE: If atomic_dec_32 and atomic_dec_32_nv are ever
 215          * separated, you need to also edit the libc sparc platform
 216          * specific mapfile and remove the NODYNSORT attribute
 217          * from atomic_dec_32_nv.
 218          */
 219         ENTRY(atomic_dec_32)
 220         ALTENTRY(atomic_dec_32_nv)
 221         ALTENTRY(atomic_dec_uint)
 222         ALTENTRY(atomic_dec_uint_nv)
 223         ALTENTRY(atomic_dec_ulong)
 224         ALTENTRY(atomic_dec_ulong_nv)
 225         ba      add_32
 226           sub   %g0, 1, %o1
 227         SET_SIZE(atomic_dec_ulong_nv)
 228         SET_SIZE(atomic_dec_ulong)
 229         SET_SIZE(atomic_dec_uint_nv)
 230         SET_SIZE(atomic_dec_uint)
 231         SET_SIZE(atomic_dec_32_nv)
 232         SET_SIZE(atomic_dec_32)
 233 
 234         /*
 235          * NOTE: If atomic_add_32 and atomic_add_32_nv are ever
 236          * separated, you need to also edit the libc sparc platform
 237          * specific mapfile and remove the NODYNSORT attribute
 238          * from atomic_add_32_nv.
 239          */
 240         ENTRY(atomic_add_32)
 241         ALTENTRY(atomic_add_32_nv)
 242         ALTENTRY(atomic_add_int)
 243         ALTENTRY(atomic_add_int_nv)
 244         ALTENTRY(atomic_add_ptr)
 245         ALTENTRY(atomic_add_ptr_nv)
 246         ALTENTRY(atomic_add_long)
 247         ALTENTRY(atomic_add_long_nv)
 248 add_32:
 249         ld      [%o0], %o2
 250 1:
 251         add     %o2, %o1, %o3
 252         cas     [%o0], %o2, %o3
 253         cmp     %o2, %o3
 254         bne,a,pn %icc, 1b
 255           mov   %o3, %o2
 256         retl
 257         add     %o2, %o1, %o0           ! return new value
 258         SET_SIZE(atomic_add_long_nv)
 259         SET_SIZE(atomic_add_long)
 260         SET_SIZE(atomic_add_ptr_nv)
 261         SET_SIZE(atomic_add_ptr)
 262         SET_SIZE(atomic_add_int_nv)
 263         SET_SIZE(atomic_add_int)
 264         SET_SIZE(atomic_add_32_nv)
 265         SET_SIZE(atomic_add_32)
 266 
 267         /*
 268          * NOTE: If atomic_inc_64 and atomic_inc_64_nv are ever
 269          * separated, you need to also edit the libc sparc platform
 270          * specific mapfile and remove the NODYNSORT attribute
 271          * from atomic_inc_64_nv.
 272          */
 273         ENTRY(atomic_inc_64)
 274         ALTENTRY(atomic_inc_64_nv)
 275         ba      add_64
 276           add   %g0, 1, %o1
 277         SET_SIZE(atomic_inc_64_nv)
 278         SET_SIZE(atomic_inc_64)
 279 
 280         /*
 281          * NOTE: If atomic_dec_64 and atomic_dec_64_nv are ever
 282          * separated, you need to also edit the libc sparc platform
 283          * specific mapfile and remove the NODYNSORT attribute
 284          * from atomic_dec_64_nv.
 285          */
 286         ENTRY(atomic_dec_64)
 287         ALTENTRY(atomic_dec_64_nv)
 288         ba      add_64
 289           sub   %g0, 1, %o1
 290         SET_SIZE(atomic_dec_64_nv)
 291         SET_SIZE(atomic_dec_64)
 292 
 293         /*
 294          * NOTE: If atomic_add_64 and atomic_add_64_nv are ever
 295          * separated, you need to also edit the libc sparc platform
 296          * specific mapfile and remove the NODYNSORT attribute
 297          * from atomic_add_64_nv.
 298          */
 299         ENTRY(atomic_add_64)
 300         ALTENTRY(atomic_add_64_nv)
 301         sllx    %o1, 32, %o1            ! upper 32 in %o1, lower in %o2
 302         srl     %o2, 0, %o2
 303         add     %o1, %o2, %o1           ! convert 2 32-bit args into 1 64-bit
 304 add_64:
 305         ldx     [%o0], %o2
 306 1:
 307         add     %o2, %o1, %o3
 308         casx    [%o0], %o2, %o3
 309         cmp     %o2, %o3
 310         bne,a,pn %xcc, 1b
 311           mov   %o3, %o2
 312         add     %o2, %o1, %o1           ! return lower 32-bits in %o1
 313         retl
 314         srlx    %o1, 32, %o0            ! return upper 32-bits in %o0
 315         SET_SIZE(atomic_add_64_nv)
 316         SET_SIZE(atomic_add_64)
 317 
 318         /*
 319          * NOTE: If atomic_or_8 and atomic_or_8_nv are ever
 320          * separated, you need to also edit the libc sparc platform
 321          * specific mapfile and remove the NODYNSORT attribute
 322          * from atomic_or_8_nv.
 323          */
 324         ENTRY(atomic_or_8)
 325         ALTENTRY(atomic_or_8_nv)
 326         ALTENTRY(atomic_or_uchar)
 327         ALTENTRY(atomic_or_uchar_nv)
 328         and     %o0, 0x3, %o4           ! %o4 = byte offset, left-to-right
 329         xor     %o4, 0x3, %g1           ! %g1 = byte offset, right-to-left
 330         sll     %g1, 3, %g1             ! %g1 = bit offset, right-to-left
 331         set     0xff, %o3               ! %o3 = mask
 332         sll     %o3, %g1, %o3           ! %o3 = shifted to bit offset
 333         sll     %o1, %g1, %o1           ! %o1 = shifted to bit offset
 334         and     %o1, %o3, %o1           ! %o1 = single byte value
 335         andn    %o0, 0x3, %o0           ! %o0 = word address
 336         ld      [%o0], %o2              ! read old value
 337 1:
 338         or      %o2, %o1, %o5           ! or in the new value
 339         cas     [%o0], %o2, %o5
 340         cmp     %o2, %o5
 341         bne,a,pn %icc, 1b
 342           mov   %o5, %o2                ! %o2 = old value
 343         or      %o2, %o1, %o5
 344         and     %o5, %o3, %o5
 345         retl
 346         srl     %o5, %g1, %o0           ! %o0 = new value
 347         SET_SIZE(atomic_or_uchar_nv)
 348         SET_SIZE(atomic_or_uchar)
 349         SET_SIZE(atomic_or_8_nv)
 350         SET_SIZE(atomic_or_8)
 351 
 352         /*
 353          * NOTE: If atomic_or_16 and atomic_or_16_nv are ever
 354          * separated, you need to also edit the libc sparc platform
 355          * specific mapfile and remove the NODYNSORT attribute
 356          * from atomic_or_16_nv.
 357          */
 358         ENTRY(atomic_or_16)
 359         ALTENTRY(atomic_or_16_nv)
 360         ALTENTRY(atomic_or_ushort)
 361         ALTENTRY(atomic_or_ushort_nv)
 362         and     %o0, 0x2, %o4           ! %o4 = byte offset, left-to-right
 363         xor     %o4, 0x2, %g1           ! %g1 = byte offset, right-to-left
 364         sll     %o4, 3, %o4             ! %o4 = bit offset, left-to-right
 365         sll     %g1, 3, %g1             ! %g1 = bit offset, right-to-left
 366         sethi   %hi(0xffff0000), %o3    ! %o3 = mask
 367         srl     %o3, %o4, %o3           ! %o3 = shifted to bit offset
 368         sll     %o1, %g1, %o1           ! %o1 = shifted to bit offset
 369         and     %o1, %o3, %o1           ! %o1 = single short value
 370         andn    %o0, 0x2, %o0           ! %o0 = word address
 371         ! if low-order bit is 1, we will properly get an alignment fault here
 372         ld      [%o0], %o2              ! read old value
 373 1:
 374         or      %o2, %o1, %o5           ! or in the new value
 375         cas     [%o0], %o2, %o5
 376         cmp     %o2, %o5
 377         bne,a,pn %icc, 1b
 378           mov   %o5, %o2                ! %o2 = old value
 379         or      %o2, %o1, %o5           ! or in the new value
 380         and     %o5, %o3, %o5
 381         retl
 382         srl     %o5, %g1, %o0           ! %o0 = new value
 383         SET_SIZE(atomic_or_ushort_nv)
 384         SET_SIZE(atomic_or_ushort)
 385         SET_SIZE(atomic_or_16_nv)
 386         SET_SIZE(atomic_or_16)
 387 
 388         /*
 389          * NOTE: If atomic_or_32 and atomic_or_32_nv are ever
 390          * separated, you need to also edit the libc sparc platform
 391          * specific mapfile and remove the NODYNSORT attribute
 392          * from atomic_or_32_nv.
 393          */
 394         ENTRY(atomic_or_32)
 395         ALTENTRY(atomic_or_32_nv)
 396         ALTENTRY(atomic_or_uint)
 397         ALTENTRY(atomic_or_uint_nv)
 398         ALTENTRY(atomic_or_ulong)
 399         ALTENTRY(atomic_or_ulong_nv)
 400         ld      [%o0], %o2
 401 1:
 402         or      %o2, %o1, %o3
 403         cas     [%o0], %o2, %o3
 404         cmp     %o2, %o3
 405         bne,a,pn %icc, 1b
 406           mov   %o3, %o2
 407         retl
 408         or      %o2, %o1, %o0           ! return new value
 409         SET_SIZE(atomic_or_ulong_nv)
 410         SET_SIZE(atomic_or_ulong)
 411         SET_SIZE(atomic_or_uint_nv)
 412         SET_SIZE(atomic_or_uint)
 413         SET_SIZE(atomic_or_32_nv)
 414         SET_SIZE(atomic_or_32)
 415 
 416         /*
 417          * NOTE: If atomic_or_64 and atomic_or_64_nv are ever
 418          * separated, you need to also edit the libc sparc platform
 419          * specific mapfile and remove the NODYNSORT attribute
 420          * from atomic_or_64_nv.
 421          */
 422         ENTRY(atomic_or_64)
 423         ALTENTRY(atomic_or_64_nv)
 424         sllx    %o1, 32, %o1            ! upper 32 in %o1, lower in %o2
 425         srl     %o2, 0, %o2
 426         add     %o1, %o2, %o1           ! convert 2 32-bit args into 1 64-bit
 427         ldx     [%o0], %o2
 428 1:
 429         or      %o2, %o1, %o3
 430         casx    [%o0], %o2, %o3
 431         cmp     %o2, %o3
 432         bne,a,pn %xcc, 1b
 433           mov   %o3, %o2
 434         or      %o2, %o1, %o1           ! return lower 32-bits in %o1
 435         retl
 436         srlx    %o1, 32, %o0            ! return upper 32-bits in %o0
 437         SET_SIZE(atomic_or_64_nv)
 438         SET_SIZE(atomic_or_64)
 439 
 440         /*
 441          * NOTE: If atomic_and_8 and atomic_and_8_nv are ever
 442          * separated, you need to also edit the libc sparc platform
 443          * specific mapfile and remove the NODYNSORT attribute
 444          * from atomic_and_8_nv.
 445          */
 446         ENTRY(atomic_and_8)
 447         ALTENTRY(atomic_and_8_nv)
 448         ALTENTRY(atomic_and_uchar)
 449         ALTENTRY(atomic_and_uchar_nv)
 450         and     %o0, 0x3, %o4           ! %o4 = byte offset, left-to-right
 451         xor     %o4, 0x3, %g1           ! %g1 = byte offset, right-to-left
 452         sll     %g1, 3, %g1             ! %g1 = bit offset, right-to-left
 453         set     0xff, %o3               ! %o3 = mask
 454         sll     %o3, %g1, %o3           ! %o3 = shifted to bit offset
 455         sll     %o1, %g1, %o1           ! %o1 = shifted to bit offset
 456         orn     %o1, %o3, %o1           ! all ones in other bytes
 457         andn    %o0, 0x3, %o0           ! %o0 = word address
 458         ld      [%o0], %o2              ! read old value
 459 1:
 460         and     %o2, %o1, %o5           ! and in the new value
 461         cas     [%o0], %o2, %o5
 462         cmp     %o2, %o5
 463         bne,a,pn %icc, 1b
 464           mov   %o5, %o2                ! %o2 = old value
 465         and     %o2, %o1, %o5
 466         and     %o5, %o3, %o5
 467         retl
 468         srl     %o5, %g1, %o0           ! %o0 = new value
 469         SET_SIZE(atomic_and_uchar_nv)
 470         SET_SIZE(atomic_and_uchar)
 471         SET_SIZE(atomic_and_8_nv)
 472         SET_SIZE(atomic_and_8)
 473 
 474         /*
 475          * NOTE: If atomic_and_16 and atomic_and_16_nv are ever
 476          * separated, you need to also edit the libc sparc platform
 477          * specific mapfile and remove the NODYNSORT attribute
 478          * from atomic_and_16_nv.
 479          */
 480         ENTRY(atomic_and_16)
 481         ALTENTRY(atomic_and_16_nv)
 482         ALTENTRY(atomic_and_ushort)
 483         ALTENTRY(atomic_and_ushort_nv)
 484         and     %o0, 0x2, %o4           ! %o4 = byte offset, left-to-right
 485         xor     %o4, 0x2, %g1           ! %g1 = byte offset, right-to-left
 486         sll     %o4, 3, %o4             ! %o4 = bit offset, left-to-right
 487         sll     %g1, 3, %g1             ! %g1 = bit offset, right-to-left
 488         sethi   %hi(0xffff0000), %o3    ! %o3 = mask
 489         srl     %o3, %o4, %o3           ! %o3 = shifted to bit offset
 490         sll     %o1, %g1, %o1           ! %o1 = shifted to bit offset
 491         orn     %o1, %o3, %o1           ! all ones in the other half
 492         andn    %o0, 0x2, %o0           ! %o0 = word address
 493         ! if low-order bit is 1, we will properly get an alignment fault here
 494         ld      [%o0], %o2              ! read old value
 495 1:
 496         and     %o2, %o1, %o5           ! and in the new value
 497         cas     [%o0], %o2, %o5
 498         cmp     %o2, %o5
 499         bne,a,pn %icc, 1b
 500           mov   %o5, %o2                ! %o2 = old value
 501         and     %o2, %o1, %o5
 502         and     %o5, %o3, %o5
 503         retl
 504         srl     %o5, %g1, %o0           ! %o0 = new value
 505         SET_SIZE(atomic_and_ushort_nv)
 506         SET_SIZE(atomic_and_ushort)
 507         SET_SIZE(atomic_and_16_nv)
 508         SET_SIZE(atomic_and_16)
 509 
 510         /*
 511          * NOTE: If atomic_and_32 and atomic_and_32_nv are ever
 512          * separated, you need to also edit the libc sparc platform
 513          * specific mapfile and remove the NODYNSORT attribute
 514          * from atomic_and_32_nv.
 515          */
 516         ENTRY(atomic_and_32)
 517         ALTENTRY(atomic_and_32_nv)
 518         ALTENTRY(atomic_and_uint)
 519         ALTENTRY(atomic_and_uint_nv)
 520         ALTENTRY(atomic_and_ulong)
 521         ALTENTRY(atomic_and_ulong_nv)
 522         ld      [%o0], %o2
 523 1:
 524         and     %o2, %o1, %o3
 525         cas     [%o0], %o2, %o3
 526         cmp     %o2, %o3
 527         bne,a,pn %icc, 1b
 528           mov   %o3, %o2
 529         retl
 530         and     %o2, %o1, %o0           ! return new value
 531         SET_SIZE(atomic_and_ulong_nv)
 532         SET_SIZE(atomic_and_ulong)
 533         SET_SIZE(atomic_and_uint_nv)
 534         SET_SIZE(atomic_and_uint)
 535         SET_SIZE(atomic_and_32_nv)
 536         SET_SIZE(atomic_and_32)
 537 
 538         /*
 539          * NOTE: If atomic_and_64 and atomic_and_64_nv are ever
 540          * separated, you need to also edit the libc sparc platform
 541          * specific mapfile and remove the NODYNSORT attribute
 542          * from atomic_and_64_nv.
 543          */
 544         ENTRY(atomic_and_64)
 545         ALTENTRY(atomic_and_64_nv)
 546         sllx    %o1, 32, %o1            ! upper 32 in %o1, lower in %o2
 547         srl     %o2, 0, %o2
 548         add     %o1, %o2, %o1           ! convert 2 32-bit args into 1 64-bit
 549         ldx     [%o0], %o2
 550 1:
 551         and     %o2, %o1, %o3
 552         casx    [%o0], %o2, %o3
 553         cmp     %o2, %o3
 554         bne,a,pn %xcc, 1b
 555           mov   %o3, %o2
 556         and     %o2, %o1, %o1           ! return lower 32-bits in %o1
 557         retl
 558         srlx    %o1, 32, %o0            ! return upper 32-bits in %o0
 559         SET_SIZE(atomic_and_64_nv)
 560         SET_SIZE(atomic_and_64)
 561 
 562         ENTRY(atomic_cas_8)
 563         ALTENTRY(atomic_cas_uchar)
 564         and     %o0, 0x3, %o4           ! %o4 = byte offset, left-to-right
 565         xor     %o4, 0x3, %g1           ! %g1 = byte offset, right-to-left
 566         sll     %g1, 3, %g1             ! %g1 = bit offset, right-to-left
 567         set     0xff, %o3               ! %o3 = mask
 568         sll     %o3, %g1, %o3           ! %o3 = shifted to bit offset
 569         sll     %o1, %g1, %o1           ! %o1 = shifted to bit offset
 570         and     %o1, %o3, %o1           ! %o1 = single byte value
 571         sll     %o2, %g1, %o2           ! %o2 = shifted to bit offset
 572         and     %o2, %o3, %o2           ! %o2 = single byte value
 573         andn    %o0, 0x3, %o0           ! %o0 = word address
 574         ld      [%o0], %o4              ! read old value
 575 1:
 576         andn    %o4, %o3, %o4           ! clear target bits
 577         or      %o4, %o2, %o5           ! insert the new value
 578         or      %o4, %o1, %o4           ! insert the comparison value
 579         cas     [%o0], %o4, %o5
 580         cmp     %o4, %o5                ! did we succeed?
 581         be,pt   %icc, 2f
 582           and   %o5, %o3, %o4           ! isolate the old value
 583         cmp     %o1, %o4                ! should we have succeeded?
 584         be,a,pt %icc, 1b                ! yes, try again
 585           mov   %o5, %o4                ! %o4 = old value
 586 2:
 587         retl
 588         srl     %o4, %g1, %o0           ! %o0 = old value
 589         SET_SIZE(atomic_cas_uchar)
 590         SET_SIZE(atomic_cas_8)
 591 
 592         ENTRY(atomic_cas_16)
 593         ALTENTRY(atomic_cas_ushort)
 594         and     %o0, 0x2, %o4           ! %o4 = byte offset, left-to-right
 595         xor     %o4, 0x2, %g1           ! %g1 = byte offset, right-to-left
 596         sll     %o4, 3, %o4             ! %o4 = bit offset, left-to-right
 597         sll     %g1, 3, %g1             ! %g1 = bit offset, right-to-left
 598         sethi   %hi(0xffff0000), %o3    ! %o3 = mask
 599         srl     %o3, %o4, %o3           ! %o3 = shifted to bit offset
 600         sll     %o1, %g1, %o1           ! %o1 = shifted to bit offset
 601         and     %o1, %o3, %o1           ! %o1 = single short value
 602         sll     %o2, %g1, %o2           ! %o2 = shifted to bit offset
 603         and     %o2, %o3, %o2           ! %o2 = single short value
 604         andn    %o0, 0x2, %o0           ! %o0 = word address
 605         ! if low-order bit is 1, we will properly get an alignment fault here
 606         ld      [%o0], %o4              ! read old value
 607 1:
 608         andn    %o4, %o3, %o4           ! clear target bits
 609         or      %o4, %o2, %o5           ! insert the new value
 610         or      %o4, %o1, %o4           ! insert the comparison value
 611         cas     [%o0], %o4, %o5
 612         cmp     %o4, %o5                ! did we succeed?
 613         be,pt   %icc, 2f
 614           and   %o5, %o3, %o4           ! isolate the old value
 615         cmp     %o1, %o4                ! should we have succeeded?
 616         be,a,pt %icc, 1b                ! yes, try again
 617           mov   %o5, %o4                ! %o4 = old value
 618 2:
 619         retl
 620         srl     %o4, %g1, %o0           ! %o0 = old value
 621         SET_SIZE(atomic_cas_ushort)
 622         SET_SIZE(atomic_cas_16)
 623 
 624         ENTRY(atomic_cas_32)
 625         ALTENTRY(atomic_cas_uint)
 626         ALTENTRY(atomic_cas_ptr)
 627         ALTENTRY(atomic_cas_ulong)
 628         cas     [%o0], %o1, %o2
 629         retl
 630         mov     %o2, %o0
 631         SET_SIZE(atomic_cas_ulong)
 632         SET_SIZE(atomic_cas_ptr)
 633         SET_SIZE(atomic_cas_uint)
 634         SET_SIZE(atomic_cas_32)
 635 
 636         ENTRY(atomic_cas_64)
 637         sllx    %o1, 32, %o1            ! cmp's upper 32 in %o1, lower in %o2
 638         srl     %o2, 0, %o2             ! convert 2 32-bit args into 1 64-bit
 639         add     %o1, %o2, %o1
 640         sllx    %o3, 32, %o2            ! newval upper 32 in %o3, lower in %o4
 641         srl     %o4, 0, %o4             ! setup %o2 to have newval
 642         add     %o2, %o4, %o2
 643         casx    [%o0], %o1, %o2
 644         srl     %o2, 0, %o1             ! return lower 32-bits in %o1
 645         retl
 646         srlx    %o2, 32, %o0            ! return upper 32-bits in %o0
 647         SET_SIZE(atomic_cas_64)
 648 
 649         ENTRY(atomic_swap_8)
 650         ALTENTRY(atomic_swap_uchar)
 651         and     %o0, 0x3, %o4           ! %o4 = byte offset, left-to-right
 652         xor     %o4, 0x3, %g1           ! %g1 = byte offset, right-to-left
 653         sll     %g1, 3, %g1             ! %g1 = bit offset, right-to-left
 654         set     0xff, %o3               ! %o3 = mask
 655         sll     %o3, %g1, %o3           ! %o3 = shifted to bit offset
 656         sll     %o1, %g1, %o1           ! %o1 = shifted to bit offset
 657         and     %o1, %o3, %o1           ! %o1 = single byte value
 658         andn    %o0, 0x3, %o0           ! %o0 = word address
 659         ld      [%o0], %o2              ! read old value
 660 1:
 661         andn    %o2, %o3, %o5           ! clear target bits
 662         or      %o5, %o1, %o5           ! insert the new value
 663         cas     [%o0], %o2, %o5
 664         cmp     %o2, %o5
 665         bne,a,pn %icc, 1b
 666           mov   %o5, %o2                ! %o2 = old value
 667         and     %o5, %o3, %o5
 668         retl
 669         srl     %o5, %g1, %o0           ! %o0 = old value
 670         SET_SIZE(atomic_swap_uchar)
 671         SET_SIZE(atomic_swap_8)
 672 
 673         ENTRY(atomic_swap_16)
 674         ALTENTRY(atomic_swap_ushort)
 675         and     %o0, 0x2, %o4           ! %o4 = byte offset, left-to-right
 676         xor     %o4, 0x2, %g1           ! %g1 = byte offset, right-to-left
 677         sll     %o4, 3, %o4             ! %o4 = bit offset, left-to-right
 678         sll     %g1, 3, %g1             ! %g1 = bit offset, right-to-left
 679         sethi   %hi(0xffff0000), %o3    ! %o3 = mask
 680         srl     %o3, %o4, %o3           ! %o3 = shifted to bit offset
 681         sll     %o1, %g1, %o1           ! %o1 = shifted to bit offset
 682         and     %o1, %o3, %o1           ! %o1 = single short value
 683         andn    %o0, 0x2, %o0           ! %o0 = word address
 684         ! if low-order bit is 1, we will properly get an alignment fault here
 685         ld      [%o0], %o2              ! read old value
 686 1:
 687         andn    %o2, %o3, %o5           ! clear target bits
 688         or      %o5, %o1, %o5           ! insert the new value
 689         cas     [%o0], %o2, %o5
 690         cmp     %o2, %o5
 691         bne,a,pn %icc, 1b
 692           mov   %o5, %o2                ! %o2 = old value
 693         and     %o5, %o3, %o5
 694         retl
 695         srl     %o5, %g1, %o0           ! %o0 = old value
 696         SET_SIZE(atomic_swap_ushort)
 697         SET_SIZE(atomic_swap_16)
 698 
 699         ENTRY(atomic_swap_32)
 700         ALTENTRY(atomic_swap_uint)
 701         ALTENTRY(atomic_swap_ptr)
 702         ALTENTRY(atomic_swap_ulong)
 703         ld      [%o0], %o2
 704 1:
 705         mov     %o1, %o3
 706         cas     [%o0], %o2, %o3
 707         cmp     %o2, %o3
 708         bne,a,pn %icc, 1b
 709           mov   %o3, %o2
 710         retl
 711         mov     %o3, %o0
 712         SET_SIZE(atomic_swap_ulong)
 713         SET_SIZE(atomic_swap_ptr)
 714         SET_SIZE(atomic_swap_uint)
 715         SET_SIZE(atomic_swap_32)
 716 
 717         ENTRY(atomic_swap_64)
 718         sllx    %o1, 32, %o1            ! upper 32 in %o1, lower in %o2
 719         srl     %o2, 0, %o2
 720         add     %o1, %o2, %o1           ! convert 2 32-bit args into 1 64-bit
 721         ldx     [%o0], %o2
 722 1:
 723         mov     %o1, %o3
 724         casx    [%o0], %o2, %o3
 725         cmp     %o2, %o3
 726         bne,a,pn %xcc, 1b
 727           mov   %o3, %o2
 728         srl     %o3, 0, %o1             ! return lower 32-bits in %o1
 729         retl
 730         srlx    %o3, 32, %o0            ! return upper 32-bits in %o0
 731         SET_SIZE(atomic_swap_64)
 732 
 733         ENTRY(atomic_set_long_excl)
 734         mov     1, %o3
 735         slln    %o3, %o1, %o3
 736         ldn     [%o0], %o2
 737 1:
 738         andcc   %o2, %o3, %g0           ! test if the bit is set
 739         bnz,a,pn %ncc, 2f               ! if so, then fail out
 740           mov   -1, %o0
 741         or      %o2, %o3, %o4           ! set the bit, and try to commit it
 742         casn    [%o0], %o2, %o4
 743         cmp     %o2, %o4
 744         bne,a,pn %ncc, 1b               ! failed to commit, try again
 745           mov   %o4, %o2
 746         mov     %g0, %o0
 747 2:
 748         retl
 749         nop
 750         SET_SIZE(atomic_set_long_excl)
 751 
 752         ENTRY(atomic_clear_long_excl)
 753         mov     1, %o3
 754         slln    %o3, %o1, %o3
 755         ldn     [%o0], %o2
 756 1:
 757         andncc  %o3, %o2, %g0           ! test if the bit is clear
 758         bnz,a,pn %ncc, 2f               ! if so, then fail out
 759           mov   -1, %o0
 760         andn    %o2, %o3, %o4           ! clear the bit, and try to commit it
 761         casn    [%o0], %o2, %o4
 762         cmp     %o2, %o4
 763         bne,a,pn %ncc, 1b               ! failed to commit, try again
 764           mov   %o4, %o2
 765         mov     %g0, %o0
 766 2:
 767         retl
 768         nop
 769         SET_SIZE(atomic_clear_long_excl)
 770 
 771 #if !defined(_KERNEL)
 772 
 773         /*
 774          * Spitfires and Blackbirds have a problem with membars in the
 775          * delay slot (SF_ERRATA_51).  For safety's sake, we assume
 776          * that the whole world needs the workaround.
 777          */
 778         ENTRY(membar_enter)
 779         membar  #StoreLoad|#StoreStore
 780         retl
 781         nop
 782         SET_SIZE(membar_enter)
 783 
 784         ENTRY(membar_exit)
 785         membar  #LoadStore|#StoreStore
 786         retl
 787         nop
 788         SET_SIZE(membar_exit)
 789 
 790         ENTRY(membar_producer)
 791         membar  #StoreStore
 792         retl
 793         nop
 794         SET_SIZE(membar_producer)
 795 
 796         ENTRY(membar_consumer)
 797         membar  #LoadLoad
 798         retl
 799         nop
 800         SET_SIZE(membar_consumer)
 801 
 802 #endif  /* !_KERNEL */