osfmk/i386/i386_lock.s

   1 /*
   2  * Copyright (c) 2000-2008 Apple Inc. All rights reserved.
   3  *
   4  * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
   5  *
   6  * This file contains Original Code and/or Modifications of Original Code
   7  * as defined in and that are subject to the Apple Public Source License
   8  * Version 2.0 (the 'License'). You may not use this file except in
   9  * compliance with the License. The rights granted to you under the License
  10  * may not be used to create, or enable the creation or redistribution of,
  11  * unlawful or unlicensed copies of an Apple operating system, or to
  12  * circumvent, violate, or enable the circumvention or violation of, any
  13  * terms of an Apple operating system software license agreement.
  14  *
  15  * Please obtain a copy of the License at
  16  * http://www.opensource.apple.com/apsl/ and read it before using this file.
  17  *
  18  * The Original Code and all software distributed under the License are
  19  * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  20  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  21  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
  22  * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
  23  * Please see the License for the specific language governing rights and
  24  * limitations under the License.
  25  *
  26  * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  27  */
  28 /*
  29  * @OSF_COPYRIGHT@
  30  */
  31 /*
  32  * Mach Operating System
  33  * Copyright (c) 1989 Carnegie-Mellon University
  34  * All rights reserved.  The CMU software License Agreement specifies
  35  * the terms and conditions for use and redistribution.
  36  */
  37
  38 #include <mach_rt.h>
  39 #include <platforms.h>
  40 #include <mach_ldebug.h>
  41 #include <i386/asm.h>
  42 #include <i386/eflags.h>
  43 #include <i386/trap.h>
  44 #include <config_dtrace.h>
  45 #include <i386/mp.h>
  46
  47 #include "assym.s"
  48
  49 #define PAUSE           rep; nop
  50
  51
  52 #define PUSHF pushf
  53 #define POPF  popf
  54 #define CLI   cli
  55
  56
  57 /*
  58  *      When performance isn't the only concern, it's
  59  *      nice to build stack frames...
  60  */
  61 #define BUILD_STACK_FRAMES   (GPROF || \
  62                                 ((MACH_LDEBUG) && MACH_KDB))
  63
  64 #if     BUILD_STACK_FRAMES
  65
  66 /* Stack-frame-relative: */
  67 #define L_PC            B_PC
  68 #define L_ARG0          B_ARG0
  69 #define L_ARG1          B_ARG1
  70
  71 #define LEAF_ENTRY(name)        \
  72         Entry(name);            \
  73         FRAME;                  \
  74         MCOUNT
  75
  76 #define LEAF_ENTRY2(n1,n2)      \
  77         Entry(n1);              \
  78         Entry(n2);              \
  79         FRAME;                  \
  80         MCOUNT
  81
  82 #define LEAF_RET                \
  83         EMARF;                  \
  84         ret
  85
  86 #else   /* BUILD_STACK_FRAMES */
  87
  88 /* Stack-pointer-relative: */
  89 #define L_PC            S_PC
  90 #define L_ARG0          S_ARG0
  91 #define L_ARG1          S_ARG1
  92
  93 #define LEAF_ENTRY(name)        \
  94         Entry(name)
  95
  96 #define LEAF_ENTRY2(n1,n2)      \
  97         Entry(n1);              \
  98         Entry(n2)
  99
 100 #define LEAF_RET                \
 101         ret
 102
 103 #endif  /* BUILD_STACK_FRAMES */
 104
 105
 106 /* Non-leaf routines always have a stack frame: */
 107
 108 #define NONLEAF_ENTRY(name)     \
 109         Entry(name);            \
 110         FRAME;                  \
 111         MCOUNT
 112
 113 #define NONLEAF_ENTRY2(n1,n2)   \
 114         Entry(n1);              \
 115         Entry(n2);              \
 116         FRAME;                  \
 117         MCOUNT
 118
 119 #define NONLEAF_RET             \
 120         EMARF;                  \
 121         ret
 122
 123
 124 /* For x86_64, the varargs ABI requires that %al indicate
 125  * how many SSE register contain arguments. In our case, 0 */
 126 #if __i386__
 127 #define LOAD_STRING_ARG0(label) pushl $##label ;
 128 #define LOAD_ARG1(x)            pushl x ;
 129 #define CALL_PANIC()            call EXT(panic) ;
 130 #else
 131 #define LOAD_STRING_ARG0(label) leaq label(%rip), %rdi ;
 132 #define LOAD_ARG1(x)            movq x, %rsi ;
 133 #define CALL_PANIC()            xorb %al,%al ; call EXT(panic) ;
 134 #endif
 135
 136 #define CHECK_UNLOCK(current, owner)                            \
 137         cmp     current, owner                          ;       \
 138         je      1f                                      ;       \
 139         LOAD_STRING_ARG0(2f)                            ;       \
 140         CALL_PANIC()                                    ;       \
 141         hlt                                             ;       \
 142         .data                                           ;       \
 143 2:      String  "Mutex unlock attempted from non-owner thread"; \
 144         .text                                           ;       \
 145 1:
 146
 147 #if     MACH_LDEBUG
 148 /*
 149  *  Routines for general lock debugging.
 150  */
 151
 152 /*
 153  * Checks for expected lock types and calls "panic" on
 154  * mismatch.  Detects calls to Mutex functions with
 155  * type simplelock and vice versa.
 156  */
 157 #define CHECK_MUTEX_TYPE()                                      \
 158         cmpl    $ MUTEX_TAG,M_TYPE                      ;       \
 159         je      1f                                      ;       \
 160         LOAD_STRING_ARG0(2f)                            ;       \
 161         CALL_PANIC()                                    ;       \
 162         hlt                                             ;       \
 163         .data                                           ;       \
 164 2:      String  "not a mutex!"                          ;       \
 165         .text                                           ;       \
 166 1:
 167
 168 /*
 169  * If one or more simplelocks are currently held by a thread,
 170  * an attempt to acquire a mutex will cause this check to fail
 171  * (since a mutex lock may context switch, holding a simplelock
 172  * is not a good thing).
 173  */
 174 #if     MACH_RT
 175 #define CHECK_PREEMPTION_LEVEL()                                \
 176         cmpl    $0,%gs:CPU_HIBERNATE                    ;       \
 177         jne     1f                                      ;       \
 178         cmpl    $0,%gs:CPU_PREEMPTION_LEVEL             ;       \
 179         je      1f                                      ;       \
 180         LOAD_ARG1(%gs:CPU_PREEMPTION_LEVEL)             ;       \
 181         LOAD_STRING_ARG0(2f)                            ;       \
 182         CALL_PANIC()                                    ;       \
 183         hlt                                             ;       \
 184         .data                                           ;       \
 185 2:      String  "preemption_level(%d) != 0!"            ;       \
 186         .text                                           ;       \
 187 1:
 188 #else   /* MACH_RT */
 189 #define CHECK_PREEMPTION_LEVEL()
 190 #endif  /* MACH_RT */
 191
 192 #define CHECK_MYLOCK(current, owner)                            \
 193         cmp     current, owner                          ;       \
 194         jne     1f                                      ;       \
 195         LOAD_STRING_ARG0(2f)                            ;       \
 196         CALL_PANIC()                                    ;       \
 197         hlt                                             ;       \
 198         .data                                           ;       \
 199 2:      String  "Attempt to recursively lock a non-recursive lock";     \
 200         .text                                           ;       \
 201 1:
 202
 203 #else   /* MACH_LDEBUG */
 204 #define CHECK_MUTEX_TYPE()
 205 #define CHECK_PREEMPTION_LEVEL()
 206 #define CHECK_MYLOCK(thd)
 207 #endif  /* MACH_LDEBUG */
 208
 209
 210 #define PREEMPTION_DISABLE                              \
 211         incl    %gs:CPU_PREEMPTION_LEVEL
 212
 213
 214 #define PREEMPTION_ENABLE                               \
 215         decl    %gs:CPU_PREEMPTION_LEVEL        ;       \
 216         jne     9f                              ;       \
 217         PUSHF                                   ;       \
 218         testl   $ EFL_IF,S_PC                   ;       \
 219         je      8f                              ;       \
 220         CLI                                     ;       \
 221         movl    %gs:CPU_PENDING_AST,%eax        ;       \
 222         testl   $ AST_URGENT,%eax               ;       \
 223         je      8f                              ;       \
 224         movl    %gs:CPU_INTERRUPT_LEVEL,%eax    ;       \
 225         testl   %eax,%eax                       ;       \
 226         jne     8f                              ;       \
 227         POPF                                    ;       \
 228         int     $(T_PREEMPT)                    ;       \
 229         jmp     9f                              ;       \
 230 8:                                                      \
 231         POPF                                    ;       \
 232 9:
 233
 234
 235
 236 #if     CONFIG_DTRACE
 237
 238        .globl  _lockstat_probe
 239        .globl  _lockstat_probemap
 240
 241 /*
 242  * LOCKSTAT_LABEL creates a dtrace symbol which contains
 243  * a pointer into the lock code function body. At that
 244  * point is a "ret" instruction that can be patched into
 245  * a "nop"
 246  */
 247
 248 #if defined(__i386__)
 249
 250 #define LOCKSTAT_LABEL(lab) \
 251         .data                           ;\
 252         .globl  lab                     ;\
 253         lab:                            ;\
 254         .long 9f                        ;\
 255         .text                           ;\
 256         9:
 257
 258 #define LOCKSTAT_RECORD(id, lck) \
 259         push    %ebp                                    ;       \
 260         mov     %esp,%ebp                               ;       \
 261         sub     $0x38,%esp      /* size of dtrace_probe args */ ; \
 262         movl    _lockstat_probemap + (id * 4),%eax      ;       \
 263         test    %eax,%eax                               ;       \
 264         je      9f                                      ;       \
 265         movl    $0,36(%esp)                             ;       \
 266         movl    $0,40(%esp)                             ;       \
 267         movl    $0,28(%esp)                             ;       \
 268         movl    $0,32(%esp)                             ;       \
 269         movl    $0,20(%esp)                             ;       \
 270         movl    $0,24(%esp)                             ;       \
 271         movl    $0,12(%esp)                             ;       \
 272         movl    $0,16(%esp)                             ;       \
 273         movl    lck,4(%esp)     /* copy lock pointer to arg 1 */ ; \
 274         movl    $0,8(%esp)                              ;       \
 275         movl    %eax,(%esp)                             ;       \
 276         call    *_lockstat_probe                        ;       \
 277 9:      leave
 278         /* ret - left to subsequent code, e.g. return values */
 279
 280 #elif defined(__x86_64__)
 281 #define        LOCKSTAT_LABEL(lab) \
 282        .data                                       ;\
 283        .globl  lab                                 ;\
 284        lab:                                        ;\
 285        .quad 9f                                    ;\
 286        .text                                       ;\
 287        9:
 288
 289 #define LOCKSTAT_RECORD(id, lck) \
 290        push    %rbp                                ;       \
 291        mov     %rsp,%rbp                           ;       \
 292        movl    _lockstat_probemap + (id * 4)(%rip),%eax ;  \
 293        test    %eax,%eax                           ;       \
 294        je              9f                          ;       \
 295        mov             lck, %rsi                   ;       \
 296        mov             %rax, %rdi                  ;       \
 297        mov             $0, %rdx                    ;       \
 298        mov             $0, %rcx                    ;       \
 299        mov             $0, %r8                     ;       \
 300        mov             $0, %r9                     ;       \
 301        call    *_lockstat_probe(%rip)              ;       \
 302 9:      leave
 303         /* ret - left to subsequent code, e.g. return values */
 304 #else
 305 #error Unsupported architecture
 306 #endif
 307 #endif /* CONFIG_DTRACE */
 308
 309 /*
 310  * For most routines, the hw_lock_t pointer is loaded into a
 311  * register initially, and then either a byte or register-sized
 312  * word is loaded/stored to the pointer
 313  */
 314
 315 #if defined(__i386__)
 316 #define HW_LOCK_REGISTER        %edx
 317 #define LOAD_HW_LOCK_REGISTER mov L_ARG0, HW_LOCK_REGISTER
 318 #define HW_LOCK_THREAD_REGISTER %ecx
 319 #define LOAD_HW_LOCK_THREAD_REGISTER mov %gs:CPU_ACTIVE_THREAD, HW_LOCK_THREAD_REGISTER
 320 #define HW_LOCK_MOV_WORD        movl
 321 #define HW_LOCK_EXAM_REGISTER   %eax
 322 #elif defined(__x86_64__)
 323 #define HW_LOCK_REGISTER        %rdi
 324 #define LOAD_HW_LOCK_REGISTER
 325 #define HW_LOCK_THREAD_REGISTER %rcx
 326 #define LOAD_HW_LOCK_THREAD_REGISTER mov %gs:CPU_ACTIVE_THREAD, HW_LOCK_THREAD_REGISTER
 327 #define HW_LOCK_MOV_WORD        movq
 328 #define HW_LOCK_EXAM_REGISTER   %rax
 329 #else
 330 #error Unsupported architecture
 331 #endif
 332
 333 /*
 334  *      void hw_lock_init(hw_lock_t)
 335  *
 336  *      Initialize a hardware lock.
 337  */
 338 LEAF_ENTRY(hw_lock_init)
 339         LOAD_HW_LOCK_REGISTER           /* fetch lock pointer */
 340         HW_LOCK_MOV_WORD $0, (HW_LOCK_REGISTER)         /* clear the lock */
 341         LEAF_RET
 342
 343
 344 /*
 345  *      void hw_lock_byte_init(uint8_t *)
 346  *
 347  *      Initialize a hardware byte lock.
 348  */
 349 LEAF_ENTRY(hw_lock_byte_init)
 350         LOAD_HW_LOCK_REGISTER           /* fetch lock pointer */
 351         movb $0, (HW_LOCK_REGISTER)             /* clear the lock */
 352         LEAF_RET
 353
 354 /*
 355  *      void hw_lock_lock(hw_lock_t)
 356  *
 357  *      Acquire lock, spinning until it becomes available.
 358  *      MACH_RT:  also return with preemption disabled.
 359  */
 360 LEAF_ENTRY(hw_lock_lock)
 361         LOAD_HW_LOCK_REGISTER           /* fetch lock pointer */
 362         LOAD_HW_LOCK_THREAD_REGISTER    /* get thread pointer */
 363
 364         PREEMPTION_DISABLE
 365 1:
 366         mov     (HW_LOCK_REGISTER), HW_LOCK_EXAM_REGISTER
 367         test    HW_LOCK_EXAM_REGISTER,HW_LOCK_EXAM_REGISTER             /* lock locked? */
 368         jne     3f                      /* branch if so */
 369         lock; cmpxchg   HW_LOCK_THREAD_REGISTER,(HW_LOCK_REGISTER)      /* try to acquire the HW lock */
 370         jne     3f
 371         movl    $1,%eax                 /* In case this was a timeout call */
 372         LEAF_RET                        /* if yes, then nothing left to do */
 373 3:
 374         PAUSE                           /* pause for hyper-threading */
 375         jmp     1b                      /* try again */
 376
 377 /*
 378  *      void    hw_lock_byte_lock(uint8_t *lock_byte)
 379  *
 380  *      Acquire byte sized lock operand, spinning until it becomes available.
 381  *      MACH_RT:  also return with preemption disabled.
 382  */
 383
 384 LEAF_ENTRY(hw_lock_byte_lock)
 385         LOAD_HW_LOCK_REGISTER           /* Load lock pointer */
 386         PREEMPTION_DISABLE
 387         movl    $1, %ecx                /* Set lock value */
 388 1:
 389         movb    (HW_LOCK_REGISTER), %al         /* Load byte at address */
 390         testb   %al,%al                 /* lock locked? */
 391         jne     3f                      /* branch if so */
 392         lock; cmpxchg   %cl,(HW_LOCK_REGISTER)  /* attempt atomic compare exchange */
 393         jne     3f
 394         LEAF_RET                        /* if yes, then nothing left to do */
 395 3:
 396         PAUSE                           /* pause for hyper-threading */
 397         jmp     1b                      /* try again */
 398
 399 /*
 400  *      unsigned int hw_lock_to(hw_lock_t, unsigned int)
 401  *
 402  *      Acquire lock, spinning until it becomes available or timeout.
 403  *      MACH_RT:  also return with preemption disabled.
 404  */
 405 LEAF_ENTRY(hw_lock_to)
 406 1:
 407         LOAD_HW_LOCK_REGISTER           /* fetch lock pointer */
 408         LOAD_HW_LOCK_THREAD_REGISTER
 409
 410         /*
 411          * Attempt to grab the lock immediately
 412          * - fastpath without timeout nonsense.
 413          */
 414         PREEMPTION_DISABLE
 415
 416         mov     (HW_LOCK_REGISTER), HW_LOCK_EXAM_REGISTER
 417         test    HW_LOCK_EXAM_REGISTER,HW_LOCK_EXAM_REGISTER             /* lock locked? */
 418         jne     2f                      /* branch if so */
 419         lock; cmpxchg   HW_LOCK_THREAD_REGISTER,(HW_LOCK_REGISTER)      /* try to acquire the HW lock */
 420         jne     2f                      /* branch on failure */
 421         movl    $1,%eax
 422         LEAF_RET
 423
 424 2:
 425 #define INNER_LOOP_COUNT        1000
 426         /*
 427          * Failed to get the lock so set the timeout
 428          * and then spin re-checking the lock but pausing
 429          * every so many (INNER_LOOP_COUNT) spins to check for timeout.
 430          */
 431 #if __i386__
 432         movl    L_ARG1,%ecx             /* fetch timeout */
 433         push    %edi
 434         push    %ebx
 435         mov     %edx,%edi
 436
 437         lfence
 438         rdtsc                           /* read cyclecount into %edx:%eax */
 439         lfence
 440         addl    %ecx,%eax               /* fetch and timeout */
 441         adcl    $0,%edx                 /* add carry */
 442         mov     %edx,%ecx
 443         mov     %eax,%ebx               /* %ecx:%ebx is the timeout expiry */
 444         mov     %edi, %edx              /* load lock back into %edx */
 445 #else
 446         push    %r9
 447         lfence
 448         rdtsc                           /* read cyclecount into %edx:%eax */
 449         lfence
 450         shlq    $32, %rdx
 451         orq     %rdx, %rax              /* load 64-bit quantity into %rax */
 452         addq    %rax, %rsi              /* %rsi is the timeout expiry */
 453 #endif
 454
 455 4:
 456         /*
 457          * The inner-loop spin to look for the lock being freed.
 458          */
 459 #if __i386__
 460         mov     $(INNER_LOOP_COUNT),%edi
 461 #else
 462         mov     $(INNER_LOOP_COUNT),%r9
 463 #endif
 464 5:
 465         PAUSE                           /* pause for hyper-threading */
 466         mov     (HW_LOCK_REGISTER),HW_LOCK_EXAM_REGISTER                /* spin checking lock value in cache */
 467         test    HW_LOCK_EXAM_REGISTER,HW_LOCK_EXAM_REGISTER
 468         je      6f                      /* zero => unlocked, try to grab it */
 469 #if __i386__
 470         decl    %edi                    /* decrement inner loop count */
 471 #else
 472         decq    %r9                     /* decrement inner loop count */
 473 #endif
 474         jnz     5b                      /* time to check for timeout? */
 475
 476         /*
 477          * Here after spinning INNER_LOOP_COUNT times, check for timeout
 478          */
 479 #if __i386__
 480         mov     %edx,%edi               /* Save %edx */
 481         lfence
 482         rdtsc                           /* cyclecount into %edx:%eax */
 483         lfence
 484         xchg    %edx,%edi               /* cyclecount into %edi:%eax */
 485         cmpl    %ecx,%edi               /* compare high-order 32-bits */
 486         jb      4b                      /* continue spinning if less, or */
 487         cmpl    %ebx,%eax               /* compare low-order 32-bits */
 488         jb      4b                      /* continue if less, else bail */
 489         xor     %eax,%eax               /* with 0 return value */
 490         pop     %ebx
 491         pop     %edi
 492 #else
 493         lfence
 494         rdtsc                           /* cyclecount into %edx:%eax */
 495         lfence
 496         shlq    $32, %rdx
 497         orq     %rdx, %rax              /* load 64-bit quantity into %rax */
 498         cmpq    %rsi, %rax              /* compare to timeout */
 499         jb      4b                      /* continue spinning if less, or */
 500         xor     %rax,%rax               /* with 0 return value */
 501         pop     %r9
 502 #endif
 503         LEAF_RET
 504
 505 6:
 506         /*
 507          * Here to try to grab the lock that now appears to be free
 508          * after contention.
 509          */
 510         LOAD_HW_LOCK_THREAD_REGISTER
 511         lock; cmpxchg   HW_LOCK_THREAD_REGISTER,(HW_LOCK_REGISTER)      /* try to acquire the HW lock */
 512         jne     4b                      /* no - spin again */
 513         movl    $1,%eax                 /* yes */
 514 #if __i386__
 515         pop     %ebx
 516         pop     %edi
 517 #else
 518         pop     %r9
 519 #endif
 520         LEAF_RET
 521
 522 /*
 523  *      void hw_lock_unlock(hw_lock_t)
 524  *
 525  *      Unconditionally release lock.
 526  *      MACH_RT:  release preemption level.
 527  */
 528 LEAF_ENTRY(hw_lock_unlock)
 529         LOAD_HW_LOCK_REGISTER           /* fetch lock pointer */
 530         HW_LOCK_MOV_WORD $0, (HW_LOCK_REGISTER)         /* clear the lock */
 531         PREEMPTION_ENABLE
 532         LEAF_RET
 533
 534 /*
 535  *      void hw_lock_byte_unlock(uint8_t *lock_byte)
 536  *
 537  *      Unconditionally release byte sized lock operand.
 538  *      MACH_RT:  release preemption level.
 539  */
 540
 541 LEAF_ENTRY(hw_lock_byte_unlock)
 542         LOAD_HW_LOCK_REGISTER           /* Load lock pointer */
 543         movb $0, (HW_LOCK_REGISTER)             /* Clear the lock byte */
 544         PREEMPTION_ENABLE
 545         LEAF_RET
 546
 547 /*
 548  *      unsigned int hw_lock_try(hw_lock_t)
 549  *      MACH_RT:  returns with preemption disabled on success.
 550  */
 551 LEAF_ENTRY(hw_lock_try)
 552         LOAD_HW_LOCK_REGISTER           /* fetch lock pointer */
 553         LOAD_HW_LOCK_THREAD_REGISTER
 554         PREEMPTION_DISABLE
 555
 556         mov     (HW_LOCK_REGISTER),HW_LOCK_EXAM_REGISTER
 557         test    HW_LOCK_EXAM_REGISTER,HW_LOCK_EXAM_REGISTER
 558         jne     1f
 559         lock; cmpxchg   HW_LOCK_THREAD_REGISTER,(HW_LOCK_REGISTER)      /* try to acquire the HW lock */
 560         jne     1f
 561
 562         movl    $1,%eax                 /* success */
 563         LEAF_RET
 564
 565 1:
 566         PREEMPTION_ENABLE               /* failure:  release preemption... */
 567         xorl    %eax,%eax               /* ...and return failure */
 568         LEAF_RET
 569
 570 /*
 571  *      unsigned int hw_lock_held(hw_lock_t)
 572  *      MACH_RT:  doesn't change preemption state.
 573  *      N.B.  Racy, of course.
 574  */
 575 LEAF_ENTRY(hw_lock_held)
 576         LOAD_HW_LOCK_REGISTER           /* fetch lock pointer */
 577         mov     (HW_LOCK_REGISTER),HW_LOCK_EXAM_REGISTER                /* check lock value */
 578         test    HW_LOCK_EXAM_REGISTER,HW_LOCK_EXAM_REGISTER
 579         movl    $1,%ecx
 580         cmovne  %ecx,%eax               /* 0 => unlocked, 1 => locked */
 581         LEAF_RET
 582
 583
 584 /*
 585  * Reader-writer lock fastpaths. These currently exist for the
 586  * shared lock acquire, the exclusive lock acquire, the shared to
 587  * exclusive upgrade and the release paths (where they reduce overhead
 588  * considerably) -- these are by far the most frequently used routines
 589  *
 590  * The following should reflect the layout of the bitfield embedded within
 591  * the lck_rw_t structure (see i386/locks.h).
 592  */
 593 #define LCK_RW_INTERLOCK        (0x1 << 16)
 594
 595 #define LCK_RW_PRIV_EXCL        (0x1 << 24)
 596 #define LCK_RW_WANT_UPGRADE     (0x2 << 24)
 597 #define LCK_RW_WANT_WRITE       (0x4 << 24)
 598 #define LCK_R_WAITING           (0x8 << 24)
 599 #define LCK_W_WAITING           (0x10 << 24)
 600
 601 #define LCK_RW_SHARED_MASK      (0xffff)
 602
 603 /*
 604  * For most routines, the lck_rw_t pointer is loaded into a
 605  * register initially, and the flags bitfield loaded into another
 606  * register and examined
 607  */
 608
 609 #if defined(__i386__)
 610 #define LCK_RW_REGISTER %edx
 611 #define LOAD_LCK_RW_REGISTER mov S_ARG0, LCK_RW_REGISTER
 612 #define LCK_RW_FLAGS_REGISTER   %eax
 613 #define LOAD_LCK_RW_FLAGS_REGISTER mov (LCK_RW_REGISTER), LCK_RW_FLAGS_REGISTER
 614 #elif defined(__x86_64__)
 615 #define LCK_RW_REGISTER %rdi
 616 #define LOAD_LCK_RW_REGISTER
 617 #define LCK_RW_FLAGS_REGISTER   %eax
 618 #define LOAD_LCK_RW_FLAGS_REGISTER mov (LCK_RW_REGISTER), LCK_RW_FLAGS_REGISTER
 619 #else
 620 #error Unsupported architecture
 621 #endif
 622
 623 #define RW_LOCK_SHARED_MASK (LCK_RW_INTERLOCK | LCK_RW_WANT_UPGRADE | LCK_RW_WANT_WRITE)
 624 /*
 625  *      void lck_rw_lock_shared(lck_rw_t *)
 626  *
 627  */
 628 Entry(lck_rw_lock_shared)
 629         LOAD_LCK_RW_REGISTER
 630 1:
 631         LOAD_LCK_RW_FLAGS_REGISTER              /* Load state bitfield and interlock */
 632         testl   $(RW_LOCK_SHARED_MASK), %eax    /* Eligible for fastpath? */
 633         jne     3f
 634
 635         movl    %eax, %ecx                      /* original value in %eax for cmpxchgl */
 636         incl    %ecx                            /* Increment reader refcount */
 637         lock
 638         cmpxchgl %ecx, (LCK_RW_REGISTER)                        /* Attempt atomic exchange */
 639         jne     2f
 640
 641 #if     CONFIG_DTRACE
 642         /*
 643          * Dtrace lockstat event: LS_LCK_RW_LOCK_SHARED_ACQUIRE
 644          * Implemented by swapping between return and no-op instructions.
 645          * See bsd/dev/dtrace/lockstat.c.
 646          */
 647         LOCKSTAT_LABEL(_lck_rw_lock_shared_lockstat_patch_point)
 648         ret
 649     /* Fall thru when patched, counting on lock pointer in LCK_RW_REGISTER  */
 650     LOCKSTAT_RECORD(LS_LCK_RW_LOCK_SHARED_ACQUIRE, LCK_RW_REGISTER)
 651 #endif
 652         ret
 653 2:
 654         PAUSE
 655         jmp     1b
 656 3:
 657         jmp     EXT(lck_rw_lock_shared_gen)
 658
 659
 660
 661 #define RW_TRY_LOCK_SHARED_MASK (LCK_RW_WANT_UPGRADE | LCK_RW_WANT_WRITE)
 662 /*
 663  *      void lck_rw_try_lock_shared(lck_rw_t *)
 664  *
 665  */
 666 Entry(lck_rw_try_lock_shared)
 667         LOAD_LCK_RW_REGISTER
 668 1:
 669         LOAD_LCK_RW_FLAGS_REGISTER              /* Load state bitfield and interlock */
 670         testl   $(LCK_RW_INTERLOCK), %eax
 671         jne     2f
 672         testl   $(RW_TRY_LOCK_SHARED_MASK), %eax
 673         jne     3f                      /* lock is busy */
 674
 675         movl    %eax, %ecx                      /* original value in %eax for cmpxchgl */
 676         incl    %ecx                            /* Increment reader refcount */
 677         lock
 678         cmpxchgl %ecx, (LCK_RW_REGISTER)                        /* Attempt atomic exchange */
 679         jne     2f
 680
 681 #if     CONFIG_DTRACE
 682         movl    $1, %eax
 683         /*
 684          * Dtrace lockstat event: LS_LCK_RW_TRY_LOCK_SHARED_ACQUIRE
 685          * Implemented by swapping between return and no-op instructions.
 686          * See bsd/dev/dtrace/lockstat.c.
 687          */
 688         LOCKSTAT_LABEL(_lck_rw_try_lock_shared_lockstat_patch_point)
 689         ret
 690     /* Fall thru when patched, counting on lock pointer in LCK_RW_REGISTER  */
 691     LOCKSTAT_RECORD(LS_LCK_RW_LOCK_SHARED_ACQUIRE, LCK_RW_REGISTER)
 692 #endif
 693         movl    $1, %eax                        /* return TRUE */
 694         ret
 695 2:
 696         PAUSE
 697         jmp     1b
 698 3:
 699         xorl    %eax, %eax
 700         ret
 701
 702
 703 #define RW_LOCK_EXCLUSIVE_HELD  (LCK_RW_WANT_WRITE | LCK_RW_WANT_UPGRADE)
 704 /*
 705  *      int lck_rw_grab_shared(lck_rw_t *)
 706  *
 707  */
 708 Entry(lck_rw_grab_shared)
 709         LOAD_LCK_RW_REGISTER
 710 1:
 711         LOAD_LCK_RW_FLAGS_REGISTER              /* Load state bitfield and interlock */
 712         testl   $(LCK_RW_INTERLOCK), %eax
 713         jne     5f
 714         testl   $(RW_LOCK_EXCLUSIVE_HELD), %eax
 715         jne     3f
 716 2:
 717         movl    %eax, %ecx                      /* original value in %eax for cmpxchgl */
 718         incl    %ecx                            /* Increment reader refcount */
 719         lock
 720         cmpxchgl %ecx, (LCK_RW_REGISTER)                        /* Attempt atomic exchange */
 721         jne     4f
 722
 723         movl    $1, %eax                        /* return success */
 724         ret
 725 3:
 726         testl   $(LCK_RW_SHARED_MASK), %eax
 727         je      4f
 728         testl   $(LCK_RW_PRIV_EXCL), %eax
 729         je      2b
 730 4:
 731         xorl    %eax, %eax                      /* return failure */
 732         ret
 733 5:
 734         PAUSE
 735         jmp     1b
 736
 737
 738
 739 #define RW_LOCK_EXCLUSIVE_MASK (LCK_RW_SHARED_MASK | LCK_RW_INTERLOCK | \
 740                                 LCK_RW_WANT_UPGRADE | LCK_RW_WANT_WRITE)
 741 /*
 742  *      void lck_rw_lock_exclusive(lck_rw_t*)
 743  *
 744  */
 745 Entry(lck_rw_lock_exclusive)
 746         LOAD_LCK_RW_REGISTER
 747 1:
 748         LOAD_LCK_RW_FLAGS_REGISTER              /* Load state bitfield, interlock and shared count */
 749         testl   $(RW_LOCK_EXCLUSIVE_MASK), %eax         /* Eligible for fastpath? */
 750         jne     3f                                      /* no, go slow */
 751
 752         movl    %eax, %ecx                              /* original value in %eax for cmpxchgl */
 753         orl     $(LCK_RW_WANT_WRITE), %ecx
 754         lock
 755         cmpxchgl %ecx, (LCK_RW_REGISTER)                        /* Attempt atomic exchange */
 756         jne     2f
 757
 758 #if     CONFIG_DTRACE
 759         /*
 760          * Dtrace lockstat event: LS_LCK_RW_LOCK_EXCL_ACQUIRE
 761          * Implemented by swapping between return and no-op instructions.
 762          * See bsd/dev/dtrace/lockstat.c.
 763          */
 764         LOCKSTAT_LABEL(_lck_rw_lock_exclusive_lockstat_patch_point)
 765         ret
 766     /* Fall thru when patched, counting on lock pointer in LCK_RW_REGISTER  */
 767     LOCKSTAT_RECORD(LS_LCK_RW_LOCK_SHARED_ACQUIRE, LCK_RW_REGISTER)
 768 #endif
 769         ret
 770 2:
 771         PAUSE
 772         jmp     1b
 773 3:
 774         jmp     EXT(lck_rw_lock_exclusive_gen)
 775
 776
 777
 778 #define RW_TRY_LOCK_EXCLUSIVE_MASK (LCK_RW_SHARED_MASK | LCK_RW_WANT_UPGRADE | LCK_RW_WANT_WRITE)
 779 /*
 780  *      void lck_rw_try_lock_exclusive(lck_rw_t *)
 781  *
 782  *              Tries to get a write lock.
 783  *
 784  *              Returns FALSE if the lock is not held on return.
 785  */
 786 Entry(lck_rw_try_lock_exclusive)
 787         LOAD_LCK_RW_REGISTER
 788 1:
 789         LOAD_LCK_RW_FLAGS_REGISTER              /* Load state bitfield, interlock and shared count */
 790         testl   $(LCK_RW_INTERLOCK), %eax
 791         jne     2f
 792         testl   $(RW_TRY_LOCK_EXCLUSIVE_MASK), %eax
 793         jne     3f                                      /* can't get it */
 794
 795         movl    %eax, %ecx                              /* original value in %eax for cmpxchgl */
 796         orl     $(LCK_RW_WANT_WRITE), %ecx
 797         lock
 798         cmpxchgl %ecx, (LCK_RW_REGISTER)                        /* Attempt atomic exchange */
 799         jne     2f
 800
 801 #if     CONFIG_DTRACE
 802         movl    $1, %eax
 803         /*
 804          * Dtrace lockstat event: LS_LCK_RW_TRY_LOCK_EXCL_ACQUIRE
 805          * Implemented by swapping between return and no-op instructions.
 806          * See bsd/dev/dtrace/lockstat.c.
 807          */
 808         LOCKSTAT_LABEL(_lck_rw_try_lock_exclusive_lockstat_patch_point)
 809         ret
 810     /* Fall thru when patched, counting on lock pointer in LCK_RW_REGISTER  */
 811     LOCKSTAT_RECORD(LS_LCK_RW_LOCK_SHARED_ACQUIRE, LCK_RW_REGISTER)
 812 #endif
 813         movl    $1, %eax                        /* return TRUE */
 814         ret
 815 2:
 816         PAUSE
 817         jmp     1b
 818 3:
 819         xorl    %eax, %eax                      /* return FALSE */
 820         ret
 821
 822
 823
 824 /*
 825  *      void lck_rw_lock_shared_to_exclusive(lck_rw_t*)
 826  *
 827  *      fastpath can be taken if
 828  *      the current rw_shared_count == 1
 829  *      AND the interlock is clear
 830  *      AND RW_WANT_UPGRADE is not set
 831  *
 832  *      note that RW_WANT_WRITE could be set, but will not
 833  *      be indicative of an exclusive hold since we have
 834  *      a read count on the lock that we have not yet released
 835  *      we can blow by that state since the lck_rw_lock_exclusive
 836  *      function will block until rw_shared_count == 0 and
 837  *      RW_WANT_UPGRADE is clear... it does this check behind
 838  *      the interlock which we are also checking for
 839  *
 840  *      to make the transition we must be able to atomically
 841  *      set RW_WANT_UPGRADE and get rid of the read count we hold
 842  */
 843 Entry(lck_rw_lock_shared_to_exclusive)
 844         LOAD_LCK_RW_REGISTER
 845 1:
 846         LOAD_LCK_RW_FLAGS_REGISTER              /* Load state bitfield, interlock and shared count */
 847         testl   $(LCK_RW_INTERLOCK), %eax
 848         jne     7f
 849         testl   $(LCK_RW_WANT_UPGRADE), %eax
 850         jne     2f
 851
 852         movl    %eax, %ecx                      /* original value in %eax for cmpxchgl */
 853         orl     $(LCK_RW_WANT_UPGRADE), %ecx    /* ask for WANT_UPGRADE */
 854         decl    %ecx                            /* and shed our read count */
 855         lock
 856         cmpxchgl %ecx, (LCK_RW_REGISTER)                        /* Attempt atomic exchange */
 857         jne     7f
 858                                                 /* we now own the WANT_UPGRADE */
 859         testl   $(LCK_RW_SHARED_MASK), %ecx     /* check to see if all of the readers are drained */
 860         jne     8f                              /* if not, we need to go wait */
 861
 862 #if     CONFIG_DTRACE
 863         movl    $1, %eax
 864         /*
 865          * Dtrace lockstat event: LS_LCK_RW_LOCK_SHARED_TO_EXCL_UPGRADE
 866          * Implemented by swapping between return and no-op instructions.
 867          * See bsd/dev/dtrace/lockstat.c.
 868          */
 869         LOCKSTAT_LABEL(_lck_rw_lock_shared_to_exclusive_lockstat_patch_point)
 870         ret
 871     /* Fall thru when patched, counting on lock pointer in LCK_RW_REGISTER  */
 872     LOCKSTAT_RECORD(LS_LCK_RW_LOCK_SHARED_ACQUIRE, LCK_RW_REGISTER)
 873 #endif
 874         movl    $1, %eax                        /* return success */
 875         ret
 876
 877 2:                                              /* someone else already holds WANT_UPGRADE */
 878         movl    %eax, %ecx                      /* original value in %eax for cmpxchgl */
 879         decl    %ecx                            /* shed our read count */
 880         testl   $(LCK_RW_SHARED_MASK), %ecx
 881         jne     3f                              /* we were the last reader */
 882         andl    $(~LCK_W_WAITING), %ecx         /* so clear the wait indicator */
 883 3:
 884         lock
 885         cmpxchgl %ecx, (LCK_RW_REGISTER)                        /* Attempt atomic exchange */
 886         jne     7f
 887
 888 #if __i386__
 889         pushl   %eax                            /* go check to see if we need to */
 890         push    %edx                            /* wakeup anyone */
 891         call    EXT(lck_rw_lock_shared_to_exclusive_failure)
 892         addl    $8, %esp
 893 #else
 894         mov     %eax, %esi                      /* put old flags as second arg */
 895                                                 /* lock is alread in %rdi */
 896         call    EXT(lck_rw_lock_shared_to_exclusive_failure)
 897 #endif
 898         ret                                     /* and pass the failure return along */
 899 7:
 900         PAUSE
 901         jmp     1b
 902 8:
 903         jmp     EXT(lck_rw_lock_shared_to_exclusive_success)
 904
 905
 906
 907         .cstring
 908 rwl_release_error_str:
 909         .asciz  "Releasing non-exclusive RW lock without a reader refcount!"
 910         .text
 911
 912 /*
 913  *      lck_rw_type_t lck_rw_done(lck_rw_t *)
 914  *
 915  */
 916 Entry(lck_rw_done)
 917         LOAD_LCK_RW_REGISTER
 918 1:
 919         LOAD_LCK_RW_FLAGS_REGISTER              /* Load state bitfield, interlock and reader count */
 920         testl   $(LCK_RW_INTERLOCK), %eax
 921         jne     7f                              /* wait for interlock to clear */
 922
 923         movl    %eax, %ecx                      /* keep original value in %eax for cmpxchgl */
 924         testl   $(LCK_RW_SHARED_MASK), %ecx     /* if reader count == 0, must be exclusive lock */
 925         je      2f
 926         decl    %ecx                            /* Decrement reader count */
 927         testl   $(LCK_RW_SHARED_MASK), %ecx     /* if reader count has now gone to 0, check for waiters */
 928         je      4f
 929         jmp     6f
 930 2:
 931         testl   $(LCK_RW_WANT_UPGRADE), %ecx
 932         je      3f
 933         andl    $(~LCK_RW_WANT_UPGRADE), %ecx
 934         jmp     4f
 935 3:
 936         testl   $(LCK_RW_WANT_WRITE), %ecx
 937         je      8f                              /* lock is not 'owned', go panic */
 938         andl    $(~LCK_RW_WANT_WRITE), %ecx
 939 4:
 940         /*
 941          * test the original values to match what
 942          * lck_rw_done_gen is going to do to determine
 943          * which wakeups need to happen...
 944          *
 945          * if !(fake_lck->lck_rw_priv_excl && fake_lck->lck_w_waiting)
 946          */
 947         testl   $(LCK_W_WAITING), %eax
 948         je      5f
 949         andl    $(~LCK_W_WAITING), %ecx
 950
 951         testl   $(LCK_RW_PRIV_EXCL), %eax
 952         jne     6f
 953 5:
 954         andl    $(~LCK_R_WAITING), %ecx
 955 6:
 956         lock
 957         cmpxchgl %ecx, (LCK_RW_REGISTER)                        /* Attempt atomic exchange */
 958         jne     7f
 959
 960 #if __i386__
 961         pushl   %eax
 962         push    %edx
 963         call    EXT(lck_rw_done_gen)
 964         addl    $8, %esp
 965 #else
 966         mov     %eax,%esi       /* old flags in %rsi */
 967                                 /* lock is in %rdi already */
 968         call    EXT(lck_rw_done_gen)
 969 #endif
 970         ret
 971 7:
 972         PAUSE
 973         jmp     1b
 974 8:
 975         LOAD_STRING_ARG0(rwl_release_error_str)
 976         CALL_PANIC()
 977
 978
 979
 980 /*
 981  *      lck_rw_type_t lck_rw_lock_exclusive_to_shared(lck_rw_t *)
 982  *
 983  */
 984 Entry(lck_rw_lock_exclusive_to_shared)
 985         LOAD_LCK_RW_REGISTER
 986 1:
 987         LOAD_LCK_RW_FLAGS_REGISTER              /* Load state bitfield, interlock and reader count */
 988         testl   $(LCK_RW_INTERLOCK), %eax
 989         jne     6f                              /* wait for interlock to clear */
 990
 991         movl    %eax, %ecx                      /* keep original value in %eax for cmpxchgl */
 992         incl    %ecx                            /* Increment reader count */
 993
 994         testl   $(LCK_RW_WANT_UPGRADE), %ecx
 995         je      2f
 996         andl    $(~LCK_RW_WANT_UPGRADE), %ecx
 997         jmp     3f
 998 2:
 999         andl    $(~LCK_RW_WANT_WRITE), %ecx
1000 3:
1001         /*
1002          * test the original values to match what
1003          * lck_rw_lock_exclusive_to_shared_gen is going to do to determine
1004          * which wakeups need to happen...
1005          *
1006          * if !(fake_lck->lck_rw_priv_excl && fake_lck->lck_w_waiting)
1007          */
1008         testl   $(LCK_W_WAITING), %eax
1009         je      4f
1010         testl   $(LCK_RW_PRIV_EXCL), %eax
1011         jne     5f
1012 4:
1013         andl    $(~LCK_R_WAITING), %ecx
1014 5:
1015         lock
1016         cmpxchgl %ecx, (LCK_RW_REGISTER)                        /* Attempt atomic exchange */
1017         jne     6f
1018
1019 #if __i386__
1020         pushl   %eax
1021         push    %edx
1022         call    EXT(lck_rw_lock_exclusive_to_shared_gen)
1023         addl    $8, %esp
1024 #else
1025         mov     %eax,%esi
1026         call    EXT(lck_rw_lock_exclusive_to_shared_gen)
1027 #endif
1028         ret
1029 6:
1030         PAUSE
1031         jmp     1b
1032
1033
1034
1035 /*
1036  *      int lck_rw_grab_want(lck_rw_t *)
1037  *
1038  */
1039 Entry(lck_rw_grab_want)
1040         LOAD_LCK_RW_REGISTER
1041 1:
1042         LOAD_LCK_RW_FLAGS_REGISTER              /* Load state bitfield, interlock and reader count */
1043         testl   $(LCK_RW_INTERLOCK), %eax
1044         jne     3f                              /* wait for interlock to clear */
1045         testl   $(LCK_RW_WANT_WRITE), %eax      /* want_write has been grabbed by someone else */
1046         jne     2f                              /* go return failure */
1047
1048         movl    %eax, %ecx                      /* original value in %eax for cmpxchgl */
1049         orl     $(LCK_RW_WANT_WRITE), %ecx
1050         lock
1051         cmpxchgl %ecx, (LCK_RW_REGISTER)                        /* Attempt atomic exchange */
1052         jne     2f
1053                                                 /* we now own want_write */
1054         movl    $1, %eax                        /* return success */
1055         ret
1056 2:
1057         xorl    %eax, %eax                      /* return failure */
1058         ret
1059 3:
1060         PAUSE
1061         jmp     1b
1062
1063
1064 #define RW_LOCK_SHARED_OR_UPGRADE_MASK (LCK_RW_SHARED_MASK | LCK_RW_INTERLOCK | LCK_RW_WANT_UPGRADE)
1065 /*
1066  *      int lck_rw_held_read_or_upgrade(lck_rw_t *)
1067  *
1068  */
1069 Entry(lck_rw_held_read_or_upgrade)
1070         LOAD_LCK_RW_REGISTER
1071         LOAD_LCK_RW_FLAGS_REGISTER              /* Load state bitfield, interlock and reader count */
1072         andl    $(RW_LOCK_SHARED_OR_UPGRADE_MASK), %eax
1073         ret
1074
1075
1076
1077 /*
1078  * N.B.: On x86, statistics are currently recorded for all indirect mutexes.
1079  * Also, only the acquire attempt count (GRP_MTX_STAT_UTIL) is maintained
1080  * as a 64-bit quantity (this matches the existing PowerPC implementation,
1081  * and the new x86 specific statistics are also maintained as 32-bit
1082  * quantities).
1083  *
1084  *
1085  * Enable this preprocessor define to record the first miss alone
1086  * By default, we count every miss, hence multiple misses may be
1087  * recorded for a single lock acquire attempt via lck_mtx_lock
1088  */
1089 #undef LOG_FIRST_MISS_ALONE
1090
1091 /*
1092  * This preprocessor define controls whether the R-M-W update of the
1093  * per-group statistics elements are atomic (LOCK-prefixed)
1094  * Enabled by default.
1095  */
1096 #define ATOMIC_STAT_UPDATES 1
1097
1098 #if defined(ATOMIC_STAT_UPDATES)
1099 #define LOCK_IF_ATOMIC_STAT_UPDATES lock
1100 #else
1101 #define LOCK_IF_ATOMIC_STAT_UPDATES
1102 #endif /* ATOMIC_STAT_UPDATES */
1103
1104
1105 /*
1106  * For most routines, the lck_mtx_t pointer is loaded into a
1107  * register initially, and the owner field checked for indirection.
1108  * Eventually the lock owner is loaded into a register and examined.
1109  */
1110
1111 #define M_OWNER         MUTEX_OWNER
1112 #define M_PTR           MUTEX_PTR
1113 #define M_STATE         MUTEX_STATE
1114
1115 #if defined(__i386__)
1116
1117 #define LMTX_ARG0       B_ARG0
1118 #define LMTX_ARG1       B_ARG1
1119 #define LMTX_REG        %edx
1120 #define LMTX_A_REG      %eax
1121 #define LMTX_A_REG32    %eax
1122 #define LMTX_C_REG      %ecx
1123 #define LMTX_C_REG32    %ecx
1124 #define LMTX_D_REG      %edx
1125 #define LMTX_RET_REG    %eax
1126 #define LMTX_LGROUP_REG %esi
1127 #define LMTX_SSTATE_REG %edi
1128 #define LOAD_LMTX_REG(arg)      mov arg, LMTX_REG
1129 #define LOAD_REG_ARG0(reg)      push reg
1130 #define LOAD_REG_ARG1(reg)      push reg
1131 #define LMTX_CHK_EXTENDED       cmp LMTX_REG, LMTX_ARG0
1132 #define LMTX_ASSERT_OWNED       cmpl $(MUTEX_ASSERT_OWNED), LMTX_ARG1
1133
1134 #define LMTX_ENTER_EXTENDED                                     \
1135         mov     M_PTR(LMTX_REG), LMTX_REG               ;       \
1136         push    LMTX_LGROUP_REG                         ;       \
1137         push    LMTX_SSTATE_REG                         ;       \
1138         xor     LMTX_SSTATE_REG, LMTX_SSTATE_REG        ;       \
1139         mov     MUTEX_GRP(LMTX_REG), LMTX_LGROUP_REG    ;       \
1140         LOCK_IF_ATOMIC_STAT_UPDATES                     ;       \
1141         addl    $1, GRP_MTX_STAT_UTIL(LMTX_LGROUP_REG)  ;       \
1142         jnc     11f                                     ;       \
1143         incl    GRP_MTX_STAT_UTIL+4(LMTX_LGROUP_REG)    ;       \
1144 11:
1145
1146 #define LMTX_EXIT_EXTENDED              \
1147         pop     LMTX_SSTATE_REG ;       \
1148         pop     LMTX_LGROUP_REG
1149
1150
1151 #define LMTX_CHK_EXTENDED_EXIT                  \
1152         cmp     LMTX_REG, LMTX_ARG0     ;       \
1153         je      12f                     ;       \
1154         pop     LMTX_SSTATE_REG         ;       \
1155         pop     LMTX_LGROUP_REG         ;       \
1156 12:
1157
1158
1159 #if     LOG_FIRST_MISS_ALONE
1160 #define LMTX_UPDATE_MISS                                        \
1161         test    $1, LMTX_SSTATE_REG                     ;       \
1162         jnz     11f                                     ;       \
1163         LOCK_IF_ATOMIC_STAT_UPDATES                     ;       \
1164         incl    GRP_MTX_STAT_MISS(LMTX_LGROUP_REG)      ;       \
1165         or      $1, LMTX_SSTATE_REG                     ;       \
1166 11:
1167 #else
1168 #define LMTX_UPDATE_MISS                                        \
1169         LOCK_IF_ATOMIC_STAT_UPDATES                     ;       \
1170         incl    GRP_MTX_STAT_MISS(LMTX_LGROUP_REG)
1171 #endif
1172
1173
1174 #if     LOG_FIRST_MISS_ALONE
1175 #define LMTX_UPDATE_WAIT                                        \
1176         test    $2, LMTX_SSTATE_REG                     ;       \
1177         jnz     11f                                     ;       \
1178         LOCK_IF_ATOMIC_STAT_UPDATES                     ;       \
1179         incl    GRP_MTX_STAT_WAIT(LMTX_LGROUP_REG)      ;       \
1180         or      $2, LMTX_SSTATE_REG                     ;       \
1181 11:
1182 #else
1183 #define LMTX_UPDATE_WAIT                                        \
1184         LOCK_IF_ATOMIC_STAT_UPDATES                     ;       \
1185         incl    GRP_MTX_STAT_WAIT(LMTX_LGROUP_REG)
1186 #endif
1187
1188
1189 /*
1190  * Record the "direct wait" statistic, which indicates if a
1191  * miss proceeded to block directly without spinning--occurs
1192  * if the owner of the mutex isn't running on another processor
1193  * at the time of the check.
1194  */
1195 #define LMTX_UPDATE_DIRECT_WAIT                                 \
1196         LOCK_IF_ATOMIC_STAT_UPDATES                     ;       \
1197         incl    GRP_MTX_STAT_DIRECT_WAIT(LMTX_LGROUP_REG)
1198
1199
1200 #define LMTX_CALLEXT1(func_name)        \
1201         push    LMTX_REG        ;       \
1202         push    LMTX_REG        ;       \
1203         call    EXT(func_name)  ;       \
1204         add     $4, %esp        ;       \
1205         pop     LMTX_REG
1206
1207 #define LMTX_CALLEXT2(func_name, reg)   \
1208         push    LMTX_REG        ;       \
1209         push    reg             ;       \
1210         push    LMTX_REG        ;       \
1211         call    EXT(func_name)  ;       \
1212         add     $8, %esp        ;       \
1213         pop     LMTX_REG
1214
1215 #elif defined(__x86_64__)
1216
1217 #define LMTX_ARG0       %rdi
1218 #define LMTX_ARG1       %rsi
1219 #define LMTX_REG_ORIG   %rdi
1220 #define LMTX_REG        %rdx
1221 #define LMTX_A_REG      %rax
1222 #define LMTX_A_REG32    %eax
1223 #define LMTX_C_REG      %rcx
1224 #define LMTX_C_REG32    %ecx
1225 #define LMTX_D_REG      %rdx
1226 #define LMTX_RET_REG    %rax
1227 #define LMTX_LGROUP_REG %r10
1228 #define LMTX_SSTATE_REG %r11
1229 #define LOAD_LMTX_REG(arg)      mov %rdi, %rdx
1230 #define LOAD_REG_ARG0(reg)      mov reg, %rdi
1231 #define LOAD_REG_ARG1(reg)      mov reg, %rsi
1232 #define LMTX_CHK_EXTENDED       cmp LMTX_REG, LMTX_REG_ORIG
1233 #define LMTX_ASSERT_OWNED       cmp $(MUTEX_ASSERT_OWNED), LMTX_ARG1
1234
1235 #define LMTX_ENTER_EXTENDED                                     \
1236         mov     M_PTR(LMTX_REG), LMTX_REG               ;       \
1237         xor     LMTX_SSTATE_REG, LMTX_SSTATE_REG        ;       \
1238         mov     MUTEX_GRP(LMTX_REG), LMTX_LGROUP_REG    ;       \
1239         LOCK_IF_ATOMIC_STAT_UPDATES                     ;       \
1240         incq    GRP_MTX_STAT_UTIL(LMTX_LGROUP_REG)
1241
1242 #define LMTX_EXIT_EXTENDED
1243
1244 #define LMTX_CHK_EXTENDED_EXIT
1245
1246
1247 #if     LOG_FIRST_MISS_ALONE
1248 #define LMTX_UPDATE_MISS                                        \
1249         test    $1, LMTX_SSTATE_REG                     ;       \
1250         jnz     11f                                     ;       \
1251         LOCK_IF_ATOMIC_STAT_UPDATES                     ;       \
1252         incl    GRP_MTX_STAT_MISS(LMTX_LGROUP_REG)      ;       \
1253         or      $1, LMTX_SSTATE_REG                     ;       \
1254 11:
1255 #else
1256 #define LMTX_UPDATE_MISS                                        \
1257         LOCK_IF_ATOMIC_STAT_UPDATES                     ;       \
1258         incl    GRP_MTX_STAT_MISS(LMTX_LGROUP_REG)
1259 #endif
1260
1261
1262 #if     LOG_FIRST_MISS_ALONE
1263 #define LMTX_UPDATE_WAIT                                        \
1264         test    $2, LMTX_SSTATE_REG                     ;       \
1265         jnz     11f                                     ;       \
1266         LOCK_IF_ATOMIC_STAT_UPDATES                     ;       \
1267         incl    GRP_MTX_STAT_WAIT(LMTX_LGROUP_REG)      ;       \
1268         or      $2, LMTX_SSTATE_REG                     ;       \
1269 11:
1270 #else
1271 #define LMTX_UPDATE_WAIT                                        \
1272         LOCK_IF_ATOMIC_STAT_UPDATES                     ;       \
1273         incl    GRP_MTX_STAT_WAIT(LMTX_LGROUP_REG)
1274 #endif
1275
1276
1277 /*
1278  * Record the "direct wait" statistic, which indicates if a
1279  * miss proceeded to block directly without spinning--occurs
1280  * if the owner of the mutex isn't running on another processor
1281  * at the time of the check.
1282  */
1283 #define LMTX_UPDATE_DIRECT_WAIT                                 \
1284         LOCK_IF_ATOMIC_STAT_UPDATES                     ;       \
1285         incl    GRP_MTX_STAT_DIRECT_WAIT(LMTX_LGROUP_REG)
1286
1287
1288 #define LMTX_CALLEXT1(func_name)                \
1289         LMTX_CHK_EXTENDED               ;       \
1290         je      12f                     ;       \
1291         push    LMTX_LGROUP_REG         ;       \
1292         push    LMTX_SSTATE_REG         ;       \
1293 12:     push    LMTX_REG_ORIG           ;       \
1294         push    LMTX_REG                ;       \
1295         mov     LMTX_REG, LMTX_ARG0     ;       \
1296         call    EXT(func_name)          ;       \
1297         pop     LMTX_REG                ;       \
1298         pop     LMTX_REG_ORIG           ;       \
1299         LMTX_CHK_EXTENDED               ;       \
1300         je      12f                     ;       \
1301         pop     LMTX_SSTATE_REG         ;       \
1302         pop     LMTX_LGROUP_REG         ;       \
1303 12:
1304
1305 #define LMTX_CALLEXT2(func_name, reg)           \
1306         LMTX_CHK_EXTENDED               ;       \
1307         je      12f                     ;       \
1308         push    LMTX_LGROUP_REG         ;       \
1309         push    LMTX_SSTATE_REG         ;       \
1310 12:     push    LMTX_REG_ORIG           ;       \
1311         push    LMTX_REG                ;       \
1312         mov     reg, LMTX_ARG1          ;       \
1313         mov     LMTX_REG, LMTX_ARG0     ;       \
1314         call    EXT(func_name)          ;       \
1315         pop     LMTX_REG                ;       \
1316         pop     LMTX_REG_ORIG           ;       \
1317         LMTX_CHK_EXTENDED               ;       \
1318         je      12f                     ;       \
1319         pop     LMTX_SSTATE_REG         ;       \
1320         pop     LMTX_LGROUP_REG         ;       \
1321 12:
1322
1323 #else
1324 #error Unsupported architecture
1325 #endif
1326
1327
1328 #define M_WAITERS_MSK           0x0000ffff
1329 #define M_PRIORITY_MSK          0x00ff0000
1330 #define M_ILOCKED_MSK           0x01000000
1331 #define M_MLOCKED_MSK           0x02000000
1332 #define M_PROMOTED_MSK          0x04000000
1333 #define M_SPIN_MSK              0x08000000
1334
1335
1336
1337 /*
1338  *      void lck_mtx_assert(lck_mtx_t* l, unsigned int)
1339  *      Takes the address of a lock, and an assertion type as parameters.
1340  *      The assertion can take one of two forms determine by the type
1341  *      parameter: either the lock is held by the current thread, and the
1342  *      type is LCK_MTX_ASSERT_OWNED, or it isn't and the type is
1343  *      LCK_MTX_ASSERT_NOTOWNED. Calls panic on assertion failure.
1344  *
1345  */
1346
1347 NONLEAF_ENTRY(lck_mtx_assert)
1348         LOAD_LMTX_REG(B_ARG0)                           /* Load lock address */
1349         mov     %gs:CPU_ACTIVE_THREAD, LMTX_A_REG       /* Load current thread */
1350
1351         mov     M_OWNER(LMTX_REG), LMTX_C_REG
1352         cmp     $(MUTEX_IND), LMTX_C_REG        /* Is this an indirect mutex? */
1353         cmove   M_PTR(LMTX_REG), LMTX_REG       /* If so, take indirection */
1354
1355         mov     M_OWNER(LMTX_REG), LMTX_C_REG   /* Load owner */
1356         LMTX_ASSERT_OWNED
1357         jne     2f                              /* Assert ownership? */
1358         cmp     LMTX_A_REG, LMTX_C_REG          /* Current thread match? */
1359         jne     3f                              /* no, go panic */
1360         testl   $(M_ILOCKED_MSK | M_MLOCKED_MSK), M_STATE(LMTX_REG)
1361         je      3f
1362 1:                                              /* yes, we own it */
1363         NONLEAF_RET
1364 2:
1365         cmp     LMTX_A_REG, LMTX_C_REG          /* Current thread match? */
1366         jne     1b                              /* No, return */
1367         LOAD_REG_ARG1(LMTX_REG)
1368         LOAD_STRING_ARG0(mutex_assert_owned_str)
1369         jmp     4f
1370 3:
1371         LOAD_REG_ARG1(LMTX_REG)
1372         LOAD_STRING_ARG0(mutex_assert_not_owned_str)
1373 4:
1374         CALL_PANIC()
1375
1376
1377 lck_mtx_destroyed:
1378         LOAD_REG_ARG1(LMTX_REG)
1379         LOAD_STRING_ARG0(mutex_interlock_destroyed_str)
1380         CALL_PANIC()
1381
1382
1383 .data
1384 mutex_assert_not_owned_str:
1385         .asciz  "mutex (%p) not owned\n"
1386 mutex_assert_owned_str:
1387         .asciz  "mutex (%p) owned\n"
1388 mutex_interlock_destroyed_str:
1389         .asciz  "trying to interlock destroyed mutex (%p)"
1390 .text
1391
1392
1393
1394 /*
1395  * lck_mtx_lock()
1396  * lck_mtx_try_lock()
1397  * lck_mtx_unlock()
1398  * lck_mtx_lock_spin()
1399  * lck_mtx_convert_spin()
1400  */
1401
1402 NONLEAF_ENTRY(lck_mtx_lock_spin)
1403         LOAD_LMTX_REG(B_ARG0)           /* fetch lock pointer */
1404
1405         CHECK_PREEMPTION_LEVEL()
1406
1407         mov     M_STATE(LMTX_REG), LMTX_C_REG32
1408         test    $(M_ILOCKED_MSK), LMTX_C_REG    /* is the interlock held */
1409         je      Llmls_enter                     /* no - can't be INDIRECT or DESTROYED */
1410
1411         mov     M_OWNER(LMTX_REG), LMTX_A_REG
1412         cmp     $(MUTEX_DESTROYED), LMTX_A_REG  /* check to see if its marked destroyed */
1413         je      lck_mtx_destroyed
1414         cmp     $(MUTEX_IND), LMTX_A_REG        /* Is this an indirect mutex */
1415         jne     Llmls_loop
1416
1417         LMTX_ENTER_EXTENDED
1418
1419         mov     M_STATE(LMTX_REG), LMTX_C_REG32
1420         test    $(M_SPIN_MSK), LMTX_C_REG
1421         je      Llmls_loop
1422
1423         LMTX_UPDATE_MISS
1424 Llmls_loop:
1425         PAUSE
1426         mov     M_STATE(LMTX_REG), LMTX_C_REG32
1427
1428         test    $(M_ILOCKED_MSK), LMTX_C_REG    /* is the interlock held */
1429         jne     Llmls_loop
1430 Llmls_enter:
1431         test    $(M_MLOCKED_MSK), LMTX_C_REG    /* is the mutex locked */
1432         jne     Llml_contended                  /* fall back to normal mutex handling */
1433
1434         PUSHF                                   /* save interrupt state */
1435         mov     LMTX_C_REG, LMTX_A_REG          /* eax contains snapshot for cmpxchgl */
1436         or      $(M_ILOCKED_MSK | M_SPIN_MSK), LMTX_C_REG
1437         CLI                                     /* disable interrupts */
1438         lock
1439         cmpxchg LMTX_C_REG32, M_STATE(LMTX_REG) /* atomic compare and exchange */
1440         jne     1f
1441
1442         mov     %gs:CPU_ACTIVE_THREAD, LMTX_A_REG
1443         mov     LMTX_A_REG, M_OWNER(LMTX_REG)   /* record owner of interlock */
1444
1445         PREEMPTION_DISABLE
1446         POPF                            /* restore interrupt state */
1447
1448         LMTX_CHK_EXTENDED_EXIT
1449         /* return with the interlock held and preemption disabled */
1450         leave
1451 #if     CONFIG_DTRACE
1452         LOCKSTAT_LABEL(_lck_mtx_lock_spin_lockstat_patch_point)
1453         ret
1454         /* inherit lock pointer in LMTX_REG above */
1455         LOCKSTAT_RECORD(LS_LCK_MTX_LOCK_SPIN_ACQUIRE, LMTX_REG)
1456 #endif
1457         ret
1458
1459 1:
1460         POPF                            /* restore interrupt state */
1461         jmp     Llmls_loop
1462
1463
1464
1465 NONLEAF_ENTRY(lck_mtx_lock)
1466         LOAD_LMTX_REG(B_ARG0)           /* fetch lock pointer */
1467
1468         CHECK_PREEMPTION_LEVEL()
1469
1470         mov     M_STATE(LMTX_REG), LMTX_C_REG32
1471         test    $(M_ILOCKED_MSK), LMTX_C_REG    /* is the interlock held */
1472         je      Llml_enter                      /* no - can't be INDIRECT or DESTROYED */
1473
1474         mov     M_OWNER(LMTX_REG), LMTX_A_REG
1475         cmp     $(MUTEX_DESTROYED), LMTX_A_REG  /* check to see if its marked destroyed */
1476         je      lck_mtx_destroyed
1477         cmp     $(MUTEX_IND), LMTX_A_REG        /* Is this an indirect mutex? */
1478         jne     Llml_loop
1479
1480         LMTX_ENTER_EXTENDED
1481
1482         mov     M_STATE(LMTX_REG), LMTX_C_REG32
1483         test    $(M_SPIN_MSK), LMTX_C_REG
1484         je      Llml_loop
1485
1486         LMTX_UPDATE_MISS
1487 Llml_loop:
1488         PAUSE
1489         mov     M_STATE(LMTX_REG), LMTX_C_REG32
1490
1491         test    $(M_ILOCKED_MSK), LMTX_C_REG
1492         jne     Llml_loop
1493 Llml_enter:
1494         test    $(M_MLOCKED_MSK), LMTX_C_REG
1495         jne     Llml_contended                  /* mutex owned by someone else, go contend for it */
1496
1497         mov     LMTX_C_REG, LMTX_A_REG          /* eax contains snapshot for cmpxchgl */
1498         or      $(M_MLOCKED_MSK), LMTX_C_REG
1499         lock
1500         cmpxchg LMTX_C_REG32, M_STATE(LMTX_REG) /* atomic compare and exchange */
1501         jne     Llml_loop
1502
1503         mov     %gs:CPU_ACTIVE_THREAD, LMTX_A_REG
1504         mov     LMTX_A_REG, M_OWNER(LMTX_REG)   /* record owner of mutex */
1505
1506 Llml_acquired:
1507         testl   $(M_WAITERS_MSK), M_STATE(LMTX_REG)
1508         je      1f
1509
1510         LMTX_CALLEXT1(lck_mtx_lock_acquire_x86)
1511 1:
1512         LMTX_CHK_EXTENDED               /* is this an extended mutex */
1513         jne     2f
1514
1515         leave
1516 #if     CONFIG_DTRACE
1517         LOCKSTAT_LABEL(_lck_mtx_lock_lockstat_patch_point)
1518         ret
1519         /* inherit lock pointer in LMTX_REG above */
1520         LOCKSTAT_RECORD(LS_LCK_MTX_LOCK_ACQUIRE, LMTX_REG)
1521 #endif
1522         ret
1523 2:
1524         LMTX_EXIT_EXTENDED
1525         leave
1526 #if     CONFIG_DTRACE
1527         LOCKSTAT_LABEL(_lck_mtx_lock_ext_lockstat_patch_point)
1528         ret
1529         /* inherit lock pointer in LMTX_REG above */
1530         LOCKSTAT_RECORD(LS_LCK_MTX_EXT_LOCK_ACQUIRE, LMTX_REG)
1531 #endif
1532         ret
1533
1534
1535 Llml_contended:
1536         LMTX_CHK_EXTENDED               /* is this an extended mutex */
1537         je      0f
1538         LMTX_UPDATE_MISS
1539 0:
1540         LMTX_CALLEXT1(lck_mtx_lock_spinwait_x86)
1541
1542         test    LMTX_RET_REG, LMTX_RET_REG
1543         je      Llml_acquired           /* acquired mutex */
1544         cmp     $1, LMTX_RET_REG        /* check for direct wait status */
1545         je      2f
1546         LMTX_CHK_EXTENDED               /* is this an extended mutex */
1547         je      2f
1548         LMTX_UPDATE_DIRECT_WAIT
1549 2:
1550         mov     M_STATE(LMTX_REG), LMTX_C_REG32
1551         test    $(M_ILOCKED_MSK), LMTX_C_REG
1552         jne     6f
1553
1554         PUSHF                                   /* save state of interrupt mask */
1555         mov     LMTX_C_REG, LMTX_A_REG          /* eax contains snapshot for cmpxchgl */
1556         or      $(M_ILOCKED_MSK), LMTX_C_REG    /* try to take the interlock */
1557         CLI                                     /* disable interrupts */
1558         lock
1559         cmpxchg LMTX_C_REG32, M_STATE(LMTX_REG) /* atomic compare and exchange */
1560         jne     5f
1561
1562         test    $(M_MLOCKED_MSK), LMTX_C_REG    /* we've got the interlock and */
1563         jne     3f
1564         or      $(M_MLOCKED_MSK), LMTX_C_REG    /* the mutex is free... grab it directly */
1565         and     $(~M_ILOCKED_MSK), LMTX_C_REG
1566
1567         mov     %gs:CPU_ACTIVE_THREAD, LMTX_A_REG
1568         mov     LMTX_A_REG, M_OWNER(LMTX_REG)   /* record owner of mutex */
1569         mov     LMTX_C_REG32, M_STATE(LMTX_REG) /* now drop the interlock */
1570
1571         POPF                            /* restore interrupt state */
1572         jmp     Llml_acquired
1573 3:                                      /* interlock held, mutex busy */
1574         PREEMPTION_DISABLE
1575         POPF                            /* restore interrupt state */
1576
1577         LMTX_CHK_EXTENDED               /* is this an extended mutex */
1578         je      4f
1579         LMTX_UPDATE_WAIT
1580 4:
1581         LMTX_CALLEXT1(lck_mtx_lock_wait_x86)
1582         jmp     Llml_contended
1583 5:
1584         POPF                            /* restore interrupt state */
1585 6:
1586         PAUSE
1587         jmp     2b
1588
1589
1590
1591 NONLEAF_ENTRY(lck_mtx_try_lock_spin)
1592         LOAD_LMTX_REG(B_ARG0)                   /* fetch lock pointer */
1593
1594         mov     M_STATE(LMTX_REG), LMTX_C_REG32
1595         test    $(M_ILOCKED_MSK), LMTX_C_REG    /* is the interlock held */
1596         je      Llmts_enter                     /* no - can't be INDIRECT or DESTROYED */
1597
1598         mov     M_OWNER(LMTX_REG), LMTX_A_REG
1599         cmp     $(MUTEX_DESTROYED), LMTX_A_REG  /* check to see if its marked destroyed */
1600         je      lck_mtx_destroyed
1601         cmp     $(MUTEX_IND), LMTX_A_REG        /* Is this an indirect mutex? */
1602         jne     Llmts_enter
1603
1604         LMTX_ENTER_EXTENDED
1605 Llmts_loop:
1606         PAUSE
1607         mov     M_STATE(LMTX_REG), LMTX_C_REG32
1608 Llmts_enter:
1609         test    $(M_MLOCKED_MSK | M_SPIN_MSK), LMTX_C_REG
1610         jne     Llmts_fail
1611         test    $(M_ILOCKED_MSK), LMTX_C_REG
1612         jne     Llmts_loop
1613
1614         PUSHF                                   /* save interrupt state */
1615         mov     LMTX_C_REG, LMTX_A_REG          /* eax contains snapshot for cmpxchgl */
1616         or      $(M_ILOCKED_MSK | M_SPIN_MSK), LMTX_C_REG
1617         CLI                                     /* disable interrupts */
1618         lock
1619         cmpxchg LMTX_C_REG32, M_STATE(LMTX_REG) /* atomic compare and exchange */
1620         jne     3f
1621
1622         mov     %gs:CPU_ACTIVE_THREAD, LMTX_A_REG
1623         mov     LMTX_A_REG, M_OWNER(LMTX_REG)   /* record owner of mutex */
1624
1625         PREEMPTION_DISABLE
1626         POPF                            /* restore interrupt state */
1627
1628         LMTX_CHK_EXTENDED_EXIT
1629         leave
1630
1631 #if     CONFIG_DTRACE
1632         mov     $1, LMTX_RET_REG        /* return success */
1633         LOCKSTAT_LABEL(_lck_mtx_try_lock_spin_lockstat_patch_point)
1634         ret
1635         /* inherit lock pointer in LMTX_REG above */
1636         LOCKSTAT_RECORD(LS_LCK_MTX_TRY_SPIN_LOCK_ACQUIRE, LMTX_REG)
1637 #endif
1638         mov     $1, LMTX_RET_REG        /* return success */
1639         ret
1640 3:
1641         POPF                            /* restore interrupt state */
1642         jmp     Llmts_loop
1643
1644
1645
1646 NONLEAF_ENTRY(lck_mtx_try_lock)
1647         LOAD_LMTX_REG(B_ARG0)                   /* fetch lock pointer */
1648
1649         mov     M_STATE(LMTX_REG), LMTX_C_REG32
1650         test    $(M_ILOCKED_MSK), LMTX_C_REG    /* is the interlock held */
1651         je      Llmt_enter                      /* no - can't be INDIRECT or DESTROYED */
1652
1653         mov     M_OWNER(LMTX_REG), LMTX_A_REG
1654         cmp     $(MUTEX_DESTROYED), LMTX_A_REG  /* check to see if its marked destroyed */
1655         je      lck_mtx_destroyed
1656         cmp     $(MUTEX_IND), LMTX_A_REG        /* Is this an indirect mutex? */
1657         jne     Llmt_enter
1658
1659         LMTX_ENTER_EXTENDED
1660 Llmt_loop:
1661         PAUSE
1662         mov     M_STATE(LMTX_REG), LMTX_C_REG32
1663 Llmt_enter:
1664         test    $(M_MLOCKED_MSK | M_SPIN_MSK), LMTX_C_REG
1665         jne     Llmt_fail
1666         test    $(M_ILOCKED_MSK), LMTX_C_REG
1667         jne     Llmt_loop
1668
1669         mov     LMTX_C_REG, LMTX_A_REG          /* eax contains snapshot for cmpxchgl */
1670         or      $(M_MLOCKED_MSK), LMTX_C_REG
1671         lock
1672         cmpxchg LMTX_C_REG32, M_STATE(LMTX_REG) /* atomic compare and exchange */
1673         jne     Llmt_loop
1674
1675         mov     %gs:CPU_ACTIVE_THREAD, LMTX_A_REG
1676         mov     LMTX_A_REG, M_OWNER(LMTX_REG)   /* record owner of mutex */
1677
1678         LMTX_CHK_EXTENDED_EXIT
1679
1680         test    $(M_WAITERS_MSK), LMTX_C_REG
1681         je      2f
1682         LMTX_CALLEXT1(lck_mtx_lock_acquire_x86)
1683 2:
1684         leave
1685
1686 #if     CONFIG_DTRACE
1687         mov     $1, LMTX_RET_REG                /* return success */
1688         /* Dtrace probe: LS_LCK_MTX_TRY_LOCK_ACQUIRE */
1689         LOCKSTAT_LABEL(_lck_mtx_try_lock_lockstat_patch_point)
1690         ret
1691         /* inherit lock pointer in LMTX_REG from above */
1692         LOCKSTAT_RECORD(LS_LCK_MTX_TRY_LOCK_ACQUIRE, LMTX_REG)
1693 #endif
1694         mov     $1, LMTX_RET_REG                /* return success */
1695         ret
1696
1697
1698 Llmt_fail:
1699 Llmts_fail:
1700         LMTX_CHK_EXTENDED               /* is this an extended mutex */
1701         je      0f
1702         LMTX_UPDATE_MISS
1703         LMTX_EXIT_EXTENDED
1704 0:
1705         xor     LMTX_RET_REG, LMTX_RET_REG
1706         NONLEAF_RET
1707
1708
1709
1710 NONLEAF_ENTRY(lck_mtx_convert_spin)
1711         LOAD_LMTX_REG(B_ARG0)                   /* fetch lock pointer */
1712
1713         mov     M_OWNER(LMTX_REG), LMTX_A_REG
1714         cmp     $(MUTEX_IND), LMTX_A_REG        /* Is this an indirect mutex? */
1715         cmove   M_PTR(LMTX_REG), LMTX_REG       /* If so, take indirection */
1716
1717         mov     M_STATE(LMTX_REG), LMTX_C_REG32
1718         test    $(M_MLOCKED_MSK), LMTX_C_REG    /* already owned as a mutex, just return */
1719         jne     2f
1720 1:
1721         and     $(~(M_ILOCKED_MSK | M_SPIN_MSK)), LMTX_C_REG    /* convert from spin version to mutex */
1722         or      $(M_MLOCKED_MSK), LMTX_C_REG
1723         mov     LMTX_C_REG32, M_STATE(LMTX_REG)         /* since I own the interlock, I don't need an atomic update */
1724
1725         PREEMPTION_ENABLE                       /* only %eax is consumed */
1726
1727         test    $(M_WAITERS_MSK), LMTX_C_REG    /* are there any waiters? */
1728         je      2f
1729
1730         LMTX_CALLEXT1(lck_mtx_lock_acquire_x86)
1731 2:
1732         NONLEAF_RET
1733
1734
1735 #if     defined(__i386__)
1736 NONLEAF_ENTRY(lck_mtx_unlock)
1737         LOAD_LMTX_REG(B_ARG0)                   /* fetch lock pointer */
1738         mov     M_OWNER(LMTX_REG), LMTX_A_REG
1739         test    LMTX_A_REG, LMTX_A_REG
1740         jnz     Llmu_prim
1741         leave
1742         ret
1743 NONLEAF_ENTRY(lck_mtx_unlock_darwin10)
1744 #else
1745 NONLEAF_ENTRY(lck_mtx_unlock)
1746 #endif
1747         LOAD_LMTX_REG(B_ARG0)                   /* fetch lock pointer */
1748         mov     M_OWNER(LMTX_REG), LMTX_A_REG
1749 Llmu_prim:
1750         cmp     $(MUTEX_IND), LMTX_A_REG        /* Is this an indirect mutex? */
1751         je      Llmu_ext
1752 0:
1753         mov     M_STATE(LMTX_REG), LMTX_C_REG32
1754         test    $(M_MLOCKED_MSK), LMTX_C_REG    /* check for full mutex */
1755         jne     1f
1756
1757         xor     LMTX_A_REG, LMTX_A_REG
1758         mov     LMTX_A_REG, M_OWNER(LMTX_REG)
1759         mov     LMTX_C_REG, LMTX_A_REG                  /* keep original state in %ecx for later evaluation */
1760         and     $(~(M_ILOCKED_MSK | M_SPIN_MSK | M_PROMOTED_MSK)), LMTX_A_REG
1761         mov     LMTX_A_REG32, M_STATE(LMTX_REG)         /* since I own the interlock, I don't need an atomic update */
1762
1763         PREEMPTION_ENABLE                       /* need to re-enable preemption - clobbers eax */
1764         jmp     2f
1765 1:
1766         test    $(M_ILOCKED_MSK), LMTX_C_REG    /* have to wait for interlock to clear */
1767         jne     7f
1768
1769         PUSHF                                   /* save interrupt state */
1770         mov     LMTX_C_REG, LMTX_A_REG          /* eax contains snapshot for cmpxchgl */
1771         and     $(~M_MLOCKED_MSK), LMTX_C_REG   /* drop mutex */
1772         or      $(M_ILOCKED_MSK), LMTX_C_REG    /* pick up interlock */
1773         CLI
1774         lock
1775         cmpxchg LMTX_C_REG32, M_STATE(LMTX_REG) /* atomic compare and exchange */
1776         jne     6f                              /* branch on failure to spin loop */
1777
1778         xor     LMTX_A_REG, LMTX_A_REG
1779         mov     LMTX_A_REG, M_OWNER(LMTX_REG)
1780         mov     LMTX_C_REG, LMTX_A_REG                  /* keep original state in %ecx for later evaluation */
1781         and     $(~(M_ILOCKED_MSK | M_PROMOTED_MSK)), LMTX_A_REG
1782         mov     LMTX_A_REG32, M_STATE(LMTX_REG)         /* since I own the interlock, I don't need an atomic update */
1783         POPF                                            /* restore interrupt state */
1784 2:
1785         test    $(M_PROMOTED_MSK | M_WAITERS_MSK), LMTX_C_REG
1786         je      3f
1787         and     $(M_PROMOTED_MSK), LMTX_C_REG
1788
1789         LMTX_CALLEXT2(lck_mtx_unlock_wakeup_x86, LMTX_C_REG)
1790 3:
1791         LMTX_CHK_EXTENDED
1792         jne     4f
1793
1794         leave
1795 #if     CONFIG_DTRACE
1796         /* Dtrace: LS_LCK_MTX_UNLOCK_RELEASE */
1797         LOCKSTAT_LABEL(_lck_mtx_unlock_lockstat_patch_point)
1798         ret
1799         /* inherit lock pointer in LMTX_REG from above */
1800         LOCKSTAT_RECORD(LS_LCK_MTX_UNLOCK_RELEASE, LMTX_REG)
1801 #endif
1802         ret
1803 4:
1804         leave
1805 #if     CONFIG_DTRACE
1806         /* Dtrace: LS_LCK_MTX_EXT_UNLOCK_RELEASE */
1807         LOCKSTAT_LABEL(_lck_mtx_ext_unlock_lockstat_patch_point)
1808         ret
1809         /* inherit lock pointer in LMTX_REG from above */
1810         LOCKSTAT_RECORD(LS_LCK_MTX_EXT_UNLOCK_RELEASE, LMTX_REG)
1811 #endif
1812         ret
1813 6:
1814         POPF                            /* restore interrupt state */
1815 7:
1816         PAUSE
1817         mov     M_STATE(LMTX_REG), LMTX_C_REG32
1818         jmp     1b
1819 Llmu_ext:
1820         mov     M_PTR(LMTX_REG), LMTX_REG
1821         mov     M_OWNER(LMTX_REG), LMTX_A_REG
1822         mov     %gs:CPU_ACTIVE_THREAD, LMTX_C_REG
1823         CHECK_UNLOCK(LMTX_C_REG, LMTX_A_REG)
1824         jmp 0b
1825
1826
1827 LEAF_ENTRY(lck_mtx_lock_decr_waiter)
1828         LOAD_LMTX_REG(L_ARG0)                   /* fetch lock pointer - no indirection here */
1829 1:
1830         mov     M_STATE(LMTX_REG), LMTX_C_REG32
1831
1832         test    $(M_WAITERS_MSK), LMTX_C_REG
1833         je      2f
1834         test    $(M_ILOCKED_MSK), LMTX_C_REG    /* have to wait for interlock to clear */
1835         jne     3f
1836
1837         mov     LMTX_C_REG, LMTX_A_REG          /* eax contains snapshot for cmpxchgl */
1838         dec     LMTX_C_REG                      /* decrement waiter count */
1839         lock
1840         cmpxchg LMTX_C_REG32, M_STATE(LMTX_REG) /* atomic compare and exchange */
1841         jne     3f                              /* branch on failure to spin loop */
1842
1843         mov     $1, LMTX_RET_REG
1844         LEAF_RET
1845 2:
1846         xor     LMTX_RET_REG, LMTX_RET_REG
1847         LEAF_RET
1848 3:
1849         PAUSE
1850         jmp     1b
1851
1852
1853
1854 LEAF_ENTRY(lck_mtx_lock_get_pri)
1855         LOAD_LMTX_REG(L_ARG0)                   /* fetch lock pointer - no indirection here */
1856 1:
1857         mov     M_STATE(LMTX_REG), LMTX_C_REG32
1858
1859         test    $(M_WAITERS_MSK), LMTX_C_REG
1860         jne     2f
1861         test    $(M_ILOCKED_MSK), LMTX_C_REG    /* have to wait for interlock to clear */
1862         jne     3f
1863
1864         mov     LMTX_C_REG, LMTX_A_REG          /* eax contains snapshot for cmpxchgl */
1865         and     $(~M_PRIORITY_MSK), LMTX_C_REG  /* no waiters, reset mutex priority to 0 */
1866         lock
1867         cmpxchg LMTX_C_REG32, M_STATE(LMTX_REG) /* atomic compare and exchange */
1868         jne     3f                              /* branch on failure to spin loop */
1869
1870         xor     LMTX_RET_REG, LMTX_RET_REG      /* return mutex priority == 0 */
1871         LEAF_RET
1872 2:
1873         mov     LMTX_C_REG, LMTX_RET_REG
1874         and     $(M_PRIORITY_MSK), LMTX_RET_REG
1875         shr     $16, LMTX_RET_REG               /* return current mutex priority */
1876         LEAF_RET
1877 3:
1878         PAUSE
1879         jmp     1b
1880
1881
1882
1883
1884 LEAF_ENTRY(lck_mtx_ilk_unlock)
1885         LOAD_LMTX_REG(L_ARG0)                   /* fetch lock pointer - no indirection here */
1886
1887         andl    $(~M_ILOCKED_MSK), M_STATE(LMTX_REG)
1888
1889         PREEMPTION_ENABLE                       /* need to re-enable preemption */
1890
1891         LEAF_RET
1892
1893
1894
1895 LEAF_ENTRY(lck_mtx_lock_grab_mutex)
1896         LOAD_LMTX_REG(L_ARG0)                   /* fetch lock pointer - no indirection here */
1897
1898         mov     M_STATE(LMTX_REG), LMTX_C_REG32
1899
1900         test    $(M_ILOCKED_MSK | M_MLOCKED_MSK), LMTX_C_REG    /* can't have the mutex yet */
1901         jne     2f
1902
1903         mov     LMTX_C_REG, LMTX_A_REG          /* eax contains snapshot for cmpxchgl */
1904         or      $(M_MLOCKED_MSK), LMTX_C_REG
1905         lock
1906         cmpxchg LMTX_C_REG32, M_STATE(LMTX_REG) /* atomic compare and exchange */
1907         jne     2f                              /* branch on failure to spin loop */
1908
1909         mov     %gs:CPU_ACTIVE_THREAD, LMTX_A_REG
1910         mov     LMTX_A_REG, M_OWNER(LMTX_REG)   /* record owner of mutex */
1911
1912         mov     $1, LMTX_RET_REG                /* return success */
1913         LEAF_RET
1914 2:
1915         xor     LMTX_RET_REG, LMTX_RET_REG      /* return failure */
1916         LEAF_RET
1917
1918
1919
1920 LEAF_ENTRY(lck_mtx_lock_mark_promoted)
1921         LOAD_LMTX_REG(L_ARG0)                   /* fetch lock pointer - no indirection here */
1922 1:
1923         mov     M_STATE(LMTX_REG), LMTX_C_REG32
1924
1925         test    $(M_PROMOTED_MSK), LMTX_C_REG
1926         jne     3f
1927         test    $(M_ILOCKED_MSK), LMTX_C_REG    /* have to wait for interlock to clear */
1928         jne     2f
1929
1930         mov     LMTX_C_REG, LMTX_A_REG          /* eax contains snapshot for cmpxchgl */
1931         or      $(M_PROMOTED_MSK), LMTX_C_REG
1932         lock
1933         cmpxchg LMTX_C_REG32, M_STATE(LMTX_REG) /* atomic compare and exchange */
1934         jne     2f                              /* branch on failure to spin loop */
1935
1936         mov     $1, LMTX_RET_REG
1937         LEAF_RET
1938 2:
1939         PAUSE
1940         jmp     1b
1941 3:
1942         xor     LMTX_RET_REG, LMTX_RET_REG
1943         LEAF_RET
1944
1945
1946
1947 LEAF_ENTRY(lck_mtx_lock_mark_destroyed)
1948         LOAD_LMTX_REG(L_ARG0)
1949 1:
1950         mov     M_OWNER(LMTX_REG), LMTX_A_REG
1951
1952         cmp     $(MUTEX_DESTROYED), LMTX_A_REG  /* check to see if its marked destroyed */
1953         je      3f
1954         cmp     $(MUTEX_IND), LMTX_A_REG        /* Is this an indirect mutex? */
1955         jne     2f
1956
1957         movl    $(MUTEX_DESTROYED), M_OWNER(LMTX_REG)   /* convert to destroyed state */
1958         jmp     3f
1959 2:
1960         mov     M_STATE(LMTX_REG), LMTX_C_REG32
1961
1962         test    $(M_ILOCKED_MSK), LMTX_C_REG    /* have to wait for interlock to clear */
1963         jne     5f
1964
1965         PUSHF                                   /* save interrupt state */
1966         mov     LMTX_C_REG, LMTX_A_REG          /* eax contains snapshot for cmpxchgl */
1967         or      $(M_ILOCKED_MSK), LMTX_C_REG
1968         CLI
1969         lock
1970         cmpxchg LMTX_C_REG32, M_STATE(LMTX_REG) /* atomic compare and exchange */
1971         jne     4f                              /* branch on failure to spin loop */
1972         movl    $(MUTEX_DESTROYED), M_OWNER(LMTX_REG)   /* convert to destroyed state */
1973         POPF                                    /* restore interrupt state */
1974 3:
1975         LEAF_RET                                /* return with M_ILOCKED set */
1976 4:
1977         POPF                                    /* restore interrupt state */
1978 5:
1979         PAUSE
1980         jmp     1b
1981
1982
1983
1984 LEAF_ENTRY(_disable_preemption)
1985 #if     MACH_RT
1986         _DISABLE_PREEMPTION
1987 #endif  /* MACH_RT */
1988         LEAF_RET
1989
1990 LEAF_ENTRY(_enable_preemption)
1991 #if     MACH_RT
1992 #if     MACH_ASSERT
1993         cmpl    $0,%gs:CPU_PREEMPTION_LEVEL
1994         jg      1f
1995 #if __i386__
1996         pushl   %gs:CPU_PREEMPTION_LEVEL
1997 #else
1998         movl    %gs:CPU_PREEMPTION_LEVEL,%esi
1999 #endif
2000         LOAD_STRING_ARG0(_enable_preemption_less_than_zero)
2001         CALL_PANIC()
2002         hlt
2003         .cstring
2004 _enable_preemption_less_than_zero:
2005         .asciz  "_enable_preemption: preemption_level(%d)  < 0!"
2006         .text
2007 1:
2008 #endif  /* MACH_ASSERT */
2009         _ENABLE_PREEMPTION
2010 #endif  /* MACH_RT */
2011         LEAF_RET
2012
2013 LEAF_ENTRY(_enable_preemption_no_check)
2014 #if     MACH_RT
2015 #if     MACH_ASSERT
2016         cmpl    $0,%gs:CPU_PREEMPTION_LEVEL
2017         jg      1f
2018         LOAD_STRING_ARG0(_enable_preemption_no_check_less_than_zero)
2019         CALL_PANIC()
2020         hlt
2021         .cstring
2022 _enable_preemption_no_check_less_than_zero:
2023         .asciz  "_enable_preemption_no_check: preemption_level <= 0!"
2024         .text
2025 1:
2026 #endif  /* MACH_ASSERT */
2027         _ENABLE_PREEMPTION_NO_CHECK
2028 #endif  /* MACH_RT */
2029         LEAF_RET
2030
2031
2032 LEAF_ENTRY(_mp_disable_preemption)
2033 #if     MACH_RT
2034         _DISABLE_PREEMPTION
2035 #endif  /* MACH_RT */
2036         LEAF_RET
2037
2038 LEAF_ENTRY(_mp_enable_preemption)
2039 #if     MACH_RT
2040 #if     MACH_ASSERT
2041         cmpl    $0,%gs:CPU_PREEMPTION_LEVEL
2042         jg      1f
2043 #if __i386__
2044         pushl   %gs:CPU_PREEMPTION_LEVEL
2045 #else
2046         movl    %gs:CPU_PREEMPTION_LEVEL,%esi
2047 #endif
2048         LOAD_STRING_ARG0(_mp_enable_preemption_less_than_zero)
2049         CALL_PANIC()
2050         hlt
2051         .cstring
2052 _mp_enable_preemption_less_than_zero:
2053         .asciz "_mp_enable_preemption: preemption_level (%d) <= 0!"
2054         .text
2055 1:
2056 #endif  /* MACH_ASSERT */
2057         _ENABLE_PREEMPTION
2058 #endif  /* MACH_RT */
2059         LEAF_RET
2060
2061 LEAF_ENTRY(_mp_enable_preemption_no_check)
2062 #if     MACH_RT
2063 #if     MACH_ASSERT
2064         cmpl    $0,%gs:CPU_PREEMPTION_LEVEL
2065         jg      1f
2066         LOAD_STRING_ARG0(_mp_enable_preemption_no_check_less_than_zero)
2067         CALL_PANIC()
2068         hlt
2069         .cstring
2070 _mp_enable_preemption_no_check_less_than_zero:
2071         .asciz "_mp_enable_preemption_no_check: preemption_level <= 0!"
2072         .text
2073 1:
2074 #endif  /* MACH_ASSERT */
2075         _ENABLE_PREEMPTION_NO_CHECK
2076 #endif  /* MACH_RT */
2077         LEAF_RET
2078
2079 #if __i386__
2080
2081 LEAF_ENTRY(i_bit_set)
2082         movl    L_ARG0,%edx
2083         movl    L_ARG1,%eax
2084         lock
2085         bts     %edx,(%eax)
2086         LEAF_RET
2087
2088 LEAF_ENTRY(i_bit_clear)
2089         movl    L_ARG0,%edx
2090         movl    L_ARG1,%eax
2091         lock
2092         btr     %edx,(%eax)
2093         LEAF_RET
2094
2095
2096 LEAF_ENTRY(bit_lock)
2097         movl    L_ARG0,%ecx
2098         movl    L_ARG1,%eax
2099 1:
2100         lock
2101         bts     %ecx,(%eax)
2102         jb      1b
2103         LEAF_RET
2104
2105
2106 LEAF_ENTRY(bit_lock_try)
2107         movl    L_ARG0,%ecx
2108         movl    L_ARG1,%eax
2109         lock
2110         bts     %ecx,(%eax)
2111         jb      bit_lock_failed
2112         LEAF_RET                /* %eax better not be null ! */
2113 bit_lock_failed:
2114         xorl    %eax,%eax
2115         LEAF_RET
2116
2117 LEAF_ENTRY(bit_unlock)
2118         movl    L_ARG0,%ecx
2119         movl    L_ARG1,%eax
2120         lock
2121         btr     %ecx,(%eax)
2122         LEAF_RET
2123
2124 /*
2125  * Atomic primitives, prototyped in kern/simple_lock.h
2126  */
2127 LEAF_ENTRY(hw_atomic_add)
2128         movl    L_ARG0, %ecx            /* Load address of operand */
2129         movl    L_ARG1, %eax            /* Load addend */
2130         movl    %eax, %edx
2131         lock
2132         xaddl   %eax, (%ecx)            /* Atomic exchange and add */
2133         addl    %edx, %eax              /* Calculate result */
2134         LEAF_RET
2135
2136 LEAF_ENTRY(hw_atomic_sub)
2137         movl    L_ARG0, %ecx            /* Load address of operand */
2138         movl    L_ARG1, %eax            /* Load subtrahend */
2139         negl    %eax
2140         movl    %eax, %edx
2141         lock
2142         xaddl   %eax, (%ecx)            /* Atomic exchange and add */
2143         addl    %edx, %eax              /* Calculate result */
2144         LEAF_RET
2145
2146 LEAF_ENTRY(hw_atomic_or)
2147         movl    L_ARG0, %ecx            /* Load address of operand */
2148         movl    (%ecx), %eax
2149 1:
2150         movl    L_ARG1, %edx            /* Load mask */
2151         orl     %eax, %edx
2152         lock
2153         cmpxchgl        %edx, (%ecx)    /* Atomic CAS */
2154         jne     1b
2155         movl    %edx, %eax              /* Result */
2156         LEAF_RET
2157 /*
2158  * A variant of hw_atomic_or which doesn't return a value.
2159  * The implementation is thus comparatively more efficient.
2160  */
2161
2162 LEAF_ENTRY(hw_atomic_or_noret)
2163         movl    L_ARG0, %ecx            /* Load address of operand */
2164         movl    L_ARG1, %edx            /* Load mask */
2165         lock
2166         orl     %edx, (%ecx)            /* Atomic OR */
2167         LEAF_RET
2168
2169 LEAF_ENTRY(hw_atomic_and)
2170         movl    L_ARG0, %ecx            /* Load address of operand */
2171         movl    (%ecx), %eax
2172 1:
2173         movl    L_ARG1, %edx            /* Load mask */
2174         andl    %eax, %edx
2175         lock
2176         cmpxchgl        %edx, (%ecx)    /* Atomic CAS */
2177         jne     1b
2178         movl    %edx, %eax              /* Result */
2179         LEAF_RET
2180 /*
2181  * A variant of hw_atomic_and which doesn't return a value.
2182  * The implementation is thus comparatively more efficient.
2183  */
2184
2185 LEAF_ENTRY(hw_atomic_and_noret)
2186         movl    L_ARG0, %ecx            /* Load address of operand */
2187         movl    L_ARG1, %edx            /* Load mask */
2188         lock
2189         andl    %edx, (%ecx)            /* Atomic AND */
2190         LEAF_RET
2191
2192 #else /* !__i386__ */
2193
2194 LEAF_ENTRY(i_bit_set)
2195         lock
2196         bts     %edi,(%rsi)
2197         LEAF_RET
2198
2199 LEAF_ENTRY(i_bit_clear)
2200         lock
2201         btr     %edi,(%rsi)
2202         LEAF_RET
2203
2204
2205 LEAF_ENTRY(bit_lock)
2206 1:
2207         lock
2208         bts     %edi,(%rsi)
2209         jb      1b
2210         LEAF_RET
2211
2212
2213 LEAF_ENTRY(bit_lock_try)
2214         lock
2215         bts     %edi,(%rsi)
2216         jb      bit_lock_failed
2217         movl    $1, %eax
2218         LEAF_RET
2219 bit_lock_failed:
2220         xorl    %eax,%eax
2221         LEAF_RET
2222
2223 LEAF_ENTRY(bit_unlock)
2224         lock
2225         btr     %edi,(%rsi)
2226         LEAF_RET
2227
2228
2229 /*
2230  * Atomic primitives, prototyped in kern/simple_lock.h
2231  */
2232 LEAF_ENTRY(hw_atomic_add)
2233         movl    %esi, %eax              /* Load addend */
2234         lock
2235         xaddl   %eax, (%rdi)            /* Atomic exchange and add */
2236         addl    %esi, %eax              /* Calculate result */
2237         LEAF_RET
2238
2239 LEAF_ENTRY(hw_atomic_sub)
2240         negl    %esi
2241         movl    %esi, %eax
2242         lock
2243         xaddl   %eax, (%rdi)            /* Atomic exchange and add */
2244         addl    %esi, %eax              /* Calculate result */
2245         LEAF_RET
2246
2247 LEAF_ENTRY(hw_atomic_or)
2248         movl    (%rdi), %eax
2249 1:
2250         movl    %esi, %edx              /* Load mask */
2251         orl     %eax, %edx
2252         lock
2253         cmpxchgl        %edx, (%rdi)    /* Atomic CAS */
2254         jne     1b
2255         movl    %edx, %eax              /* Result */
2256         LEAF_RET
2257 /*
2258  * A variant of hw_atomic_or which doesn't return a value.
2259  * The implementation is thus comparatively more efficient.
2260  */
2261
2262 LEAF_ENTRY(hw_atomic_or_noret)
2263         lock
2264         orl     %esi, (%rdi)            /* Atomic OR */
2265         LEAF_RET
2266
2267
2268 LEAF_ENTRY(hw_atomic_and)
2269         movl    (%rdi), %eax
2270 1:
2271         movl    %esi, %edx              /* Load mask */
2272         andl    %eax, %edx
2273         lock
2274         cmpxchgl        %edx, (%rdi)    /* Atomic CAS */
2275         jne     1b
2276         movl    %edx, %eax              /* Result */
2277         LEAF_RET
2278 /*
2279  * A variant of hw_atomic_and which doesn't return a value.
2280  * The implementation is thus comparatively more efficient.
2281  */
2282
2283 LEAF_ENTRY(hw_atomic_and_noret)
2284         lock
2285         andl    %esi, (%rdi)            /* Atomic OR */
2286         LEAF_RET
2287
2288 #endif /* !__i386 __ */