osfmk/i386/i386_lock.s

   1 /*
   2  * Copyright (c) 2000 Apple Computer, Inc. All rights reserved.
   3  *
   4  * @APPLE_LICENSE_HEADER_START@
   5  *
   6  * The contents of this file constitute Original Code as defined in and
   7  * are subject to the Apple Public Source License Version 1.1 (the
   8  * "License").  You may not use this file except in compliance with the
   9  * License.  Please obtain a copy of the License at
  10  * http://www.apple.com/publicsource and read it before using this file.
  11  *
  12  * This Original Code and all software distributed under the License are
  13  * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  14  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  15  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
  16  * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
  17  * License for the specific language governing rights and limitations
  18  * under the License.
  19  *
  20  * @APPLE_LICENSE_HEADER_END@
  21  */
  22 /*
  23  * @OSF_COPYRIGHT@
  24  */
  25 /*
  26  * Mach Operating System
  27  * Copyright (c) 1989 Carnegie-Mellon University
  28  * All rights reserved.  The CMU software License Agreement specifies
  29  * the terms and conditions for use and redistribution.
  30  */
  31
  32 #include <mach_rt.h>
  33 #include <platforms.h>
  34 #include <mach_ldebug.h>
  35 #include <i386/asm.h>
  36
  37 #include "assym.s"
  38
  39 #define PAUSE           rep; nop
  40
  41 /*
  42  *      When performance isn't the only concern, it's
  43  *      nice to build stack frames...
  44  */
  45 #define BUILD_STACK_FRAMES   (GPROF || \
  46                                 ((MACH_LDEBUG || ETAP_LOCK_TRACE) && MACH_KDB))
  47
  48 #if     BUILD_STACK_FRAMES
  49
  50 /* STack-frame-relative: */
  51 #define L_PC            B_PC
  52 #define L_ARG0          B_ARG0
  53 #define L_ARG1          B_ARG1
  54
  55 #define LEAF_ENTRY(name)        \
  56         Entry(name);            \
  57         FRAME;                  \
  58         MCOUNT
  59
  60 #define LEAF_ENTRY2(n1,n2)      \
  61         Entry(n1);              \
  62         Entry(n2);              \
  63         FRAME;                  \
  64         MCOUNT
  65
  66 #define LEAF_RET                \
  67         EMARF;                  \
  68         ret
  69
  70 #else   /* BUILD_STACK_FRAMES */
  71
  72 /* Stack-pointer-relative: */
  73 #define L_PC            S_PC
  74 #define L_ARG0          S_ARG0
  75 #define L_ARG1          S_ARG1
  76
  77 #define LEAF_ENTRY(name)        \
  78         Entry(name)
  79
  80 #define LEAF_ENTRY2(n1,n2)      \
  81         Entry(n1);              \
  82         Entry(n2)
  83
  84 #define LEAF_RET                \
  85         ret
  86
  87 #endif  /* BUILD_STACK_FRAMES */
  88
  89
  90 /* Non-leaf routines always have a stack frame: */
  91
  92 #define NONLEAF_ENTRY(name)     \
  93         Entry(name);            \
  94         FRAME;                  \
  95         MCOUNT
  96
  97 #define NONLEAF_ENTRY2(n1,n2)   \
  98         Entry(n1);              \
  99         Entry(n2);              \
 100         FRAME;                  \
 101         MCOUNT
 102
 103 #define NONLEAF_RET             \
 104         EMARF;                  \
 105         ret
 106
 107
 108 #define M_ILK           (%edx)
 109 #define M_LOCKED        MUTEX_LOCKED(%edx)
 110 #define M_WAITERS       MUTEX_WAITERS(%edx)
 111 #define M_PROMOTED_PRI  MUTEX_PROMOTED_PRI(%edx)
 112 #define M_ITAG          MUTEX_ITAG(%edx)
 113 #define M_PTR           MUTEX_PTR(%edx)
 114 #if     MACH_LDEBUG
 115 #define M_TYPE          MUTEX_TYPE(%edx)
 116 #define M_PC            MUTEX_PC(%edx)
 117 #define M_THREAD        MUTEX_THREAD(%edx)
 118 #endif  /* MACH_LDEBUG */
 119
 120 #include <i386/mp.h>
 121 #define CX(addr,reg)    addr(,reg,4)
 122
 123 #if     MACH_LDEBUG
 124 /*
 125  *  Routines for general lock debugging.
 126  */
 127
 128 /*
 129  * Checks for expected lock types and calls "panic" on
 130  * mismatch.  Detects calls to Mutex functions with
 131  * type simplelock and vice versa.
 132  */
 133 #define CHECK_MUTEX_TYPE()                                      \
 134         cmpl    $ MUTEX_TAG,M_TYPE                      ;       \
 135         je      1f                                      ;       \
 136         pushl   $2f                                     ;       \
 137         call    EXT(panic)                              ;       \
 138         hlt                                             ;       \
 139         .data                                           ;       \
 140 2:      String  "not a mutex!"                          ;       \
 141         .text                                           ;       \
 142 1:
 143
 144 /*
 145  * If one or more simplelocks are currently held by a thread,
 146  * an attempt to acquire a mutex will cause this check to fail
 147  * (since a mutex lock may context switch, holding a simplelock
 148  * is not a good thing).
 149  */
 150 #if     MACH_RT
 151 #define CHECK_PREEMPTION_LEVEL()                                \
 152         cmpl    $0,%gs:CPU_PREEMPTION_LEVEL             ;       \
 153         je      1f                                      ;       \
 154         pushl   $2f                                     ;       \
 155         call    EXT(panic)                              ;       \
 156         hlt                                             ;       \
 157         .data                                           ;       \
 158 2:      String  "preemption_level != 0!"                ;       \
 159         .text                                           ;       \
 160 1:
 161 #else   /* MACH_RT */
 162 #define CHECK_PREEMPTION_LEVEL()
 163 #endif  /* MACH_RT */
 164
 165 #define CHECK_NO_SIMPLELOCKS()                                  \
 166         cmpl    $0,%gs:CPU_SIMPLE_LOCK_COUNT            ;       \
 167         je      1f                                      ;       \
 168         pushl   $2f                                     ;       \
 169         call    EXT(panic)                              ;       \
 170         hlt                                             ;       \
 171         .data                                           ;       \
 172 2:      String  "simple_locks_held!"                    ;       \
 173         .text                                           ;       \
 174 1:
 175
 176 /*
 177  * Verifies return to the correct thread in "unlock" situations.
 178  */
 179 #define CHECK_THREAD(thd)                                       \
 180         movl    %gs:CPU_ACTIVE_THREAD,%ecx              ;       \
 181         testl   %ecx,%ecx                               ;       \
 182         je      1f                                      ;       \
 183         cmpl    %ecx,thd                                ;       \
 184         je      1f                                      ;       \
 185         pushl   $2f                                     ;       \
 186         call    EXT(panic)                              ;       \
 187         hlt                                             ;       \
 188         .data                                           ;       \
 189 2:      String  "wrong thread!"                         ;       \
 190         .text                                           ;       \
 191 1:
 192
 193 #define CHECK_MYLOCK(thd)                                       \
 194         movl    %gs:CPU_ACTIVE_THREAD,%ecx              ;       \
 195         testl   %ecx,%ecx                               ;       \
 196         je      1f                                      ;       \
 197         cmpl    %ecx,thd                                ;       \
 198         jne     1f                                      ;       \
 199         pushl   $2f                                     ;       \
 200         call    EXT(panic)                              ;       \
 201         hlt                                             ;       \
 202         .data                                           ;       \
 203 2:      String  "mylock attempt!"                       ;       \
 204         .text                                           ;       \
 205 1:
 206
 207 #define METER_SIMPLE_LOCK_LOCK(reg)                             \
 208         pushl   reg                                     ;       \
 209         call    EXT(meter_simple_lock)                  ;       \
 210         popl    reg
 211
 212 #define METER_SIMPLE_LOCK_UNLOCK(reg)                           \
 213         pushl   reg                                     ;       \
 214         call    EXT(meter_simple_unlock)                ;       \
 215         popl    reg
 216
 217 #else   /* MACH_LDEBUG */
 218 #define CHECK_MUTEX_TYPE()
 219 #define CHECK_SIMPLE_LOCK_TYPE
 220 #define CHECK_THREAD(thd)
 221 #define CHECK_PREEMPTION_LEVEL()
 222 #define CHECK_NO_SIMPLELOCKS()
 223 #define CHECK_MYLOCK(thd)
 224 #define METER_SIMPLE_LOCK_LOCK(reg)
 225 #define METER_SIMPLE_LOCK_UNLOCK(reg)
 226 #endif  /* MACH_LDEBUG */
 227
 228
 229 /*
 230  *      void hw_lock_init(hw_lock_t)
 231  *
 232  *      Initialize a hardware lock.
 233  */
 234 LEAF_ENTRY(hw_lock_init)
 235         movl    L_ARG0,%edx             /* fetch lock pointer */
 236         movl    $0,0(%edx)              /* clear the lock */
 237         LEAF_RET
 238
 239 /*
 240  *      void hw_lock_lock(hw_lock_t)
 241  *
 242  *      Acquire lock, spinning until it becomes available.
 243  *      MACH_RT:  also return with preemption disabled.
 244  */
 245 LEAF_ENTRY(hw_lock_lock)
 246         movl    L_ARG0,%edx             /* fetch lock pointer */
 247
 248         movl    %gs:CPU_ACTIVE_THREAD,%ecx
 249         DISABLE_PREEMPTION
 250 1:
 251         movl    0(%edx), %eax
 252         testl   %eax,%eax               /* lock locked? */
 253         jne     3f                      /* branch if so */
 254         lock; cmpxchgl  %ecx,0(%edx)    /* try to acquire the HW lock */
 255         jne     3f
 256         movl    $1,%eax                 /* In case this was a timeout call */
 257         LEAF_RET                        /* if yes, then nothing left to do */
 258 3:
 259         PAUSE                           /* pause for hyper-threading */
 260         jmp     1b                      /* try again */
 261
 262 /*
 263  *      unsigned int hw_lock_to(hw_lock_t, unsigned int)
 264  *
 265  *      Acquire lock, spinning until it becomes available or timeout.
 266  *      MACH_RT:  also return with preemption disabled.
 267  */
 268 LEAF_ENTRY(hw_lock_to)
 269 1:
 270         movl    L_ARG0,%edx             /* fetch lock pointer */
 271         movl    %gs:CPU_ACTIVE_THREAD,%ecx
 272         /*
 273          * Attempt to grab the lock immediately
 274          * - fastpath without timeout nonsense.
 275          */
 276         DISABLE_PREEMPTION
 277         movl    0(%edx), %eax
 278         testl   %eax,%eax               /* lock locked? */
 279         jne     2f                      /* branch if so */
 280         lock; cmpxchgl  %ecx,0(%edx)    /* try to acquire the HW lock */
 281         jne     2f                      /* branch on failure */
 282         movl    $1,%eax
 283         LEAF_RET
 284
 285 2:
 286 #define INNER_LOOP_COUNT        1000
 287         /*
 288          * Failed to get the lock so set the timeout
 289          * and then spin re-checking the lock but pausing
 290          * every so many (INNER_LOOP_COUNT) spins to check for timeout.
 291          */
 292         movl    L_ARG1,%ecx             /* fetch timeout */
 293         push    %edi
 294         push    %ebx
 295         mov     %edx,%edi
 296
 297         rdtsc                           /* read cyclecount into %edx:%eax */
 298         addl    %ecx,%eax               /* fetch and timeout */
 299         adcl    $0,%edx                 /* add carry */
 300         mov     %edx,%ecx
 301         mov     %eax,%ebx               /* %ecx:%ebx is the timeout expiry */
 302 4:
 303         /*
 304          * The inner-loop spin to look for the lock being freed.
 305          */
 306         mov     $(INNER_LOOP_COUNT),%edx
 307 5:
 308         PAUSE                           /* pause for hyper-threading */
 309         movl    0(%edi),%eax            /* spin checking lock value in cache */
 310         testl   %eax,%eax
 311         je      6f                      /* zero => unlocked, try to grab it */
 312         decl    %edx                    /* decrement inner loop count */
 313         jnz     5b                      /* time to check for timeout? */
 314
 315         /*
 316          * Here after spinning INNER_LOOP_COUNT times, check for timeout
 317          */
 318         rdtsc                           /* cyclecount into %edx:%eax */
 319         cmpl    %ecx,%edx               /* compare high-order 32-bits */
 320         jb      4b                      /* continue spinning if less, or */
 321         cmpl    %ebx,%eax               /* compare low-order 32-bits */
 322         jb      4b                      /* continue if less, else bail */
 323         xor     %eax,%eax               /* with 0 return value */
 324         pop     %ebx
 325         pop     %edi
 326         LEAF_RET
 327
 328 6:
 329         /*
 330          * Here to try to grab the lock that now appears to be free
 331          * after contention.
 332          */
 333         movl    %gs:CPU_ACTIVE_THREAD,%edx
 334         lock; cmpxchgl  %edx,0(%edi)    /* try to acquire the HW lock */
 335         jne     4b                      /* no - spin again */
 336         movl    $1,%eax                 /* yes */
 337         pop     %ebx
 338         pop     %edi
 339         LEAF_RET
 340
 341 /*
 342  *      void hw_lock_unlock(hw_lock_t)
 343  *
 344  *      Unconditionally release lock.
 345  *      MACH_RT:  release preemption level.
 346  */
 347 LEAF_ENTRY(hw_lock_unlock)
 348         movl    L_ARG0,%edx             /* fetch lock pointer */
 349         movl    $0,0(%edx)              /* clear the lock */
 350         ENABLE_PREEMPTION
 351         LEAF_RET
 352
 353 /*
 354  *      unsigned int hw_lock_try(hw_lock_t)
 355  *      MACH_RT:  returns with preemption disabled on success.
 356  */
 357 LEAF_ENTRY(hw_lock_try)
 358         movl    L_ARG0,%edx             /* fetch lock pointer */
 359
 360         movl    %gs:CPU_ACTIVE_THREAD,%ecx
 361         DISABLE_PREEMPTION
 362         movl    0(%edx),%eax
 363         testl   %eax,%eax
 364         jne     1f
 365         lock; cmpxchgl  %ecx,0(%edx)    /* try to acquire the HW lock */
 366         jne     1f
 367
 368         movl    $1,%eax                 /* success */
 369         LEAF_RET
 370
 371 1:
 372         ENABLE_PREEMPTION               /* failure:  release preemption... */
 373         xorl    %eax,%eax               /* ...and return failure */
 374         LEAF_RET
 375
 376 /*
 377  *      unsigned int hw_lock_held(hw_lock_t)
 378  *      MACH_RT:  doesn't change preemption state.
 379  *      N.B.  Racy, of course.
 380  */
 381 LEAF_ENTRY(hw_lock_held)
 382         movl    L_ARG0,%edx             /* fetch lock pointer */
 383
 384         movl    0(%edx),%eax            /* check lock value */
 385         testl   %eax,%eax
 386         movl    $1,%ecx
 387         cmovne  %ecx,%eax               /* 0 => unlocked, 1 => locked */
 388         LEAF_RET
 389
 390 LEAF_ENTRY(mutex_init)
 391         movl    L_ARG0,%edx             /* fetch lock pointer */
 392         xorl    %eax,%eax
 393         movl    %eax,M_ILK              /* clear interlock */
 394         movl    %eax,M_LOCKED           /* clear locked flag */
 395         movw    %ax,M_WAITERS           /* init waiter count */
 396         movw    %ax,M_PROMOTED_PRI
 397
 398 #if     MACH_LDEBUG
 399         movl    $ MUTEX_TAG,M_TYPE      /* set lock type */
 400         movl    %eax,M_PC               /* init caller pc */
 401         movl    %eax,M_THREAD           /* and owning thread */
 402 #endif
 403
 404         LEAF_RET
 405
 406 NONLEAF_ENTRY2(mutex_lock,_mutex_lock)
 407
 408         movl    B_ARG0,%edx             /* fetch lock pointer */
 409
 410         CHECK_MUTEX_TYPE()
 411         CHECK_NO_SIMPLELOCKS()
 412         CHECK_PREEMPTION_LEVEL()
 413
 414         pushf                           /* save interrupt state */
 415         cli                             /* disable interrupts */
 416 Lml_retry:
 417         movl    %gs:CPU_ACTIVE_THREAD,%ecx
 418
 419 Lml_get_hw:
 420         movl    M_ILK,%eax              /* read interlock */
 421         testl   %eax,%eax               /* unlocked? */
 422         jne     Lml_ilk_fail            /* no - take the slow path */
 423
 424         lock; cmpxchgl  %ecx,M_ILK      /* atomic compare and exchange */
 425         jne     Lml_get_hw              /* branch on failure to retry */
 426
 427         movl    M_LOCKED,%ecx           /* get lock owner */
 428         testl   %ecx,%ecx               /* is the mutex locked? */
 429         jne     Lml_fail                /* yes, we lose */
 430 Lml_acquire:
 431         movl    %gs:CPU_ACTIVE_THREAD,%ecx
 432         movl    %ecx,M_LOCKED
 433
 434 #if     MACH_LDEBUG
 435         movl    %ecx,M_THREAD
 436         movl    B_PC,%ecx
 437         movl    %ecx,M_PC
 438 #endif
 439
 440         cmpw    $0,M_WAITERS            /* are there any waiters? */
 441         jne     Lml_waiters             /* yes, more work to do */
 442 Lml_return:
 443         xorl    %eax,%eax
 444         movl    %eax,M_ILK
 445
 446         popf                            /* restore interrupt state */
 447
 448         NONLEAF_RET
 449
 450 Lml_waiters:
 451         pushl   %edx                    /* save mutex address */
 452         pushl   %edx
 453         call    EXT(lck_mtx_lock_acquire)
 454         addl    $4,%esp
 455         popl    %edx                    /* restore mutex address */
 456         jmp     Lml_return
 457
 458 Lml_ilk_fail:
 459         /*
 460          * Slow path: call out to do the spinning.
 461          */
 462         pushl   %edx                    /* lock address */
 463         call    EXT(lck_mtx_interlock_spin)
 464         popl    %edx                    /* lock pointer */
 465         jmp     Lml_retry               /* try again */
 466
 467 Lml_fail:
 468         /*
 469          n Check if the owner is on another processor and therefore
 470          * we should try to spin before blocking.
 471          */
 472         testl   $(OnProc),ACT_SPF(%ecx)
 473         jz      Lml_block
 474
 475         /*
 476          * Here if owner is on another processor:
 477          *  - release the interlock
 478          *  - spin on the holder until release or timeout
 479          *  - in either case re-acquire the interlock
 480          *  - if released, acquire it
 481          *  - otherwise drop thru to block.
 482          */
 483         xorl    %eax,%eax
 484         movl    %eax,M_ILK              /* zero interlock */
 485         popf
 486         pushf                           /* restore interrupt state */
 487
 488         push    %edx                    /* lock address */
 489         call    EXT(lck_mtx_lock_spin)  /* call out to do spinning */
 490         addl    $4,%esp
 491         movl    B_ARG0,%edx             /* refetch mutex address */
 492
 493         /* Re-acquire interlock */
 494         cli                             /* disable interrupts */
 495 Lml_reget_retry:
 496         movl    %gs:CPU_ACTIVE_THREAD,%ecx
 497
 498 Lml_reget_hw:
 499         movl    M_ILK,%eax              /* read interlock */
 500         testl   %eax,%eax               /* unlocked? */
 501         jne     Lml_ilk_refail          /* no - slow path */
 502
 503         lock; cmpxchgl  %ecx,M_ILK      /* atomic compare and exchange */
 504         jne     Lml_reget_hw            /* branch on failure to retry */
 505
 506         movl    M_LOCKED,%ecx           /* get lock owner */
 507         testl   %ecx,%ecx               /* is the mutex free? */
 508         je      Lml_acquire             /* yes, acquire */
 509
 510 Lml_block:
 511         CHECK_MYLOCK(M_THREAD)
 512         pushl   M_LOCKED
 513         pushl   %edx                    /* push mutex address */
 514         call    EXT(lck_mtx_lock_wait)  /* wait for the lock */
 515         addl    $8,%esp
 516         movl    B_ARG0,%edx             /* refetch mutex address */
 517         cli                             /* ensure interrupts disabled */
 518         jmp     Lml_retry               /* and try again */
 519
 520 Lml_ilk_refail:
 521         /*
 522          * Slow path: call out to do the spinning.
 523          */
 524         pushl   %edx                    /* lock address */
 525         call    EXT(lck_mtx_interlock_spin)
 526         popl    %edx                    /* lock pointer */
 527         jmp     Lml_reget_retry         /* try again */
 528
 529 NONLEAF_ENTRY2(mutex_try,_mutex_try)
 530
 531         movl    B_ARG0,%edx             /* fetch lock pointer */
 532
 533         CHECK_MUTEX_TYPE()
 534         CHECK_NO_SIMPLELOCKS()
 535
 536         pushf                           /* save interrupt state */
 537         cli                             /* disable interrupts */
 538 Lmt_retry:
 539         movl    %gs:CPU_ACTIVE_THREAD,%ecx
 540
 541 Lmt_get_hw:
 542         movl    M_ILK,%eax              /* read interlock */
 543         testl   %eax,%eax               /* unlocked? */
 544         jne     Lmt_ilk_fail            /* no - slow path */
 545
 546         lock; cmpxchgl  %ecx,M_ILK      /* atomic compare and exchange */
 547         jne     Lmt_get_hw              /* branch on failure to retry */
 548
 549         movl    M_LOCKED,%ecx           /* get lock owner */
 550         testl   %ecx,%ecx               /* is the mutex locked? */
 551         jne     Lmt_fail                /* yes, we lose */
 552         movl    %gs:CPU_ACTIVE_THREAD,%ecx
 553         movl    %ecx,M_LOCKED
 554
 555 #if     MACH_LDEBUG
 556         movl    %ecx,M_THREAD
 557         movl    B_PC,%ecx
 558         movl    %ecx,M_PC
 559 #endif
 560
 561         cmpl    $0,M_WAITERS            /* are there any waiters? */
 562         jne     Lmt_waiters             /* yes, more work to do */
 563 Lmt_return:
 564         xorl    %eax,%eax
 565         movl    %eax,M_ILK
 566         popf                            /* restore interrupt state */
 567
 568         movl    $1,%eax
 569
 570         NONLEAF_RET
 571
 572 Lmt_waiters:
 573         pushl   %edx                    /* save mutex address */
 574         pushl   %edx
 575         call    EXT(lck_mtx_lock_acquire)
 576         addl    $4,%esp
 577         popl    %edx                    /* restore mutex address */
 578         jmp     Lmt_return
 579
 580 Lmt_ilk_fail:
 581         /*
 582          * Slow path: call out to do the spinning.
 583          */
 584         pushl   %edx                    /* lock address */
 585         call    EXT(lck_mtx_interlock_spin)
 586         popl    %edx                    /* lock pointer */
 587         jmp     Lmt_retry               /* try again */
 588
 589 Lmt_fail:
 590         xorl    %eax,%eax
 591         movl    %eax,M_ILK
 592
 593         popf                            /* restore interrupt state */
 594
 595         xorl    %eax,%eax
 596
 597         NONLEAF_RET
 598
 599 NONLEAF_ENTRY(mutex_unlock)
 600         movl    B_ARG0,%edx             /* fetch lock pointer */
 601
 602         CHECK_MUTEX_TYPE()
 603         CHECK_THREAD(M_THREAD)
 604
 605         pushf                           /* save interrupt state */
 606         cli                             /* disable interrupts */
 607 Lmu_retry:
 608         movl    %gs:CPU_ACTIVE_THREAD,%ecx
 609
 610 Lmu_get_hw:
 611         movl    M_ILK,%eax              /* read interlock */
 612         testl   %eax,%eax               /* unlocked? */
 613         jne     Lmu_ilk_fail            /* no - slow path */
 614
 615         lock; cmpxchgl  %ecx,M_ILK      /* atomic compare and exchange */
 616         jne     Lmu_get_hw              /* branch on failure to retry */
 617
 618         cmpw    $0,M_WAITERS            /* are there any waiters? */
 619         jne     Lmu_wakeup              /* yes, more work to do */
 620
 621 Lmu_doit:
 622
 623 #if     MACH_LDEBUG
 624         movl    $0,M_THREAD             /* disown thread */
 625 #endif
 626
 627         xorl    %ecx,%ecx
 628         movl    %ecx,M_LOCKED           /* unlock the mutex */
 629
 630         movl    %ecx,M_ILK
 631
 632         popf                            /* restore interrupt state */
 633
 634         NONLEAF_RET
 635
 636 Lmu_ilk_fail:
 637         /*
 638          * Slow path: call out to do the spinning.
 639          */
 640         pushl   %edx                    /* lock address */
 641         call    EXT(lck_mtx_interlock_spin)
 642         popl    %edx                    /* lock pointer */
 643         jmp     Lmu_retry               /* try again */
 644
 645 Lmu_wakeup:
 646         pushl   M_LOCKED
 647         pushl   %edx                    /* push mutex address */
 648         call    EXT(lck_mtx_unlock_wakeup)/* yes, wake a thread */
 649         addl    $8,%esp
 650         movl    B_ARG0,%edx             /* restore lock pointer */
 651         jmp     Lmu_doit
 652
 653 /*
 654  * lck_mtx_lock()
 655  * lck_mtx_try_lock()
 656  * lck_mutex_unlock()
 657  *
 658  * These are variants of mutex_lock(), mutex_try() and mutex_unlock() without
 659  * DEBUG checks (which require fields not present in lck_mtx_t's).
 660  */
 661 NONLEAF_ENTRY(lck_mtx_lock)
 662
 663         movl    B_ARG0,%edx             /* fetch lock pointer */
 664         cmpl    $(MUTEX_IND),M_ITAG     /* is this indirect? */
 665         cmove   M_PTR,%edx              /* yes - take indirection */
 666
 667         CHECK_NO_SIMPLELOCKS()
 668         CHECK_PREEMPTION_LEVEL()
 669
 670         pushf                           /* save interrupt state */
 671         cli                             /* disable interrupts */
 672 Llml_retry:
 673         movl    %gs:CPU_ACTIVE_THREAD,%ecx
 674
 675 Llml_get_hw:
 676         movl    M_ILK,%eax              /* read interlock */
 677         testl   %eax,%eax               /* unlocked? */
 678         jne     Llml_ilk_fail           /* no - slow path */
 679
 680         lock; cmpxchgl  %ecx,M_ILK      /* atomic compare and exchange */
 681         jne     Llml_get_hw             /* branch on failure to retry */
 682
 683         movl    M_LOCKED,%ecx           /* get lock owner */
 684         testl   %ecx,%ecx               /* is the mutex locked? */
 685         jne     Llml_fail               /* yes, we lose */
 686 Llml_acquire:
 687         movl    %gs:CPU_ACTIVE_THREAD,%ecx
 688         movl    %ecx,M_LOCKED
 689
 690         cmpl    $0,M_WAITERS            /* are there any waiters? */
 691         jne     Llml_waiters            /* yes, more work to do */
 692 Llml_return:
 693         xorl    %eax,%eax
 694         movl    %eax,M_ILK
 695
 696         popf                            /* restore interrupt state */
 697
 698         NONLEAF_RET
 699
 700 Llml_waiters:
 701         pushl   %edx                    /* save mutex address */
 702         pushl   %edx
 703         call    EXT(lck_mtx_lock_acquire)
 704         addl    $4,%esp
 705         popl    %edx                    /* restore mutex address */
 706         jmp     Llml_return
 707
 708 Llml_ilk_fail:
 709         /*
 710          * Slow path: call out to do the spinning.
 711          */
 712         pushl   %edx                    /* lock address */
 713         call    EXT(lck_mtx_interlock_spin)
 714         popl    %edx                    /* lock pointer */
 715         jmp     Llml_retry              /* try again */
 716
 717 Llml_fail:
 718         /*
 719          * Check if the owner is on another processor and therefore
 720          * we should try to spin before blocking.
 721          */
 722         testl   $(OnProc),ACT_SPF(%ecx)
 723         jz      Llml_block
 724
 725         /*
 726          * Here if owner is on another processor:
 727          *  - release the interlock
 728          *  - spin on the holder until release or timeout
 729          *  - in either case re-acquire the interlock
 730          *  - if released, acquire it
 731          *  - otherwise drop thru to block.
 732          */
 733         xorl    %eax,%eax
 734         movl    %eax,M_ILK              /* zero interlock */
 735         popf
 736         pushf                           /* restore interrupt state */
 737
 738         pushl   %edx                    /* save mutex address */
 739         pushl   %edx
 740         call    EXT(lck_mtx_lock_spin)
 741         addl    $4,%esp
 742         popl    %edx                    /* restore mutex address */
 743
 744         /* Re-acquire interlock */
 745         cli                             /* disable interrupts */
 746 Llml_reget_retry:
 747         movl    %gs:CPU_ACTIVE_THREAD,%ecx
 748
 749 Llml_reget_hw:
 750         movl    M_ILK,%eax              /* read interlock */
 751         testl   %eax,%eax               /* unlocked? */
 752         jne     Llml_ilk_refail         /* no - slow path */
 753
 754         lock; cmpxchgl  %ecx,M_ILK      /* atomic compare and exchange */
 755         jne     Llml_reget_hw           /* branch on failure to retry */
 756
 757         movl    M_LOCKED,%ecx           /* get lock owner */
 758         testl   %ecx,%ecx               /* is the mutex free? */
 759         je      Llml_acquire            /* yes, acquire */
 760
 761 Llml_block:
 762         CHECK_MYLOCK(M_THREAD)
 763         pushl   %edx                    /* save mutex address */
 764         pushl   M_LOCKED
 765         pushl   %edx                    /* push mutex address */
 766         call    EXT(lck_mtx_lock_wait)  /* wait for the lock */
 767         addl    $8,%esp
 768         popl    %edx                    /* restore mutex address */
 769         cli                             /* ensure interrupts disabled */
 770         jmp     Llml_retry              /* and try again */
 771
 772 Llml_ilk_refail:
 773         /*
 774          * Slow path: call out to do the spinning.
 775          */
 776         pushl   %edx                    /* lock address */
 777         call    EXT(lck_mtx_interlock_spin)
 778         popl    %edx                    /* lock pointer */
 779         jmp     Llml_reget_retry        /* try again */
 780
 781 NONLEAF_ENTRY(lck_mtx_try_lock)
 782
 783         movl    B_ARG0,%edx             /* fetch lock pointer */
 784         cmpl    $(MUTEX_IND),M_ITAG     /* is this indirect? */
 785         cmove   M_PTR,%edx              /* yes - take indirection */
 786
 787         CHECK_NO_SIMPLELOCKS()
 788         CHECK_PREEMPTION_LEVEL()
 789
 790         pushf                           /* save interrupt state */
 791         cli                             /* disable interrupts */
 792 Llmt_retry:
 793         movl    %gs:CPU_ACTIVE_THREAD,%ecx
 794
 795 Llmt_get_hw:
 796         movl    M_ILK,%eax              /* read interlock */
 797         testl   %eax,%eax               /* unlocked? */
 798         jne     Llmt_ilk_fail           /* no - slow path */
 799
 800         lock; cmpxchgl  %ecx,M_ILK      /* atomic compare and exchange */
 801         jne     Llmt_get_hw             /* branch on failure to retry */
 802
 803         movl    M_LOCKED,%ecx           /* get lock owner */
 804         testl   %ecx,%ecx               /* is the mutex locked? */
 805         jne     Llmt_fail               /* yes, we lose */
 806         movl    %gs:CPU_ACTIVE_THREAD,%ecx
 807         movl    %ecx,M_LOCKED
 808
 809         cmpl    $0,M_WAITERS            /* are there any waiters? */
 810         jne     Llmt_waiters            /* yes, more work to do */
 811 Llmt_return:
 812         xorl    %eax,%eax
 813         movl    %eax,M_ILK
 814
 815         popf                            /* restore interrupt state */
 816
 817         movl    $1,%eax                 /* return success */
 818         NONLEAF_RET
 819
 820 Llmt_waiters:
 821         pushl   %edx                    /* save mutex address */
 822         pushl   %edx
 823         call    EXT(lck_mtx_lock_acquire)
 824         addl    $4,%esp
 825         popl    %edx                    /* restore mutex address */
 826         jmp     Llmt_return
 827
 828 Llmt_ilk_fail:
 829         /*
 830          * Slow path: call out to do the spinning.
 831          */
 832         pushl   %edx                    /* lock address */
 833         call    EXT(lck_mtx_interlock_spin)
 834         popl    %edx                    /* lock pointer */
 835         jmp     Llmt_retry              /* try again */
 836
 837 Llmt_fail:
 838         xorl    %eax,%eax
 839         movl    %eax,M_ILK
 840
 841         popf                            /* restore interrupt state */
 842
 843         xorl    %eax,%eax               /* return failure */
 844         NONLEAF_RET
 845
 846 NONLEAF_ENTRY(lck_mtx_unlock)
 847
 848         movl    B_ARG0,%edx             /* fetch lock pointer */
 849         cmpl    $(MUTEX_IND),M_ITAG     /* is this indirect? */
 850         cmove   M_PTR,%edx              /* yes - take indirection */
 851
 852         pushf                           /* save interrupt state */
 853         cli                             /* disable interrupts */
 854 Llmu_retry:
 855         movl    %gs:CPU_ACTIVE_THREAD,%ecx
 856
 857 Llmu_get_hw:
 858         movl    M_ILK,%eax              /* read interlock */
 859         testl   %eax,%eax               /* unlocked? */
 860         jne     Llmu_ilk_fail           /* no - slow path */
 861
 862         lock; cmpxchgl  %ecx,M_ILK      /* atomic compare and exchange */
 863         jne     Llmu_get_hw             /* branch on failure to retry */
 864
 865         cmpw    $0,M_WAITERS            /* are there any waiters? */
 866         jne     Llmu_wakeup             /* yes, more work to do */
 867
 868 Llmu_doit:
 869         xorl    %ecx,%ecx
 870         movl    %ecx,M_LOCKED           /* unlock the mutex */
 871
 872         movl    %ecx,M_ILK
 873
 874         popf                            /* restore interrupt state */
 875
 876         NONLEAF_RET
 877
 878 Llmu_ilk_fail:
 879         /*
 880          * Slow path: call out to do the spinning.
 881          */
 882         pushl   %edx                    /* lock address */
 883         call    EXT(lck_mtx_interlock_spin)
 884         popl    %edx                    /* lock pointer */
 885         jmp     Llmu_retry              /* try again */
 886
 887 Llmu_wakeup:
 888         pushl   %edx                    /* save mutex address */
 889         pushl   M_LOCKED
 890         pushl   %edx                    /* push mutex address */
 891         call    EXT(lck_mtx_unlock_wakeup)/* yes, wake a thread */
 892         addl    $8,%esp
 893         popl    %edx                    /* restore mutex pointer */
 894         jmp     Llmu_doit
 895
 896 LEAF_ENTRY(lck_mtx_ilk_unlock)
 897         movl    L_ARG0,%edx             /* no indirection here */
 898
 899         xorl    %eax,%eax
 900         movl    %eax,M_ILK
 901
 902         LEAF_RET
 903
 904 LEAF_ENTRY(_disable_preemption)
 905 #if     MACH_RT
 906         _DISABLE_PREEMPTION
 907 #endif  /* MACH_RT */
 908         LEAF_RET
 909
 910 LEAF_ENTRY(_enable_preemption)
 911 #if     MACH_RT
 912 #if     MACH_ASSERT
 913         cmpl    $0,%gs:CPU_PREEMPTION_LEVEL
 914         jg      1f
 915         pushl   %gs:CPU_PREEMPTION_LEVEL
 916         pushl   $2f
 917         call    EXT(panic)
 918         hlt
 919         .data
 920 2:      String  "_enable_preemption: preemption_level(%d)  < 0!"
 921         .text
 922 1:
 923 #endif  /* MACH_ASSERT */
 924         _ENABLE_PREEMPTION
 925 #endif  /* MACH_RT */
 926         LEAF_RET
 927
 928 LEAF_ENTRY(_enable_preemption_no_check)
 929 #if     MACH_RT
 930 #if     MACH_ASSERT
 931         cmpl    $0,%gs:CPU_PREEMPTION_LEVEL
 932         jg      1f
 933         pushl   $2f
 934         call    EXT(panic)
 935         hlt
 936         .data
 937 2:      String  "_enable_preemption_no_check: preemption_level <= 0!"
 938         .text
 939 1:
 940 #endif  /* MACH_ASSERT */
 941         _ENABLE_PREEMPTION_NO_CHECK
 942 #endif  /* MACH_RT */
 943         LEAF_RET
 944
 945
 946 LEAF_ENTRY(_mp_disable_preemption)
 947 #if     MACH_RT
 948         _DISABLE_PREEMPTION
 949 #endif  /* MACH_RT */
 950         LEAF_RET
 951
 952 LEAF_ENTRY(_mp_enable_preemption)
 953 #if     MACH_RT
 954 #if     MACH_ASSERT
 955         cmpl    $0,%gs:CPU_PREEMPTION_LEVEL
 956         jg      1f
 957         pushl   %gs:CPU_PREEMPTION_LEVEL
 958         pushl   $2f
 959         call    EXT(panic)
 960         hlt
 961         .data
 962 2:      String  "_mp_enable_preemption: preemption_level (%d) <= 0!"
 963         .text
 964 1:
 965 #endif  /* MACH_ASSERT */
 966         _ENABLE_PREEMPTION
 967 #endif  /* MACH_RT */
 968         LEAF_RET
 969
 970 LEAF_ENTRY(_mp_enable_preemption_no_check)
 971 #if     MACH_RT
 972 #if     MACH_ASSERT
 973         cmpl    $0,%gs:CPU_PREEMPTION_LEVEL
 974         jg      1f
 975         pushl   $2f
 976         call    EXT(panic)
 977         hlt
 978         .data
 979 2:      String  "_mp_enable_preemption_no_check: preemption_level <= 0!"
 980         .text
 981 1:
 982 #endif  /* MACH_ASSERT */
 983         _ENABLE_PREEMPTION_NO_CHECK
 984 #endif  /* MACH_RT */
 985         LEAF_RET
 986
 987
 988 LEAF_ENTRY(i_bit_set)
 989         movl    L_ARG0,%edx
 990         movl    L_ARG1,%eax
 991         lock
 992         bts     %edx,(%eax)
 993         LEAF_RET
 994
 995 LEAF_ENTRY(i_bit_clear)
 996         movl    L_ARG0,%edx
 997         movl    L_ARG1,%eax
 998         lock
 999         btr     %edx,(%eax)
1000         LEAF_RET
1001
1002 LEAF_ENTRY(bit_lock)
1003         movl    L_ARG0,%ecx
1004         movl    L_ARG1,%eax
1005 1:
1006         lock
1007         bts     %ecx,(%eax)
1008         jb      1b
1009         LEAF_RET
1010
1011 LEAF_ENTRY(bit_lock_try)
1012         movl    L_ARG0,%ecx
1013         movl    L_ARG1,%eax
1014         lock
1015         bts     %ecx,(%eax)
1016         jb      bit_lock_failed
1017         LEAF_RET                /* %eax better not be null ! */
1018 bit_lock_failed:
1019         xorl    %eax,%eax
1020         LEAF_RET
1021
1022 LEAF_ENTRY(bit_unlock)
1023         movl    L_ARG0,%ecx
1024         movl    L_ARG1,%eax
1025         lock
1026         btr     %ecx,(%eax)
1027         LEAF_RET