osfmk/i386/i386_lock.s

   1 /*
   2  * Copyright (c) 2000 Apple Computer, Inc. All rights reserved.
   3  *
   4  * @APPLE_LICENSE_HEADER_START@
   5  *
   6  * The contents of this file constitute Original Code as defined in and
   7  * are subject to the Apple Public Source License Version 1.1 (the
   8  * "License").  You may not use this file except in compliance with the
   9  * License.  Please obtain a copy of the License at
  10  * http://www.apple.com/publicsource and read it before using this file.
  11  *
  12  * This Original Code and all software distributed under the License are
  13  * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  14  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  15  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
  16  * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
  17  * License for the specific language governing rights and limitations
  18  * under the License.
  19  *
  20  * @APPLE_LICENSE_HEADER_END@
  21  */
  22 /*
  23  * @OSF_COPYRIGHT@
  24  */
  25 /*
  26  * Mach Operating System
  27  * Copyright (c) 1989 Carnegie-Mellon University
  28  * All rights reserved.  The CMU software License Agreement specifies
  29  * the terms and conditions for use and redistribution.
  30  */
  31
  32 #include <mach_rt.h>
  33 #include <platforms.h>
  34 #include <mach_ldebug.h>
  35 #include <i386/asm.h>
  36
  37 #include "assym.s"
  38
  39 #define PAUSE           rep; nop
  40
  41 /*
  42  *      When performance isn't the only concern, it's
  43  *      nice to build stack frames...
  44  */
  45 #define BUILD_STACK_FRAMES   (GPROF || \
  46                                 ((MACH_LDEBUG || ETAP_LOCK_TRACE) && MACH_KDB))
  47
  48 #if     BUILD_STACK_FRAMES
  49
  50 /* STack-frame-relative: */
  51 #define L_PC            B_PC
  52 #define L_ARG0          B_ARG0
  53 #define L_ARG1          B_ARG1
  54
  55 #define LEAF_ENTRY(name)        \
  56         Entry(name);            \
  57         FRAME;                  \
  58         MCOUNT
  59
  60 #define LEAF_ENTRY2(n1,n2)      \
  61         Entry(n1);              \
  62         Entry(n2);              \
  63         FRAME;                  \
  64         MCOUNT
  65
  66 #define LEAF_RET                \
  67         EMARF;                  \
  68         ret
  69
  70 #else   /* BUILD_STACK_FRAMES */
  71
  72 /* Stack-pointer-relative: */
  73 #define L_PC            S_PC
  74 #define L_ARG0          S_ARG0
  75 #define L_ARG1          S_ARG1
  76
  77 #define LEAF_ENTRY(name)        \
  78         Entry(name)
  79
  80 #define LEAF_ENTRY2(n1,n2)      \
  81         Entry(n1);              \
  82         Entry(n2)
  83
  84 #define LEAF_RET                \
  85         ret
  86
  87 #endif  /* BUILD_STACK_FRAMES */
  88
  89
  90 /* Non-leaf routines always have a stack frame: */
  91
  92 #define NONLEAF_ENTRY(name)     \
  93         Entry(name);            \
  94         FRAME;                  \
  95         MCOUNT
  96
  97 #define NONLEAF_ENTRY2(n1,n2)   \
  98         Entry(n1);              \
  99         Entry(n2);              \
 100         FRAME;                  \
 101         MCOUNT
 102
 103 #define NONLEAF_RET             \
 104         EMARF;                  \
 105         ret
 106
 107
 108 #define M_ILK           (%edx)
 109 #define M_LOCKED        MUTEX_LOCKED(%edx)
 110 #define M_WAITERS       MUTEX_WAITERS(%edx)
 111 #define M_PROMOTED_PRI  MUTEX_PROMOTED_PRI(%edx)
 112 #define M_ITAG          MUTEX_ITAG(%edx)
 113 #define M_PTR           MUTEX_PTR(%edx)
 114 #if     MACH_LDEBUG
 115 #define M_TYPE          MUTEX_TYPE(%edx)
 116 #define M_PC            MUTEX_PC(%edx)
 117 #define M_THREAD        MUTEX_THREAD(%edx)
 118 #endif  /* MACH_LDEBUG */
 119
 120 #include <i386/mp.h>
 121 #define CX(addr,reg)    addr(,reg,4)
 122
 123 #if     MACH_LDEBUG
 124 /*
 125  *  Routines for general lock debugging.
 126  */
 127 #define S_TYPE          SLOCK_TYPE(%edx)
 128 #define S_PC            SLOCK_PC(%edx)
 129 #define S_THREAD        SLOCK_THREAD(%edx)
 130 #define S_DURATIONH     SLOCK_DURATIONH(%edx)
 131 #define S_DURATIONL     SLOCK_DURATIONL(%edx)
 132
 133 /*
 134  * Checks for expected lock types and calls "panic" on
 135  * mismatch.  Detects calls to Mutex functions with
 136  * type simplelock and vice versa.
 137  */
 138 #define CHECK_MUTEX_TYPE()                                      \
 139         cmpl    $ MUTEX_TAG,M_TYPE                      ;       \
 140         je      1f                                      ;       \
 141         pushl   $2f                                     ;       \
 142         call    EXT(panic)                              ;       \
 143         hlt                                             ;       \
 144         .data                                           ;       \
 145 2:      String  "not a mutex!"                          ;       \
 146         .text                                           ;       \
 147 1:
 148
 149 #define CHECK_SIMPLE_LOCK_TYPE()                                \
 150         cmpl    $ USLOCK_TAG,S_TYPE                     ;       \
 151         je      1f                                      ;       \
 152         pushl   $2f                                     ;       \
 153         call    EXT(panic)                              ;       \
 154         hlt                                             ;       \
 155         .data                                           ;       \
 156 2:      String  "not a simple lock!"                    ;       \
 157         .text                                           ;       \
 158 1:
 159
 160 /*
 161  * If one or more simplelocks are currently held by a thread,
 162  * an attempt to acquire a mutex will cause this check to fail
 163  * (since a mutex lock may context switch, holding a simplelock
 164  * is not a good thing).
 165  */
 166 #if     MACH_RT
 167 #define CHECK_PREEMPTION_LEVEL()                                \
 168         cmpl    $0,%gs:CPU_PREEMPTION_LEVEL             ;       \
 169         je      1f                                      ;       \
 170         pushl   $2f                                     ;       \
 171         call    EXT(panic)                              ;       \
 172         hlt                                             ;       \
 173         .data                                           ;       \
 174 2:      String  "preemption_level != 0!"                ;       \
 175         .text                                           ;       \
 176 1:
 177 #else   /* MACH_RT */
 178 #define CHECK_PREEMPTION_LEVEL()
 179 #endif  /* MACH_RT */
 180
 181 #define CHECK_NO_SIMPLELOCKS()                                  \
 182         cmpl    $0,%gs:CPU_SIMPLE_LOCK_COUNT            ;       \
 183         je      1f                                      ;       \
 184         pushl   $2f                                     ;       \
 185         call    EXT(panic)                              ;       \
 186         hlt                                             ;       \
 187         .data                                           ;       \
 188 2:      String  "simple_locks_held!"                    ;       \
 189         .text                                           ;       \
 190 1:
 191
 192 /*
 193  * Verifies return to the correct thread in "unlock" situations.
 194  */
 195 #define CHECK_THREAD(thd)                                       \
 196         movl    %gs:CPU_ACTIVE_THREAD,%ecx              ;       \
 197         testl   %ecx,%ecx                               ;       \
 198         je      1f                                      ;       \
 199         cmpl    %ecx,thd                                ;       \
 200         je      1f                                      ;       \
 201         pushl   $2f                                     ;       \
 202         call    EXT(panic)                              ;       \
 203         hlt                                             ;       \
 204         .data                                           ;       \
 205 2:      String  "wrong thread!"                         ;       \
 206         .text                                           ;       \
 207 1:
 208
 209 #define CHECK_MYLOCK(thd)                                       \
 210         movl    %gs:CPU_ACTIVE_THREAD,%ecx              ;       \
 211         testl   %ecx,%ecx                               ;       \
 212         je      1f                                      ;       \
 213         cmpl    %ecx,thd                                ;       \
 214         jne     1f                                      ;       \
 215         pushl   $2f                                     ;       \
 216         call    EXT(panic)                              ;       \
 217         hlt                                             ;       \
 218         .data                                           ;       \
 219 2:      String  "mylock attempt!"                       ;       \
 220         .text                                           ;       \
 221 1:
 222
 223 #define METER_SIMPLE_LOCK_LOCK(reg)                             \
 224         pushl   reg                                     ;       \
 225         call    EXT(meter_simple_lock)                  ;       \
 226         popl    reg
 227
 228 #define METER_SIMPLE_LOCK_UNLOCK(reg)                           \
 229         pushl   reg                                     ;       \
 230         call    EXT(meter_simple_unlock)                ;       \
 231         popl    reg
 232
 233 #else   /* MACH_LDEBUG */
 234 #define CHECK_MUTEX_TYPE()
 235 #define CHECK_SIMPLE_LOCK_TYPE
 236 #define CHECK_THREAD(thd)
 237 #define CHECK_PREEMPTION_LEVEL()
 238 #define CHECK_NO_SIMPLELOCKS()
 239 #define CHECK_MYLOCK(thd)
 240 #define METER_SIMPLE_LOCK_LOCK(reg)
 241 #define METER_SIMPLE_LOCK_UNLOCK(reg)
 242 #endif  /* MACH_LDEBUG */
 243
 244
 245 /*
 246  *      void hw_lock_init(hw_lock_t)
 247  *
 248  *      Initialize a hardware lock.
 249  */
 250 LEAF_ENTRY(hw_lock_init)
 251         movl    L_ARG0,%edx             /* fetch lock pointer */
 252         movl    $0,0(%edx)              /* clear the lock */
 253         LEAF_RET
 254
 255 /*
 256  *      void hw_lock_lock(hw_lock_t)
 257  *
 258  *      Acquire lock, spinning until it becomes available.
 259  *      MACH_RT:  also return with preemption disabled.
 260  */
 261 LEAF_ENTRY(hw_lock_lock)
 262         movl    L_ARG0,%edx             /* fetch lock pointer */
 263
 264         movl    L_PC,%ecx
 265 1:      DISABLE_PREEMPTION
 266         movl    0(%edx), %eax
 267         testl   %eax,%eax               /* lock locked? */
 268         jne     3f                      /* branch if so */
 269         lock; cmpxchgl  %ecx,0(%edx)    /* try to acquire the HW lock */
 270         jne     3f
 271         movl    $1,%eax                 /* In case this was a timeout call */
 272         LEAF_RET                        /* if yes, then nothing left to do */
 273
 274 3:      ENABLE_PREEMPTION               /* no reason we can't be preemptable */
 275         PAUSE                           /* pause for hyper-threading */
 276         jmp     1b                      /* try again */
 277
 278 /*
 279  *      unsigned int hw_lock_to(hw_lock_t, unsigned int)
 280  *
 281  *      Acquire lock, spinning until it becomes available or timeout.
 282  *      MACH_RT:  also return with preemption disabled.
 283  */
 284 LEAF_ENTRY(hw_lock_to)
 285 1:
 286         movl    L_ARG0,%edx             /* fetch lock pointer */
 287         movl    L_PC,%ecx
 288         /*
 289          * Attempt to grab the lock immediately
 290          * - fastpath without timeout nonsense.
 291          */
 292         DISABLE_PREEMPTION
 293         movl    0(%edx), %eax
 294         testl   %eax,%eax               /* lock locked? */
 295         jne     2f                      /* branch if so */
 296         lock; cmpxchgl  %ecx,0(%edx)    /* try to acquire the HW lock */
 297         jne     2f                      /* branch on failure */
 298         movl    $1,%eax
 299         LEAF_RET
 300
 301 2:
 302 #define INNER_LOOP_COUNT        1000
 303         /*
 304          * Failed to get the lock so set the timeout
 305          * and then spin re-checking the lock but pausing
 306          * every so many (INNER_LOOP_COUNT) spins to check for timeout.
 307          */
 308         movl    L_ARG1,%ecx             /* fetch timeout */
 309         push    %edi
 310         push    %ebx
 311         mov     %edx,%edi
 312
 313         rdtsc                           /* read cyclecount into %edx:%eax */
 314         addl    %ecx,%eax               /* fetch and timeout */
 315         adcl    $0,%edx                 /* add carry */
 316         mov     %edx,%ecx
 317         mov     %eax,%ebx               /* %ecx:%ebx is the timeout expiry */
 318 3:
 319         ENABLE_PREEMPTION               /* no reason not to be preempted now */
 320 4:
 321         /*
 322          * The inner-loop spin to look for the lock being freed.
 323          */
 324         mov     $(INNER_LOOP_COUNT),%edx
 325 5:
 326         PAUSE                           /* pause for hyper-threading */
 327         movl    0(%edi),%eax            /* spin checking lock value in cache */
 328         testl   %eax,%eax
 329         je      6f                      /* zero => unlocked, try to grab it */
 330         decl    %edx                    /* decrement inner loop count */
 331         jnz     5b                      /* time to check for timeout? */
 332
 333         /*
 334          * Here after spinning INNER_LOOP_COUNT times, check for timeout
 335          */
 336         rdtsc                           /* cyclecount into %edx:%eax */
 337         cmpl    %ecx,%edx               /* compare high-order 32-bits */
 338         jb      4b                      /* continue spinning if less, or */
 339         cmpl    %ebx,%eax               /* compare low-order 32-bits */
 340         jb      5b                      /* continue if less, else bail */
 341         xor     %eax,%eax               /* with 0 return value */
 342         pop     %ebx
 343         pop     %edi
 344         LEAF_RET
 345
 346 6:
 347         /*
 348          * Here to try to grab the lock that now appears to be free
 349          * after contention.
 350          */
 351         movl    8+L_PC,%edx             /* calling pc (8+ for pushed regs) */
 352         DISABLE_PREEMPTION
 353         lock; cmpxchgl  %edx,0(%edi)    /* try to acquire the HW lock */
 354         jne     3b                      /* no - spin again */
 355         movl    $1,%eax                 /* yes */
 356         pop     %ebx
 357         pop     %edi
 358         LEAF_RET
 359
 360 /*
 361  *      void hw_lock_unlock(hw_lock_t)
 362  *
 363  *      Unconditionally release lock.
 364  *      MACH_RT:  release preemption level.
 365  */
 366 LEAF_ENTRY(hw_lock_unlock)
 367         movl    L_ARG0,%edx             /* fetch lock pointer */
 368         movl    $0,0(%edx)              /* clear the lock */
 369         ENABLE_PREEMPTION
 370         LEAF_RET
 371
 372 /*
 373  *      unsigned int hw_lock_try(hw_lock_t)
 374  *      MACH_RT:  returns with preemption disabled on success.
 375  */
 376 LEAF_ENTRY(hw_lock_try)
 377         movl    L_ARG0,%edx             /* fetch lock pointer */
 378
 379         movl    L_PC,%ecx
 380         DISABLE_PREEMPTION
 381         movl    0(%edx),%eax
 382         testl   %eax,%eax
 383         jne     1f
 384         lock; cmpxchgl  %ecx,0(%edx)    /* try to acquire the HW lock */
 385         jne     1f
 386
 387         movl    $1,%eax                 /* success */
 388         LEAF_RET
 389
 390 1:      ENABLE_PREEMPTION               /* failure:  release preemption... */
 391         xorl    %eax,%eax               /* ...and return failure */
 392         LEAF_RET
 393
 394 /*
 395  *      unsigned int hw_lock_held(hw_lock_t)
 396  *      MACH_RT:  doesn't change preemption state.
 397  *      N.B.  Racy, of course.
 398  */
 399 LEAF_ENTRY(hw_lock_held)
 400         movl    L_ARG0,%edx             /* fetch lock pointer */
 401
 402         movl    0(%edx),%eax            /* check lock value */
 403         testl   %eax,%eax
 404         movl    $1,%ecx
 405         cmovne  %ecx,%eax               /* 0 => unlocked, 1 => locked */
 406         LEAF_RET
 407
 408 LEAF_ENTRY(mutex_init)
 409         movl    L_ARG0,%edx             /* fetch lock pointer */
 410         xorl    %eax,%eax
 411         movl    %eax,M_ILK              /* clear interlock */
 412         movl    %eax,M_LOCKED           /* clear locked flag */
 413         movw    %ax,M_WAITERS           /* init waiter count */
 414         movw    %ax,M_PROMOTED_PRI
 415
 416 #if     MACH_LDEBUG
 417         movl    $ MUTEX_TAG,M_TYPE      /* set lock type */
 418         movl    %eax,M_PC               /* init caller pc */
 419         movl    %eax,M_THREAD           /* and owning thread */
 420 #endif
 421
 422         LEAF_RET
 423
 424 NONLEAF_ENTRY2(mutex_lock,_mutex_lock)
 425
 426         movl    B_ARG0,%edx             /* fetch lock pointer */
 427
 428         CHECK_MUTEX_TYPE()
 429         CHECK_NO_SIMPLELOCKS()
 430         CHECK_PREEMPTION_LEVEL()
 431
 432         pushf                           /* save interrupt state */
 433         cli                             /* disable interrupts */
 434
 435 ml_retry:
 436         movl    B_PC,%ecx
 437
 438 ml_get_hw:
 439         movl    M_ILK,%eax              /* read interlock */
 440         testl   %eax,%eax               /* unlocked? */
 441         je      1f                      /* yes - attempt to lock it */
 442         PAUSE                           /* no  - pause */
 443         jmp     ml_get_hw               /* try again */
 444 1:
 445         lock; cmpxchgl  %ecx,M_ILK      /* atomic compare and exchange */
 446         jne     ml_get_hw               /* branch on failure to retry */
 447
 448         movl    M_LOCKED,%ecx           /* get lock owner */
 449         testl   %ecx,%ecx               /* is the mutex locked? */
 450         jne     ml_fail                 /* yes, we lose */
 451         movl    %gs:CPU_ACTIVE_THREAD,%ecx
 452         movl    %ecx,M_LOCKED
 453
 454 #if     MACH_LDEBUG
 455         movl    %ecx,M_THREAD
 456         movl    B_PC,%ecx
 457         movl    %ecx,M_PC
 458 #endif
 459
 460         pushl   %edx                    /* save mutex address */
 461         pushl   %edx
 462         call    EXT(lck_mtx_lock_acquire)
 463         addl    $4,%esp
 464         popl    %edx                    /* restore mutex address */
 465
 466         xorl    %eax,%eax
 467         movl    %eax,M_ILK
 468
 469         popf                            /* restore interrupt state */
 470
 471         NONLEAF_RET
 472
 473 ml_fail:
 474 ml_block:
 475         CHECK_MYLOCK(M_THREAD)
 476         pushl   M_LOCKED
 477         pushl   %edx                    /* push mutex address */
 478         call    EXT(lck_mtx_lock_wait)  /* wait for the lock */
 479         addl    $8,%esp
 480         movl    B_ARG0,%edx             /* refetch mutex address */
 481         jmp     ml_retry                /* and try again */
 482
 483 NONLEAF_ENTRY2(mutex_try,_mutex_try)
 484
 485         movl    B_ARG0,%edx             /* fetch lock pointer */
 486
 487         CHECK_MUTEX_TYPE()
 488         CHECK_NO_SIMPLELOCKS()
 489
 490         movl    B_PC,%ecx
 491
 492         pushf                           /* save interrupt state */
 493         cli                             /* disable interrupts */
 494
 495 mt_get_hw:
 496         movl    M_ILK,%eax              /* read interlock */
 497         testl   %eax,%eax               /* unlocked? */
 498         je      1f                      /* yes - attempt to lock it */
 499         PAUSE                           /* no  - pause */
 500         jmp     mt_get_hw               /* try again */
 501 1:
 502         lock; cmpxchgl  %ecx,M_ILK      /* atomic compare and exchange */
 503         jne     mt_get_hw               /* branch on failure to retry */
 504
 505         movl    M_LOCKED,%ecx           /* get lock owner */
 506         testl   %ecx,%ecx               /* is the mutex locked? */
 507         jne     mt_fail                 /* yes, we lose */
 508         movl    %gs:CPU_ACTIVE_THREAD,%ecx
 509         movl    %ecx,M_LOCKED
 510
 511 #if     MACH_LDEBUG
 512         movl    %ecx,M_THREAD
 513         movl    B_PC,%ecx
 514         movl    %ecx,M_PC
 515 #endif
 516
 517         pushl   %edx                    /* save mutex address */
 518         pushl   %edx
 519         call    EXT(lck_mtx_lock_acquire)
 520         addl    $4,%esp
 521         popl    %edx                    /* restore mutex address */
 522
 523         xorl    %eax,%eax
 524         movl    %eax,M_ILK
 525
 526         popf                            /* restore interrupt state */
 527
 528         movl    $1,%eax
 529
 530         NONLEAF_RET
 531
 532 mt_fail:
 533         xorl    %eax,%eax
 534         movl    %eax,M_ILK
 535
 536         popf                            /* restore interrupt state */
 537
 538         xorl    %eax,%eax
 539
 540         NONLEAF_RET
 541
 542 NONLEAF_ENTRY(mutex_unlock)
 543         movl    B_ARG0,%edx             /* fetch lock pointer */
 544
 545         CHECK_MUTEX_TYPE()
 546         CHECK_THREAD(M_THREAD)
 547
 548         movl    B_PC,%ecx
 549
 550         pushf                           /* save interrupt state */
 551         cli                             /* disable interrupts */
 552
 553 mu_get_hw:
 554         movl    M_ILK,%eax              /* read interlock */
 555         testl   %eax,%eax               /* unlocked? */
 556         je      1f                      /* yes - attempt to lock it */
 557         PAUSE                           /* no  - pause */
 558         jmp     mu_get_hw               /* try again */
 559 1:
 560         lock; cmpxchgl  %ecx,M_ILK      /* atomic compare and exchange */
 561         jne     mu_get_hw               /* branch on failure to retry */
 562
 563         cmpw    $0,M_WAITERS            /* are there any waiters? */
 564         jne     mu_wakeup               /* yes, more work to do */
 565
 566 mu_doit:
 567
 568 #if     MACH_LDEBUG
 569         movl    $0,M_THREAD             /* disown thread */
 570 #endif
 571
 572         xorl    %ecx,%ecx
 573         movl    %ecx,M_LOCKED           /* unlock the mutex */
 574
 575         movl    %ecx,M_ILK
 576
 577         popf                            /* restore interrupt state */
 578
 579         NONLEAF_RET
 580
 581 mu_wakeup:
 582         pushl   M_LOCKED
 583         pushl   %edx                    /* push mutex address */
 584         call    EXT(lck_mtx_unlock_wakeup)/* yes, wake a thread */
 585         addl    $8,%esp
 586         movl    B_ARG0,%edx             /* restore lock pointer */
 587         jmp     mu_doit
 588
 589 /*
 590  * lck_mtx_lock()
 591  * lck_mtx_try_lock()
 592  * lck_mutex_unlock()
 593  *
 594  * These are variants of mutex_lock(), mutex_try() and mutex_unlock() without
 595  * DEBUG checks (which require fields not present in lck_mtx_t's).
 596  */
 597 NONLEAF_ENTRY(lck_mtx_lock)
 598
 599         movl    B_ARG0,%edx             /* fetch lock pointer */
 600         cmpl    $(MUTEX_IND),M_ITAG     /* is this indirect? */
 601         cmove   M_PTR,%edx              /* yes - take indirection */
 602
 603         CHECK_NO_SIMPLELOCKS()
 604         CHECK_PREEMPTION_LEVEL()
 605
 606         pushf                           /* save interrupt state */
 607         cli                             /* disable interrupts */
 608
 609 lml_retry:
 610         movl    B_PC,%ecx
 611
 612 lml_get_hw:
 613         movl    M_ILK,%eax              /* read interlock */
 614         testl   %eax,%eax               /* unlocked? */
 615         je      1f                      /* yes - attempt to lock it */
 616         PAUSE                           /* no  - pause */
 617         jmp     lml_get_hw              /* try again */
 618 1:
 619         lock; cmpxchgl  %ecx,M_ILK      /* atomic compare and exchange */
 620         jne     lml_get_hw              /* branch on failure to retry */
 621
 622         movl    M_LOCKED,%ecx           /* get lock owner */
 623         testl   %ecx,%ecx               /* is the mutex locked? */
 624         jne     lml_fail                /* yes, we lose */
 625         movl    %gs:CPU_ACTIVE_THREAD,%ecx
 626         movl    %ecx,M_LOCKED
 627
 628         pushl   %edx                    /* save mutex address */
 629         pushl   %edx
 630         call    EXT(lck_mtx_lock_acquire)
 631         addl    $4,%esp
 632         popl    %edx                    /* restore mutex address */
 633
 634         xorl    %eax,%eax
 635         movl    %eax,M_ILK
 636
 637         popf                            /* restore interrupt state */
 638
 639         NONLEAF_RET
 640
 641 lml_fail:
 642         CHECK_MYLOCK(M_THREAD)
 643         pushl   %edx                    /* save mutex address */
 644         pushl   M_LOCKED
 645         pushl   %edx                    /* push mutex address */
 646         call    EXT(lck_mtx_lock_wait)  /* wait for the lock */
 647         addl    $8,%esp
 648         popl    %edx                    /* restore mutex address */
 649         jmp     lml_retry               /* and try again */
 650
 651 NONLEAF_ENTRY(lck_mtx_try_lock)
 652
 653         movl    B_ARG0,%edx             /* fetch lock pointer */
 654         cmpl    $(MUTEX_IND),M_ITAG     /* is this indirect? */
 655         cmove   M_PTR,%edx              /* yes - take indirection */
 656
 657         CHECK_NO_SIMPLELOCKS()
 658         CHECK_PREEMPTION_LEVEL()
 659
 660         movl    B_PC,%ecx
 661
 662         pushf                           /* save interrupt state */
 663         cli                             /* disable interrupts */
 664
 665 lmt_get_hw:
 666         movl    M_ILK,%eax              /* read interlock */
 667         testl   %eax,%eax               /* unlocked? */
 668         je      1f                      /* yes - attempt to lock it */
 669         PAUSE                           /* no  - pause */
 670         jmp     lmt_get_hw              /* try again */
 671 1:
 672         lock; cmpxchgl  %ecx,M_ILK      /* atomic compare and exchange */
 673         jne     lmt_get_hw              /* branch on failure to retry */
 674
 675         movl    M_LOCKED,%ecx           /* get lock owner */
 676         testl   %ecx,%ecx               /* is the mutex locked? */
 677         jne     lmt_fail                /* yes, we lose */
 678         movl    %gs:CPU_ACTIVE_THREAD,%ecx
 679         movl    %ecx,M_LOCKED
 680
 681         pushl   %edx                    /* save mutex address */
 682         pushl   %edx
 683         call    EXT(lck_mtx_lock_acquire)
 684         addl    $4,%esp
 685         popl    %edx                    /* restore mutex address */
 686
 687         xorl    %eax,%eax
 688         movl    %eax,M_ILK
 689
 690         popf                            /* restore interrupt state */
 691
 692         movl    $1,%eax                 /* return success */
 693         NONLEAF_RET
 694
 695 lmt_fail:
 696         xorl    %eax,%eax
 697         movl    %eax,M_ILK
 698
 699         popf                            /* restore interrupt state */
 700
 701         xorl    %eax,%eax               /* return failure */
 702         NONLEAF_RET
 703
 704 NONLEAF_ENTRY(lck_mtx_unlock)
 705
 706         movl    B_ARG0,%edx             /* fetch lock pointer */
 707         cmpl    $(MUTEX_IND),M_ITAG     /* is this indirect? */
 708         cmove   M_PTR,%edx              /* yes - take indirection */
 709
 710         movl    B_PC,%ecx
 711
 712         pushf                           /* save interrupt state */
 713         cli                             /* disable interrupts */
 714
 715 lmu_get_hw:
 716         movl    M_ILK,%eax              /* read interlock */
 717         testl   %eax,%eax               /* unlocked? */
 718         je      1f                      /* yes - attempt to lock it */
 719         PAUSE                           /* no  - pause */
 720         jmp     lmu_get_hw              /* try again */
 721 1:
 722         lock; cmpxchgl  %ecx,M_ILK      /* atomic compare and exchange */
 723         jne     lmu_get_hw              /* branch on failure to retry */
 724
 725         cmpw    $0,M_WAITERS            /* are there any waiters? */
 726         jne     lmu_wakeup              /* yes, more work to do */
 727
 728 lmu_doit:
 729         xorl    %ecx,%ecx
 730         movl    %ecx,M_LOCKED           /* unlock the mutex */
 731
 732         movl    %ecx,M_ILK
 733
 734         popf                            /* restore interrupt state */
 735
 736         NONLEAF_RET
 737
 738 lmu_wakeup:
 739         pushl   %edx                    /* save mutex address */
 740         pushl   M_LOCKED
 741         pushl   %edx                    /* push mutex address */
 742         call    EXT(lck_mtx_unlock_wakeup)/* yes, wake a thread */
 743         addl    $8,%esp
 744         popl    %edx                    /* restore mutex pointer */
 745         jmp     lmu_doit
 746
 747 LEAF_ENTRY(lck_mtx_ilk_unlock)
 748         movl    L_ARG0,%edx             /* no indirection here */
 749
 750         xorl    %eax,%eax
 751         movl    %eax,M_ILK
 752
 753         LEAF_RET
 754
 755 LEAF_ENTRY(_disable_preemption)
 756 #if     MACH_RT
 757         _DISABLE_PREEMPTION
 758 #endif  /* MACH_RT */
 759         LEAF_RET
 760
 761 LEAF_ENTRY(_enable_preemption)
 762 #if     MACH_RT
 763 #if     MACH_ASSERT
 764         cmpl    $0,%gs:CPU_PREEMPTION_LEVEL
 765         jg      1f
 766         pushl   %gs:CPU_PREEMPTION_LEVEL
 767         pushl   $2f
 768         call    EXT(panic)
 769         hlt
 770         .data
 771 2:      String  "_enable_preemption: preemption_level(%d)  < 0!"
 772         .text
 773 1:
 774 #endif  /* MACH_ASSERT */
 775         _ENABLE_PREEMPTION
 776 #endif  /* MACH_RT */
 777         LEAF_RET
 778
 779 LEAF_ENTRY(_enable_preemption_no_check)
 780 #if     MACH_RT
 781 #if     MACH_ASSERT
 782         cmpl    $0,%gs:CPU_PREEMPTION_LEVEL
 783         jg      1f
 784         pushl   $2f
 785         call    EXT(panic)
 786         hlt
 787         .data
 788 2:      String  "_enable_preemption_no_check: preemption_level <= 0!"
 789         .text
 790 1:
 791 #endif  /* MACH_ASSERT */
 792         _ENABLE_PREEMPTION_NO_CHECK
 793 #endif  /* MACH_RT */
 794         LEAF_RET
 795
 796
 797 LEAF_ENTRY(_mp_disable_preemption)
 798 #if     MACH_RT
 799         _DISABLE_PREEMPTION
 800 #endif  /* MACH_RT */
 801         LEAF_RET
 802
 803 LEAF_ENTRY(_mp_enable_preemption)
 804 #if     MACH_RT
 805 #if     MACH_ASSERT
 806         cmpl    $0,%gs:CPU_PREEMPTION_LEVEL
 807         jg      1f
 808         pushl   %gs:CPU_PREEMPTION_LEVEL
 809         pushl   $2f
 810         call    EXT(panic)
 811         hlt
 812         .data
 813 2:      String  "_mp_enable_preemption: preemption_level (%d) <= 0!"
 814         .text
 815 1:
 816 #endif  /* MACH_ASSERT */
 817         _ENABLE_PREEMPTION
 818 #endif  /* MACH_RT */
 819         LEAF_RET
 820
 821 LEAF_ENTRY(_mp_enable_preemption_no_check)
 822 #if     MACH_RT
 823 #if     MACH_ASSERT
 824         cmpl    $0,%gs:CPU_PREEMPTION_LEVEL
 825         jg      1f
 826         pushl   $2f
 827         call    EXT(panic)
 828         hlt
 829         .data
 830 2:      String  "_mp_enable_preemption_no_check: preemption_level <= 0!"
 831         .text
 832 1:
 833 #endif  /* MACH_ASSERT */
 834         _ENABLE_PREEMPTION_NO_CHECK
 835 #endif  /* MACH_RT */
 836         LEAF_RET
 837
 838
 839 LEAF_ENTRY(i_bit_set)
 840         movl    L_ARG0,%edx
 841         movl    L_ARG1,%eax
 842         lock
 843         bts     %dl,(%eax)
 844         LEAF_RET
 845
 846 LEAF_ENTRY(i_bit_clear)
 847         movl    L_ARG0,%edx
 848         movl    L_ARG1,%eax
 849         lock
 850         btr     %dl,(%eax)
 851         LEAF_RET
 852
 853 LEAF_ENTRY(bit_lock)
 854         movl    L_ARG0,%ecx
 855         movl    L_ARG1,%eax
 856 1:
 857         lock
 858         bts     %ecx,(%eax)
 859         jb      1b
 860         LEAF_RET
 861
 862 LEAF_ENTRY(bit_lock_try)
 863         movl    L_ARG0,%ecx
 864         movl    L_ARG1,%eax
 865         lock
 866         bts     %ecx,(%eax)
 867         jb      bit_lock_failed
 868         LEAF_RET                /* %eax better not be null ! */
 869 bit_lock_failed:
 870         xorl    %eax,%eax
 871         LEAF_RET
 872
 873 LEAF_ENTRY(bit_unlock)
 874         movl    L_ARG0,%ecx
 875         movl    L_ARG1,%eax
 876         lock
 877         btr     %ecx,(%eax)
 878         LEAF_RET