]> git.saurik.com Git - apple/xnu.git/blob - osfmk/i386/i386_lock.s
projects / apple / xnu.git / blob
? search:
summary | shortlog | log | commit | commitdiff | tree
blame | history | raw | HEAD
xnu-3789.70.16.tar.gz
[apple/xnu.git] / osfmk / i386 / i386_lock.s
1 /*
2 * Copyright (c) 2000-2012 Apple Inc. All rights reserved.
3 *
4 * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
5 *
6 * This file contains Original Code and/or Modifications of Original Code
7 * as defined in and that are subject to the Apple Public Source License
8 * Version 2.0 (the 'License'). You may not use this file except in
9 * compliance with the License. The rights granted to you under the License
10 * may not be used to create, or enable the creation or redistribution of,
11 * unlawful or unlicensed copies of an Apple operating system, or to
12 * circumvent, violate, or enable the circumvention or violation of, any
13 * terms of an Apple operating system software license agreement.
14 *
15 * Please obtain a copy of the License at
16 * http://www.opensource.apple.com/apsl/ and read it before using this file.
17 *
18 * The Original Code and all software distributed under the License are
19 * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
20 * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
21 * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
22 * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
23 * Please see the License for the specific language governing rights and
24 * limitations under the License.
25 *
26 * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
27 */
28 /*
29 * @OSF_COPYRIGHT@
30 */
31 /*
32 * Mach Operating System
33 * Copyright (c) 1989 Carnegie-Mellon University
34 * All rights reserved. The CMU software License Agreement specifies
35 * the terms and conditions for use and redistribution.
36 */
37
38 #include <mach_rt.h>
39 #include <mach_ldebug.h>
40 #include <i386/asm.h>
41 #include <i386/eflags.h>
42 #include <i386/trap.h>
43 #include <config_dtrace.h>
44 #include <i386/mp.h>
45
46 #include "assym.s"
47
48 #define PAUSE rep; nop
49
50 #include <i386/pal_lock_asm.h>
51
52 #define LEAF_ENTRY(name) \
53 Entry(name)
54
55 #define LEAF_ENTRY2(n1,n2) \
56 Entry(n1); \
57 Entry(n2)
58
59 #define LEAF_RET \
60 ret
61
62 /* Non-leaf routines always have a stack frame: */
63
64 #define NONLEAF_ENTRY(name) \
65 Entry(name); \
66 FRAME
67
68 #define NONLEAF_ENTRY2(n1,n2) \
69 Entry(n1); \
70 Entry(n2); \
71 FRAME
72
73 #define NONLEAF_RET \
74 EMARF; \
75 ret
76
77
78 /* For x86_64, the varargs ABI requires that %al indicate
79 * how many SSE register contain arguments. In our case, 0 */
80 #define ALIGN_STACK() and $0xFFFFFFFFFFFFFFF0, %rsp ;
81 #define LOAD_STRING_ARG0(label) leaq label(%rip), %rdi ;
82 #define LOAD_ARG1(x) mov x, %esi ;
83 #define LOAD_PTR_ARG1(x) mov x, %rsi ;
84 #define CALL_PANIC() xorb %al,%al ; call EXT(panic) ;
85
86 #define CHECK_UNLOCK(current, owner) \
87 cmp current, owner ; \
88 je 1f ; \
89 ALIGN_STACK() ; \
90 LOAD_STRING_ARG0(2f) ; \
91 CALL_PANIC() ; \
92 hlt ; \
93 .data ; \
94 2: String "Mutex unlock attempted from non-owner thread"; \
95 .text ; \
96 1:
97
98 #if MACH_LDEBUG
99 /*
100 * Routines for general lock debugging.
101 */
102
103 /*
104 * Checks for expected lock types and calls "panic" on
105 * mismatch. Detects calls to Mutex functions with
106 * type simplelock and vice versa.
107 */
108 #define CHECK_MUTEX_TYPE() \
109 cmpl $ MUTEX_TAG,M_TYPE ; \
110 je 1f ; \
111 ALIGN_STACK() ; \
112 LOAD_STRING_ARG0(2f) ; \
113 CALL_PANIC() ; \
114 hlt ; \
115 .data ; \
116 2: String "not a mutex!" ; \
117 .text ; \
118 1:
119
120 /*
121 * If one or more simplelocks are currently held by a thread,
122 * an attempt to acquire a mutex will cause this check to fail
123 * (since a mutex lock may context switch, holding a simplelock
124 * is not a good thing).
125 */
126 #if MACH_RT
127 #define CHECK_PREEMPTION_LEVEL() \
128 cmpl $0,%gs:CPU_HIBERNATE ; \
129 jne 1f ; \
130 cmpl $0,%gs:CPU_PREEMPTION_LEVEL ; \
131 je 1f ; \
132 ALIGN_STACK() ; \
133 movl %gs:CPU_PREEMPTION_LEVEL, %eax ; \
134 LOAD_ARG1(%eax) ; \
135 LOAD_STRING_ARG0(2f) ; \
136 CALL_PANIC() ; \
137 hlt ; \
138 .data ; \
139 2: String "preemption_level(%d) != 0!" ; \
140 .text ; \
141 1:
142 #else /* MACH_RT */
143 #define CHECK_PREEMPTION_LEVEL()
144 #endif /* MACH_RT */
145
146 #define CHECK_MYLOCK(current, owner) \
147 cmp current, owner ; \
148 jne 1f ; \
149 ALIGN_STACK() ; \
150 LOAD_STRING_ARG0(2f) ; \
151 CALL_PANIC() ; \
152 hlt ; \
153 .data ; \
154 2: String "Attempt to recursively lock a non-recursive lock"; \
155 .text ; \
156 1:
157
158 #else /* MACH_LDEBUG */
159 #define CHECK_MUTEX_TYPE()
160 #define CHECK_PREEMPTION_LEVEL()
161 #define CHECK_MYLOCK(thd)
162 #endif /* MACH_LDEBUG */
163
164 #define PREEMPTION_DISABLE \
165 incl %gs:CPU_PREEMPTION_LEVEL
166
167 #define PREEMPTION_LEVEL_DEBUG 1
168 #if PREEMPTION_LEVEL_DEBUG
169 #define PREEMPTION_ENABLE \
170 decl %gs:CPU_PREEMPTION_LEVEL ; \
171 js 17f ; \
172 jnz 19f ; \
173 testl $AST_URGENT,%gs:CPU_PENDING_AST ; \
174 jz 19f ; \
175 PUSHF ; \
176 testl $EFL_IF, S_PC ; \
177 jz 18f ; \
178 POPF ; \
179 int $(T_PREEMPT) ; \
180 jmp 19f ; \
181 17: \
182 call _preemption_underflow_panic ; \
183 18: \
184 POPF ; \
185 19:
186 #else
187 #define PREEMPTION_ENABLE \
188 decl %gs:CPU_PREEMPTION_LEVEL ; \
189 jnz 19f ; \
190 testl $AST_URGENT,%gs:CPU_PENDING_AST ; \
191 jz 19f ; \
192 PUSHF ; \
193 testl $EFL_IF, S_PC ; \
194 jz 18f ; \
195 POPF ; \
196 int $(T_PREEMPT) ; \
197 jmp 19f ; \
198 18: \
199 POPF ; \
200 19:
201 #endif
202
203
204 #if CONFIG_DTRACE
205
206 .globl _lockstat_probe
207 .globl _lockstat_probemap
208
209 /*
210 * LOCKSTAT_LABEL creates a dtrace symbol which contains
211 * a pointer into the lock code function body. At that
212 * point is a "ret" instruction that can be patched into
213 * a "nop"
214 */
215
216 #define LOCKSTAT_LABEL(lab) \
217 .data ;\
218 .globl lab ;\
219 lab: ;\
220 .quad 9f ;\
221 .text ;\
222 9:
223
224 #define LOCKSTAT_RECORD(id, lck) \
225 push %rbp ; \
226 mov %rsp,%rbp ; \
227 movl _lockstat_probemap + (id * 4)(%rip),%eax ; \
228 test %eax,%eax ; \
229 je 9f ; \
230 mov lck, %rsi ; \
231 mov %rax, %rdi ; \
232 mov $0, %rdx ; \
233 mov $0, %rcx ; \
234 mov $0, %r8 ; \
235 mov $0, %r9 ; \
236 call *_lockstat_probe(%rip) ; \
237 9: leave
238 /* ret - left to subsequent code, e.g. return values */
239
240 #endif /* CONFIG_DTRACE */
241
242 /*
243 * For most routines, the hw_lock_t pointer is loaded into a
244 * register initially, and then either a byte or register-sized
245 * word is loaded/stored to the pointer
246 */
247
248 /*
249 * void hw_lock_byte_init(volatile uint8_t *)
250 *
251 * Initialize a hardware byte lock.
252 */
253 LEAF_ENTRY(hw_lock_byte_init)
254 movb $0, (%rdi) /* clear the lock */
255 LEAF_RET
256
257 /*
258 * void hw_lock_byte_lock(uint8_t *lock_byte)
259 *
260 * Acquire byte sized lock operand, spinning until it becomes available.
261 * MACH_RT: also return with preemption disabled.
262 */
263
264 LEAF_ENTRY(hw_lock_byte_lock)
265 PREEMPTION_DISABLE
266 movl $1, %ecx /* Set lock value */
267 1:
268 movb (%rdi), %al /* Load byte at address */
269 testb %al,%al /* lock locked? */
270 jne 3f /* branch if so */
271 lock; cmpxchg %cl,(%rdi) /* attempt atomic compare exchange */
272 jne 3f
273 LEAF_RET /* if yes, then nothing left to do */
274 3:
275 PAUSE /* pause for hyper-threading */
276 jmp 1b /* try again */
277
278 /*
279 * void hw_lock_byte_unlock(uint8_t *lock_byte)
280 *
281 * Unconditionally release byte sized lock operand.
282 * MACH_RT: release preemption level.
283 */
284
285 LEAF_ENTRY(hw_lock_byte_unlock)
286 movb $0, (%rdi) /* Clear the lock byte */
287 PREEMPTION_ENABLE
288 LEAF_RET
289
290 /*
291 * Reader-writer lock fastpaths. These currently exist for the
292 * shared lock acquire, the exclusive lock acquire, the shared to
293 * exclusive upgrade and the release paths (where they reduce overhead
294 * considerably) -- these are by far the most frequently used routines
295 *
296 * The following should reflect the layout of the bitfield embedded within
297 * the lck_rw_t structure (see i386/locks.h).
298 */
299 #define LCK_RW_INTERLOCK (0x1 << 16)
300
301 #define LCK_RW_PRIV_EXCL (0x1 << 24)
302 #define LCK_RW_WANT_UPGRADE (0x2 << 24)
303 #define LCK_RW_WANT_WRITE (0x4 << 24)
304 #define LCK_R_WAITING (0x8 << 24)
305 #define LCK_W_WAITING (0x10 << 24)
306
307 #define LCK_RW_SHARED_MASK (0xffff)
308
309 /*
310 * For most routines, the lck_rw_t pointer is loaded into a
311 * register initially, and the flags bitfield loaded into another
312 * register and examined
313 */
314
315 #define RW_LOCK_SHARED_MASK (LCK_RW_INTERLOCK | LCK_RW_WANT_UPGRADE | LCK_RW_WANT_WRITE)
316 /*
317 * void lck_rw_lock_shared(lck_rw_t *)
318 *
319 */
320 Entry(lck_rw_lock_shared)
321 mov %gs:CPU_ACTIVE_THREAD, %rcx /* Load thread pointer */
322 incl TH_RWLOCK_COUNT(%rcx) /* Increment count before atomic CAS */
323 1:
324 mov (%rdi), %eax /* Load state bitfield and interlock */
325 testl $(RW_LOCK_SHARED_MASK), %eax /* Eligible for fastpath? */
326 jne 3f
327
328 movl %eax, %ecx /* original value in %eax for cmpxchgl */
329 incl %ecx /* Increment reader refcount */
330 lock
331 cmpxchgl %ecx, (%rdi) /* Attempt atomic exchange */
332 jne 2f
333
334 #if CONFIG_DTRACE
335 /*
336 * Dtrace lockstat event: LS_LCK_RW_LOCK_SHARED_ACQUIRE
337 * Implemented by swapping between return and no-op instructions.
338 * See bsd/dev/dtrace/lockstat.c.
339 */
340 LOCKSTAT_LABEL(_lck_rw_lock_shared_lockstat_patch_point)
341 ret
342 /*
343 Fall thru when patched, counting on lock pointer in %rdi
344 */
345 LOCKSTAT_RECORD(LS_LCK_RW_LOCK_SHARED_ACQUIRE, %rdi)
346 #endif
347 ret
348 2:
349 PAUSE
350 jmp 1b
351 3:
352 jmp EXT(lck_rw_lock_shared_gen)
353
354
355
356 #define RW_TRY_LOCK_SHARED_MASK (LCK_RW_WANT_UPGRADE | LCK_RW_WANT_WRITE)
357 /*
358 * void lck_rw_try_lock_shared(lck_rw_t *)
359 *
360 */
361 Entry(lck_rw_try_lock_shared)
362 1:
363 mov (%rdi), %eax /* Load state bitfield and interlock */
364 testl $(LCK_RW_INTERLOCK), %eax
365 jne 2f
366 testl $(RW_TRY_LOCK_SHARED_MASK), %eax
367 jne 3f /* lock is busy */
368
369 movl %eax, %ecx /* original value in %eax for cmpxchgl */
370 incl %ecx /* Increment reader refcount */
371 lock
372 cmpxchgl %ecx, (%rdi) /* Attempt atomic exchange */
373 jne 2f
374
375 mov %gs:CPU_ACTIVE_THREAD, %rcx /* Load thread pointer */
376 incl TH_RWLOCK_COUNT(%rcx) /* Increment count on success. */
377 /* There is a 3 instr window where preemption may not notice rwlock_count after cmpxchg */
378
379 #if CONFIG_DTRACE
380 movl $1, %eax
381 /*
382 * Dtrace lockstat event: LS_LCK_RW_TRY_LOCK_SHARED_ACQUIRE
383 * Implemented by swapping between return and no-op instructions.
384 * See bsd/dev/dtrace/lockstat.c.
385 */
386 LOCKSTAT_LABEL(_lck_rw_try_lock_shared_lockstat_patch_point)
387 ret
388 /* Fall thru when patched, counting on lock pointer in %rdi */
389 LOCKSTAT_RECORD(LS_LCK_RW_TRY_LOCK_SHARED_ACQUIRE, %rdi)
390 #endif
391 movl $1, %eax /* return TRUE */
392 ret
393 2:
394 PAUSE
395 jmp 1b
396 3:
397 xorl %eax, %eax
398 ret
399
400
401 #define RW_LOCK_EXCLUSIVE_HELD (LCK_RW_WANT_WRITE | LCK_RW_WANT_UPGRADE)
402 /*
403 * int lck_rw_grab_shared(lck_rw_t *)
404 *
405 */
406 Entry(lck_rw_grab_shared)
407 1:
408 mov (%rdi), %eax /* Load state bitfield and interlock */
409 testl $(LCK_RW_INTERLOCK), %eax
410 jne 5f
411 testl $(RW_LOCK_EXCLUSIVE_HELD), %eax
412 jne 3f
413 2:
414 movl %eax, %ecx /* original value in %eax for cmpxchgl */
415 incl %ecx /* Increment reader refcount */
416 lock
417 cmpxchgl %ecx, (%rdi) /* Attempt atomic exchange */
418 jne 4f
419
420 movl $1, %eax /* return success */
421 ret
422 3:
423 testl $(LCK_RW_SHARED_MASK), %eax
424 je 4f
425 testl $(LCK_RW_PRIV_EXCL), %eax
426 je 2b
427 4:
428 xorl %eax, %eax /* return failure */
429 ret
430 5:
431 PAUSE
432 jmp 1b
433
434
435
436 #define RW_LOCK_EXCLUSIVE_MASK (LCK_RW_SHARED_MASK | LCK_RW_INTERLOCK | \
437 LCK_RW_WANT_UPGRADE | LCK_RW_WANT_WRITE)
438 /*
439 * void lck_rw_lock_exclusive(lck_rw_t*)
440 *
441 */
442 Entry(lck_rw_lock_exclusive)
443 mov %gs:CPU_ACTIVE_THREAD, %rcx /* Load thread pointer */
444 incl TH_RWLOCK_COUNT(%rcx) /* Increment count before atomic CAS */
445 1:
446 mov (%rdi), %eax /* Load state bitfield, interlock and shared count */
447 testl $(RW_LOCK_EXCLUSIVE_MASK), %eax /* Eligible for fastpath? */
448 jne 3f /* no, go slow */
449
450 movl %eax, %ecx /* original value in %eax for cmpxchgl */
451 orl $(LCK_RW_WANT_WRITE), %ecx
452 lock
453 cmpxchgl %ecx, (%rdi) /* Attempt atomic exchange */
454 jne 2f
455
456 #if CONFIG_DTRACE
457 /*
458 * Dtrace lockstat event: LS_LCK_RW_LOCK_EXCL_ACQUIRE
459 * Implemented by swapping between return and no-op instructions.
460 * See bsd/dev/dtrace/lockstat.c.
461 */
462 LOCKSTAT_LABEL(_lck_rw_lock_exclusive_lockstat_patch_point)
463 ret
464 /* Fall thru when patched, counting on lock pointer in %rdi */
465 LOCKSTAT_RECORD(LS_LCK_RW_LOCK_EXCL_ACQUIRE, %rdi)
466 #endif
467 ret
468 2:
469 PAUSE
470 jmp 1b
471 3:
472 jmp EXT(lck_rw_lock_exclusive_gen)
473
474
475
476 #define RW_TRY_LOCK_EXCLUSIVE_MASK (LCK_RW_SHARED_MASK | LCK_RW_WANT_UPGRADE | LCK_RW_WANT_WRITE)
477 /*
478 * void lck_rw_try_lock_exclusive(lck_rw_t *)
479 *
480 * Tries to get a write lock.
481 *
482 * Returns FALSE if the lock is not held on return.
483 */
484 Entry(lck_rw_try_lock_exclusive)
485 1:
486 mov (%rdi), %eax /* Load state bitfield, interlock and shared count */
487 testl $(LCK_RW_INTERLOCK), %eax
488 jne 2f
489 testl $(RW_TRY_LOCK_EXCLUSIVE_MASK), %eax
490 jne 3f /* can't get it */
491
492 movl %eax, %ecx /* original value in %eax for cmpxchgl */
493 orl $(LCK_RW_WANT_WRITE), %ecx
494 lock
495 cmpxchgl %ecx, (%rdi) /* Attempt atomic exchange */
496 jne 2f
497
498 mov %gs:CPU_ACTIVE_THREAD, %rcx /* Load thread pointer */
499 incl TH_RWLOCK_COUNT(%rcx) /* Increment count on success. */
500 /* There is a 3 instr window where preemption may not notice rwlock_count after cmpxchg */
501
502 #if CONFIG_DTRACE
503 movl $1, %eax
504 /*
505 * Dtrace lockstat event: LS_LCK_RW_TRY_LOCK_EXCL_ACQUIRE
506 * Implemented by swapping between return and no-op instructions.
507 * See bsd/dev/dtrace/lockstat.c.
508 */
509 LOCKSTAT_LABEL(_lck_rw_try_lock_exclusive_lockstat_patch_point)
510 ret
511 /* Fall thru when patched, counting on lock pointer in %rdi */
512 LOCKSTAT_RECORD(LS_LCK_RW_TRY_LOCK_EXCL_ACQUIRE, %rdi)
513 #endif
514 movl $1, %eax /* return TRUE */
515 ret
516 2:
517 PAUSE
518 jmp 1b
519 3:
520 xorl %eax, %eax /* return FALSE */
521 ret
522
523
524
525 /*
526 * void lck_rw_lock_shared_to_exclusive(lck_rw_t*)
527 *
528 * fastpath can be taken if
529 * the current rw_shared_count == 1
530 * AND the interlock is clear
531 * AND RW_WANT_UPGRADE is not set
532 *
533 * note that RW_WANT_WRITE could be set, but will not
534 * be indicative of an exclusive hold since we have
535 * a read count on the lock that we have not yet released
536 * we can blow by that state since the lck_rw_lock_exclusive
537 * function will block until rw_shared_count == 0 and
538 * RW_WANT_UPGRADE is clear... it does this check behind
539 * the interlock which we are also checking for
540 *
541 * to make the transition we must be able to atomically
542 * set RW_WANT_UPGRADE and get rid of the read count we hold
543 */
544 Entry(lck_rw_lock_shared_to_exclusive)
545 1:
546 mov (%rdi), %eax /* Load state bitfield, interlock and shared count */
547 testl $(LCK_RW_INTERLOCK), %eax
548 jne 7f
549 testl $(LCK_RW_WANT_UPGRADE), %eax
550 jne 2f
551
552 movl %eax, %ecx /* original value in %eax for cmpxchgl */
553 orl $(LCK_RW_WANT_UPGRADE), %ecx /* ask for WANT_UPGRADE */
554 decl %ecx /* and shed our read count */
555 lock
556 cmpxchgl %ecx, (%rdi) /* Attempt atomic exchange */
557 jne 7f
558 /* we now own the WANT_UPGRADE */
559 testl $(LCK_RW_SHARED_MASK), %ecx /* check to see if all of the readers are drained */
560 jne 8f /* if not, we need to go wait */
561
562 #if CONFIG_DTRACE
563 movl $1, %eax
564 /*
565 * Dtrace lockstat event: LS_LCK_RW_LOCK_SHARED_TO_EXCL_UPGRADE
566 * Implemented by swapping between return and no-op instructions.
567 * See bsd/dev/dtrace/lockstat.c.
568 */
569 LOCKSTAT_LABEL(_lck_rw_lock_shared_to_exclusive_lockstat_patch_point)
570 ret
571 /* Fall thru when patched, counting on lock pointer in %rdi */
572 LOCKSTAT_RECORD(LS_LCK_RW_LOCK_SHARED_TO_EXCL_UPGRADE, %rdi)
573 #endif
574 movl $1, %eax /* return success */
575 ret
576
577 2: /* someone else already holds WANT_UPGRADE */
578 movl %eax, %ecx /* original value in %eax for cmpxchgl */
579 decl %ecx /* shed our read count */
580 testl $(LCK_RW_SHARED_MASK), %ecx
581 jne 3f /* we were the last reader */
582 andl $(~LCK_W_WAITING), %ecx /* so clear the wait indicator */
583 3:
584 lock
585 cmpxchgl %ecx, (%rdi) /* Attempt atomic exchange */
586 jne 7f
587
588 mov %eax, %esi /* put old flags as second arg */
589 /* lock is alread in %rdi */
590 call EXT(lck_rw_lock_shared_to_exclusive_failure)
591 ret /* and pass the failure return along */
592 7:
593 PAUSE
594 jmp 1b
595 8:
596 jmp EXT(lck_rw_lock_shared_to_exclusive_success)
597
598
599
600 .cstring
601 rwl_release_error_str:
602 .asciz "Releasing non-exclusive RW lock without a reader refcount!"
603 .text
604
605 /*
606 * lck_rw_type_t lck_rw_done(lck_rw_t *)
607 *
608 */
609 Entry(lck_rw_done)
610 1:
611 mov (%rdi), %eax /* Load state bitfield, interlock and reader count */
612 testl $(LCK_RW_INTERLOCK), %eax
613 jne 7f /* wait for interlock to clear */
614
615 movl %eax, %ecx /* keep original value in %eax for cmpxchgl */
616 testl $(LCK_RW_SHARED_MASK), %ecx /* if reader count == 0, must be exclusive lock */
617 je 2f
618 decl %ecx /* Decrement reader count */
619 testl $(LCK_RW_SHARED_MASK), %ecx /* if reader count has now gone to 0, check for waiters */
620 je 4f
621 jmp 6f
622 2:
623 testl $(LCK_RW_WANT_UPGRADE), %ecx
624 je 3f
625 andl $(~LCK_RW_WANT_UPGRADE), %ecx
626 jmp 4f
627 3:
628 testl $(LCK_RW_WANT_WRITE), %ecx
629 je 8f /* lock is not 'owned', go panic */
630 andl $(~LCK_RW_WANT_WRITE), %ecx
631 4:
632 /*
633 * test the original values to match what
634 * lck_rw_done_gen is going to do to determine
635 * which wakeups need to happen...
636 *
637 * if !(fake_lck->lck_rw_priv_excl && fake_lck->lck_w_waiting)
638 */
639 testl $(LCK_W_WAITING), %eax
640 je 5f
641 andl $(~LCK_W_WAITING), %ecx
642
643 testl $(LCK_RW_PRIV_EXCL), %eax
644 jne 6f
645 5:
646 andl $(~LCK_R_WAITING), %ecx
647 6:
648 lock
649 cmpxchgl %ecx, (%rdi) /* Attempt atomic exchange */
650 jne 7f
651
652 mov %eax,%esi /* old flags in %rsi */
653 /* lock is in %rdi already */
654 call EXT(lck_rw_done_gen)
655 ret
656 7:
657 PAUSE
658 jmp 1b
659 8:
660 ALIGN_STACK()
661 LOAD_STRING_ARG0(rwl_release_error_str)
662 CALL_PANIC()
663
664
665
666 /*
667 * lck_rw_type_t lck_rw_lock_exclusive_to_shared(lck_rw_t *)
668 *
669 */
670 Entry(lck_rw_lock_exclusive_to_shared)
671 1:
672 mov (%rdi), %eax /* Load state bitfield, interlock and reader count */
673 testl $(LCK_RW_INTERLOCK), %eax
674 jne 6f /* wait for interlock to clear */
675
676 movl %eax, %ecx /* keep original value in %eax for cmpxchgl */
677 incl %ecx /* Increment reader count */
678
679 testl $(LCK_RW_WANT_UPGRADE), %ecx
680 je 2f
681 andl $(~LCK_RW_WANT_UPGRADE), %ecx
682 jmp 3f
683 2:
684 andl $(~LCK_RW_WANT_WRITE), %ecx
685 3:
686 /*
687 * test the original values to match what
688 * lck_rw_lock_exclusive_to_shared_gen is going to do to determine
689 * which wakeups need to happen...
690 *
691 * if !(fake_lck->lck_rw_priv_excl && fake_lck->lck_w_waiting)
692 */
693 testl $(LCK_W_WAITING), %eax
694 je 4f
695 testl $(LCK_RW_PRIV_EXCL), %eax
696 jne 5f
697 4:
698 andl $(~LCK_R_WAITING), %ecx
699 5:
700 lock
701 cmpxchgl %ecx, (%rdi) /* Attempt atomic exchange */
702 jne 6f
703
704 mov %eax,%esi
705 call EXT(lck_rw_lock_exclusive_to_shared_gen)
706 ret
707 6:
708 PAUSE
709 jmp 1b
710
711
712
713 /*
714 * int lck_rw_grab_want(lck_rw_t *)
715 *
716 */
717 Entry(lck_rw_grab_want)
718 1:
719 mov (%rdi), %eax /* Load state bitfield, interlock and reader count */
720 testl $(LCK_RW_INTERLOCK), %eax
721 jne 3f /* wait for interlock to clear */
722 testl $(LCK_RW_WANT_WRITE), %eax /* want_write has been grabbed by someone else */
723 jne 2f /* go return failure */
724
725 movl %eax, %ecx /* original value in %eax for cmpxchgl */
726 orl $(LCK_RW_WANT_WRITE), %ecx
727 lock
728 cmpxchgl %ecx, (%rdi) /* Attempt atomic exchange */
729 jne 2f
730 /* we now own want_write */
731 movl $1, %eax /* return success */
732 ret
733 2:
734 xorl %eax, %eax /* return failure */
735 ret
736 3:
737 PAUSE
738 jmp 1b
739
740
741 #define RW_LOCK_SHARED_OR_UPGRADE_MASK (LCK_RW_SHARED_MASK | LCK_RW_INTERLOCK | LCK_RW_WANT_UPGRADE)
742 /*
743 * int lck_rw_held_read_or_upgrade(lck_rw_t *)
744 *
745 */
746 Entry(lck_rw_held_read_or_upgrade)
747 mov (%rdi), %eax
748 andl $(RW_LOCK_SHARED_OR_UPGRADE_MASK), %eax
749 ret
750
751
752
753 /*
754 * N.B.: On x86, statistics are currently recorded for all indirect mutexes.
755 * Also, only the acquire attempt count (GRP_MTX_STAT_UTIL) is maintained
756 * as a 64-bit quantity (this matches the existing PowerPC implementation,
757 * and the new x86 specific statistics are also maintained as 32-bit
758 * quantities).
759 *
760 *
761 * Enable this preprocessor define to record the first miss alone
762 * By default, we count every miss, hence multiple misses may be
763 * recorded for a single lock acquire attempt via lck_mtx_lock
764 */
765 #undef LOG_FIRST_MISS_ALONE
766
767 /*
768 * This preprocessor define controls whether the R-M-W update of the
769 * per-group statistics elements are atomic (LOCK-prefixed)
770 * Enabled by default.
771 */
772 #define ATOMIC_STAT_UPDATES 1
773
774 #if defined(ATOMIC_STAT_UPDATES)
775 #define LOCK_IF_ATOMIC_STAT_UPDATES lock
776 #else
777 #define LOCK_IF_ATOMIC_STAT_UPDATES
778 #endif /* ATOMIC_STAT_UPDATES */
779
780
781 /*
782 * For most routines, the lck_mtx_t pointer is loaded into a
783 * register initially, and the owner field checked for indirection.
784 * Eventually the lock owner is loaded into a register and examined.
785 */
786
787 #define M_OWNER MUTEX_OWNER
788 #define M_PTR MUTEX_PTR
789 #define M_STATE MUTEX_STATE
790
791
792 #define LMTX_ENTER_EXTENDED \
793 mov M_PTR(%rdx), %rdx ; \
794 xor %r11, %r11 ; \
795 mov MUTEX_GRP(%rdx), %r10 ; \
796 LOCK_IF_ATOMIC_STAT_UPDATES ; \
797 incq GRP_MTX_STAT_UTIL(%r10)
798
799
800 #if LOG_FIRST_MISS_ALONE
801 #define LMTX_UPDATE_MISS \
802 test $1, %r11 ; \
803 jnz 11f ; \
804 LOCK_IF_ATOMIC_STAT_UPDATES ; \
805 incl GRP_MTX_STAT_MISS(%r10) ; \
806 or $1, %r11 ; \
807 11:
808 #else
809 #define LMTX_UPDATE_MISS \
810 LOCK_IF_ATOMIC_STAT_UPDATES ; \
811 incl GRP_MTX_STAT_MISS(%r10)
812 #endif
813
814
815 #if LOG_FIRST_MISS_ALONE
816 #define LMTX_UPDATE_WAIT \
817 test $2, %r11 ; \
818 jnz 11f ; \
819 LOCK_IF_ATOMIC_STAT_UPDATES ; \
820 incl GRP_MTX_STAT_WAIT(%r10) ; \
821 or $2, %r11 ; \
822 11:
823 #else
824 #define LMTX_UPDATE_WAIT \
825 LOCK_IF_ATOMIC_STAT_UPDATES ; \
826 incl GRP_MTX_STAT_WAIT(%r10)
827 #endif
828
829
830 /*
831 * Record the "direct wait" statistic, which indicates if a
832 * miss proceeded to block directly without spinning--occurs
833 * if the owner of the mutex isn't running on another processor
834 * at the time of the check.
835 */
836 #define LMTX_UPDATE_DIRECT_WAIT \
837 LOCK_IF_ATOMIC_STAT_UPDATES ; \
838 incl GRP_MTX_STAT_DIRECT_WAIT(%r10)
839
840
841 #define LMTX_CALLEXT1(func_name) \
842 cmp %rdx, %rdi ; \
843 je 12f ; \
844 push %r10 ; \
845 push %r11 ; \
846 12: push %rdi ; \
847 push %rdx ; \
848 mov %rdx, %rdi ; \
849 call EXT(func_name) ; \
850 pop %rdx ; \
851 pop %rdi ; \
852 cmp %rdx, %rdi ; \
853 je 12f ; \
854 pop %r11 ; \
855 pop %r10 ; \
856 12:
857
858 #define LMTX_CALLEXT2(func_name, reg) \
859 cmp %rdx, %rdi ; \
860 je 12f ; \
861 push %r10 ; \
862 push %r11 ; \
863 12: push %rdi ; \
864 push %rdx ; \
865 mov reg, %rsi ; \
866 mov %rdx, %rdi ; \
867 call EXT(func_name) ; \
868 pop %rdx ; \
869 pop %rdi ; \
870 cmp %rdx, %rdi ; \
871 je 12f ; \
872 pop %r11 ; \
873 pop %r10 ; \
874 12:
875
876
877 #define M_WAITERS_MSK 0x0000ffff
878 #define M_PRIORITY_MSK 0x00ff0000
879 #define M_ILOCKED_MSK 0x01000000
880 #define M_MLOCKED_MSK 0x02000000
881 #define M_PROMOTED_MSK 0x04000000
882 #define M_SPIN_MSK 0x08000000
883
884 /*
885 * void lck_mtx_assert(lck_mtx_t* l, unsigned int)
886 * Takes the address of a lock, and an assertion type as parameters.
887 * The assertion can take one of two forms determine by the type
888 * parameter: either the lock is held by the current thread, and the
889 * type is LCK_MTX_ASSERT_OWNED, or it isn't and the type is
890 * LCK_MTX_ASSERT_NOTOWNED. Calls panic on assertion failure.
891 *
892 */
893
894 NONLEAF_ENTRY(lck_mtx_assert)
895 mov %rdi, %rdx /* Load lock address */
896 mov %gs:CPU_ACTIVE_THREAD, %rax /* Load current thread */
897
898 mov M_STATE(%rdx), %ecx
899 cmp $(MUTEX_IND), %ecx /* Is this an indirect mutex? */
900 jne 0f
901 mov M_PTR(%rdx), %rdx /* If so, take indirection */
902 0:
903 mov M_OWNER(%rdx), %rcx /* Load owner */
904 cmp $(MUTEX_ASSERT_OWNED), %rsi
905 jne 2f /* Assert ownership? */
906 cmp %rax, %rcx /* Current thread match? */
907 jne 3f /* no, go panic */
908 testl $(M_ILOCKED_MSK | M_MLOCKED_MSK), M_STATE(%rdx)
909 je 3f
910 1: /* yes, we own it */
911 NONLEAF_RET
912 2:
913 cmp %rax, %rcx /* Current thread match? */
914 jne 1b /* No, return */
915 ALIGN_STACK()
916 LOAD_PTR_ARG1(%rdx)
917 LOAD_STRING_ARG0(mutex_assert_owned_str)
918 jmp 4f
919 3:
920 ALIGN_STACK()
921 LOAD_PTR_ARG1(%rdx)
922 LOAD_STRING_ARG0(mutex_assert_not_owned_str)
923 4:
924 CALL_PANIC()
925
926
927 lck_mtx_destroyed:
928 ALIGN_STACK()
929 LOAD_PTR_ARG1(%rdx)
930 LOAD_STRING_ARG0(mutex_interlock_destroyed_str)
931 CALL_PANIC()
932
933
934 .data
935 mutex_assert_not_owned_str:
936 .asciz "mutex (%p) not owned\n"
937 mutex_assert_owned_str:
938 .asciz "mutex (%p) owned\n"
939 mutex_interlock_destroyed_str:
940 .asciz "trying to interlock destroyed mutex (%p)"
941 .text
942
943
944
945 /*
946 * lck_mtx_lock()
947 * lck_mtx_try_lock()
948 * lck_mtx_unlock()
949 * lck_mtx_lock_spin()
950 * lck_mtx_lock_spin_always()
951 * lck_mtx_try_lock_spin()
952 * lck_mtx_try_lock_spin_always()
953 * lck_mtx_convert_spin()
954 */
955 NONLEAF_ENTRY(lck_mtx_lock_spin_always)
956 mov %rdi, %rdx /* fetch lock pointer */
957 jmp Llmls_avoid_check
958
959 NONLEAF_ENTRY(lck_mtx_lock_spin)
960 mov %rdi, %rdx /* fetch lock pointer */
961
962 CHECK_PREEMPTION_LEVEL()
963 Llmls_avoid_check:
964 mov M_STATE(%rdx), %ecx
965 test $(M_ILOCKED_MSK | M_MLOCKED_MSK), %ecx /* is the interlock or mutex held */
966 jnz Llmls_slow
967 Llmls_try: /* no - can't be INDIRECT, DESTROYED or locked */
968 mov %rcx, %rax /* eax contains snapshot for cmpxchgl */
969 or $(M_ILOCKED_MSK | M_SPIN_MSK), %ecx
970
971 PREEMPTION_DISABLE
972 lock
973 cmpxchg %ecx, M_STATE(%rdx) /* atomic compare and exchange */
974 jne Llmls_busy_disabled
975
976 mov %gs:CPU_ACTIVE_THREAD, %rax
977 mov %rax, M_OWNER(%rdx) /* record owner of interlock */
978 #if MACH_LDEBUG
979 test %rax, %rax
980 jz 1f
981 incl TH_MUTEX_COUNT(%rax) /* lock statistic */
982 1:
983 #endif /* MACH_LDEBUG */
984
985 /* return with the interlock held and preemption disabled */
986 leave
987 #if CONFIG_DTRACE
988 LOCKSTAT_LABEL(_lck_mtx_lock_spin_lockstat_patch_point)
989 ret
990 /* inherit lock pointer in %rdx above */
991 LOCKSTAT_RECORD(LS_LCK_MTX_LOCK_SPIN_ACQUIRE, %rdx)
992 #endif
993 ret
994
995 Llmls_slow:
996 test $M_ILOCKED_MSK, %ecx /* is the interlock held */
997 jz Llml_contended /* no, must have been the mutex */
998
999 cmp $(MUTEX_DESTROYED), %ecx /* check to see if its marked destroyed */
1000 je lck_mtx_destroyed
1001 cmp $(MUTEX_IND), %ecx /* Is this an indirect mutex */
1002 jne Llmls_loop /* no... must be interlocked */
1003
1004 LMTX_ENTER_EXTENDED
1005
1006 mov M_STATE(%rdx), %ecx
1007 test $(M_SPIN_MSK), %ecx
1008 jz Llmls_loop1
1009
1010 LMTX_UPDATE_MISS /* M_SPIN_MSK was set, so M_ILOCKED_MSK must also be present */
1011 Llmls_loop:
1012 PAUSE
1013 mov M_STATE(%rdx), %ecx
1014 Llmls_loop1:
1015 test $(M_ILOCKED_MSK | M_MLOCKED_MSK), %ecx
1016 jz Llmls_try
1017 test $(M_MLOCKED_MSK), %ecx
1018 jnz Llml_contended /* mutex owned by someone else, go contend for it */
1019 jmp Llmls_loop
1020
1021 Llmls_busy_disabled:
1022 PREEMPTION_ENABLE
1023 jmp Llmls_loop
1024
1025
1026
1027 NONLEAF_ENTRY(lck_mtx_lock)
1028 mov %rdi, %rdx /* fetch lock pointer */
1029
1030 CHECK_PREEMPTION_LEVEL()
1031
1032 mov M_STATE(%rdx), %ecx
1033 test $(M_ILOCKED_MSK | M_MLOCKED_MSK), %ecx /* is the interlock or mutex held */
1034 jnz Llml_slow
1035 Llml_try: /* no - can't be INDIRECT, DESTROYED or locked */
1036 mov %rcx, %rax /* eax contains snapshot for cmpxchgl */
1037 or $(M_ILOCKED_MSK | M_MLOCKED_MSK), %ecx
1038
1039 PREEMPTION_DISABLE
1040 lock
1041 cmpxchg %ecx, M_STATE(%rdx) /* atomic compare and exchange */
1042 jne Llml_busy_disabled
1043
1044 mov %gs:CPU_ACTIVE_THREAD, %rax
1045 mov %rax, M_OWNER(%rdx) /* record owner of mutex */
1046 #if MACH_LDEBUG
1047 test %rax, %rax
1048 jz 1f
1049 incl TH_MUTEX_COUNT(%rax) /* lock statistic */
1050 1:
1051 #endif /* MACH_LDEBUG */
1052
1053 testl $(M_WAITERS_MSK), M_STATE(%rdx)
1054 jz Llml_finish
1055
1056 LMTX_CALLEXT1(lck_mtx_lock_acquire_x86)
1057
1058 Llml_finish:
1059 andl $(~M_ILOCKED_MSK), M_STATE(%rdx)
1060 PREEMPTION_ENABLE
1061
1062 cmp %rdx, %rdi /* is this an extended mutex */
1063 jne 2f
1064
1065 leave
1066 #if CONFIG_DTRACE
1067 LOCKSTAT_LABEL(_lck_mtx_lock_lockstat_patch_point)
1068 ret
1069 /* inherit lock pointer in %rdx above */
1070 LOCKSTAT_RECORD(LS_LCK_MTX_LOCK_ACQUIRE, %rdx)
1071 #endif
1072 ret
1073 2:
1074 leave
1075 #if CONFIG_DTRACE
1076 LOCKSTAT_LABEL(_lck_mtx_lock_ext_lockstat_patch_point)
1077 ret
1078 /* inherit lock pointer in %rdx above */
1079 LOCKSTAT_RECORD(LS_LCK_MTX_EXT_LOCK_ACQUIRE, %rdx)
1080 #endif
1081 ret
1082
1083
1084 Llml_slow:
1085 test $M_ILOCKED_MSK, %ecx /* is the interlock held */
1086 jz Llml_contended /* no, must have been the mutex */
1087
1088 cmp $(MUTEX_DESTROYED), %ecx /* check to see if its marked destroyed */
1089 je lck_mtx_destroyed
1090 cmp $(MUTEX_IND), %ecx /* Is this an indirect mutex? */
1091 jne Llml_loop /* no... must be interlocked */
1092
1093 LMTX_ENTER_EXTENDED
1094
1095 mov M_STATE(%rdx), %ecx
1096 test $(M_SPIN_MSK), %ecx
1097 jz Llml_loop1
1098
1099 LMTX_UPDATE_MISS /* M_SPIN_MSK was set, so M_ILOCKED_MSK must also be present */
1100 Llml_loop:
1101 PAUSE
1102 mov M_STATE(%rdx), %ecx
1103 Llml_loop1:
1104 test $(M_ILOCKED_MSK | M_MLOCKED_MSK), %ecx
1105 jz Llml_try
1106 test $(M_MLOCKED_MSK), %ecx
1107 jnz Llml_contended /* mutex owned by someone else, go contend for it */
1108 jmp Llml_loop
1109
1110 Llml_busy_disabled:
1111 PREEMPTION_ENABLE
1112 jmp Llml_loop
1113
1114
1115 Llml_contended:
1116 cmp %rdx, %rdi /* is this an extended mutex */
1117 je 0f
1118 LMTX_UPDATE_MISS
1119 0:
1120 LMTX_CALLEXT1(lck_mtx_lock_spinwait_x86)
1121
1122 test %rax, %rax
1123 jz Llml_acquired /* acquired mutex, interlock held and preemption disabled */
1124
1125 cmp $1, %rax /* check for direct wait status */
1126 je 2f
1127 cmp %rdx, %rdi /* is this an extended mutex */
1128 je 2f
1129 LMTX_UPDATE_DIRECT_WAIT
1130 2:
1131 mov M_STATE(%rdx), %ecx
1132 test $(M_ILOCKED_MSK), %ecx
1133 jnz 6f
1134
1135 mov %rcx, %rax /* eax contains snapshot for cmpxchgl */
1136 or $(M_ILOCKED_MSK), %ecx /* try to take the interlock */
1137
1138 PREEMPTION_DISABLE
1139 lock
1140 cmpxchg %ecx, M_STATE(%rdx) /* atomic compare and exchange */
1141 jne 5f
1142
1143 test $(M_MLOCKED_MSK), %ecx /* we've got the interlock and */
1144 jnz 3f
1145 or $(M_MLOCKED_MSK), %ecx /* the mutex is free... grab it directly */
1146 mov %ecx, M_STATE(%rdx)
1147
1148 mov %gs:CPU_ACTIVE_THREAD, %rax
1149 mov %rax, M_OWNER(%rdx) /* record owner of mutex */
1150 #if MACH_LDEBUG
1151 test %rax, %rax
1152 jz 1f
1153 incl TH_MUTEX_COUNT(%rax) /* lock statistic */
1154 1:
1155 #endif /* MACH_LDEBUG */
1156
1157 Llml_acquired:
1158 testl $(M_WAITERS_MSK), M_STATE(%rdx)
1159 jnz 1f
1160 mov M_OWNER(%rdx), %rax
1161 mov TH_WAS_PROMOTED_ON_WAKEUP(%rax), %eax
1162 test %eax, %eax
1163 jz Llml_finish
1164 1:
1165 LMTX_CALLEXT1(lck_mtx_lock_acquire_x86)
1166 jmp Llml_finish
1167
1168 3: /* interlock held, mutex busy */
1169 cmp %rdx, %rdi /* is this an extended mutex */
1170 je 4f
1171 LMTX_UPDATE_WAIT
1172 4:
1173 LMTX_CALLEXT1(lck_mtx_lock_wait_x86)
1174 jmp Llml_contended
1175 5:
1176 PREEMPTION_ENABLE
1177 6:
1178 PAUSE
1179 jmp 2b
1180
1181
1182 NONLEAF_ENTRY(lck_mtx_try_lock_spin_always)
1183 mov %rdi, %rdx /* fetch lock pointer */
1184 jmp Llmts_avoid_check
1185
1186 NONLEAF_ENTRY(lck_mtx_try_lock_spin)
1187 mov %rdi, %rdx /* fetch lock pointer */
1188
1189 Llmts_avoid_check:
1190 mov M_STATE(%rdx), %ecx
1191 test $(M_ILOCKED_MSK | M_MLOCKED_MSK), %ecx /* is the interlock or mutex held */
1192 jnz Llmts_slow
1193 Llmts_try: /* no - can't be INDIRECT, DESTROYED or locked */
1194 mov %rcx, %rax /* eax contains snapshot for cmpxchgl */
1195 or $(M_ILOCKED_MSK | M_SPIN_MSK), %rcx
1196
1197 PREEMPTION_DISABLE
1198 lock
1199 cmpxchg %ecx, M_STATE(%rdx) /* atomic compare and exchange */
1200 jne Llmts_busy_disabled
1201
1202 mov %gs:CPU_ACTIVE_THREAD, %rax
1203 mov %rax, M_OWNER(%rdx) /* record owner of mutex */
1204 #if MACH_LDEBUG
1205 test %rax, %rax
1206 jz 1f
1207 incl TH_MUTEX_COUNT(%rax) /* lock statistic */
1208 1:
1209 #endif /* MACH_LDEBUG */
1210
1211 leave
1212
1213 #if CONFIG_DTRACE
1214 mov $1, %rax /* return success */
1215 LOCKSTAT_LABEL(_lck_mtx_try_lock_spin_lockstat_patch_point)
1216 ret
1217 /* inherit lock pointer in %rdx above */
1218 LOCKSTAT_RECORD(LS_LCK_MTX_TRY_SPIN_LOCK_ACQUIRE, %rdx)
1219 #endif
1220 mov $1, %rax /* return success */
1221 ret
1222
1223 Llmts_slow:
1224 test $(M_ILOCKED_MSK), %ecx /* is the interlock held */
1225 jz Llmts_fail /* no, must be held as a mutex */
1226
1227 cmp $(MUTEX_DESTROYED), %ecx /* check to see if its marked destroyed */
1228 je lck_mtx_destroyed
1229 cmp $(MUTEX_IND), %ecx /* Is this an indirect mutex? */
1230 jne Llmts_loop1
1231
1232 LMTX_ENTER_EXTENDED
1233 Llmts_loop:
1234 PAUSE
1235 mov M_STATE(%rdx), %ecx
1236 Llmts_loop1:
1237 test $(M_MLOCKED_MSK | M_SPIN_MSK), %ecx
1238 jnz Llmts_fail
1239 test $(M_ILOCKED_MSK), %ecx
1240 jz Llmts_try
1241 jmp Llmts_loop
1242
1243 Llmts_busy_disabled:
1244 PREEMPTION_ENABLE
1245 jmp Llmts_loop
1246
1247
1248
1249 NONLEAF_ENTRY(lck_mtx_try_lock)
1250 mov %rdi, %rdx /* fetch lock pointer */
1251
1252 mov M_STATE(%rdx), %ecx
1253 test $(M_ILOCKED_MSK | M_MLOCKED_MSK), %ecx /* is the interlock or mutex held */
1254 jnz Llmt_slow
1255 Llmt_try: /* no - can't be INDIRECT, DESTROYED or locked */
1256 mov %rcx, %rax /* eax contains snapshot for cmpxchgl */
1257 or $(M_ILOCKED_MSK | M_MLOCKED_MSK), %ecx
1258
1259 PREEMPTION_DISABLE
1260 lock
1261 cmpxchg %ecx, M_STATE(%rdx) /* atomic compare and exchange */
1262 jne Llmt_busy_disabled
1263
1264 mov %gs:CPU_ACTIVE_THREAD, %rax
1265 mov %rax, M_OWNER(%rdx) /* record owner of mutex */
1266 #if MACH_LDEBUG
1267 test %rax, %rax
1268 jz 1f
1269 incl TH_MUTEX_COUNT(%rax) /* lock statistic */
1270 1:
1271 #endif /* MACH_LDEBUG */
1272
1273 test $(M_WAITERS_MSK), %ecx
1274 jz 0f
1275
1276 LMTX_CALLEXT1(lck_mtx_lock_acquire_x86)
1277 0:
1278 andl $(~M_ILOCKED_MSK), M_STATE(%rdx)
1279 PREEMPTION_ENABLE
1280
1281 leave
1282 #if CONFIG_DTRACE
1283 mov $1, %rax /* return success */
1284 /* Dtrace probe: LS_LCK_MTX_TRY_LOCK_ACQUIRE */
1285 LOCKSTAT_LABEL(_lck_mtx_try_lock_lockstat_patch_point)
1286 ret
1287 /* inherit lock pointer in %rdx from above */
1288 LOCKSTAT_RECORD(LS_LCK_MTX_TRY_LOCK_ACQUIRE, %rdx)
1289 #endif
1290 mov $1, %rax /* return success */
1291 ret
1292
1293 Llmt_slow:
1294 test $(M_ILOCKED_MSK), %ecx /* is the interlock held */
1295 jz Llmt_fail /* no, must be held as a mutex */
1296
1297 cmp $(MUTEX_DESTROYED), %ecx /* check to see if its marked destroyed */
1298 je lck_mtx_destroyed
1299 cmp $(MUTEX_IND), %ecx /* Is this an indirect mutex? */
1300 jne Llmt_loop
1301
1302 LMTX_ENTER_EXTENDED
1303 Llmt_loop:
1304 PAUSE
1305 mov M_STATE(%rdx), %ecx
1306 Llmt_loop1:
1307 test $(M_MLOCKED_MSK | M_SPIN_MSK), %ecx
1308 jnz Llmt_fail
1309 test $(M_ILOCKED_MSK), %ecx
1310 jz Llmt_try
1311 jmp Llmt_loop
1312
1313 Llmt_busy_disabled:
1314 PREEMPTION_ENABLE
1315 jmp Llmt_loop
1316
1317
1318 Llmt_fail:
1319 Llmts_fail:
1320 cmp %rdx, %rdi /* is this an extended mutex */
1321 je 0f
1322 LMTX_UPDATE_MISS
1323 0:
1324 xor %rax, %rax
1325 NONLEAF_RET
1326
1327
1328
1329 NONLEAF_ENTRY(lck_mtx_convert_spin)
1330 mov %rdi, %rdx /* fetch lock pointer */
1331
1332 mov M_STATE(%rdx), %ecx
1333 cmp $(MUTEX_IND), %ecx /* Is this an indirect mutex? */
1334 jne 0f
1335 mov M_PTR(%rdx), %rdx /* If so, take indirection */
1336 mov M_STATE(%rdx), %ecx
1337 0:
1338 test $(M_MLOCKED_MSK), %ecx /* already owned as a mutex, just return */
1339 jnz 2f
1340 test $(M_WAITERS_MSK), %ecx /* are there any waiters? */
1341 jz 1f
1342
1343 LMTX_CALLEXT1(lck_mtx_lock_acquire_x86)
1344 mov M_STATE(%rdx), %ecx
1345 1:
1346 and $(~(M_ILOCKED_MSK | M_SPIN_MSK)), %ecx /* convert from spin version to mutex */
1347 or $(M_MLOCKED_MSK), %ecx
1348 mov %ecx, M_STATE(%rdx) /* since I own the interlock, I don't need an atomic update */
1349
1350 PREEMPTION_ENABLE
1351 2:
1352 NONLEAF_RET
1353
1354
1355
1356 NONLEAF_ENTRY(lck_mtx_unlock)
1357 mov %rdi, %rdx /* fetch lock pointer */
1358 Llmu_entry:
1359 mov M_STATE(%rdx), %ecx
1360 Llmu_prim:
1361 cmp $(MUTEX_IND), %ecx /* Is this an indirect mutex? */
1362 je Llmu_ext
1363
1364 Llmu_chktype:
1365 test $(M_MLOCKED_MSK), %ecx /* check for full mutex */
1366 jz Llmu_unlock
1367 Llmu_mutex:
1368 test $(M_ILOCKED_MSK), %rcx /* have to wait for interlock to clear */
1369 jnz Llmu_busy
1370
1371 mov %rcx, %rax /* eax contains snapshot for cmpxchgl */
1372 and $(~M_MLOCKED_MSK), %ecx /* drop mutex */
1373 or $(M_ILOCKED_MSK), %ecx /* pick up interlock */
1374
1375 PREEMPTION_DISABLE
1376 lock
1377 cmpxchg %ecx, M_STATE(%rdx) /* atomic compare and exchange */
1378 jne Llmu_busy_disabled /* branch on failure to spin loop */
1379
1380 Llmu_unlock:
1381 xor %rax, %rax
1382 mov %rax, M_OWNER(%rdx)
1383 mov %rcx, %rax /* keep original state in %ecx for later evaluation */
1384 and $(~(M_ILOCKED_MSK | M_SPIN_MSK | M_PROMOTED_MSK)), %rax
1385
1386 test $(M_WAITERS_MSK), %eax
1387 jz 2f
1388 dec %eax /* decrement waiter count */
1389 2:
1390 mov %eax, M_STATE(%rdx) /* since I own the interlock, I don't need an atomic update */
1391
1392 #if MACH_LDEBUG
1393 /* perform lock statistics after drop to prevent delay */
1394 mov %gs:CPU_ACTIVE_THREAD, %rax
1395 test %rax, %rax
1396 jz 1f
1397 decl TH_MUTEX_COUNT(%rax) /* lock statistic */
1398 1:
1399 #endif /* MACH_LDEBUG */
1400
1401 test $(M_PROMOTED_MSK | M_WAITERS_MSK), %ecx
1402 jz 3f
1403
1404 LMTX_CALLEXT2(lck_mtx_unlock_wakeup_x86, %rcx)
1405 3:
1406 PREEMPTION_ENABLE
1407
1408 cmp %rdx, %rdi
1409 jne 4f
1410
1411 leave
1412 #if CONFIG_DTRACE
1413 /* Dtrace: LS_LCK_MTX_UNLOCK_RELEASE */
1414 LOCKSTAT_LABEL(_lck_mtx_unlock_lockstat_patch_point)
1415 ret
1416 /* inherit lock pointer in %rdx from above */
1417 LOCKSTAT_RECORD(LS_LCK_MTX_UNLOCK_RELEASE, %rdx)
1418 #endif
1419 ret
1420 4:
1421 leave
1422 #if CONFIG_DTRACE
1423 /* Dtrace: LS_LCK_MTX_EXT_UNLOCK_RELEASE */
1424 LOCKSTAT_LABEL(_lck_mtx_ext_unlock_lockstat_patch_point)
1425 ret
1426 /* inherit lock pointer in %rdx from above */
1427 LOCKSTAT_RECORD(LS_LCK_MTX_EXT_UNLOCK_RELEASE, %rdx)
1428 #endif
1429 ret
1430
1431
1432 Llmu_busy_disabled:
1433 PREEMPTION_ENABLE
1434 Llmu_busy:
1435 PAUSE
1436 mov M_STATE(%rdx), %ecx
1437 jmp Llmu_mutex
1438
1439 Llmu_ext:
1440 mov M_PTR(%rdx), %rdx
1441 mov M_OWNER(%rdx), %rax
1442 mov %gs:CPU_ACTIVE_THREAD, %rcx
1443 CHECK_UNLOCK(%rcx, %rax)
1444 mov M_STATE(%rdx), %ecx
1445 jmp Llmu_chktype
1446
1447
1448
1449 LEAF_ENTRY(lck_mtx_ilk_try_lock)
1450 mov %rdi, %rdx /* fetch lock pointer - no indirection here */
1451
1452 mov M_STATE(%rdx), %ecx
1453
1454 test $(M_ILOCKED_MSK), %ecx /* can't have the interlock yet */
1455 jnz 3f
1456
1457 mov %rcx, %rax /* eax contains snapshot for cmpxchgl */
1458 or $(M_ILOCKED_MSK), %ecx
1459
1460 PREEMPTION_DISABLE
1461 lock
1462 cmpxchg %ecx, M_STATE(%rdx) /* atomic compare and exchange */
1463 jne 2f /* return failure after re-enabling preemption */
1464
1465 mov $1, %rax /* return success with preemption disabled */
1466 LEAF_RET
1467 2:
1468 PREEMPTION_ENABLE /* need to re-enable preemption */
1469 3:
1470 xor %rax, %rax /* return failure */
1471 LEAF_RET
1472
1473
1474 LEAF_ENTRY(lck_mtx_ilk_unlock)
1475 mov %rdi, %rdx /* fetch lock pointer - no indirection here */
1476
1477 andl $(~M_ILOCKED_MSK), M_STATE(%rdx)
1478
1479 PREEMPTION_ENABLE /* need to re-enable preemption */
1480
1481 LEAF_RET
1482
1483
1484 LEAF_ENTRY(lck_mtx_lock_grab_mutex)
1485 mov %rdi, %rdx /* fetch lock pointer - no indirection here */
1486
1487 mov M_STATE(%rdx), %ecx
1488
1489 test $(M_ILOCKED_MSK | M_MLOCKED_MSK), %ecx /* can't have the mutex yet */
1490 jnz 3f
1491
1492 mov %rcx, %rax /* eax contains snapshot for cmpxchgl */
1493 or $(M_ILOCKED_MSK | M_MLOCKED_MSK), %ecx
1494
1495 PREEMPTION_DISABLE
1496 lock
1497 cmpxchg %ecx, M_STATE(%rdx) /* atomic compare and exchange */
1498 jne 2f /* branch on failure to spin loop */
1499
1500 mov %gs:CPU_ACTIVE_THREAD, %rax
1501 mov %rax, M_OWNER(%rdx) /* record owner of mutex */
1502 #if MACH_LDEBUG
1503 test %rax, %rax
1504 jz 1f
1505 incl TH_MUTEX_COUNT(%rax) /* lock statistic */
1506 1:
1507 #endif /* MACH_LDEBUG */
1508
1509 mov $1, %rax /* return success */
1510 LEAF_RET
1511 2:
1512 PREEMPTION_ENABLE
1513 3:
1514 xor %rax, %rax /* return failure */
1515 LEAF_RET
1516
1517
1518
1519 LEAF_ENTRY(lck_mtx_lock_mark_destroyed)
1520 mov %rdi, %rdx
1521 1:
1522 mov M_STATE(%rdx), %ecx
1523 cmp $(MUTEX_IND), %ecx /* Is this an indirect mutex? */
1524 jne 2f
1525
1526 movl $(MUTEX_DESTROYED), M_STATE(%rdx) /* convert to destroyed state */
1527 jmp 3f
1528 2:
1529 test $(M_ILOCKED_MSK), %rcx /* have to wait for interlock to clear */
1530 jnz 5f
1531
1532 PREEMPTION_DISABLE
1533 mov %rcx, %rax /* eax contains snapshot for cmpxchgl */
1534 or $(M_ILOCKED_MSK), %ecx
1535 lock
1536 cmpxchg %ecx, M_STATE(%rdx) /* atomic compare and exchange */
1537 jne 4f /* branch on failure to spin loop */
1538 movl $(MUTEX_DESTROYED), M_STATE(%rdx) /* convert to destroyed state */
1539 PREEMPTION_ENABLE
1540 3:
1541 LEAF_RET /* return with M_ILOCKED set */
1542 4:
1543 PREEMPTION_ENABLE
1544 5:
1545 PAUSE
1546 jmp 1b
1547
1548 LEAF_ENTRY(preemption_underflow_panic)
1549 FRAME
1550 incl %gs:CPU_PREEMPTION_LEVEL
1551 ALIGN_STACK()
1552 LOAD_STRING_ARG0(16f)
1553 CALL_PANIC()
1554 hlt
1555 .data
1556 16: String "Preemption level underflow, possible cause unlocking an unlocked mutex or spinlock"
1557 .text
1558
1559
mirror of XNU