]> git.saurik.com Git - apple/xnu.git/blob - osfmk/i386/i386_lock.s
projects / apple / xnu.git / blob
? search:
summary | shortlog | log | commit | commitdiff | tree
blame | history | raw | HEAD
xnu-2782.40.9.tar.gz
[apple/xnu.git] / osfmk / i386 / i386_lock.s
1 /*
2 * Copyright (c) 2000-2012 Apple Inc. All rights reserved.
3 *
4 * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
5 *
6 * This file contains Original Code and/or Modifications of Original Code
7 * as defined in and that are subject to the Apple Public Source License
8 * Version 2.0 (the 'License'). You may not use this file except in
9 * compliance with the License. The rights granted to you under the License
10 * may not be used to create, or enable the creation or redistribution of,
11 * unlawful or unlicensed copies of an Apple operating system, or to
12 * circumvent, violate, or enable the circumvention or violation of, any
13 * terms of an Apple operating system software license agreement.
14 *
15 * Please obtain a copy of the License at
16 * http://www.opensource.apple.com/apsl/ and read it before using this file.
17 *
18 * The Original Code and all software distributed under the License are
19 * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
20 * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
21 * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
22 * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
23 * Please see the License for the specific language governing rights and
24 * limitations under the License.
25 *
26 * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
27 */
28 /*
29 * @OSF_COPYRIGHT@
30 */
31 /*
32 * Mach Operating System
33 * Copyright (c) 1989 Carnegie-Mellon University
34 * All rights reserved. The CMU software License Agreement specifies
35 * the terms and conditions for use and redistribution.
36 */
37
38 #include <mach_rt.h>
39 #include <mach_ldebug.h>
40 #include <i386/asm.h>
41 #include <i386/eflags.h>
42 #include <i386/trap.h>
43 #include <config_dtrace.h>
44 #include <i386/mp.h>
45
46 #include "assym.s"
47
48 #define PAUSE rep; nop
49
50 #include <i386/pal_lock_asm.h>
51
52 #define LEAF_ENTRY(name) \
53 Entry(name)
54
55 #define LEAF_ENTRY2(n1,n2) \
56 Entry(n1); \
57 Entry(n2)
58
59 #define LEAF_RET \
60 ret
61
62 /* Non-leaf routines always have a stack frame: */
63
64 #define NONLEAF_ENTRY(name) \
65 Entry(name); \
66 FRAME
67
68 #define NONLEAF_ENTRY2(n1,n2) \
69 Entry(n1); \
70 Entry(n2); \
71 FRAME
72
73 #define NONLEAF_RET \
74 EMARF; \
75 ret
76
77
78 /* For x86_64, the varargs ABI requires that %al indicate
79 * how many SSE register contain arguments. In our case, 0 */
80 #define ALIGN_STACK() and $0xFFFFFFFFFFFFFFF0, %rsp ;
81 #define LOAD_STRING_ARG0(label) leaq label(%rip), %rdi ;
82 #define LOAD_ARG1(x) mov x, %esi ;
83 #define LOAD_PTR_ARG1(x) mov x, %rsi ;
84 #define CALL_PANIC() xorb %al,%al ; call EXT(panic) ;
85
86 #define CHECK_UNLOCK(current, owner) \
87 cmp current, owner ; \
88 je 1f ; \
89 ALIGN_STACK() ; \
90 LOAD_STRING_ARG0(2f) ; \
91 CALL_PANIC() ; \
92 hlt ; \
93 .data ; \
94 2: String "Mutex unlock attempted from non-owner thread"; \
95 .text ; \
96 1:
97
98 #if MACH_LDEBUG
99 /*
100 * Routines for general lock debugging.
101 */
102
103 /*
104 * Checks for expected lock types and calls "panic" on
105 * mismatch. Detects calls to Mutex functions with
106 * type simplelock and vice versa.
107 */
108 #define CHECK_MUTEX_TYPE() \
109 cmpl $ MUTEX_TAG,M_TYPE ; \
110 je 1f ; \
111 ALIGN_STACK() ; \
112 LOAD_STRING_ARG0(2f) ; \
113 CALL_PANIC() ; \
114 hlt ; \
115 .data ; \
116 2: String "not a mutex!" ; \
117 .text ; \
118 1:
119
120 /*
121 * If one or more simplelocks are currently held by a thread,
122 * an attempt to acquire a mutex will cause this check to fail
123 * (since a mutex lock may context switch, holding a simplelock
124 * is not a good thing).
125 */
126 #if MACH_RT
127 #define CHECK_PREEMPTION_LEVEL() \
128 cmpl $0,%gs:CPU_HIBERNATE ; \
129 jne 1f ; \
130 cmpl $0,%gs:CPU_PREEMPTION_LEVEL ; \
131 je 1f ; \
132 ALIGN_STACK() ; \
133 movl %gs:CPU_PREEMPTION_LEVEL, %eax ; \
134 LOAD_ARG1(%eax) ; \
135 LOAD_STRING_ARG0(2f) ; \
136 CALL_PANIC() ; \
137 hlt ; \
138 .data ; \
139 2: String "preemption_level(%d) != 0!" ; \
140 .text ; \
141 1:
142 #else /* MACH_RT */
143 #define CHECK_PREEMPTION_LEVEL()
144 #endif /* MACH_RT */
145
146 #define CHECK_MYLOCK(current, owner) \
147 cmp current, owner ; \
148 jne 1f ; \
149 ALIGN_STACK() ; \
150 LOAD_STRING_ARG0(2f) ; \
151 CALL_PANIC() ; \
152 hlt ; \
153 .data ; \
154 2: String "Attempt to recursively lock a non-recursive lock"; \
155 .text ; \
156 1:
157
158 #else /* MACH_LDEBUG */
159 #define CHECK_MUTEX_TYPE()
160 #define CHECK_PREEMPTION_LEVEL()
161 #define CHECK_MYLOCK(thd)
162 #endif /* MACH_LDEBUG */
163
164 #define PREEMPTION_DISABLE \
165 incl %gs:CPU_PREEMPTION_LEVEL
166
167 #define PREEMPTION_LEVEL_DEBUG 1
168 #if PREEMPTION_LEVEL_DEBUG
169 #define PREEMPTION_ENABLE \
170 decl %gs:CPU_PREEMPTION_LEVEL ; \
171 js 17f ; \
172 jnz 19f ; \
173 testl $AST_URGENT,%gs:CPU_PENDING_AST ; \
174 jz 19f ; \
175 PUSHF ; \
176 testl $EFL_IF, S_PC ; \
177 jz 18f ; \
178 POPF ; \
179 int $(T_PREEMPT) ; \
180 jmp 19f ; \
181 17: \
182 call _preemption_underflow_panic ; \
183 18: \
184 POPF ; \
185 19:
186 #else
187 #define PREEMPTION_ENABLE \
188 decl %gs:CPU_PREEMPTION_LEVEL ; \
189 jnz 19f ; \
190 testl $AST_URGENT,%gs:CPU_PENDING_AST ; \
191 jz 19f ; \
192 PUSHF ; \
193 testl $EFL_IF, S_PC ; \
194 jz 18f ; \
195 POPF ; \
196 int $(T_PREEMPT) ; \
197 jmp 19f ; \
198 18: \
199 POPF ; \
200 19:
201 #endif
202
203
204 #if CONFIG_DTRACE
205
206 .globl _lockstat_probe
207 .globl _lockstat_probemap
208
209 /*
210 * LOCKSTAT_LABEL creates a dtrace symbol which contains
211 * a pointer into the lock code function body. At that
212 * point is a "ret" instruction that can be patched into
213 * a "nop"
214 */
215
216 #define LOCKSTAT_LABEL(lab) \
217 .data ;\
218 .globl lab ;\
219 lab: ;\
220 .quad 9f ;\
221 .text ;\
222 9:
223
224 #define LOCKSTAT_RECORD(id, lck) \
225 push %rbp ; \
226 mov %rsp,%rbp ; \
227 movl _lockstat_probemap + (id * 4)(%rip),%eax ; \
228 test %eax,%eax ; \
229 je 9f ; \
230 mov lck, %rsi ; \
231 mov %rax, %rdi ; \
232 mov $0, %rdx ; \
233 mov $0, %rcx ; \
234 mov $0, %r8 ; \
235 mov $0, %r9 ; \
236 call *_lockstat_probe(%rip) ; \
237 9: leave
238 /* ret - left to subsequent code, e.g. return values */
239
240 #endif /* CONFIG_DTRACE */
241
242 /*
243 * For most routines, the hw_lock_t pointer is loaded into a
244 * register initially, and then either a byte or register-sized
245 * word is loaded/stored to the pointer
246 */
247
248 /*
249 * void hw_lock_init(hw_lock_t)
250 *
251 * Initialize a hardware lock.
252 */
253 LEAF_ENTRY(hw_lock_init)
254 movq $0, (%rdi) /* clear the lock */
255 LEAF_RET
256
257
258 /*
259 * void hw_lock_byte_init(volatile uint8_t *)
260 *
261 * Initialize a hardware byte lock.
262 */
263 LEAF_ENTRY(hw_lock_byte_init)
264 movb $0, (%rdi) /* clear the lock */
265 LEAF_RET
266
267 /*
268 * void hw_lock_lock(hw_lock_t)
269 *
270 * Acquire lock, spinning until it becomes available.
271 * MACH_RT: also return with preemption disabled.
272 */
273 LEAF_ENTRY(hw_lock_lock)
274 mov %gs:CPU_ACTIVE_THREAD, %rcx /* get thread pointer */
275
276 PREEMPTION_DISABLE
277 1:
278 mov (%rdi), %rax
279 test %rax,%rax /* lock locked? */
280 jne 3f /* branch if so */
281 lock; cmpxchg %rcx,(%rdi) /* try to acquire the HW lock */
282 jne 3f
283 movl $1,%eax /* In case this was a timeout call */
284 LEAF_RET /* if yes, then nothing left to do */
285 3:
286 PAUSE /* pause for hyper-threading */
287 jmp 1b /* try again */
288
289 /*
290 * void hw_lock_byte_lock(uint8_t *lock_byte)
291 *
292 * Acquire byte sized lock operand, spinning until it becomes available.
293 * MACH_RT: also return with preemption disabled.
294 */
295
296 LEAF_ENTRY(hw_lock_byte_lock)
297 PREEMPTION_DISABLE
298 movl $1, %ecx /* Set lock value */
299 1:
300 movb (%rdi), %al /* Load byte at address */
301 testb %al,%al /* lock locked? */
302 jne 3f /* branch if so */
303 lock; cmpxchg %cl,(%rdi) /* attempt atomic compare exchange */
304 jne 3f
305 LEAF_RET /* if yes, then nothing left to do */
306 3:
307 PAUSE /* pause for hyper-threading */
308 jmp 1b /* try again */
309
310 /*
311 * unsigned int hw_lock_to(hw_lock_t, unsigned int)
312 *
313 * Acquire lock, spinning until it becomes available or timeout.
314 * MACH_RT: also return with preemption disabled.
315 */
316 LEAF_ENTRY(hw_lock_to)
317 1:
318 mov %gs:CPU_ACTIVE_THREAD, %rcx
319
320 /*
321 * Attempt to grab the lock immediately
322 * - fastpath without timeout nonsense.
323 */
324 PREEMPTION_DISABLE
325
326 mov (%rdi), %rax
327 test %rax,%rax /* lock locked? */
328 jne 2f /* branch if so */
329 lock; cmpxchg %rcx,(%rdi) /* try to acquire the HW lock */
330 jne 2f /* branch on failure */
331 movl $1,%eax
332 LEAF_RET
333
334 2:
335 #define INNER_LOOP_COUNT 1000
336 /*
337 * Failed to get the lock so set the timeout
338 * and then spin re-checking the lock but pausing
339 * every so many (INNER_LOOP_COUNT) spins to check for timeout.
340 */
341 push %r9
342 lfence
343 rdtsc /* read cyclecount into %edx:%eax */
344 shlq $32, %rdx
345 orq %rdx, %rax /* load 64-bit quantity into %rax */
346 addq %rax, %rsi /* %rsi is the timeout expiry */
347
348 4:
349 /*
350 * The inner-loop spin to look for the lock being freed.
351 */
352 mov $(INNER_LOOP_COUNT),%r9
353 5:
354 PAUSE /* pause for hyper-threading */
355 mov (%rdi),%rax /* spin checking lock value in cache */
356 test %rax,%rax
357 je 6f /* zero => unlocked, try to grab it */
358 decq %r9 /* decrement inner loop count */
359 jnz 5b /* time to check for timeout? */
360
361 /*
362 * Here after spinning INNER_LOOP_COUNT times, check for timeout
363 */
364 lfence
365 rdtsc /* cyclecount into %edx:%eax */
366 shlq $32, %rdx
367 orq %rdx, %rax /* load 64-bit quantity into %rax */
368 cmpq %rsi, %rax /* compare to timeout */
369 jb 4b /* continue spinning if less, or */
370 xor %rax,%rax /* with 0 return value */
371 pop %r9
372 LEAF_RET
373
374 6:
375 /*
376 * Here to try to grab the lock that now appears to be free
377 * after contention.
378 */
379 mov %gs:CPU_ACTIVE_THREAD, %rcx
380 lock; cmpxchg %rcx,(%rdi) /* try to acquire the HW lock */
381 jne 4b /* no - spin again */
382 movl $1,%eax /* yes */
383 pop %r9
384 LEAF_RET
385
386 /*
387 * void hw_lock_unlock(hw_lock_t)
388 *
389 * Unconditionally release lock.
390 * MACH_RT: release preemption level.
391 */
392 LEAF_ENTRY(hw_lock_unlock)
393 movq $0, (%rdi) /* clear the lock */
394 PREEMPTION_ENABLE
395 LEAF_RET
396
397 /*
398 * void hw_lock_byte_unlock(uint8_t *lock_byte)
399 *
400 * Unconditionally release byte sized lock operand.
401 * MACH_RT: release preemption level.
402 */
403
404 LEAF_ENTRY(hw_lock_byte_unlock)
405 movb $0, (%rdi) /* Clear the lock byte */
406 PREEMPTION_ENABLE
407 LEAF_RET
408
409 /*
410 * unsigned int hw_lock_try(hw_lock_t)
411 * MACH_RT: returns with preemption disabled on success.
412 */
413 LEAF_ENTRY(hw_lock_try)
414 mov %gs:CPU_ACTIVE_THREAD, %rcx
415 PREEMPTION_DISABLE
416
417 mov (%rdi),%rax
418 test %rax,%rax
419 jne 1f
420 lock; cmpxchg %rcx,(%rdi) /* try to acquire the HW lock */
421 jne 1f
422
423 movl $1,%eax /* success */
424 LEAF_RET
425
426 1:
427 PREEMPTION_ENABLE /* failure: release preemption... */
428 xorl %eax,%eax /* ...and return failure */
429 LEAF_RET
430
431 /*
432 * unsigned int hw_lock_held(hw_lock_t)
433 * MACH_RT: doesn't change preemption state.
434 * N.B. Racy, of course.
435 */
436 LEAF_ENTRY(hw_lock_held)
437 mov (%rdi),%rax /* check lock value */
438 test %rax,%rax
439 movl $1,%ecx
440 cmovne %ecx,%eax /* 0 => unlocked, 1 => locked */
441 LEAF_RET
442
443
444 /*
445 * Reader-writer lock fastpaths. These currently exist for the
446 * shared lock acquire, the exclusive lock acquire, the shared to
447 * exclusive upgrade and the release paths (where they reduce overhead
448 * considerably) -- these are by far the most frequently used routines
449 *
450 * The following should reflect the layout of the bitfield embedded within
451 * the lck_rw_t structure (see i386/locks.h).
452 */
453 #define LCK_RW_INTERLOCK (0x1 << 16)
454
455 #define LCK_RW_PRIV_EXCL (0x1 << 24)
456 #define LCK_RW_WANT_UPGRADE (0x2 << 24)
457 #define LCK_RW_WANT_WRITE (0x4 << 24)
458 #define LCK_R_WAITING (0x8 << 24)
459 #define LCK_W_WAITING (0x10 << 24)
460
461 #define LCK_RW_SHARED_MASK (0xffff)
462
463 /*
464 * For most routines, the lck_rw_t pointer is loaded into a
465 * register initially, and the flags bitfield loaded into another
466 * register and examined
467 */
468
469 #define RW_LOCK_SHARED_MASK (LCK_RW_INTERLOCK | LCK_RW_WANT_UPGRADE | LCK_RW_WANT_WRITE)
470 /*
471 * void lck_rw_lock_shared(lck_rw_t *)
472 *
473 */
474 Entry(lck_rw_lock_shared)
475 mov %gs:CPU_ACTIVE_THREAD, %rcx /* Load thread pointer */
476 incl TH_RWLOCK_COUNT(%rcx) /* Increment count before atomic CAS */
477 1:
478 mov (%rdi), %eax /* Load state bitfield and interlock */
479 testl $(RW_LOCK_SHARED_MASK), %eax /* Eligible for fastpath? */
480 jne 3f
481
482 movl %eax, %ecx /* original value in %eax for cmpxchgl */
483 incl %ecx /* Increment reader refcount */
484 lock
485 cmpxchgl %ecx, (%rdi) /* Attempt atomic exchange */
486 jne 2f
487
488 #if CONFIG_DTRACE
489 /*
490 * Dtrace lockstat event: LS_LCK_RW_LOCK_SHARED_ACQUIRE
491 * Implemented by swapping between return and no-op instructions.
492 * See bsd/dev/dtrace/lockstat.c.
493 */
494 LOCKSTAT_LABEL(_lck_rw_lock_shared_lockstat_patch_point)
495 ret
496 /*
497 Fall thru when patched, counting on lock pointer in %rdi
498 */
499 LOCKSTAT_RECORD(LS_LCK_RW_LOCK_SHARED_ACQUIRE, %rdi)
500 #endif
501 ret
502 2:
503 PAUSE
504 jmp 1b
505 3:
506 jmp EXT(lck_rw_lock_shared_gen)
507
508
509
510 #define RW_TRY_LOCK_SHARED_MASK (LCK_RW_WANT_UPGRADE | LCK_RW_WANT_WRITE)
511 /*
512 * void lck_rw_try_lock_shared(lck_rw_t *)
513 *
514 */
515 Entry(lck_rw_try_lock_shared)
516 1:
517 mov (%rdi), %eax /* Load state bitfield and interlock */
518 testl $(LCK_RW_INTERLOCK), %eax
519 jne 2f
520 testl $(RW_TRY_LOCK_SHARED_MASK), %eax
521 jne 3f /* lock is busy */
522
523 movl %eax, %ecx /* original value in %eax for cmpxchgl */
524 incl %ecx /* Increment reader refcount */
525 lock
526 cmpxchgl %ecx, (%rdi) /* Attempt atomic exchange */
527 jne 2f
528
529 mov %gs:CPU_ACTIVE_THREAD, %rcx /* Load thread pointer */
530 incl TH_RWLOCK_COUNT(%rcx) /* Increment count on success. */
531 /* There is a 3 instr window where preemption may not notice rwlock_count after cmpxchg */
532
533 #if CONFIG_DTRACE
534 movl $1, %eax
535 /*
536 * Dtrace lockstat event: LS_LCK_RW_TRY_LOCK_SHARED_ACQUIRE
537 * Implemented by swapping between return and no-op instructions.
538 * See bsd/dev/dtrace/lockstat.c.
539 */
540 LOCKSTAT_LABEL(_lck_rw_try_lock_shared_lockstat_patch_point)
541 ret
542 /* Fall thru when patched, counting on lock pointer in %rdi */
543 LOCKSTAT_RECORD(LS_LCK_RW_TRY_LOCK_SHARED_ACQUIRE, %rdi)
544 #endif
545 movl $1, %eax /* return TRUE */
546 ret
547 2:
548 PAUSE
549 jmp 1b
550 3:
551 xorl %eax, %eax
552 ret
553
554
555 #define RW_LOCK_EXCLUSIVE_HELD (LCK_RW_WANT_WRITE | LCK_RW_WANT_UPGRADE)
556 /*
557 * int lck_rw_grab_shared(lck_rw_t *)
558 *
559 */
560 Entry(lck_rw_grab_shared)
561 1:
562 mov (%rdi), %eax /* Load state bitfield and interlock */
563 testl $(LCK_RW_INTERLOCK), %eax
564 jne 5f
565 testl $(RW_LOCK_EXCLUSIVE_HELD), %eax
566 jne 3f
567 2:
568 movl %eax, %ecx /* original value in %eax for cmpxchgl */
569 incl %ecx /* Increment reader refcount */
570 lock
571 cmpxchgl %ecx, (%rdi) /* Attempt atomic exchange */
572 jne 4f
573
574 movl $1, %eax /* return success */
575 ret
576 3:
577 testl $(LCK_RW_SHARED_MASK), %eax
578 je 4f
579 testl $(LCK_RW_PRIV_EXCL), %eax
580 je 2b
581 4:
582 xorl %eax, %eax /* return failure */
583 ret
584 5:
585 PAUSE
586 jmp 1b
587
588
589
590 #define RW_LOCK_EXCLUSIVE_MASK (LCK_RW_SHARED_MASK | LCK_RW_INTERLOCK | \
591 LCK_RW_WANT_UPGRADE | LCK_RW_WANT_WRITE)
592 /*
593 * void lck_rw_lock_exclusive(lck_rw_t*)
594 *
595 */
596 Entry(lck_rw_lock_exclusive)
597 mov %gs:CPU_ACTIVE_THREAD, %rcx /* Load thread pointer */
598 incl TH_RWLOCK_COUNT(%rcx) /* Increment count before atomic CAS */
599 1:
600 mov (%rdi), %eax /* Load state bitfield, interlock and shared count */
601 testl $(RW_LOCK_EXCLUSIVE_MASK), %eax /* Eligible for fastpath? */
602 jne 3f /* no, go slow */
603
604 movl %eax, %ecx /* original value in %eax for cmpxchgl */
605 orl $(LCK_RW_WANT_WRITE), %ecx
606 lock
607 cmpxchgl %ecx, (%rdi) /* Attempt atomic exchange */
608 jne 2f
609
610 #if CONFIG_DTRACE
611 /*
612 * Dtrace lockstat event: LS_LCK_RW_LOCK_EXCL_ACQUIRE
613 * Implemented by swapping between return and no-op instructions.
614 * See bsd/dev/dtrace/lockstat.c.
615 */
616 LOCKSTAT_LABEL(_lck_rw_lock_exclusive_lockstat_patch_point)
617 ret
618 /* Fall thru when patched, counting on lock pointer in %rdi */
619 LOCKSTAT_RECORD(LS_LCK_RW_LOCK_EXCL_ACQUIRE, %rdi)
620 #endif
621 ret
622 2:
623 PAUSE
624 jmp 1b
625 3:
626 jmp EXT(lck_rw_lock_exclusive_gen)
627
628
629
630 #define RW_TRY_LOCK_EXCLUSIVE_MASK (LCK_RW_SHARED_MASK | LCK_RW_WANT_UPGRADE | LCK_RW_WANT_WRITE)
631 /*
632 * void lck_rw_try_lock_exclusive(lck_rw_t *)
633 *
634 * Tries to get a write lock.
635 *
636 * Returns FALSE if the lock is not held on return.
637 */
638 Entry(lck_rw_try_lock_exclusive)
639 1:
640 mov (%rdi), %eax /* Load state bitfield, interlock and shared count */
641 testl $(LCK_RW_INTERLOCK), %eax
642 jne 2f
643 testl $(RW_TRY_LOCK_EXCLUSIVE_MASK), %eax
644 jne 3f /* can't get it */
645
646 movl %eax, %ecx /* original value in %eax for cmpxchgl */
647 orl $(LCK_RW_WANT_WRITE), %ecx
648 lock
649 cmpxchgl %ecx, (%rdi) /* Attempt atomic exchange */
650 jne 2f
651
652 mov %gs:CPU_ACTIVE_THREAD, %rcx /* Load thread pointer */
653 incl TH_RWLOCK_COUNT(%rcx) /* Increment count on success. */
654 /* There is a 3 instr window where preemption may not notice rwlock_count after cmpxchg */
655
656 #if CONFIG_DTRACE
657 movl $1, %eax
658 /*
659 * Dtrace lockstat event: LS_LCK_RW_TRY_LOCK_EXCL_ACQUIRE
660 * Implemented by swapping between return and no-op instructions.
661 * See bsd/dev/dtrace/lockstat.c.
662 */
663 LOCKSTAT_LABEL(_lck_rw_try_lock_exclusive_lockstat_patch_point)
664 ret
665 /* Fall thru when patched, counting on lock pointer in %rdi */
666 LOCKSTAT_RECORD(LS_LCK_RW_TRY_LOCK_EXCL_ACQUIRE, %rdi)
667 #endif
668 movl $1, %eax /* return TRUE */
669 ret
670 2:
671 PAUSE
672 jmp 1b
673 3:
674 xorl %eax, %eax /* return FALSE */
675 ret
676
677
678
679 /*
680 * void lck_rw_lock_shared_to_exclusive(lck_rw_t*)
681 *
682 * fastpath can be taken if
683 * the current rw_shared_count == 1
684 * AND the interlock is clear
685 * AND RW_WANT_UPGRADE is not set
686 *
687 * note that RW_WANT_WRITE could be set, but will not
688 * be indicative of an exclusive hold since we have
689 * a read count on the lock that we have not yet released
690 * we can blow by that state since the lck_rw_lock_exclusive
691 * function will block until rw_shared_count == 0 and
692 * RW_WANT_UPGRADE is clear... it does this check behind
693 * the interlock which we are also checking for
694 *
695 * to make the transition we must be able to atomically
696 * set RW_WANT_UPGRADE and get rid of the read count we hold
697 */
698 Entry(lck_rw_lock_shared_to_exclusive)
699 1:
700 mov (%rdi), %eax /* Load state bitfield, interlock and shared count */
701 testl $(LCK_RW_INTERLOCK), %eax
702 jne 7f
703 testl $(LCK_RW_WANT_UPGRADE), %eax
704 jne 2f
705
706 movl %eax, %ecx /* original value in %eax for cmpxchgl */
707 orl $(LCK_RW_WANT_UPGRADE), %ecx /* ask for WANT_UPGRADE */
708 decl %ecx /* and shed our read count */
709 lock
710 cmpxchgl %ecx, (%rdi) /* Attempt atomic exchange */
711 jne 7f
712 /* we now own the WANT_UPGRADE */
713 testl $(LCK_RW_SHARED_MASK), %ecx /* check to see if all of the readers are drained */
714 jne 8f /* if not, we need to go wait */
715
716 #if CONFIG_DTRACE
717 movl $1, %eax
718 /*
719 * Dtrace lockstat event: LS_LCK_RW_LOCK_SHARED_TO_EXCL_UPGRADE
720 * Implemented by swapping between return and no-op instructions.
721 * See bsd/dev/dtrace/lockstat.c.
722 */
723 LOCKSTAT_LABEL(_lck_rw_lock_shared_to_exclusive_lockstat_patch_point)
724 ret
725 /* Fall thru when patched, counting on lock pointer in %rdi */
726 LOCKSTAT_RECORD(LS_LCK_RW_LOCK_SHARED_TO_EXCL_UPGRADE, %rdi)
727 #endif
728 movl $1, %eax /* return success */
729 ret
730
731 2: /* someone else already holds WANT_UPGRADE */
732 movl %eax, %ecx /* original value in %eax for cmpxchgl */
733 decl %ecx /* shed our read count */
734 testl $(LCK_RW_SHARED_MASK), %ecx
735 jne 3f /* we were the last reader */
736 andl $(~LCK_W_WAITING), %ecx /* so clear the wait indicator */
737 3:
738 lock
739 cmpxchgl %ecx, (%rdi) /* Attempt atomic exchange */
740 jne 7f
741
742 mov %eax, %esi /* put old flags as second arg */
743 /* lock is alread in %rdi */
744 call EXT(lck_rw_lock_shared_to_exclusive_failure)
745 ret /* and pass the failure return along */
746 7:
747 PAUSE
748 jmp 1b
749 8:
750 jmp EXT(lck_rw_lock_shared_to_exclusive_success)
751
752
753
754 .cstring
755 rwl_release_error_str:
756 .asciz "Releasing non-exclusive RW lock without a reader refcount!"
757 .text
758
759 /*
760 * lck_rw_type_t lck_rw_done(lck_rw_t *)
761 *
762 */
763 Entry(lck_rw_done)
764 1:
765 mov (%rdi), %eax /* Load state bitfield, interlock and reader count */
766 testl $(LCK_RW_INTERLOCK), %eax
767 jne 7f /* wait for interlock to clear */
768
769 movl %eax, %ecx /* keep original value in %eax for cmpxchgl */
770 testl $(LCK_RW_SHARED_MASK), %ecx /* if reader count == 0, must be exclusive lock */
771 je 2f
772 decl %ecx /* Decrement reader count */
773 testl $(LCK_RW_SHARED_MASK), %ecx /* if reader count has now gone to 0, check for waiters */
774 je 4f
775 jmp 6f
776 2:
777 testl $(LCK_RW_WANT_UPGRADE), %ecx
778 je 3f
779 andl $(~LCK_RW_WANT_UPGRADE), %ecx
780 jmp 4f
781 3:
782 testl $(LCK_RW_WANT_WRITE), %ecx
783 je 8f /* lock is not 'owned', go panic */
784 andl $(~LCK_RW_WANT_WRITE), %ecx
785 4:
786 /*
787 * test the original values to match what
788 * lck_rw_done_gen is going to do to determine
789 * which wakeups need to happen...
790 *
791 * if !(fake_lck->lck_rw_priv_excl && fake_lck->lck_w_waiting)
792 */
793 testl $(LCK_W_WAITING), %eax
794 je 5f
795 andl $(~LCK_W_WAITING), %ecx
796
797 testl $(LCK_RW_PRIV_EXCL), %eax
798 jne 6f
799 5:
800 andl $(~LCK_R_WAITING), %ecx
801 6:
802 lock
803 cmpxchgl %ecx, (%rdi) /* Attempt atomic exchange */
804 jne 7f
805
806 mov %eax,%esi /* old flags in %rsi */
807 /* lock is in %rdi already */
808 call EXT(lck_rw_done_gen)
809 ret
810 7:
811 PAUSE
812 jmp 1b
813 8:
814 ALIGN_STACK()
815 LOAD_STRING_ARG0(rwl_release_error_str)
816 CALL_PANIC()
817
818
819
820 /*
821 * lck_rw_type_t lck_rw_lock_exclusive_to_shared(lck_rw_t *)
822 *
823 */
824 Entry(lck_rw_lock_exclusive_to_shared)
825 1:
826 mov (%rdi), %eax /* Load state bitfield, interlock and reader count */
827 testl $(LCK_RW_INTERLOCK), %eax
828 jne 6f /* wait for interlock to clear */
829
830 movl %eax, %ecx /* keep original value in %eax for cmpxchgl */
831 incl %ecx /* Increment reader count */
832
833 testl $(LCK_RW_WANT_UPGRADE), %ecx
834 je 2f
835 andl $(~LCK_RW_WANT_UPGRADE), %ecx
836 jmp 3f
837 2:
838 andl $(~LCK_RW_WANT_WRITE), %ecx
839 3:
840 /*
841 * test the original values to match what
842 * lck_rw_lock_exclusive_to_shared_gen is going to do to determine
843 * which wakeups need to happen...
844 *
845 * if !(fake_lck->lck_rw_priv_excl && fake_lck->lck_w_waiting)
846 */
847 testl $(LCK_W_WAITING), %eax
848 je 4f
849 testl $(LCK_RW_PRIV_EXCL), %eax
850 jne 5f
851 4:
852 andl $(~LCK_R_WAITING), %ecx
853 5:
854 lock
855 cmpxchgl %ecx, (%rdi) /* Attempt atomic exchange */
856 jne 6f
857
858 mov %eax,%esi
859 call EXT(lck_rw_lock_exclusive_to_shared_gen)
860 ret
861 6:
862 PAUSE
863 jmp 1b
864
865
866
867 /*
868 * int lck_rw_grab_want(lck_rw_t *)
869 *
870 */
871 Entry(lck_rw_grab_want)
872 1:
873 mov (%rdi), %eax /* Load state bitfield, interlock and reader count */
874 testl $(LCK_RW_INTERLOCK), %eax
875 jne 3f /* wait for interlock to clear */
876 testl $(LCK_RW_WANT_WRITE), %eax /* want_write has been grabbed by someone else */
877 jne 2f /* go return failure */
878
879 movl %eax, %ecx /* original value in %eax for cmpxchgl */
880 orl $(LCK_RW_WANT_WRITE), %ecx
881 lock
882 cmpxchgl %ecx, (%rdi) /* Attempt atomic exchange */
883 jne 2f
884 /* we now own want_write */
885 movl $1, %eax /* return success */
886 ret
887 2:
888 xorl %eax, %eax /* return failure */
889 ret
890 3:
891 PAUSE
892 jmp 1b
893
894
895 #define RW_LOCK_SHARED_OR_UPGRADE_MASK (LCK_RW_SHARED_MASK | LCK_RW_INTERLOCK | LCK_RW_WANT_UPGRADE)
896 /*
897 * int lck_rw_held_read_or_upgrade(lck_rw_t *)
898 *
899 */
900 Entry(lck_rw_held_read_or_upgrade)
901 mov (%rdi), %eax
902 andl $(RW_LOCK_SHARED_OR_UPGRADE_MASK), %eax
903 ret
904
905
906
907 /*
908 * N.B.: On x86, statistics are currently recorded for all indirect mutexes.
909 * Also, only the acquire attempt count (GRP_MTX_STAT_UTIL) is maintained
910 * as a 64-bit quantity (this matches the existing PowerPC implementation,
911 * and the new x86 specific statistics are also maintained as 32-bit
912 * quantities).
913 *
914 *
915 * Enable this preprocessor define to record the first miss alone
916 * By default, we count every miss, hence multiple misses may be
917 * recorded for a single lock acquire attempt via lck_mtx_lock
918 */
919 #undef LOG_FIRST_MISS_ALONE
920
921 /*
922 * This preprocessor define controls whether the R-M-W update of the
923 * per-group statistics elements are atomic (LOCK-prefixed)
924 * Enabled by default.
925 */
926 #define ATOMIC_STAT_UPDATES 1
927
928 #if defined(ATOMIC_STAT_UPDATES)
929 #define LOCK_IF_ATOMIC_STAT_UPDATES lock
930 #else
931 #define LOCK_IF_ATOMIC_STAT_UPDATES
932 #endif /* ATOMIC_STAT_UPDATES */
933
934
935 /*
936 * For most routines, the lck_mtx_t pointer is loaded into a
937 * register initially, and the owner field checked for indirection.
938 * Eventually the lock owner is loaded into a register and examined.
939 */
940
941 #define M_OWNER MUTEX_OWNER
942 #define M_PTR MUTEX_PTR
943 #define M_STATE MUTEX_STATE
944
945
946 #define LMTX_ENTER_EXTENDED \
947 mov M_PTR(%rdx), %rdx ; \
948 xor %r11, %r11 ; \
949 mov MUTEX_GRP(%rdx), %r10 ; \
950 LOCK_IF_ATOMIC_STAT_UPDATES ; \
951 incq GRP_MTX_STAT_UTIL(%r10)
952
953
954 #if LOG_FIRST_MISS_ALONE
955 #define LMTX_UPDATE_MISS \
956 test $1, %r11 ; \
957 jnz 11f ; \
958 LOCK_IF_ATOMIC_STAT_UPDATES ; \
959 incl GRP_MTX_STAT_MISS(%r10) ; \
960 or $1, %r11 ; \
961 11:
962 #else
963 #define LMTX_UPDATE_MISS \
964 LOCK_IF_ATOMIC_STAT_UPDATES ; \
965 incl GRP_MTX_STAT_MISS(%r10)
966 #endif
967
968
969 #if LOG_FIRST_MISS_ALONE
970 #define LMTX_UPDATE_WAIT \
971 test $2, %r11 ; \
972 jnz 11f ; \
973 LOCK_IF_ATOMIC_STAT_UPDATES ; \
974 incl GRP_MTX_STAT_WAIT(%r10) ; \
975 or $2, %r11 ; \
976 11:
977 #else
978 #define LMTX_UPDATE_WAIT \
979 LOCK_IF_ATOMIC_STAT_UPDATES ; \
980 incl GRP_MTX_STAT_WAIT(%r10)
981 #endif
982
983
984 /*
985 * Record the "direct wait" statistic, which indicates if a
986 * miss proceeded to block directly without spinning--occurs
987 * if the owner of the mutex isn't running on another processor
988 * at the time of the check.
989 */
990 #define LMTX_UPDATE_DIRECT_WAIT \
991 LOCK_IF_ATOMIC_STAT_UPDATES ; \
992 incl GRP_MTX_STAT_DIRECT_WAIT(%r10)
993
994
995 #define LMTX_CALLEXT1(func_name) \
996 cmp %rdx, %rdi ; \
997 je 12f ; \
998 push %r10 ; \
999 push %r11 ; \
1000 12: push %rdi ; \
1001 push %rdx ; \
1002 mov %rdx, %rdi ; \
1003 call EXT(func_name) ; \
1004 pop %rdx ; \
1005 pop %rdi ; \
1006 cmp %rdx, %rdi ; \
1007 je 12f ; \
1008 pop %r11 ; \
1009 pop %r10 ; \
1010 12:
1011
1012 #define LMTX_CALLEXT2(func_name, reg) \
1013 cmp %rdx, %rdi ; \
1014 je 12f ; \
1015 push %r10 ; \
1016 push %r11 ; \
1017 12: push %rdi ; \
1018 push %rdx ; \
1019 mov reg, %rsi ; \
1020 mov %rdx, %rdi ; \
1021 call EXT(func_name) ; \
1022 pop %rdx ; \
1023 pop %rdi ; \
1024 cmp %rdx, %rdi ; \
1025 je 12f ; \
1026 pop %r11 ; \
1027 pop %r10 ; \
1028 12:
1029
1030
1031 #define M_WAITERS_MSK 0x0000ffff
1032 #define M_PRIORITY_MSK 0x00ff0000
1033 #define M_ILOCKED_MSK 0x01000000
1034 #define M_MLOCKED_MSK 0x02000000
1035 #define M_PROMOTED_MSK 0x04000000
1036 #define M_SPIN_MSK 0x08000000
1037
1038 /*
1039 * void lck_mtx_assert(lck_mtx_t* l, unsigned int)
1040 * Takes the address of a lock, and an assertion type as parameters.
1041 * The assertion can take one of two forms determine by the type
1042 * parameter: either the lock is held by the current thread, and the
1043 * type is LCK_MTX_ASSERT_OWNED, or it isn't and the type is
1044 * LCK_MTX_ASSERT_NOTOWNED. Calls panic on assertion failure.
1045 *
1046 */
1047
1048 NONLEAF_ENTRY(lck_mtx_assert)
1049 mov %rdi, %rdx /* Load lock address */
1050 mov %gs:CPU_ACTIVE_THREAD, %rax /* Load current thread */
1051
1052 mov M_STATE(%rdx), %ecx
1053 cmp $(MUTEX_IND), %ecx /* Is this an indirect mutex? */
1054 jne 0f
1055 mov M_PTR(%rdx), %rdx /* If so, take indirection */
1056 0:
1057 mov M_OWNER(%rdx), %rcx /* Load owner */
1058 cmp $(MUTEX_ASSERT_OWNED), %rsi
1059 jne 2f /* Assert ownership? */
1060 cmp %rax, %rcx /* Current thread match? */
1061 jne 3f /* no, go panic */
1062 testl $(M_ILOCKED_MSK | M_MLOCKED_MSK), M_STATE(%rdx)
1063 je 3f
1064 1: /* yes, we own it */
1065 NONLEAF_RET
1066 2:
1067 cmp %rax, %rcx /* Current thread match? */
1068 jne 1b /* No, return */
1069 ALIGN_STACK()
1070 LOAD_PTR_ARG1(%rdx)
1071 LOAD_STRING_ARG0(mutex_assert_owned_str)
1072 jmp 4f
1073 3:
1074 ALIGN_STACK()
1075 LOAD_PTR_ARG1(%rdx)
1076 LOAD_STRING_ARG0(mutex_assert_not_owned_str)
1077 4:
1078 CALL_PANIC()
1079
1080
1081 lck_mtx_destroyed:
1082 ALIGN_STACK()
1083 LOAD_PTR_ARG1(%rdx)
1084 LOAD_STRING_ARG0(mutex_interlock_destroyed_str)
1085 CALL_PANIC()
1086
1087
1088 .data
1089 mutex_assert_not_owned_str:
1090 .asciz "mutex (%p) not owned\n"
1091 mutex_assert_owned_str:
1092 .asciz "mutex (%p) owned\n"
1093 mutex_interlock_destroyed_str:
1094 .asciz "trying to interlock destroyed mutex (%p)"
1095 .text
1096
1097
1098
1099 /*
1100 * lck_mtx_lock()
1101 * lck_mtx_try_lock()
1102 * lck_mtx_unlock()
1103 * lck_mtx_lock_spin()
1104 * lck_mtx_lock_spin_always()
1105 * lck_mtx_try_lock_spin()
1106 * lck_mtx_try_lock_spin_always()
1107 * lck_mtx_convert_spin()
1108 */
1109 NONLEAF_ENTRY(lck_mtx_lock_spin_always)
1110 mov %rdi, %rdx /* fetch lock pointer */
1111 jmp Llmls_avoid_check
1112
1113 NONLEAF_ENTRY(lck_mtx_lock_spin)
1114 mov %rdi, %rdx /* fetch lock pointer */
1115
1116 CHECK_PREEMPTION_LEVEL()
1117 Llmls_avoid_check:
1118 mov M_STATE(%rdx), %ecx
1119 test $(M_ILOCKED_MSK | M_MLOCKED_MSK), %ecx /* is the interlock or mutex held */
1120 jnz Llmls_slow
1121 Llmls_try: /* no - can't be INDIRECT, DESTROYED or locked */
1122 mov %rcx, %rax /* eax contains snapshot for cmpxchgl */
1123 or $(M_ILOCKED_MSK | M_SPIN_MSK), %ecx
1124
1125 PREEMPTION_DISABLE
1126 lock
1127 cmpxchg %ecx, M_STATE(%rdx) /* atomic compare and exchange */
1128 jne Llmls_busy_disabled
1129
1130 mov %gs:CPU_ACTIVE_THREAD, %rax
1131 mov %rax, M_OWNER(%rdx) /* record owner of interlock */
1132 #if MACH_LDEBUG
1133 test %rax, %rax
1134 jz 1f
1135 incl TH_MUTEX_COUNT(%rax) /* lock statistic */
1136 1:
1137 #endif /* MACH_LDEBUG */
1138
1139 /* return with the interlock held and preemption disabled */
1140 leave
1141 #if CONFIG_DTRACE
1142 LOCKSTAT_LABEL(_lck_mtx_lock_spin_lockstat_patch_point)
1143 ret
1144 /* inherit lock pointer in %rdx above */
1145 LOCKSTAT_RECORD(LS_LCK_MTX_LOCK_SPIN_ACQUIRE, %rdx)
1146 #endif
1147 ret
1148
1149 Llmls_slow:
1150 test $M_ILOCKED_MSK, %ecx /* is the interlock held */
1151 jz Llml_contended /* no, must have been the mutex */
1152
1153 cmp $(MUTEX_DESTROYED), %ecx /* check to see if its marked destroyed */
1154 je lck_mtx_destroyed
1155 cmp $(MUTEX_IND), %ecx /* Is this an indirect mutex */
1156 jne Llmls_loop /* no... must be interlocked */
1157
1158 LMTX_ENTER_EXTENDED
1159
1160 mov M_STATE(%rdx), %ecx
1161 test $(M_SPIN_MSK), %ecx
1162 jz Llmls_loop1
1163
1164 LMTX_UPDATE_MISS /* M_SPIN_MSK was set, so M_ILOCKED_MSK must also be present */
1165 Llmls_loop:
1166 PAUSE
1167 mov M_STATE(%rdx), %ecx
1168 Llmls_loop1:
1169 test $(M_ILOCKED_MSK | M_MLOCKED_MSK), %ecx
1170 jz Llmls_try
1171 test $(M_MLOCKED_MSK), %ecx
1172 jnz Llml_contended /* mutex owned by someone else, go contend for it */
1173 jmp Llmls_loop
1174
1175 Llmls_busy_disabled:
1176 PREEMPTION_ENABLE
1177 jmp Llmls_loop
1178
1179
1180
1181 NONLEAF_ENTRY(lck_mtx_lock)
1182 mov %rdi, %rdx /* fetch lock pointer */
1183
1184 CHECK_PREEMPTION_LEVEL()
1185
1186 mov M_STATE(%rdx), %ecx
1187 test $(M_ILOCKED_MSK | M_MLOCKED_MSK), %ecx /* is the interlock or mutex held */
1188 jnz Llml_slow
1189 Llml_try: /* no - can't be INDIRECT, DESTROYED or locked */
1190 mov %rcx, %rax /* eax contains snapshot for cmpxchgl */
1191 or $(M_ILOCKED_MSK | M_MLOCKED_MSK), %ecx
1192
1193 PREEMPTION_DISABLE
1194 lock
1195 cmpxchg %ecx, M_STATE(%rdx) /* atomic compare and exchange */
1196 jne Llml_busy_disabled
1197
1198 mov %gs:CPU_ACTIVE_THREAD, %rax
1199 mov %rax, M_OWNER(%rdx) /* record owner of mutex */
1200 #if MACH_LDEBUG
1201 test %rax, %rax
1202 jz 1f
1203 incl TH_MUTEX_COUNT(%rax) /* lock statistic */
1204 1:
1205 #endif /* MACH_LDEBUG */
1206
1207 testl $(M_WAITERS_MSK), M_STATE(%rdx)
1208 jz Llml_finish
1209
1210 LMTX_CALLEXT1(lck_mtx_lock_acquire_x86)
1211
1212 Llml_finish:
1213 andl $(~M_ILOCKED_MSK), M_STATE(%rdx)
1214 PREEMPTION_ENABLE
1215
1216 cmp %rdx, %rdi /* is this an extended mutex */
1217 jne 2f
1218
1219 leave
1220 #if CONFIG_DTRACE
1221 LOCKSTAT_LABEL(_lck_mtx_lock_lockstat_patch_point)
1222 ret
1223 /* inherit lock pointer in %rdx above */
1224 LOCKSTAT_RECORD(LS_LCK_MTX_LOCK_ACQUIRE, %rdx)
1225 #endif
1226 ret
1227 2:
1228 leave
1229 #if CONFIG_DTRACE
1230 LOCKSTAT_LABEL(_lck_mtx_lock_ext_lockstat_patch_point)
1231 ret
1232 /* inherit lock pointer in %rdx above */
1233 LOCKSTAT_RECORD(LS_LCK_MTX_EXT_LOCK_ACQUIRE, %rdx)
1234 #endif
1235 ret
1236
1237
1238 Llml_slow:
1239 test $M_ILOCKED_MSK, %ecx /* is the interlock held */
1240 jz Llml_contended /* no, must have been the mutex */
1241
1242 cmp $(MUTEX_DESTROYED), %ecx /* check to see if its marked destroyed */
1243 je lck_mtx_destroyed
1244 cmp $(MUTEX_IND), %ecx /* Is this an indirect mutex? */
1245 jne Llml_loop /* no... must be interlocked */
1246
1247 LMTX_ENTER_EXTENDED
1248
1249 mov M_STATE(%rdx), %ecx
1250 test $(M_SPIN_MSK), %ecx
1251 jz Llml_loop1
1252
1253 LMTX_UPDATE_MISS /* M_SPIN_MSK was set, so M_ILOCKED_MSK must also be present */
1254 Llml_loop:
1255 PAUSE
1256 mov M_STATE(%rdx), %ecx
1257 Llml_loop1:
1258 test $(M_ILOCKED_MSK | M_MLOCKED_MSK), %ecx
1259 jz Llml_try
1260 test $(M_MLOCKED_MSK), %ecx
1261 jnz Llml_contended /* mutex owned by someone else, go contend for it */
1262 jmp Llml_loop
1263
1264 Llml_busy_disabled:
1265 PREEMPTION_ENABLE
1266 jmp Llml_loop
1267
1268
1269 Llml_contended:
1270 cmp %rdx, %rdi /* is this an extended mutex */
1271 je 0f
1272 LMTX_UPDATE_MISS
1273 0:
1274 LMTX_CALLEXT1(lck_mtx_lock_spinwait_x86)
1275
1276 test %rax, %rax
1277 jz Llml_acquired /* acquired mutex, interlock held and preemption disabled */
1278
1279 cmp $1, %rax /* check for direct wait status */
1280 je 2f
1281 cmp %rdx, %rdi /* is this an extended mutex */
1282 je 2f
1283 LMTX_UPDATE_DIRECT_WAIT
1284 2:
1285 mov M_STATE(%rdx), %ecx
1286 test $(M_ILOCKED_MSK), %ecx
1287 jnz 6f
1288
1289 mov %rcx, %rax /* eax contains snapshot for cmpxchgl */
1290 or $(M_ILOCKED_MSK), %ecx /* try to take the interlock */
1291
1292 PREEMPTION_DISABLE
1293 lock
1294 cmpxchg %ecx, M_STATE(%rdx) /* atomic compare and exchange */
1295 jne 5f
1296
1297 test $(M_MLOCKED_MSK), %ecx /* we've got the interlock and */
1298 jnz 3f
1299 or $(M_MLOCKED_MSK), %ecx /* the mutex is free... grab it directly */
1300 mov %ecx, M_STATE(%rdx)
1301
1302 mov %gs:CPU_ACTIVE_THREAD, %rax
1303 mov %rax, M_OWNER(%rdx) /* record owner of mutex */
1304 #if MACH_LDEBUG
1305 test %rax, %rax
1306 jz 1f
1307 incl TH_MUTEX_COUNT(%rax) /* lock statistic */
1308 1:
1309 #endif /* MACH_LDEBUG */
1310
1311 Llml_acquired:
1312 testl $(M_WAITERS_MSK), M_STATE(%rdx)
1313 jnz 1f
1314 mov M_OWNER(%rdx), %rax
1315 mov TH_WAS_PROMOTED_ON_WAKEUP(%rax), %eax
1316 test %eax, %eax
1317 jz Llml_finish
1318 1:
1319 LMTX_CALLEXT1(lck_mtx_lock_acquire_x86)
1320 jmp Llml_finish
1321
1322 3: /* interlock held, mutex busy */
1323 cmp %rdx, %rdi /* is this an extended mutex */
1324 je 4f
1325 LMTX_UPDATE_WAIT
1326 4:
1327 LMTX_CALLEXT1(lck_mtx_lock_wait_x86)
1328 jmp Llml_contended
1329 5:
1330 PREEMPTION_ENABLE
1331 6:
1332 PAUSE
1333 jmp 2b
1334
1335
1336 NONLEAF_ENTRY(lck_mtx_try_lock_spin_always)
1337 mov %rdi, %rdx /* fetch lock pointer */
1338 jmp Llmts_avoid_check
1339
1340 NONLEAF_ENTRY(lck_mtx_try_lock_spin)
1341 mov %rdi, %rdx /* fetch lock pointer */
1342
1343 Llmts_avoid_check:
1344 mov M_STATE(%rdx), %ecx
1345 test $(M_ILOCKED_MSK | M_MLOCKED_MSK), %ecx /* is the interlock or mutex held */
1346 jnz Llmts_slow
1347 Llmts_try: /* no - can't be INDIRECT, DESTROYED or locked */
1348 mov %rcx, %rax /* eax contains snapshot for cmpxchgl */
1349 or $(M_ILOCKED_MSK | M_SPIN_MSK), %rcx
1350
1351 PREEMPTION_DISABLE
1352 lock
1353 cmpxchg %ecx, M_STATE(%rdx) /* atomic compare and exchange */
1354 jne Llmts_busy_disabled
1355
1356 mov %gs:CPU_ACTIVE_THREAD, %rax
1357 mov %rax, M_OWNER(%rdx) /* record owner of mutex */
1358 #if MACH_LDEBUG
1359 test %rax, %rax
1360 jz 1f
1361 incl TH_MUTEX_COUNT(%rax) /* lock statistic */
1362 1:
1363 #endif /* MACH_LDEBUG */
1364
1365 leave
1366
1367 #if CONFIG_DTRACE
1368 mov $1, %rax /* return success */
1369 LOCKSTAT_LABEL(_lck_mtx_try_lock_spin_lockstat_patch_point)
1370 ret
1371 /* inherit lock pointer in %rdx above */
1372 LOCKSTAT_RECORD(LS_LCK_MTX_TRY_SPIN_LOCK_ACQUIRE, %rdx)
1373 #endif
1374 mov $1, %rax /* return success */
1375 ret
1376
1377 Llmts_slow:
1378 test $(M_ILOCKED_MSK), %ecx /* is the interlock held */
1379 jz Llmts_fail /* no, must be held as a mutex */
1380
1381 cmp $(MUTEX_DESTROYED), %ecx /* check to see if its marked destroyed */
1382 je lck_mtx_destroyed
1383 cmp $(MUTEX_IND), %ecx /* Is this an indirect mutex? */
1384 jne Llmts_loop1
1385
1386 LMTX_ENTER_EXTENDED
1387 Llmts_loop:
1388 PAUSE
1389 mov M_STATE(%rdx), %ecx
1390 Llmts_loop1:
1391 test $(M_MLOCKED_MSK | M_SPIN_MSK), %ecx
1392 jnz Llmts_fail
1393 test $(M_ILOCKED_MSK), %ecx
1394 jz Llmts_try
1395 jmp Llmts_loop
1396
1397 Llmts_busy_disabled:
1398 PREEMPTION_ENABLE
1399 jmp Llmts_loop
1400
1401
1402
1403 NONLEAF_ENTRY(lck_mtx_try_lock)
1404 mov %rdi, %rdx /* fetch lock pointer */
1405
1406 mov M_STATE(%rdx), %ecx
1407 test $(M_ILOCKED_MSK | M_MLOCKED_MSK), %ecx /* is the interlock or mutex held */
1408 jnz Llmt_slow
1409 Llmt_try: /* no - can't be INDIRECT, DESTROYED or locked */
1410 mov %rcx, %rax /* eax contains snapshot for cmpxchgl */
1411 or $(M_ILOCKED_MSK | M_MLOCKED_MSK), %ecx
1412
1413 PREEMPTION_DISABLE
1414 lock
1415 cmpxchg %ecx, M_STATE(%rdx) /* atomic compare and exchange */
1416 jne Llmt_busy_disabled
1417
1418 mov %gs:CPU_ACTIVE_THREAD, %rax
1419 mov %rax, M_OWNER(%rdx) /* record owner of mutex */
1420 #if MACH_LDEBUG
1421 test %rax, %rax
1422 jz 1f
1423 incl TH_MUTEX_COUNT(%rax) /* lock statistic */
1424 1:
1425 #endif /* MACH_LDEBUG */
1426
1427 test $(M_WAITERS_MSK), %ecx
1428 jz 0f
1429
1430 LMTX_CALLEXT1(lck_mtx_lock_acquire_x86)
1431 0:
1432 andl $(~M_ILOCKED_MSK), M_STATE(%rdx)
1433 PREEMPTION_ENABLE
1434
1435 leave
1436 #if CONFIG_DTRACE
1437 mov $1, %rax /* return success */
1438 /* Dtrace probe: LS_LCK_MTX_TRY_LOCK_ACQUIRE */
1439 LOCKSTAT_LABEL(_lck_mtx_try_lock_lockstat_patch_point)
1440 ret
1441 /* inherit lock pointer in %rdx from above */
1442 LOCKSTAT_RECORD(LS_LCK_MTX_TRY_LOCK_ACQUIRE, %rdx)
1443 #endif
1444 mov $1, %rax /* return success */
1445 ret
1446
1447 Llmt_slow:
1448 test $(M_ILOCKED_MSK), %ecx /* is the interlock held */
1449 jz Llmt_fail /* no, must be held as a mutex */
1450
1451 cmp $(MUTEX_DESTROYED), %ecx /* check to see if its marked destroyed */
1452 je lck_mtx_destroyed
1453 cmp $(MUTEX_IND), %ecx /* Is this an indirect mutex? */
1454 jne Llmt_loop
1455
1456 LMTX_ENTER_EXTENDED
1457 Llmt_loop:
1458 PAUSE
1459 mov M_STATE(%rdx), %ecx
1460 Llmt_loop1:
1461 test $(M_MLOCKED_MSK | M_SPIN_MSK), %ecx
1462 jnz Llmt_fail
1463 test $(M_ILOCKED_MSK), %ecx
1464 jz Llmt_try
1465 jmp Llmt_loop
1466
1467 Llmt_busy_disabled:
1468 PREEMPTION_ENABLE
1469 jmp Llmt_loop
1470
1471
1472 Llmt_fail:
1473 Llmts_fail:
1474 cmp %rdx, %rdi /* is this an extended mutex */
1475 je 0f
1476 LMTX_UPDATE_MISS
1477 0:
1478 xor %rax, %rax
1479 NONLEAF_RET
1480
1481
1482
1483 NONLEAF_ENTRY(lck_mtx_convert_spin)
1484 mov %rdi, %rdx /* fetch lock pointer */
1485
1486 mov M_STATE(%rdx), %ecx
1487 cmp $(MUTEX_IND), %ecx /* Is this an indirect mutex? */
1488 jne 0f
1489 mov M_PTR(%rdx), %rdx /* If so, take indirection */
1490 mov M_STATE(%rdx), %ecx
1491 0:
1492 test $(M_MLOCKED_MSK), %ecx /* already owned as a mutex, just return */
1493 jnz 2f
1494 test $(M_WAITERS_MSK), %ecx /* are there any waiters? */
1495 jz 1f
1496
1497 LMTX_CALLEXT1(lck_mtx_lock_acquire_x86)
1498 mov M_STATE(%rdx), %ecx
1499 1:
1500 and $(~(M_ILOCKED_MSK | M_SPIN_MSK)), %ecx /* convert from spin version to mutex */
1501 or $(M_MLOCKED_MSK), %ecx
1502 mov %ecx, M_STATE(%rdx) /* since I own the interlock, I don't need an atomic update */
1503
1504 PREEMPTION_ENABLE
1505 2:
1506 NONLEAF_RET
1507
1508
1509
1510 NONLEAF_ENTRY(lck_mtx_unlock)
1511 mov %rdi, %rdx /* fetch lock pointer */
1512 Llmu_entry:
1513 mov M_STATE(%rdx), %ecx
1514 Llmu_prim:
1515 cmp $(MUTEX_IND), %ecx /* Is this an indirect mutex? */
1516 je Llmu_ext
1517
1518 Llmu_chktype:
1519 test $(M_MLOCKED_MSK), %ecx /* check for full mutex */
1520 jz Llmu_unlock
1521 Llmu_mutex:
1522 test $(M_ILOCKED_MSK), %rcx /* have to wait for interlock to clear */
1523 jnz Llmu_busy
1524
1525 mov %rcx, %rax /* eax contains snapshot for cmpxchgl */
1526 and $(~M_MLOCKED_MSK), %ecx /* drop mutex */
1527 or $(M_ILOCKED_MSK), %ecx /* pick up interlock */
1528
1529 PREEMPTION_DISABLE
1530 lock
1531 cmpxchg %ecx, M_STATE(%rdx) /* atomic compare and exchange */
1532 jne Llmu_busy_disabled /* branch on failure to spin loop */
1533
1534 Llmu_unlock:
1535 xor %rax, %rax
1536 mov %rax, M_OWNER(%rdx)
1537 mov %rcx, %rax /* keep original state in %ecx for later evaluation */
1538 and $(~(M_ILOCKED_MSK | M_SPIN_MSK | M_PROMOTED_MSK)), %rax
1539
1540 test $(M_WAITERS_MSK), %eax
1541 jz 2f
1542 dec %eax /* decrement waiter count */
1543 2:
1544 mov %eax, M_STATE(%rdx) /* since I own the interlock, I don't need an atomic update */
1545
1546 #if MACH_LDEBUG
1547 /* perform lock statistics after drop to prevent delay */
1548 mov %gs:CPU_ACTIVE_THREAD, %rax
1549 test %rax, %rax
1550 jz 1f
1551 decl TH_MUTEX_COUNT(%rax) /* lock statistic */
1552 1:
1553 #endif /* MACH_LDEBUG */
1554
1555 test $(M_PROMOTED_MSK | M_WAITERS_MSK), %ecx
1556 jz 3f
1557
1558 LMTX_CALLEXT2(lck_mtx_unlock_wakeup_x86, %rcx)
1559 3:
1560 PREEMPTION_ENABLE
1561
1562 cmp %rdx, %rdi
1563 jne 4f
1564
1565 leave
1566 #if CONFIG_DTRACE
1567 /* Dtrace: LS_LCK_MTX_UNLOCK_RELEASE */
1568 LOCKSTAT_LABEL(_lck_mtx_unlock_lockstat_patch_point)
1569 ret
1570 /* inherit lock pointer in %rdx from above */
1571 LOCKSTAT_RECORD(LS_LCK_MTX_UNLOCK_RELEASE, %rdx)
1572 #endif
1573 ret
1574 4:
1575 leave
1576 #if CONFIG_DTRACE
1577 /* Dtrace: LS_LCK_MTX_EXT_UNLOCK_RELEASE */
1578 LOCKSTAT_LABEL(_lck_mtx_ext_unlock_lockstat_patch_point)
1579 ret
1580 /* inherit lock pointer in %rdx from above */
1581 LOCKSTAT_RECORD(LS_LCK_MTX_EXT_UNLOCK_RELEASE, %rdx)
1582 #endif
1583 ret
1584
1585
1586 Llmu_busy_disabled:
1587 PREEMPTION_ENABLE
1588 Llmu_busy:
1589 PAUSE
1590 mov M_STATE(%rdx), %ecx
1591 jmp Llmu_mutex
1592
1593 Llmu_ext:
1594 mov M_PTR(%rdx), %rdx
1595 mov M_OWNER(%rdx), %rax
1596 mov %gs:CPU_ACTIVE_THREAD, %rcx
1597 CHECK_UNLOCK(%rcx, %rax)
1598 mov M_STATE(%rdx), %ecx
1599 jmp Llmu_chktype
1600
1601
1602
1603 LEAF_ENTRY(lck_mtx_ilk_unlock)
1604 mov %rdi, %rdx /* fetch lock pointer - no indirection here */
1605
1606 andl $(~M_ILOCKED_MSK), M_STATE(%rdx)
1607
1608 PREEMPTION_ENABLE /* need to re-enable preemption */
1609
1610 LEAF_RET
1611
1612
1613
1614 LEAF_ENTRY(lck_mtx_lock_grab_mutex)
1615 mov %rdi, %rdx /* fetch lock pointer - no indirection here */
1616
1617 mov M_STATE(%rdx), %ecx
1618
1619 test $(M_ILOCKED_MSK | M_MLOCKED_MSK), %ecx /* can't have the mutex yet */
1620 jnz 3f
1621
1622 mov %rcx, %rax /* eax contains snapshot for cmpxchgl */
1623 or $(M_ILOCKED_MSK | M_MLOCKED_MSK), %ecx
1624
1625 PREEMPTION_DISABLE
1626 lock
1627 cmpxchg %ecx, M_STATE(%rdx) /* atomic compare and exchange */
1628 jne 2f /* branch on failure to spin loop */
1629
1630 mov %gs:CPU_ACTIVE_THREAD, %rax
1631 mov %rax, M_OWNER(%rdx) /* record owner of mutex */
1632 #if MACH_LDEBUG
1633 test %rax, %rax
1634 jz 1f
1635 incl TH_MUTEX_COUNT(%rax) /* lock statistic */
1636 1:
1637 #endif /* MACH_LDEBUG */
1638
1639 mov $1, %rax /* return success */
1640 LEAF_RET
1641 2:
1642 PREEMPTION_ENABLE
1643 3:
1644 xor %rax, %rax /* return failure */
1645 LEAF_RET
1646
1647
1648
1649 LEAF_ENTRY(lck_mtx_lock_mark_destroyed)
1650 mov %rdi, %rdx
1651 1:
1652 mov M_STATE(%rdx), %ecx
1653 cmp $(MUTEX_IND), %ecx /* Is this an indirect mutex? */
1654 jne 2f
1655
1656 movl $(MUTEX_DESTROYED), M_STATE(%rdx) /* convert to destroyed state */
1657 jmp 3f
1658 2:
1659 test $(M_ILOCKED_MSK), %rcx /* have to wait for interlock to clear */
1660 jnz 5f
1661
1662 PREEMPTION_DISABLE
1663 mov %rcx, %rax /* eax contains snapshot for cmpxchgl */
1664 or $(M_ILOCKED_MSK), %ecx
1665 lock
1666 cmpxchg %ecx, M_STATE(%rdx) /* atomic compare and exchange */
1667 jne 4f /* branch on failure to spin loop */
1668 movl $(MUTEX_DESTROYED), M_STATE(%rdx) /* convert to destroyed state */
1669 PREEMPTION_ENABLE
1670 3:
1671 LEAF_RET /* return with M_ILOCKED set */
1672 4:
1673 PREEMPTION_ENABLE
1674 5:
1675 PAUSE
1676 jmp 1b
1677
1678 LEAF_ENTRY(preemption_underflow_panic)
1679 FRAME
1680 incl %gs:CPU_PREEMPTION_LEVEL
1681 ALIGN_STACK()
1682 LOAD_STRING_ARG0(16f)
1683 CALL_PANIC()
1684 hlt
1685 .data
1686 16: String "Preemption level underflow, possible cause unlocking an unlocked mutex or spinlock"
1687 .text
1688
1689
1690 LEAF_ENTRY(_disable_preemption)
1691 #if MACH_RT
1692 PREEMPTION_DISABLE
1693 #endif /* MACH_RT */
1694 LEAF_RET
1695
1696 LEAF_ENTRY(_enable_preemption)
1697 #if MACH_RT
1698 #if MACH_ASSERT
1699 cmpl $0,%gs:CPU_PREEMPTION_LEVEL
1700 jg 1f
1701 movl %gs:CPU_PREEMPTION_LEVEL,%esi
1702 ALIGN_STACK()
1703 LOAD_STRING_ARG0(_enable_preemption_less_than_zero)
1704 CALL_PANIC()
1705 hlt
1706 .cstring
1707 _enable_preemption_less_than_zero:
1708 .asciz "_enable_preemption: preemption_level(%d) < 0!"
1709 .text
1710 1:
1711 #endif /* MACH_ASSERT */
1712 PREEMPTION_ENABLE
1713 #endif /* MACH_RT */
1714 LEAF_RET
1715
1716 LEAF_ENTRY(_enable_preemption_no_check)
1717 #if MACH_RT
1718 #if MACH_ASSERT
1719 cmpl $0,%gs:CPU_PREEMPTION_LEVEL
1720 jg 1f
1721 ALIGN_STACK()
1722 LOAD_STRING_ARG0(_enable_preemption_no_check_less_than_zero)
1723 CALL_PANIC()
1724 hlt
1725 .cstring
1726 _enable_preemption_no_check_less_than_zero:
1727 .asciz "_enable_preemption_no_check: preemption_level <= 0!"
1728 .text
1729 1:
1730 #endif /* MACH_ASSERT */
1731 _ENABLE_PREEMPTION_NO_CHECK
1732 #endif /* MACH_RT */
1733 LEAF_RET
1734
1735
1736 LEAF_ENTRY(_mp_disable_preemption)
1737 #if MACH_RT
1738 PREEMPTION_DISABLE
1739 #endif /* MACH_RT */
1740 LEAF_RET
1741
1742 LEAF_ENTRY(_mp_enable_preemption)
1743 #if MACH_RT
1744 #if MACH_ASSERT
1745 cmpl $0,%gs:CPU_PREEMPTION_LEVEL
1746 jg 1f
1747 movl %gs:CPU_PREEMPTION_LEVEL,%esi
1748 ALIGN_PANIC()
1749 LOAD_STRING_ARG0(_mp_enable_preemption_less_than_zero)
1750 CALL_PANIC()
1751 hlt
1752 .cstring
1753 _mp_enable_preemption_less_than_zero:
1754 .asciz "_mp_enable_preemption: preemption_level (%d) <= 0!"
1755 .text
1756 1:
1757 #endif /* MACH_ASSERT */
1758 PREEMPTION_ENABLE
1759 #endif /* MACH_RT */
1760 LEAF_RET
1761
1762 LEAF_ENTRY(_mp_enable_preemption_no_check)
1763 #if MACH_RT
1764 #if MACH_ASSERT
1765 cmpl $0,%gs:CPU_PREEMPTION_LEVEL
1766 jg 1f
1767 ALIGN_STACK()
1768 LOAD_STRING_ARG0(_mp_enable_preemption_no_check_less_than_zero)
1769 CALL_PANIC()
1770 hlt
1771 .cstring
1772 _mp_enable_preemption_no_check_less_than_zero:
1773 .asciz "_mp_enable_preemption_no_check: preemption_level <= 0!"
1774 .text
1775 1:
1776 #endif /* MACH_ASSERT */
1777 _ENABLE_PREEMPTION_NO_CHECK
1778 #endif /* MACH_RT */
1779 LEAF_RET
1780
1781 /*
1782 * Atomic primitives, prototyped in kern/simple_lock.h
1783 */
1784 LEAF_ENTRY(hw_atomic_add)
1785 #if MACH_LDEBUG
1786 test $3, %rdi
1787 jz 1f
1788 ud2
1789 1:
1790 #endif
1791 movl %esi, %eax /* Load addend */
1792 lock xaddl %eax, (%rdi) /* Atomic exchange and add */
1793 addl %esi, %eax /* Calculate result */
1794 LEAF_RET
1795
1796 LEAF_ENTRY(hw_atomic_sub)
1797 #if MACH_LDEBUG
1798 test $3, %rdi
1799 jz 1f
1800 ud2
1801 1:
1802 #endif
1803 negl %esi
1804 movl %esi, %eax
1805 lock xaddl %eax, (%rdi) /* Atomic exchange and add */
1806 addl %esi, %eax /* Calculate result */
1807 LEAF_RET
1808
1809 LEAF_ENTRY(hw_atomic_or)
1810 #if MACH_LDEBUG
1811 test $3, %rdi
1812 jz 1f
1813 ud2
1814 1:
1815 #endif
1816 movl (%rdi), %eax
1817 1:
1818 movl %esi, %edx /* Load mask */
1819 orl %eax, %edx
1820 lock cmpxchgl %edx, (%rdi) /* Atomic CAS */
1821 jne 1b
1822 movl %edx, %eax /* Result */
1823 LEAF_RET
1824 /*
1825 * A variant of hw_atomic_or which doesn't return a value.
1826 * The implementation is thus comparatively more efficient.
1827 */
1828
1829 LEAF_ENTRY(hw_atomic_or_noret)
1830 #if MACH_LDEBUG
1831 test $3, %rdi
1832 jz 1f
1833 ud2
1834 1:
1835 #endif
1836 lock
1837 orl %esi, (%rdi) /* Atomic OR */
1838 LEAF_RET
1839
1840
1841 LEAF_ENTRY(hw_atomic_and)
1842 #if MACH_LDEBUG
1843 test $3, %rdi
1844 jz 1f
1845 ud2
1846 1:
1847 #endif
1848 movl (%rdi), %eax
1849 1:
1850 movl %esi, %edx /* Load mask */
1851 andl %eax, %edx
1852 lock cmpxchgl %edx, (%rdi) /* Atomic CAS */
1853 jne 1b
1854 movl %edx, %eax /* Result */
1855 LEAF_RET
1856 /*
1857 * A variant of hw_atomic_and which doesn't return a value.
1858 * The implementation is thus comparatively more efficient.
1859 */
1860
1861 LEAF_ENTRY(hw_atomic_and_noret)
1862 #if MACH_LDEBUG
1863 test $3, %rdi
1864 jz 1f
1865 ud2
1866 1:
1867 #endif
1868 lock andl %esi, (%rdi) /* Atomic OR */
1869 LEAF_RET
1870
mirror of XNU