]> git.saurik.com Git - apple/xnu.git/blob - osfmk/i386/i386_lock.s
projects / apple / xnu.git / blob
? search:
summary | shortlog | log | commit | commitdiff | tree
blame | history | raw | HEAD
xnu-1699.22.73.tar.gz
[apple/xnu.git] / osfmk / i386 / i386_lock.s
1 /*
2 * Copyright (c) 2000-2009 Apple Inc. All rights reserved.
3 *
4 * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
5 *
6 * This file contains Original Code and/or Modifications of Original Code
7 * as defined in and that are subject to the Apple Public Source License
8 * Version 2.0 (the 'License'). You may not use this file except in
9 * compliance with the License. The rights granted to you under the License
10 * may not be used to create, or enable the creation or redistribution of,
11 * unlawful or unlicensed copies of an Apple operating system, or to
12 * circumvent, violate, or enable the circumvention or violation of, any
13 * terms of an Apple operating system software license agreement.
14 *
15 * Please obtain a copy of the License at
16 * http://www.opensource.apple.com/apsl/ and read it before using this file.
17 *
18 * The Original Code and all software distributed under the License are
19 * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
20 * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
21 * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
22 * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
23 * Please see the License for the specific language governing rights and
24 * limitations under the License.
25 *
26 * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
27 */
28 /*
29 * @OSF_COPYRIGHT@
30 */
31 /*
32 * Mach Operating System
33 * Copyright (c) 1989 Carnegie-Mellon University
34 * All rights reserved. The CMU software License Agreement specifies
35 * the terms and conditions for use and redistribution.
36 */
37
38 #include <mach_rt.h>
39 #include <platforms.h>
40 #include <mach_ldebug.h>
41 #include <i386/asm.h>
42 #include <i386/eflags.h>
43 #include <i386/trap.h>
44 #include <config_dtrace.h>
45 #include <i386/mp.h>
46
47 #include "assym.s"
48
49 #define PAUSE rep; nop
50
51 #include <i386/pal_lock_asm.h>
52
53 /*
54 * When performance isn't the only concern, it's
55 * nice to build stack frames...
56 */
57 #define BUILD_STACK_FRAMES (GPROF || \
58 ((MACH_LDEBUG) && MACH_KDB))
59
60 #if BUILD_STACK_FRAMES
61
62 /* Stack-frame-relative: */
63 #define L_PC B_PC
64 #define L_ARG0 B_ARG0
65 #define L_ARG1 B_ARG1
66
67 #define LEAF_ENTRY(name) \
68 Entry(name); \
69 FRAME; \
70 MCOUNT
71
72 #define LEAF_ENTRY2(n1,n2) \
73 Entry(n1); \
74 Entry(n2); \
75 FRAME; \
76 MCOUNT
77
78 #define LEAF_RET \
79 EMARF; \
80 ret
81
82 #else /* BUILD_STACK_FRAMES */
83
84 /* Stack-pointer-relative: */
85 #define L_PC S_PC
86 #define L_ARG0 S_ARG0
87 #define L_ARG1 S_ARG1
88
89 #define LEAF_ENTRY(name) \
90 Entry(name)
91
92 #define LEAF_ENTRY2(n1,n2) \
93 Entry(n1); \
94 Entry(n2)
95
96 #define LEAF_RET \
97 ret
98
99 #endif /* BUILD_STACK_FRAMES */
100
101
102 /* Non-leaf routines always have a stack frame: */
103
104 #define NONLEAF_ENTRY(name) \
105 Entry(name); \
106 FRAME; \
107 MCOUNT
108
109 #define NONLEAF_ENTRY2(n1,n2) \
110 Entry(n1); \
111 Entry(n2); \
112 FRAME; \
113 MCOUNT
114
115 #define NONLEAF_RET \
116 EMARF; \
117 ret
118
119
120 /* For x86_64, the varargs ABI requires that %al indicate
121 * how many SSE register contain arguments. In our case, 0 */
122 #if __i386__
123 #define ALIGN_STACK() subl $8, %esp; andl $0xFFFFFFF0, %esp ;
124 #define LOAD_STRING_ARG0(label) movl $##label, (%esp) ;
125 #define LOAD_ARG1(x) mov x, 4(%esp) ;
126 #define LOAD_PTR_ARG1(x) mov x, 4(%esp) ;
127 #define CALL_PANIC() call EXT(panic) ;
128 #else
129 #define ALIGN_STACK() and $0xFFFFFFFFFFFFFFF0, %rsp ;
130 #define LOAD_STRING_ARG0(label) leaq label(%rip), %rdi ;
131 #define LOAD_ARG1(x) mov x, %esi ;
132 #define LOAD_PTR_ARG1(x) mov x, %rsi ;
133 #define CALL_PANIC() xorb %al,%al ; call EXT(panic) ;
134 #endif
135
136 #define CHECK_UNLOCK(current, owner) \
137 cmp current, owner ; \
138 je 1f ; \
139 ALIGN_STACK() ; \
140 LOAD_STRING_ARG0(2f) ; \
141 CALL_PANIC() ; \
142 hlt ; \
143 .data ; \
144 2: String "Mutex unlock attempted from non-owner thread"; \
145 .text ; \
146 1:
147
148 #if MACH_LDEBUG
149 /*
150 * Routines for general lock debugging.
151 */
152
153 /*
154 * Checks for expected lock types and calls "panic" on
155 * mismatch. Detects calls to Mutex functions with
156 * type simplelock and vice versa.
157 */
158 #define CHECK_MUTEX_TYPE() \
159 cmpl $ MUTEX_TAG,M_TYPE ; \
160 je 1f ; \
161 ALIGN_STACK() ; \
162 LOAD_STRING_ARG0(2f) ; \
163 CALL_PANIC() ; \
164 hlt ; \
165 .data ; \
166 2: String "not a mutex!" ; \
167 .text ; \
168 1:
169
170 /*
171 * If one or more simplelocks are currently held by a thread,
172 * an attempt to acquire a mutex will cause this check to fail
173 * (since a mutex lock may context switch, holding a simplelock
174 * is not a good thing).
175 */
176 #if MACH_RT
177 #define CHECK_PREEMPTION_LEVEL() \
178 cmpl $0,%gs:CPU_HIBERNATE ; \
179 jne 1f ; \
180 cmpl $0,%gs:CPU_PREEMPTION_LEVEL ; \
181 je 1f ; \
182 ALIGN_STACK() ; \
183 movl %gs:CPU_PREEMPTION_LEVEL, %eax ; \
184 LOAD_ARG1(%eax) ; \
185 LOAD_STRING_ARG0(2f) ; \
186 CALL_PANIC() ; \
187 hlt ; \
188 .data ; \
189 2: String "preemption_level(%d) != 0!" ; \
190 .text ; \
191 1:
192 #else /* MACH_RT */
193 #define CHECK_PREEMPTION_LEVEL()
194 #endif /* MACH_RT */
195
196 #define CHECK_MYLOCK(current, owner) \
197 cmp current, owner ; \
198 jne 1f ; \
199 ALIGN_STACK() ; \
200 LOAD_STRING_ARG0(2f) ; \
201 CALL_PANIC() ; \
202 hlt ; \
203 .data ; \
204 2: String "Attempt to recursively lock a non-recursive lock"; \
205 .text ; \
206 1:
207
208 #else /* MACH_LDEBUG */
209 #define CHECK_MUTEX_TYPE()
210 #define CHECK_PREEMPTION_LEVEL()
211 #define CHECK_MYLOCK(thd)
212 #endif /* MACH_LDEBUG */
213
214 #define PREEMPTION_DISABLE \
215 incl %gs:CPU_PREEMPTION_LEVEL
216
217 #if MACH_LDEBUG || 1
218 #define PREEMPTION_LEVEL_DEBUG 1
219 #endif
220 #if PREEMPTION_LEVEL_DEBUG
221 #define PREEMPTION_ENABLE \
222 decl %gs:CPU_PREEMPTION_LEVEL ; \
223 js 17f ; \
224 jnz 19f ; \
225 testl $AST_URGENT,%gs:CPU_PENDING_AST ; \
226 jz 19f ; \
227 PUSHF ; \
228 testl $EFL_IF, S_PC ; \
229 jz 18f ; \
230 POPF ; \
231 int $(T_PREEMPT) ; \
232 jmp 19f ; \
233 17: \
234 call _preemption_underflow_panic ; \
235 18: \
236 POPF ; \
237 19:
238 #else
239 #define PREEMPTION_ENABLE \
240 decl %gs:CPU_PREEMPTION_LEVEL ; \
241 jnz 19f ; \
242 testl $AST_URGENT,%gs:CPU_PENDING_AST ; \
243 jz 19f ; \
244 PUSHF ; \
245 testl $EFL_IF, S_PC ; \
246 jz 18f ; \
247 POPF ; \
248 int $(T_PREEMPT) ; \
249 jmp 19f ; \
250 18: \
251 POPF ; \
252 19:
253 #endif
254
255
256 #if CONFIG_DTRACE
257
258 .globl _lockstat_probe
259 .globl _lockstat_probemap
260
261 /*
262 * LOCKSTAT_LABEL creates a dtrace symbol which contains
263 * a pointer into the lock code function body. At that
264 * point is a "ret" instruction that can be patched into
265 * a "nop"
266 */
267
268 #if defined(__i386__)
269
270 #define LOCKSTAT_LABEL(lab) \
271 .data ;\
272 .globl lab ;\
273 lab: ;\
274 .long 9f ;\
275 .text ;\
276 9:
277
278 #define LOCKSTAT_RECORD(id, lck) \
279 push %ebp ; \
280 mov %esp,%ebp ; \
281 sub $0x38,%esp /* size of dtrace_probe args */ ; \
282 movl _lockstat_probemap + (id * 4),%eax ; \
283 test %eax,%eax ; \
284 je 9f ; \
285 movl $0,36(%esp) ; \
286 movl $0,40(%esp) ; \
287 movl $0,28(%esp) ; \
288 movl $0,32(%esp) ; \
289 movl $0,20(%esp) ; \
290 movl $0,24(%esp) ; \
291 movl $0,12(%esp) ; \
292 movl $0,16(%esp) ; \
293 movl lck,4(%esp) /* copy lock pointer to arg 1 */ ; \
294 movl $0,8(%esp) ; \
295 movl %eax,(%esp) ; \
296 call *_lockstat_probe ; \
297 9: leave
298 /* ret - left to subsequent code, e.g. return values */
299
300 #elif defined(__x86_64__)
301 #define LOCKSTAT_LABEL(lab) \
302 .data ;\
303 .globl lab ;\
304 lab: ;\
305 .quad 9f ;\
306 .text ;\
307 9:
308
309 #define LOCKSTAT_RECORD(id, lck) \
310 push %rbp ; \
311 mov %rsp,%rbp ; \
312 movl _lockstat_probemap + (id * 4)(%rip),%eax ; \
313 test %eax,%eax ; \
314 je 9f ; \
315 mov lck, %rsi ; \
316 mov %rax, %rdi ; \
317 mov $0, %rdx ; \
318 mov $0, %rcx ; \
319 mov $0, %r8 ; \
320 mov $0, %r9 ; \
321 call *_lockstat_probe(%rip) ; \
322 9: leave
323 /* ret - left to subsequent code, e.g. return values */
324 #else
325 #error Unsupported architecture
326 #endif
327 #endif /* CONFIG_DTRACE */
328
329 /*
330 * For most routines, the hw_lock_t pointer is loaded into a
331 * register initially, and then either a byte or register-sized
332 * word is loaded/stored to the pointer
333 */
334
335 #if defined(__i386__)
336 #define HW_LOCK_REGISTER %edx
337 #define LOAD_HW_LOCK_REGISTER mov L_ARG0, HW_LOCK_REGISTER
338 #define HW_LOCK_THREAD_REGISTER %ecx
339 #define LOAD_HW_LOCK_THREAD_REGISTER mov %gs:CPU_ACTIVE_THREAD, HW_LOCK_THREAD_REGISTER
340 #define HW_LOCK_MOV_WORD movl
341 #define HW_LOCK_EXAM_REGISTER %eax
342 #elif defined(__x86_64__)
343 #define HW_LOCK_REGISTER %rdi
344 #define LOAD_HW_LOCK_REGISTER
345 #define HW_LOCK_THREAD_REGISTER %rcx
346 #define LOAD_HW_LOCK_THREAD_REGISTER mov %gs:CPU_ACTIVE_THREAD, HW_LOCK_THREAD_REGISTER
347 #define HW_LOCK_MOV_WORD movq
348 #define HW_LOCK_EXAM_REGISTER %rax
349 #else
350 #error Unsupported architecture
351 #endif
352
353 /*
354 * void hw_lock_init(hw_lock_t)
355 *
356 * Initialize a hardware lock.
357 */
358 LEAF_ENTRY(hw_lock_init)
359 LOAD_HW_LOCK_REGISTER /* fetch lock pointer */
360 HW_LOCK_MOV_WORD $0, (HW_LOCK_REGISTER) /* clear the lock */
361 LEAF_RET
362
363
364 /*
365 * void hw_lock_byte_init(uint8_t *)
366 *
367 * Initialize a hardware byte lock.
368 */
369 LEAF_ENTRY(hw_lock_byte_init)
370 LOAD_HW_LOCK_REGISTER /* fetch lock pointer */
371 movb $0, (HW_LOCK_REGISTER) /* clear the lock */
372 LEAF_RET
373
374 /*
375 * void hw_lock_lock(hw_lock_t)
376 *
377 * Acquire lock, spinning until it becomes available.
378 * MACH_RT: also return with preemption disabled.
379 */
380 LEAF_ENTRY(hw_lock_lock)
381 LOAD_HW_LOCK_REGISTER /* fetch lock pointer */
382 LOAD_HW_LOCK_THREAD_REGISTER /* get thread pointer */
383
384 PREEMPTION_DISABLE
385 1:
386 mov (HW_LOCK_REGISTER), HW_LOCK_EXAM_REGISTER
387 test HW_LOCK_EXAM_REGISTER,HW_LOCK_EXAM_REGISTER /* lock locked? */
388 jne 3f /* branch if so */
389 lock; cmpxchg HW_LOCK_THREAD_REGISTER,(HW_LOCK_REGISTER) /* try to acquire the HW lock */
390 jne 3f
391 movl $1,%eax /* In case this was a timeout call */
392 LEAF_RET /* if yes, then nothing left to do */
393 3:
394 PAUSE /* pause for hyper-threading */
395 jmp 1b /* try again */
396
397 /*
398 * void hw_lock_byte_lock(uint8_t *lock_byte)
399 *
400 * Acquire byte sized lock operand, spinning until it becomes available.
401 * MACH_RT: also return with preemption disabled.
402 */
403
404 LEAF_ENTRY(hw_lock_byte_lock)
405 LOAD_HW_LOCK_REGISTER /* Load lock pointer */
406 PREEMPTION_DISABLE
407 movl $1, %ecx /* Set lock value */
408 1:
409 movb (HW_LOCK_REGISTER), %al /* Load byte at address */
410 testb %al,%al /* lock locked? */
411 jne 3f /* branch if so */
412 lock; cmpxchg %cl,(HW_LOCK_REGISTER) /* attempt atomic compare exchange */
413 jne 3f
414 LEAF_RET /* if yes, then nothing left to do */
415 3:
416 PAUSE /* pause for hyper-threading */
417 jmp 1b /* try again */
418
419 /*
420 * unsigned int hw_lock_to(hw_lock_t, unsigned int)
421 *
422 * Acquire lock, spinning until it becomes available or timeout.
423 * MACH_RT: also return with preemption disabled.
424 */
425 LEAF_ENTRY(hw_lock_to)
426 1:
427 LOAD_HW_LOCK_REGISTER /* fetch lock pointer */
428 LOAD_HW_LOCK_THREAD_REGISTER
429
430 /*
431 * Attempt to grab the lock immediately
432 * - fastpath without timeout nonsense.
433 */
434 PREEMPTION_DISABLE
435
436 mov (HW_LOCK_REGISTER), HW_LOCK_EXAM_REGISTER
437 test HW_LOCK_EXAM_REGISTER,HW_LOCK_EXAM_REGISTER /* lock locked? */
438 jne 2f /* branch if so */
439 lock; cmpxchg HW_LOCK_THREAD_REGISTER,(HW_LOCK_REGISTER) /* try to acquire the HW lock */
440 jne 2f /* branch on failure */
441 movl $1,%eax
442 LEAF_RET
443
444 2:
445 #define INNER_LOOP_COUNT 1000
446 /*
447 * Failed to get the lock so set the timeout
448 * and then spin re-checking the lock but pausing
449 * every so many (INNER_LOOP_COUNT) spins to check for timeout.
450 */
451 #if __i386__
452 movl L_ARG1,%ecx /* fetch timeout */
453 push %edi
454 push %ebx
455 mov %edx,%edi
456
457 lfence
458 rdtsc /* read cyclecount into %edx:%eax */
459 lfence
460 addl %ecx,%eax /* fetch and timeout */
461 adcl $0,%edx /* add carry */
462 mov %edx,%ecx
463 mov %eax,%ebx /* %ecx:%ebx is the timeout expiry */
464 mov %edi, %edx /* load lock back into %edx */
465 #else
466 push %r9
467 lfence
468 rdtsc /* read cyclecount into %edx:%eax */
469 lfence
470 shlq $32, %rdx
471 orq %rdx, %rax /* load 64-bit quantity into %rax */
472 addq %rax, %rsi /* %rsi is the timeout expiry */
473 #endif
474
475 4:
476 /*
477 * The inner-loop spin to look for the lock being freed.
478 */
479 #if __i386__
480 mov $(INNER_LOOP_COUNT),%edi
481 #else
482 mov $(INNER_LOOP_COUNT),%r9
483 #endif
484 5:
485 PAUSE /* pause for hyper-threading */
486 mov (HW_LOCK_REGISTER),HW_LOCK_EXAM_REGISTER /* spin checking lock value in cache */
487 test HW_LOCK_EXAM_REGISTER,HW_LOCK_EXAM_REGISTER
488 je 6f /* zero => unlocked, try to grab it */
489 #if __i386__
490 decl %edi /* decrement inner loop count */
491 #else
492 decq %r9 /* decrement inner loop count */
493 #endif
494 jnz 5b /* time to check for timeout? */
495
496 /*
497 * Here after spinning INNER_LOOP_COUNT times, check for timeout
498 */
499 #if __i386__
500 mov %edx,%edi /* Save %edx */
501 lfence
502 rdtsc /* cyclecount into %edx:%eax */
503 lfence
504 xchg %edx,%edi /* cyclecount into %edi:%eax */
505 cmpl %ecx,%edi /* compare high-order 32-bits */
506 jb 4b /* continue spinning if less, or */
507 cmpl %ebx,%eax /* compare low-order 32-bits */
508 jb 4b /* continue if less, else bail */
509 xor %eax,%eax /* with 0 return value */
510 pop %ebx
511 pop %edi
512 #else
513 lfence
514 rdtsc /* cyclecount into %edx:%eax */
515 lfence
516 shlq $32, %rdx
517 orq %rdx, %rax /* load 64-bit quantity into %rax */
518 cmpq %rsi, %rax /* compare to timeout */
519 jb 4b /* continue spinning if less, or */
520 xor %rax,%rax /* with 0 return value */
521 pop %r9
522 #endif
523 LEAF_RET
524
525 6:
526 /*
527 * Here to try to grab the lock that now appears to be free
528 * after contention.
529 */
530 LOAD_HW_LOCK_THREAD_REGISTER
531 lock; cmpxchg HW_LOCK_THREAD_REGISTER,(HW_LOCK_REGISTER) /* try to acquire the HW lock */
532 jne 4b /* no - spin again */
533 movl $1,%eax /* yes */
534 #if __i386__
535 pop %ebx
536 pop %edi
537 #else
538 pop %r9
539 #endif
540 LEAF_RET
541
542 /*
543 * void hw_lock_unlock(hw_lock_t)
544 *
545 * Unconditionally release lock.
546 * MACH_RT: release preemption level.
547 */
548 LEAF_ENTRY(hw_lock_unlock)
549 LOAD_HW_LOCK_REGISTER /* fetch lock pointer */
550 HW_LOCK_MOV_WORD $0, (HW_LOCK_REGISTER) /* clear the lock */
551 PREEMPTION_ENABLE
552 LEAF_RET
553
554 /*
555 * void hw_lock_byte_unlock(uint8_t *lock_byte)
556 *
557 * Unconditionally release byte sized lock operand.
558 * MACH_RT: release preemption level.
559 */
560
561 LEAF_ENTRY(hw_lock_byte_unlock)
562 LOAD_HW_LOCK_REGISTER /* Load lock pointer */
563 movb $0, (HW_LOCK_REGISTER) /* Clear the lock byte */
564 PREEMPTION_ENABLE
565 LEAF_RET
566
567 /*
568 * unsigned int hw_lock_try(hw_lock_t)
569 * MACH_RT: returns with preemption disabled on success.
570 */
571 LEAF_ENTRY(hw_lock_try)
572 LOAD_HW_LOCK_REGISTER /* fetch lock pointer */
573 LOAD_HW_LOCK_THREAD_REGISTER
574 PREEMPTION_DISABLE
575
576 mov (HW_LOCK_REGISTER),HW_LOCK_EXAM_REGISTER
577 test HW_LOCK_EXAM_REGISTER,HW_LOCK_EXAM_REGISTER
578 jne 1f
579 lock; cmpxchg HW_LOCK_THREAD_REGISTER,(HW_LOCK_REGISTER) /* try to acquire the HW lock */
580 jne 1f
581
582 movl $1,%eax /* success */
583 LEAF_RET
584
585 1:
586 PREEMPTION_ENABLE /* failure: release preemption... */
587 xorl %eax,%eax /* ...and return failure */
588 LEAF_RET
589
590 /*
591 * unsigned int hw_lock_held(hw_lock_t)
592 * MACH_RT: doesn't change preemption state.
593 * N.B. Racy, of course.
594 */
595 LEAF_ENTRY(hw_lock_held)
596 LOAD_HW_LOCK_REGISTER /* fetch lock pointer */
597 mov (HW_LOCK_REGISTER),HW_LOCK_EXAM_REGISTER /* check lock value */
598 test HW_LOCK_EXAM_REGISTER,HW_LOCK_EXAM_REGISTER
599 movl $1,%ecx
600 cmovne %ecx,%eax /* 0 => unlocked, 1 => locked */
601 LEAF_RET
602
603
604 /*
605 * Reader-writer lock fastpaths. These currently exist for the
606 * shared lock acquire, the exclusive lock acquire, the shared to
607 * exclusive upgrade and the release paths (where they reduce overhead
608 * considerably) -- these are by far the most frequently used routines
609 *
610 * The following should reflect the layout of the bitfield embedded within
611 * the lck_rw_t structure (see i386/locks.h).
612 */
613 #define LCK_RW_INTERLOCK (0x1 << 16)
614
615 #define LCK_RW_PRIV_EXCL (0x1 << 24)
616 #define LCK_RW_WANT_UPGRADE (0x2 << 24)
617 #define LCK_RW_WANT_WRITE (0x4 << 24)
618 #define LCK_R_WAITING (0x8 << 24)
619 #define LCK_W_WAITING (0x10 << 24)
620
621 #define LCK_RW_SHARED_MASK (0xffff)
622
623 /*
624 * For most routines, the lck_rw_t pointer is loaded into a
625 * register initially, and the flags bitfield loaded into another
626 * register and examined
627 */
628
629 #if defined(__i386__)
630 #define LCK_RW_REGISTER %edx
631 #define LOAD_LCK_RW_REGISTER mov S_ARG0, LCK_RW_REGISTER
632 #define LCK_RW_FLAGS_REGISTER %eax
633 #define LOAD_LCK_RW_FLAGS_REGISTER mov (LCK_RW_REGISTER), LCK_RW_FLAGS_REGISTER
634 #elif defined(__x86_64__)
635 #define LCK_RW_REGISTER %rdi
636 #define LOAD_LCK_RW_REGISTER
637 #define LCK_RW_FLAGS_REGISTER %eax
638 #define LOAD_LCK_RW_FLAGS_REGISTER mov (LCK_RW_REGISTER), LCK_RW_FLAGS_REGISTER
639 #else
640 #error Unsupported architecture
641 #endif
642
643 #define RW_LOCK_SHARED_MASK (LCK_RW_INTERLOCK | LCK_RW_WANT_UPGRADE | LCK_RW_WANT_WRITE)
644 /*
645 * void lck_rw_lock_shared(lck_rw_t *)
646 *
647 */
648 Entry(lck_rw_lock_shared)
649 LOAD_LCK_RW_REGISTER
650 1:
651 LOAD_LCK_RW_FLAGS_REGISTER /* Load state bitfield and interlock */
652 testl $(RW_LOCK_SHARED_MASK), %eax /* Eligible for fastpath? */
653 jne 3f
654
655 movl %eax, %ecx /* original value in %eax for cmpxchgl */
656 incl %ecx /* Increment reader refcount */
657 lock
658 cmpxchgl %ecx, (LCK_RW_REGISTER) /* Attempt atomic exchange */
659 jne 2f
660
661 #if CONFIG_DTRACE
662 /*
663 * Dtrace lockstat event: LS_LCK_RW_LOCK_SHARED_ACQUIRE
664 * Implemented by swapping between return and no-op instructions.
665 * See bsd/dev/dtrace/lockstat.c.
666 */
667 LOCKSTAT_LABEL(_lck_rw_lock_shared_lockstat_patch_point)
668 ret
669 /*
670 Fall thru when patched, counting on lock pointer in LCK_RW_REGISTER
671 */
672 LOCKSTAT_RECORD(LS_LCK_RW_LOCK_SHARED_ACQUIRE, LCK_RW_REGISTER)
673 #endif
674 ret
675 2:
676 PAUSE
677 jmp 1b
678 3:
679 jmp EXT(lck_rw_lock_shared_gen)
680
681
682
683 #define RW_TRY_LOCK_SHARED_MASK (LCK_RW_WANT_UPGRADE | LCK_RW_WANT_WRITE)
684 /*
685 * void lck_rw_try_lock_shared(lck_rw_t *)
686 *
687 */
688 Entry(lck_rw_try_lock_shared)
689 LOAD_LCK_RW_REGISTER
690 1:
691 LOAD_LCK_RW_FLAGS_REGISTER /* Load state bitfield and interlock */
692 testl $(LCK_RW_INTERLOCK), %eax
693 jne 2f
694 testl $(RW_TRY_LOCK_SHARED_MASK), %eax
695 jne 3f /* lock is busy */
696
697 movl %eax, %ecx /* original value in %eax for cmpxchgl */
698 incl %ecx /* Increment reader refcount */
699 lock
700 cmpxchgl %ecx, (LCK_RW_REGISTER) /* Attempt atomic exchange */
701 jne 2f
702
703 #if CONFIG_DTRACE
704 movl $1, %eax
705 /*
706 * Dtrace lockstat event: LS_LCK_RW_TRY_LOCK_SHARED_ACQUIRE
707 * Implemented by swapping between return and no-op instructions.
708 * See bsd/dev/dtrace/lockstat.c.
709 */
710 LOCKSTAT_LABEL(_lck_rw_try_lock_shared_lockstat_patch_point)
711 ret
712 /* Fall thru when patched, counting on lock pointer in LCK_RW_REGISTER */
713 LOCKSTAT_RECORD(LS_LCK_RW_LOCK_SHARED_ACQUIRE, LCK_RW_REGISTER)
714 #endif
715 movl $1, %eax /* return TRUE */
716 ret
717 2:
718 PAUSE
719 jmp 1b
720 3:
721 xorl %eax, %eax
722 ret
723
724
725 #define RW_LOCK_EXCLUSIVE_HELD (LCK_RW_WANT_WRITE | LCK_RW_WANT_UPGRADE)
726 /*
727 * int lck_rw_grab_shared(lck_rw_t *)
728 *
729 */
730 Entry(lck_rw_grab_shared)
731 LOAD_LCK_RW_REGISTER
732 1:
733 LOAD_LCK_RW_FLAGS_REGISTER /* Load state bitfield and interlock */
734 testl $(LCK_RW_INTERLOCK), %eax
735 jne 5f
736 testl $(RW_LOCK_EXCLUSIVE_HELD), %eax
737 jne 3f
738 2:
739 movl %eax, %ecx /* original value in %eax for cmpxchgl */
740 incl %ecx /* Increment reader refcount */
741 lock
742 cmpxchgl %ecx, (LCK_RW_REGISTER) /* Attempt atomic exchange */
743 jne 4f
744
745 movl $1, %eax /* return success */
746 ret
747 3:
748 testl $(LCK_RW_SHARED_MASK), %eax
749 je 4f
750 testl $(LCK_RW_PRIV_EXCL), %eax
751 je 2b
752 4:
753 xorl %eax, %eax /* return failure */
754 ret
755 5:
756 PAUSE
757 jmp 1b
758
759
760
761 #define RW_LOCK_EXCLUSIVE_MASK (LCK_RW_SHARED_MASK | LCK_RW_INTERLOCK | \
762 LCK_RW_WANT_UPGRADE | LCK_RW_WANT_WRITE)
763 /*
764 * void lck_rw_lock_exclusive(lck_rw_t*)
765 *
766 */
767 Entry(lck_rw_lock_exclusive)
768 LOAD_LCK_RW_REGISTER
769 1:
770 LOAD_LCK_RW_FLAGS_REGISTER /* Load state bitfield, interlock and shared count */
771 testl $(RW_LOCK_EXCLUSIVE_MASK), %eax /* Eligible for fastpath? */
772 jne 3f /* no, go slow */
773
774 movl %eax, %ecx /* original value in %eax for cmpxchgl */
775 orl $(LCK_RW_WANT_WRITE), %ecx
776 lock
777 cmpxchgl %ecx, (LCK_RW_REGISTER) /* Attempt atomic exchange */
778 jne 2f
779
780 #if CONFIG_DTRACE
781 /*
782 * Dtrace lockstat event: LS_LCK_RW_LOCK_EXCL_ACQUIRE
783 * Implemented by swapping between return and no-op instructions.
784 * See bsd/dev/dtrace/lockstat.c.
785 */
786 LOCKSTAT_LABEL(_lck_rw_lock_exclusive_lockstat_patch_point)
787 ret
788 /* Fall thru when patched, counting on lock pointer in LCK_RW_REGISTER */
789 LOCKSTAT_RECORD(LS_LCK_RW_LOCK_SHARED_ACQUIRE, LCK_RW_REGISTER)
790 #endif
791 ret
792 2:
793 PAUSE
794 jmp 1b
795 3:
796 jmp EXT(lck_rw_lock_exclusive_gen)
797
798
799
800 #define RW_TRY_LOCK_EXCLUSIVE_MASK (LCK_RW_SHARED_MASK | LCK_RW_WANT_UPGRADE | LCK_RW_WANT_WRITE)
801 /*
802 * void lck_rw_try_lock_exclusive(lck_rw_t *)
803 *
804 * Tries to get a write lock.
805 *
806 * Returns FALSE if the lock is not held on return.
807 */
808 Entry(lck_rw_try_lock_exclusive)
809 LOAD_LCK_RW_REGISTER
810 1:
811 LOAD_LCK_RW_FLAGS_REGISTER /* Load state bitfield, interlock and shared count */
812 testl $(LCK_RW_INTERLOCK), %eax
813 jne 2f
814 testl $(RW_TRY_LOCK_EXCLUSIVE_MASK), %eax
815 jne 3f /* can't get it */
816
817 movl %eax, %ecx /* original value in %eax for cmpxchgl */
818 orl $(LCK_RW_WANT_WRITE), %ecx
819 lock
820 cmpxchgl %ecx, (LCK_RW_REGISTER) /* Attempt atomic exchange */
821 jne 2f
822
823 #if CONFIG_DTRACE
824 movl $1, %eax
825 /*
826 * Dtrace lockstat event: LS_LCK_RW_TRY_LOCK_EXCL_ACQUIRE
827 * Implemented by swapping between return and no-op instructions.
828 * See bsd/dev/dtrace/lockstat.c.
829 */
830 LOCKSTAT_LABEL(_lck_rw_try_lock_exclusive_lockstat_patch_point)
831 ret
832 /* Fall thru when patched, counting on lock pointer in LCK_RW_REGISTER */
833 LOCKSTAT_RECORD(LS_LCK_RW_LOCK_SHARED_ACQUIRE, LCK_RW_REGISTER)
834 #endif
835 movl $1, %eax /* return TRUE */
836 ret
837 2:
838 PAUSE
839 jmp 1b
840 3:
841 xorl %eax, %eax /* return FALSE */
842 ret
843
844
845
846 /*
847 * void lck_rw_lock_shared_to_exclusive(lck_rw_t*)
848 *
849 * fastpath can be taken if
850 * the current rw_shared_count == 1
851 * AND the interlock is clear
852 * AND RW_WANT_UPGRADE is not set
853 *
854 * note that RW_WANT_WRITE could be set, but will not
855 * be indicative of an exclusive hold since we have
856 * a read count on the lock that we have not yet released
857 * we can blow by that state since the lck_rw_lock_exclusive
858 * function will block until rw_shared_count == 0 and
859 * RW_WANT_UPGRADE is clear... it does this check behind
860 * the interlock which we are also checking for
861 *
862 * to make the transition we must be able to atomically
863 * set RW_WANT_UPGRADE and get rid of the read count we hold
864 */
865 Entry(lck_rw_lock_shared_to_exclusive)
866 LOAD_LCK_RW_REGISTER
867 1:
868 LOAD_LCK_RW_FLAGS_REGISTER /* Load state bitfield, interlock and shared count */
869 testl $(LCK_RW_INTERLOCK), %eax
870 jne 7f
871 testl $(LCK_RW_WANT_UPGRADE), %eax
872 jne 2f
873
874 movl %eax, %ecx /* original value in %eax for cmpxchgl */
875 orl $(LCK_RW_WANT_UPGRADE), %ecx /* ask for WANT_UPGRADE */
876 decl %ecx /* and shed our read count */
877 lock
878 cmpxchgl %ecx, (LCK_RW_REGISTER) /* Attempt atomic exchange */
879 jne 7f
880 /* we now own the WANT_UPGRADE */
881 testl $(LCK_RW_SHARED_MASK), %ecx /* check to see if all of the readers are drained */
882 jne 8f /* if not, we need to go wait */
883
884 #if CONFIG_DTRACE
885 movl $1, %eax
886 /*
887 * Dtrace lockstat event: LS_LCK_RW_LOCK_SHARED_TO_EXCL_UPGRADE
888 * Implemented by swapping between return and no-op instructions.
889 * See bsd/dev/dtrace/lockstat.c.
890 */
891 LOCKSTAT_LABEL(_lck_rw_lock_shared_to_exclusive_lockstat_patch_point)
892 ret
893 /* Fall thru when patched, counting on lock pointer in LCK_RW_REGISTER */
894 LOCKSTAT_RECORD(LS_LCK_RW_LOCK_SHARED_ACQUIRE, LCK_RW_REGISTER)
895 #endif
896 movl $1, %eax /* return success */
897 ret
898
899 2: /* someone else already holds WANT_UPGRADE */
900 movl %eax, %ecx /* original value in %eax for cmpxchgl */
901 decl %ecx /* shed our read count */
902 testl $(LCK_RW_SHARED_MASK), %ecx
903 jne 3f /* we were the last reader */
904 andl $(~LCK_W_WAITING), %ecx /* so clear the wait indicator */
905 3:
906 lock
907 cmpxchgl %ecx, (LCK_RW_REGISTER) /* Attempt atomic exchange */
908 jne 7f
909
910 #if __i386__
911 pushl %eax /* go check to see if we need to */
912 push %edx /* wakeup anyone */
913 call EXT(lck_rw_lock_shared_to_exclusive_failure)
914 addl $8, %esp
915 #else
916 mov %eax, %esi /* put old flags as second arg */
917 /* lock is alread in %rdi */
918 call EXT(lck_rw_lock_shared_to_exclusive_failure)
919 #endif
920 ret /* and pass the failure return along */
921 7:
922 PAUSE
923 jmp 1b
924 8:
925 jmp EXT(lck_rw_lock_shared_to_exclusive_success)
926
927
928
929 .cstring
930 rwl_release_error_str:
931 .asciz "Releasing non-exclusive RW lock without a reader refcount!"
932 .text
933
934 /*
935 * lck_rw_type_t lck_rw_done(lck_rw_t *)
936 *
937 */
938 Entry(lck_rw_done)
939 LOAD_LCK_RW_REGISTER
940 1:
941 LOAD_LCK_RW_FLAGS_REGISTER /* Load state bitfield, interlock and reader count */
942 testl $(LCK_RW_INTERLOCK), %eax
943 jne 7f /* wait for interlock to clear */
944
945 movl %eax, %ecx /* keep original value in %eax for cmpxchgl */
946 testl $(LCK_RW_SHARED_MASK), %ecx /* if reader count == 0, must be exclusive lock */
947 je 2f
948 decl %ecx /* Decrement reader count */
949 testl $(LCK_RW_SHARED_MASK), %ecx /* if reader count has now gone to 0, check for waiters */
950 je 4f
951 jmp 6f
952 2:
953 testl $(LCK_RW_WANT_UPGRADE), %ecx
954 je 3f
955 andl $(~LCK_RW_WANT_UPGRADE), %ecx
956 jmp 4f
957 3:
958 testl $(LCK_RW_WANT_WRITE), %ecx
959 je 8f /* lock is not 'owned', go panic */
960 andl $(~LCK_RW_WANT_WRITE), %ecx
961 4:
962 /*
963 * test the original values to match what
964 * lck_rw_done_gen is going to do to determine
965 * which wakeups need to happen...
966 *
967 * if !(fake_lck->lck_rw_priv_excl && fake_lck->lck_w_waiting)
968 */
969 testl $(LCK_W_WAITING), %eax
970 je 5f
971 andl $(~LCK_W_WAITING), %ecx
972
973 testl $(LCK_RW_PRIV_EXCL), %eax
974 jne 6f
975 5:
976 andl $(~LCK_R_WAITING), %ecx
977 6:
978 lock
979 cmpxchgl %ecx, (LCK_RW_REGISTER) /* Attempt atomic exchange */
980 jne 7f
981
982 #if __i386__
983 pushl %eax
984 push %edx
985 call EXT(lck_rw_done_gen)
986 addl $8, %esp
987 #else
988 mov %eax,%esi /* old flags in %rsi */
989 /* lock is in %rdi already */
990 call EXT(lck_rw_done_gen)
991 #endif
992 ret
993 7:
994 PAUSE
995 jmp 1b
996 8:
997 ALIGN_STACK()
998 LOAD_STRING_ARG0(rwl_release_error_str)
999 CALL_PANIC()
1000
1001
1002
1003 /*
1004 * lck_rw_type_t lck_rw_lock_exclusive_to_shared(lck_rw_t *)
1005 *
1006 */
1007 Entry(lck_rw_lock_exclusive_to_shared)
1008 LOAD_LCK_RW_REGISTER
1009 1:
1010 LOAD_LCK_RW_FLAGS_REGISTER /* Load state bitfield, interlock and reader count */
1011 testl $(LCK_RW_INTERLOCK), %eax
1012 jne 6f /* wait for interlock to clear */
1013
1014 movl %eax, %ecx /* keep original value in %eax for cmpxchgl */
1015 incl %ecx /* Increment reader count */
1016
1017 testl $(LCK_RW_WANT_UPGRADE), %ecx
1018 je 2f
1019 andl $(~LCK_RW_WANT_UPGRADE), %ecx
1020 jmp 3f
1021 2:
1022 andl $(~LCK_RW_WANT_WRITE), %ecx
1023 3:
1024 /*
1025 * test the original values to match what
1026 * lck_rw_lock_exclusive_to_shared_gen is going to do to determine
1027 * which wakeups need to happen...
1028 *
1029 * if !(fake_lck->lck_rw_priv_excl && fake_lck->lck_w_waiting)
1030 */
1031 testl $(LCK_W_WAITING), %eax
1032 je 4f
1033 testl $(LCK_RW_PRIV_EXCL), %eax
1034 jne 5f
1035 4:
1036 andl $(~LCK_R_WAITING), %ecx
1037 5:
1038 lock
1039 cmpxchgl %ecx, (LCK_RW_REGISTER) /* Attempt atomic exchange */
1040 jne 6f
1041
1042 #if __i386__
1043 pushl %eax
1044 push %edx
1045 call EXT(lck_rw_lock_exclusive_to_shared_gen)
1046 addl $8, %esp
1047 #else
1048 mov %eax,%esi
1049 call EXT(lck_rw_lock_exclusive_to_shared_gen)
1050 #endif
1051 ret
1052 6:
1053 PAUSE
1054 jmp 1b
1055
1056
1057
1058 /*
1059 * int lck_rw_grab_want(lck_rw_t *)
1060 *
1061 */
1062 Entry(lck_rw_grab_want)
1063 LOAD_LCK_RW_REGISTER
1064 1:
1065 LOAD_LCK_RW_FLAGS_REGISTER /* Load state bitfield, interlock and reader count */
1066 testl $(LCK_RW_INTERLOCK), %eax
1067 jne 3f /* wait for interlock to clear */
1068 testl $(LCK_RW_WANT_WRITE), %eax /* want_write has been grabbed by someone else */
1069 jne 2f /* go return failure */
1070
1071 movl %eax, %ecx /* original value in %eax for cmpxchgl */
1072 orl $(LCK_RW_WANT_WRITE), %ecx
1073 lock
1074 cmpxchgl %ecx, (LCK_RW_REGISTER) /* Attempt atomic exchange */
1075 jne 2f
1076 /* we now own want_write */
1077 movl $1, %eax /* return success */
1078 ret
1079 2:
1080 xorl %eax, %eax /* return failure */
1081 ret
1082 3:
1083 PAUSE
1084 jmp 1b
1085
1086
1087 #define RW_LOCK_SHARED_OR_UPGRADE_MASK (LCK_RW_SHARED_MASK | LCK_RW_INTERLOCK | LCK_RW_WANT_UPGRADE)
1088 /*
1089 * int lck_rw_held_read_or_upgrade(lck_rw_t *)
1090 *
1091 */
1092 Entry(lck_rw_held_read_or_upgrade)
1093 LOAD_LCK_RW_REGISTER
1094 LOAD_LCK_RW_FLAGS_REGISTER /* Load state bitfield, interlock and reader count */
1095 andl $(RW_LOCK_SHARED_OR_UPGRADE_MASK), %eax
1096 ret
1097
1098
1099
1100 /*
1101 * N.B.: On x86, statistics are currently recorded for all indirect mutexes.
1102 * Also, only the acquire attempt count (GRP_MTX_STAT_UTIL) is maintained
1103 * as a 64-bit quantity (this matches the existing PowerPC implementation,
1104 * and the new x86 specific statistics are also maintained as 32-bit
1105 * quantities).
1106 *
1107 *
1108 * Enable this preprocessor define to record the first miss alone
1109 * By default, we count every miss, hence multiple misses may be
1110 * recorded for a single lock acquire attempt via lck_mtx_lock
1111 */
1112 #undef LOG_FIRST_MISS_ALONE
1113
1114 /*
1115 * This preprocessor define controls whether the R-M-W update of the
1116 * per-group statistics elements are atomic (LOCK-prefixed)
1117 * Enabled by default.
1118 */
1119 #define ATOMIC_STAT_UPDATES 1
1120
1121 #if defined(ATOMIC_STAT_UPDATES)
1122 #define LOCK_IF_ATOMIC_STAT_UPDATES lock
1123 #else
1124 #define LOCK_IF_ATOMIC_STAT_UPDATES
1125 #endif /* ATOMIC_STAT_UPDATES */
1126
1127
1128 /*
1129 * For most routines, the lck_mtx_t pointer is loaded into a
1130 * register initially, and the owner field checked for indirection.
1131 * Eventually the lock owner is loaded into a register and examined.
1132 */
1133
1134 #define M_OWNER MUTEX_OWNER
1135 #define M_PTR MUTEX_PTR
1136 #define M_STATE MUTEX_STATE
1137
1138 #if defined(__i386__)
1139
1140 #define LMTX_ARG0 B_ARG0
1141 #define LMTX_ARG1 B_ARG1
1142 #define LMTX_REG %edx
1143 #define LMTX_A_REG %eax
1144 #define LMTX_A_REG32 %eax
1145 #define LMTX_C_REG %ecx
1146 #define LMTX_C_REG32 %ecx
1147 #define LMTX_RET_REG %eax
1148 #define LMTX_RET_REG32 %eax
1149 #define LMTX_LGROUP_REG %esi
1150 #define LMTX_SSTATE_REG %edi
1151 #define LOAD_LMTX_REG(arg) mov arg, LMTX_REG
1152 #define LMTX_CHK_EXTENDED cmp LMTX_REG, LMTX_ARG0
1153 #define LMTX_ASSERT_OWNED cmpl $(MUTEX_ASSERT_OWNED), LMTX_ARG1
1154
1155 #define LMTX_ENTER_EXTENDED \
1156 mov M_PTR(LMTX_REG), LMTX_REG ; \
1157 push LMTX_LGROUP_REG ; \
1158 push LMTX_SSTATE_REG ; \
1159 xor LMTX_SSTATE_REG, LMTX_SSTATE_REG ; \
1160 mov MUTEX_GRP(LMTX_REG), LMTX_LGROUP_REG ; \
1161 LOCK_IF_ATOMIC_STAT_UPDATES ; \
1162 addl $1, GRP_MTX_STAT_UTIL(LMTX_LGROUP_REG) ; \
1163 jnc 11f ; \
1164 incl GRP_MTX_STAT_UTIL+4(LMTX_LGROUP_REG) ; \
1165 11:
1166
1167 #define LMTX_EXIT_EXTENDED \
1168 pop LMTX_SSTATE_REG ; \
1169 pop LMTX_LGROUP_REG
1170
1171
1172 #define LMTX_CHK_EXTENDED_EXIT \
1173 cmp LMTX_REG, LMTX_ARG0 ; \
1174 je 12f ; \
1175 pop LMTX_SSTATE_REG ; \
1176 pop LMTX_LGROUP_REG ; \
1177 12:
1178
1179
1180 #if LOG_FIRST_MISS_ALONE
1181 #define LMTX_UPDATE_MISS \
1182 test $1, LMTX_SSTATE_REG ; \
1183 jnz 11f ; \
1184 LOCK_IF_ATOMIC_STAT_UPDATES ; \
1185 incl GRP_MTX_STAT_MISS(LMTX_LGROUP_REG) ; \
1186 or $1, LMTX_SSTATE_REG ; \
1187 11:
1188 #else
1189 #define LMTX_UPDATE_MISS \
1190 LOCK_IF_ATOMIC_STAT_UPDATES ; \
1191 incl GRP_MTX_STAT_MISS(LMTX_LGROUP_REG)
1192 #endif
1193
1194
1195 #if LOG_FIRST_MISS_ALONE
1196 #define LMTX_UPDATE_WAIT \
1197 test $2, LMTX_SSTATE_REG ; \
1198 jnz 11f ; \
1199 LOCK_IF_ATOMIC_STAT_UPDATES ; \
1200 incl GRP_MTX_STAT_WAIT(LMTX_LGROUP_REG) ; \
1201 or $2, LMTX_SSTATE_REG ; \
1202 11:
1203 #else
1204 #define LMTX_UPDATE_WAIT \
1205 LOCK_IF_ATOMIC_STAT_UPDATES ; \
1206 incl GRP_MTX_STAT_WAIT(LMTX_LGROUP_REG)
1207 #endif
1208
1209
1210 /*
1211 * Record the "direct wait" statistic, which indicates if a
1212 * miss proceeded to block directly without spinning--occurs
1213 * if the owner of the mutex isn't running on another processor
1214 * at the time of the check.
1215 */
1216 #define LMTX_UPDATE_DIRECT_WAIT \
1217 LOCK_IF_ATOMIC_STAT_UPDATES ; \
1218 incl GRP_MTX_STAT_DIRECT_WAIT(LMTX_LGROUP_REG)
1219
1220
1221 #define LMTX_CALLEXT1(func_name) \
1222 push LMTX_REG ; \
1223 push LMTX_REG ; \
1224 call EXT(func_name) ; \
1225 add $4, %esp ; \
1226 pop LMTX_REG
1227
1228 #define LMTX_CALLEXT2(func_name, reg) \
1229 push LMTX_REG ; \
1230 push reg ; \
1231 push LMTX_REG ; \
1232 call EXT(func_name) ; \
1233 add $8, %esp ; \
1234 pop LMTX_REG
1235
1236 #elif defined(__x86_64__)
1237
1238 #define LMTX_ARG0 %rdi
1239 #define LMTX_ARG1 %rsi
1240 #define LMTX_REG_ORIG %rdi
1241 #define LMTX_REG %rdx
1242 #define LMTX_A_REG %rax
1243 #define LMTX_A_REG32 %eax
1244 #define LMTX_C_REG %rcx
1245 #define LMTX_C_REG32 %ecx
1246 #define LMTX_RET_REG %rax
1247 #define LMTX_RET_REG32 %eax
1248 #define LMTX_LGROUP_REG %r10
1249 #define LMTX_SSTATE_REG %r11
1250 #define LOAD_LMTX_REG(arg) mov %rdi, %rdx
1251 #define LMTX_CHK_EXTENDED cmp LMTX_REG, LMTX_REG_ORIG
1252 #define LMTX_ASSERT_OWNED cmp $(MUTEX_ASSERT_OWNED), LMTX_ARG1
1253
1254 #define LMTX_ENTER_EXTENDED \
1255 mov M_PTR(LMTX_REG), LMTX_REG ; \
1256 xor LMTX_SSTATE_REG, LMTX_SSTATE_REG ; \
1257 mov MUTEX_GRP(LMTX_REG), LMTX_LGROUP_REG ; \
1258 LOCK_IF_ATOMIC_STAT_UPDATES ; \
1259 incq GRP_MTX_STAT_UTIL(LMTX_LGROUP_REG)
1260
1261 #define LMTX_EXIT_EXTENDED
1262
1263 #define LMTX_CHK_EXTENDED_EXIT
1264
1265
1266 #if LOG_FIRST_MISS_ALONE
1267 #define LMTX_UPDATE_MISS \
1268 test $1, LMTX_SSTATE_REG ; \
1269 jnz 11f ; \
1270 LOCK_IF_ATOMIC_STAT_UPDATES ; \
1271 incl GRP_MTX_STAT_MISS(LMTX_LGROUP_REG) ; \
1272 or $1, LMTX_SSTATE_REG ; \
1273 11:
1274 #else
1275 #define LMTX_UPDATE_MISS \
1276 LOCK_IF_ATOMIC_STAT_UPDATES ; \
1277 incl GRP_MTX_STAT_MISS(LMTX_LGROUP_REG)
1278 #endif
1279
1280
1281 #if LOG_FIRST_MISS_ALONE
1282 #define LMTX_UPDATE_WAIT \
1283 test $2, LMTX_SSTATE_REG ; \
1284 jnz 11f ; \
1285 LOCK_IF_ATOMIC_STAT_UPDATES ; \
1286 incl GRP_MTX_STAT_WAIT(LMTX_LGROUP_REG) ; \
1287 or $2, LMTX_SSTATE_REG ; \
1288 11:
1289 #else
1290 #define LMTX_UPDATE_WAIT \
1291 LOCK_IF_ATOMIC_STAT_UPDATES ; \
1292 incl GRP_MTX_STAT_WAIT(LMTX_LGROUP_REG)
1293 #endif
1294
1295
1296 /*
1297 * Record the "direct wait" statistic, which indicates if a
1298 * miss proceeded to block directly without spinning--occurs
1299 * if the owner of the mutex isn't running on another processor
1300 * at the time of the check.
1301 */
1302 #define LMTX_UPDATE_DIRECT_WAIT \
1303 LOCK_IF_ATOMIC_STAT_UPDATES ; \
1304 incl GRP_MTX_STAT_DIRECT_WAIT(LMTX_LGROUP_REG)
1305
1306
1307 #define LMTX_CALLEXT1(func_name) \
1308 LMTX_CHK_EXTENDED ; \
1309 je 12f ; \
1310 push LMTX_LGROUP_REG ; \
1311 push LMTX_SSTATE_REG ; \
1312 12: push LMTX_REG_ORIG ; \
1313 push LMTX_REG ; \
1314 mov LMTX_REG, LMTX_ARG0 ; \
1315 call EXT(func_name) ; \
1316 pop LMTX_REG ; \
1317 pop LMTX_REG_ORIG ; \
1318 LMTX_CHK_EXTENDED ; \
1319 je 12f ; \
1320 pop LMTX_SSTATE_REG ; \
1321 pop LMTX_LGROUP_REG ; \
1322 12:
1323
1324 #define LMTX_CALLEXT2(func_name, reg) \
1325 LMTX_CHK_EXTENDED ; \
1326 je 12f ; \
1327 push LMTX_LGROUP_REG ; \
1328 push LMTX_SSTATE_REG ; \
1329 12: push LMTX_REG_ORIG ; \
1330 push LMTX_REG ; \
1331 mov reg, LMTX_ARG1 ; \
1332 mov LMTX_REG, LMTX_ARG0 ; \
1333 call EXT(func_name) ; \
1334 pop LMTX_REG ; \
1335 pop LMTX_REG_ORIG ; \
1336 LMTX_CHK_EXTENDED ; \
1337 je 12f ; \
1338 pop LMTX_SSTATE_REG ; \
1339 pop LMTX_LGROUP_REG ; \
1340 12:
1341
1342 #else
1343 #error Unsupported architecture
1344 #endif
1345
1346
1347 #define M_WAITERS_MSK 0x0000ffff
1348 #define M_PRIORITY_MSK 0x00ff0000
1349 #define M_ILOCKED_MSK 0x01000000
1350 #define M_MLOCKED_MSK 0x02000000
1351 #define M_PROMOTED_MSK 0x04000000
1352 #define M_SPIN_MSK 0x08000000
1353
1354 /*
1355 * void lck_mtx_assert(lck_mtx_t* l, unsigned int)
1356 * Takes the address of a lock, and an assertion type as parameters.
1357 * The assertion can take one of two forms determine by the type
1358 * parameter: either the lock is held by the current thread, and the
1359 * type is LCK_MTX_ASSERT_OWNED, or it isn't and the type is
1360 * LCK_MTX_ASSERT_NOTOWNED. Calls panic on assertion failure.
1361 *
1362 */
1363
1364 NONLEAF_ENTRY(lck_mtx_assert)
1365 LOAD_LMTX_REG(B_ARG0) /* Load lock address */
1366 mov %gs:CPU_ACTIVE_THREAD, LMTX_A_REG /* Load current thread */
1367
1368 mov M_STATE(LMTX_REG), LMTX_C_REG32
1369 cmp $(MUTEX_IND), LMTX_C_REG32 /* Is this an indirect mutex? */
1370 jne 0f
1371 mov M_PTR(LMTX_REG), LMTX_REG /* If so, take indirection */
1372 0:
1373 mov M_OWNER(LMTX_REG), LMTX_C_REG /* Load owner */
1374 LMTX_ASSERT_OWNED
1375 jne 2f /* Assert ownership? */
1376 cmp LMTX_A_REG, LMTX_C_REG /* Current thread match? */
1377 jne 3f /* no, go panic */
1378 testl $(M_ILOCKED_MSK | M_MLOCKED_MSK), M_STATE(LMTX_REG)
1379 je 3f
1380 1: /* yes, we own it */
1381 NONLEAF_RET
1382 2:
1383 cmp LMTX_A_REG, LMTX_C_REG /* Current thread match? */
1384 jne 1b /* No, return */
1385 ALIGN_STACK()
1386 LOAD_PTR_ARG1(LMTX_REG)
1387 LOAD_STRING_ARG0(mutex_assert_owned_str)
1388 jmp 4f
1389 3:
1390 ALIGN_STACK()
1391 LOAD_PTR_ARG1(LMTX_REG)
1392 LOAD_STRING_ARG0(mutex_assert_not_owned_str)
1393 4:
1394 CALL_PANIC()
1395
1396
1397 lck_mtx_destroyed:
1398 ALIGN_STACK()
1399 LOAD_PTR_ARG1(LMTX_REG)
1400 LOAD_STRING_ARG0(mutex_interlock_destroyed_str)
1401 CALL_PANIC()
1402
1403
1404 .data
1405 mutex_assert_not_owned_str:
1406 .asciz "mutex (%p) not owned\n"
1407 mutex_assert_owned_str:
1408 .asciz "mutex (%p) owned\n"
1409 mutex_interlock_destroyed_str:
1410 .asciz "trying to interlock destroyed mutex (%p)"
1411 .text
1412
1413
1414
1415 /*
1416 * lck_mtx_lock()
1417 * lck_mtx_try_lock()
1418 * lck_mtx_unlock()
1419 * lck_mtx_lock_spin()
1420 * lck_mtx_lock_spin_always()
1421 * lck_mtx_convert_spin()
1422 */
1423 NONLEAF_ENTRY(lck_mtx_lock_spin_always)
1424 LOAD_LMTX_REG(B_ARG0) /* fetch lock pointer */
1425 jmp Llmls_avoid_check
1426
1427 NONLEAF_ENTRY(lck_mtx_lock_spin)
1428 LOAD_LMTX_REG(B_ARG0) /* fetch lock pointer */
1429
1430 CHECK_PREEMPTION_LEVEL()
1431 Llmls_avoid_check:
1432 mov M_STATE(LMTX_REG), LMTX_C_REG32
1433 test $(M_ILOCKED_MSK | M_MLOCKED_MSK), LMTX_C_REG32 /* is the interlock or mutex held */
1434 jnz Llmls_slow
1435 Llmls_try: /* no - can't be INDIRECT, DESTROYED or locked */
1436 mov LMTX_C_REG, LMTX_A_REG /* eax contains snapshot for cmpxchgl */
1437 or $(M_ILOCKED_MSK | M_SPIN_MSK), LMTX_C_REG32
1438
1439 PREEMPTION_DISABLE
1440 lock
1441 cmpxchg LMTX_C_REG32, M_STATE(LMTX_REG) /* atomic compare and exchange */
1442 jne Llmls_busy_disabled
1443
1444 mov %gs:CPU_ACTIVE_THREAD, LMTX_A_REG
1445 mov LMTX_A_REG, M_OWNER(LMTX_REG) /* record owner of interlock */
1446 #if MACH_LDEBUG
1447 test LMTX_A_REG, LMTX_A_REG
1448 jz 1f
1449 incl TH_MUTEX_COUNT(LMTX_A_REG) /* lock statistic */
1450 1:
1451 #endif /* MACH_LDEBUG */
1452
1453 LMTX_CHK_EXTENDED_EXIT
1454 /* return with the interlock held and preemption disabled */
1455 leave
1456 #if CONFIG_DTRACE
1457 LOCKSTAT_LABEL(_lck_mtx_lock_spin_lockstat_patch_point)
1458 ret
1459 /* inherit lock pointer in LMTX_REG above */
1460 LOCKSTAT_RECORD(LS_LCK_MTX_LOCK_SPIN_ACQUIRE, LMTX_REG)
1461 #endif
1462 ret
1463
1464 Llmls_slow:
1465 test $M_ILOCKED_MSK, LMTX_C_REG32 /* is the interlock held */
1466 jz Llml_contended /* no, must have been the mutex */
1467
1468 cmp $(MUTEX_DESTROYED), LMTX_C_REG32 /* check to see if its marked destroyed */
1469 je lck_mtx_destroyed
1470 cmp $(MUTEX_IND), LMTX_C_REG32 /* Is this an indirect mutex */
1471 jne Llmls_loop /* no... must be interlocked */
1472
1473 LMTX_ENTER_EXTENDED
1474
1475 mov M_STATE(LMTX_REG), LMTX_C_REG32
1476 test $(M_SPIN_MSK), LMTX_C_REG32
1477 jz Llmls_loop1
1478
1479 LMTX_UPDATE_MISS /* M_SPIN_MSK was set, so M_ILOCKED_MSK must also be present */
1480 Llmls_loop:
1481 PAUSE
1482 mov M_STATE(LMTX_REG), LMTX_C_REG32
1483 Llmls_loop1:
1484 test $(M_ILOCKED_MSK | M_MLOCKED_MSK), LMTX_C_REG32
1485 jz Llmls_try
1486 test $(M_MLOCKED_MSK), LMTX_C_REG32
1487 jnz Llml_contended /* mutex owned by someone else, go contend for it */
1488 jmp Llmls_loop
1489
1490 Llmls_busy_disabled:
1491 PREEMPTION_ENABLE
1492 jmp Llmls_loop
1493
1494
1495
1496 NONLEAF_ENTRY(lck_mtx_lock)
1497 LOAD_LMTX_REG(B_ARG0) /* fetch lock pointer */
1498
1499 CHECK_PREEMPTION_LEVEL()
1500
1501 mov M_STATE(LMTX_REG), LMTX_C_REG32
1502 test $(M_ILOCKED_MSK | M_MLOCKED_MSK), LMTX_C_REG32 /* is the interlock or mutex held */
1503 jnz Llml_slow
1504 Llml_try: /* no - can't be INDIRECT, DESTROYED or locked */
1505 mov LMTX_C_REG, LMTX_A_REG /* eax contains snapshot for cmpxchgl */
1506 or $(M_ILOCKED_MSK | M_MLOCKED_MSK), LMTX_C_REG32
1507
1508 PREEMPTION_DISABLE
1509 lock
1510 cmpxchg LMTX_C_REG32, M_STATE(LMTX_REG) /* atomic compare and exchange */
1511 jne Llml_busy_disabled
1512
1513 mov %gs:CPU_ACTIVE_THREAD, LMTX_A_REG
1514 mov LMTX_A_REG, M_OWNER(LMTX_REG) /* record owner of mutex */
1515 #if MACH_LDEBUG
1516 test LMTX_A_REG, LMTX_A_REG
1517 jz 1f
1518 incl TH_MUTEX_COUNT(LMTX_A_REG) /* lock statistic */
1519 1:
1520 #endif /* MACH_LDEBUG */
1521
1522 testl $(M_WAITERS_MSK), M_STATE(LMTX_REG)
1523 jz Llml_finish
1524
1525 LMTX_CALLEXT1(lck_mtx_lock_acquire_x86)
1526
1527 Llml_finish:
1528 andl $(~M_ILOCKED_MSK), M_STATE(LMTX_REG)
1529 PREEMPTION_ENABLE
1530
1531 LMTX_CHK_EXTENDED /* is this an extended mutex */
1532 jne 2f
1533
1534 leave
1535 #if CONFIG_DTRACE
1536 LOCKSTAT_LABEL(_lck_mtx_lock_lockstat_patch_point)
1537 ret
1538 /* inherit lock pointer in LMTX_REG above */
1539 LOCKSTAT_RECORD(LS_LCK_MTX_LOCK_ACQUIRE, LMTX_REG)
1540 #endif
1541 ret
1542 2:
1543 LMTX_EXIT_EXTENDED
1544 leave
1545 #if CONFIG_DTRACE
1546 LOCKSTAT_LABEL(_lck_mtx_lock_ext_lockstat_patch_point)
1547 ret
1548 /* inherit lock pointer in LMTX_REG above */
1549 LOCKSTAT_RECORD(LS_LCK_MTX_EXT_LOCK_ACQUIRE, LMTX_REG)
1550 #endif
1551 ret
1552
1553
1554 Llml_slow:
1555 test $M_ILOCKED_MSK, LMTX_C_REG32 /* is the interlock held */
1556 jz Llml_contended /* no, must have been the mutex */
1557
1558 cmp $(MUTEX_DESTROYED), LMTX_C_REG32 /* check to see if its marked destroyed */
1559 je lck_mtx_destroyed
1560 cmp $(MUTEX_IND), LMTX_C_REG32 /* Is this an indirect mutex? */
1561 jne Llml_loop /* no... must be interlocked */
1562
1563 LMTX_ENTER_EXTENDED
1564
1565 mov M_STATE(LMTX_REG), LMTX_C_REG32
1566 test $(M_SPIN_MSK), LMTX_C_REG32
1567 jz Llml_loop1
1568
1569 LMTX_UPDATE_MISS /* M_SPIN_MSK was set, so M_ILOCKED_MSK must also be present */
1570 Llml_loop:
1571 PAUSE
1572 mov M_STATE(LMTX_REG), LMTX_C_REG32
1573 Llml_loop1:
1574 test $(M_ILOCKED_MSK | M_MLOCKED_MSK), LMTX_C_REG32
1575 jz Llml_try
1576 test $(M_MLOCKED_MSK), LMTX_C_REG32
1577 jnz Llml_contended /* mutex owned by someone else, go contend for it */
1578 jmp Llml_loop
1579
1580 Llml_busy_disabled:
1581 PREEMPTION_ENABLE
1582 jmp Llml_loop
1583
1584
1585 Llml_contended:
1586 LMTX_CHK_EXTENDED /* is this an extended mutex */
1587 je 0f
1588 LMTX_UPDATE_MISS
1589 0:
1590 LMTX_CALLEXT1(lck_mtx_lock_spinwait_x86)
1591
1592 test LMTX_RET_REG, LMTX_RET_REG
1593 jz Llml_acquired /* acquired mutex, interlock held and preemption disabled */
1594
1595 cmp $1, LMTX_RET_REG /* check for direct wait status */
1596 je 2f
1597 LMTX_CHK_EXTENDED /* is this an extended mutex */
1598 je 2f
1599 LMTX_UPDATE_DIRECT_WAIT
1600 2:
1601 mov M_STATE(LMTX_REG), LMTX_C_REG32
1602 test $(M_ILOCKED_MSK), LMTX_C_REG32
1603 jnz 6f
1604
1605 mov LMTX_C_REG, LMTX_A_REG /* eax contains snapshot for cmpxchgl */
1606 or $(M_ILOCKED_MSK), LMTX_C_REG32 /* try to take the interlock */
1607
1608 PREEMPTION_DISABLE
1609 lock
1610 cmpxchg LMTX_C_REG32, M_STATE(LMTX_REG) /* atomic compare and exchange */
1611 jne 5f
1612
1613 test $(M_MLOCKED_MSK), LMTX_C_REG32 /* we've got the interlock and */
1614 jnz 3f
1615 or $(M_MLOCKED_MSK), LMTX_C_REG32 /* the mutex is free... grab it directly */
1616 mov LMTX_C_REG32, M_STATE(LMTX_REG)
1617
1618 mov %gs:CPU_ACTIVE_THREAD, LMTX_A_REG
1619 mov LMTX_A_REG, M_OWNER(LMTX_REG) /* record owner of mutex */
1620 #if MACH_LDEBUG
1621 test LMTX_A_REG, LMTX_A_REG
1622 jz 1f
1623 incl TH_MUTEX_COUNT(LMTX_A_REG) /* lock statistic */
1624 1:
1625 #endif /* MACH_LDEBUG */
1626
1627 Llml_acquired:
1628 testl $(M_WAITERS_MSK), M_STATE(LMTX_REG)
1629 jnz 1f
1630 mov M_OWNER(LMTX_REG), LMTX_A_REG
1631 mov TH_WAS_PROMOTED_ON_WAKEUP(LMTX_A_REG), LMTX_A_REG32
1632 test LMTX_A_REG32, LMTX_A_REG32
1633 jz Llml_finish
1634 1:
1635 LMTX_CALLEXT1(lck_mtx_lock_acquire_x86)
1636 jmp Llml_finish
1637
1638 3: /* interlock held, mutex busy */
1639 LMTX_CHK_EXTENDED /* is this an extended mutex */
1640 je 4f
1641 LMTX_UPDATE_WAIT
1642 4:
1643 LMTX_CALLEXT1(lck_mtx_lock_wait_x86)
1644 jmp Llml_contended
1645 5:
1646 PREEMPTION_ENABLE
1647 6:
1648 PAUSE
1649 jmp 2b
1650
1651
1652
1653 NONLEAF_ENTRY(lck_mtx_try_lock_spin)
1654 LOAD_LMTX_REG(B_ARG0) /* fetch lock pointer */
1655
1656 mov M_STATE(LMTX_REG), LMTX_C_REG32
1657 test $(M_ILOCKED_MSK | M_MLOCKED_MSK), LMTX_C_REG32 /* is the interlock or mutex held */
1658 jnz Llmts_slow
1659 Llmts_try: /* no - can't be INDIRECT, DESTROYED or locked */
1660 mov LMTX_C_REG, LMTX_A_REG /* eax contains snapshot for cmpxchgl */
1661 or $(M_ILOCKED_MSK | M_SPIN_MSK), LMTX_C_REG
1662
1663 PREEMPTION_DISABLE
1664 lock
1665 cmpxchg LMTX_C_REG32, M_STATE(LMTX_REG) /* atomic compare and exchange */
1666 jne Llmts_busy_disabled
1667
1668 mov %gs:CPU_ACTIVE_THREAD, LMTX_A_REG
1669 mov LMTX_A_REG, M_OWNER(LMTX_REG) /* record owner of mutex */
1670 #if MACH_LDEBUG
1671 test LMTX_A_REG, LMTX_A_REG
1672 jz 1f
1673 incl TH_MUTEX_COUNT(LMTX_A_REG) /* lock statistic */
1674 1:
1675 #endif /* MACH_LDEBUG */
1676
1677 LMTX_CHK_EXTENDED_EXIT
1678 leave
1679
1680 #if CONFIG_DTRACE
1681 mov $1, LMTX_RET_REG /* return success */
1682 LOCKSTAT_LABEL(_lck_mtx_try_lock_spin_lockstat_patch_point)
1683 ret
1684 /* inherit lock pointer in LMTX_REG above */
1685 LOCKSTAT_RECORD(LS_LCK_MTX_TRY_SPIN_LOCK_ACQUIRE, LMTX_REG)
1686 #endif
1687 mov $1, LMTX_RET_REG /* return success */
1688 ret
1689
1690 Llmts_slow:
1691 test $(M_ILOCKED_MSK), LMTX_C_REG32 /* is the interlock held */
1692 jz Llmts_fail /* no, must be held as a mutex */
1693
1694 cmp $(MUTEX_DESTROYED), LMTX_C_REG32 /* check to see if its marked destroyed */
1695 je lck_mtx_destroyed
1696 cmp $(MUTEX_IND), LMTX_C_REG32 /* Is this an indirect mutex? */
1697 jne Llmts_loop1
1698
1699 LMTX_ENTER_EXTENDED
1700 Llmts_loop:
1701 PAUSE
1702 mov M_STATE(LMTX_REG), LMTX_C_REG32
1703 Llmts_loop1:
1704 test $(M_MLOCKED_MSK | M_SPIN_MSK), LMTX_C_REG32
1705 jnz Llmts_fail
1706 test $(M_ILOCKED_MSK), LMTX_C_REG32
1707 jz Llmts_try
1708 jmp Llmts_loop
1709
1710 Llmts_busy_disabled:
1711 PREEMPTION_ENABLE
1712 jmp Llmts_loop
1713
1714
1715
1716 NONLEAF_ENTRY(lck_mtx_try_lock)
1717 LOAD_LMTX_REG(B_ARG0) /* fetch lock pointer */
1718
1719 mov M_STATE(LMTX_REG), LMTX_C_REG32
1720 test $(M_ILOCKED_MSK | M_MLOCKED_MSK), LMTX_C_REG32 /* is the interlock or mutex held */
1721 jnz Llmt_slow
1722 Llmt_try: /* no - can't be INDIRECT, DESTROYED or locked */
1723 mov LMTX_C_REG, LMTX_A_REG /* eax contains snapshot for cmpxchgl */
1724 or $(M_ILOCKED_MSK | M_MLOCKED_MSK), LMTX_C_REG32
1725
1726 PREEMPTION_DISABLE
1727 lock
1728 cmpxchg LMTX_C_REG32, M_STATE(LMTX_REG) /* atomic compare and exchange */
1729 jne Llmt_busy_disabled
1730
1731 mov %gs:CPU_ACTIVE_THREAD, LMTX_A_REG
1732 mov LMTX_A_REG, M_OWNER(LMTX_REG) /* record owner of mutex */
1733 #if MACH_LDEBUG
1734 test LMTX_A_REG, LMTX_A_REG
1735 jz 1f
1736 incl TH_MUTEX_COUNT(LMTX_A_REG) /* lock statistic */
1737 1:
1738 #endif /* MACH_LDEBUG */
1739
1740 LMTX_CHK_EXTENDED_EXIT
1741
1742 test $(M_WAITERS_MSK), LMTX_C_REG32
1743 jz 0f
1744
1745 LMTX_CALLEXT1(lck_mtx_lock_acquire_x86)
1746 0:
1747 andl $(~M_ILOCKED_MSK), M_STATE(LMTX_REG)
1748 PREEMPTION_ENABLE
1749
1750 leave
1751 #if CONFIG_DTRACE
1752 mov $1, LMTX_RET_REG /* return success */
1753 /* Dtrace probe: LS_LCK_MTX_TRY_LOCK_ACQUIRE */
1754 LOCKSTAT_LABEL(_lck_mtx_try_lock_lockstat_patch_point)
1755 ret
1756 /* inherit lock pointer in LMTX_REG from above */
1757 LOCKSTAT_RECORD(LS_LCK_MTX_TRY_LOCK_ACQUIRE, LMTX_REG)
1758 #endif
1759 mov $1, LMTX_RET_REG /* return success */
1760 ret
1761
1762 Llmt_slow:
1763 test $(M_ILOCKED_MSK), LMTX_C_REG32 /* is the interlock held */
1764 jz Llmt_fail /* no, must be held as a mutex */
1765
1766 cmp $(MUTEX_DESTROYED), LMTX_C_REG32 /* check to see if its marked destroyed */
1767 je lck_mtx_destroyed
1768 cmp $(MUTEX_IND), LMTX_C_REG32 /* Is this an indirect mutex? */
1769 jne Llmt_loop
1770
1771 LMTX_ENTER_EXTENDED
1772 Llmt_loop:
1773 PAUSE
1774 mov M_STATE(LMTX_REG), LMTX_C_REG32
1775 Llmt_loop1:
1776 test $(M_MLOCKED_MSK | M_SPIN_MSK), LMTX_C_REG32
1777 jnz Llmt_fail
1778 test $(M_ILOCKED_MSK), LMTX_C_REG32
1779 jz Llmt_try
1780 jmp Llmt_loop
1781
1782 Llmt_busy_disabled:
1783 PREEMPTION_ENABLE
1784 jmp Llmt_loop
1785
1786
1787 Llmt_fail:
1788 Llmts_fail:
1789 LMTX_CHK_EXTENDED /* is this an extended mutex */
1790 je 0f
1791 LMTX_UPDATE_MISS
1792 LMTX_EXIT_EXTENDED
1793 0:
1794 xor LMTX_RET_REG, LMTX_RET_REG
1795 NONLEAF_RET
1796
1797
1798
1799 NONLEAF_ENTRY(lck_mtx_convert_spin)
1800 LOAD_LMTX_REG(B_ARG0) /* fetch lock pointer */
1801
1802 mov M_STATE(LMTX_REG), LMTX_C_REG32
1803 cmp $(MUTEX_IND), LMTX_C_REG32 /* Is this an indirect mutex? */
1804 jne 0f
1805 mov M_PTR(LMTX_REG), LMTX_REG /* If so, take indirection */
1806 mov M_STATE(LMTX_REG), LMTX_C_REG32
1807 0:
1808 test $(M_MLOCKED_MSK), LMTX_C_REG32 /* already owned as a mutex, just return */
1809 jnz 2f
1810 test $(M_WAITERS_MSK), LMTX_C_REG32 /* are there any waiters? */
1811 jz 1f
1812
1813 LMTX_CALLEXT1(lck_mtx_lock_acquire_x86)
1814 mov M_STATE(LMTX_REG), LMTX_C_REG32
1815 1:
1816 and $(~(M_ILOCKED_MSK | M_SPIN_MSK)), LMTX_C_REG32 /* convert from spin version to mutex */
1817 or $(M_MLOCKED_MSK), LMTX_C_REG32
1818 mov LMTX_C_REG32, M_STATE(LMTX_REG) /* since I own the interlock, I don't need an atomic update */
1819
1820 PREEMPTION_ENABLE
1821 2:
1822 NONLEAF_RET
1823
1824
1825
1826 #if defined(__i386__)
1827 NONLEAF_ENTRY(lck_mtx_unlock)
1828 LOAD_LMTX_REG(B_ARG0) /* fetch lock pointer */
1829 mov M_OWNER(LMTX_REG), LMTX_A_REG
1830 test LMTX_A_REG, LMTX_A_REG
1831 jnz Llmu_entry
1832 leave
1833 ret
1834 NONLEAF_ENTRY(lck_mtx_unlock_darwin10)
1835 #else
1836 NONLEAF_ENTRY(lck_mtx_unlock)
1837 #endif
1838 LOAD_LMTX_REG(B_ARG0) /* fetch lock pointer */
1839 Llmu_entry:
1840 mov M_STATE(LMTX_REG), LMTX_C_REG32
1841 Llmu_prim:
1842 cmp $(MUTEX_IND), LMTX_C_REG32 /* Is this an indirect mutex? */
1843 je Llmu_ext
1844
1845 Llmu_chktype:
1846 test $(M_MLOCKED_MSK), LMTX_C_REG32 /* check for full mutex */
1847 jz Llmu_unlock
1848 Llmu_mutex:
1849 test $(M_ILOCKED_MSK), LMTX_C_REG /* have to wait for interlock to clear */
1850 jnz Llmu_busy
1851
1852 mov LMTX_C_REG, LMTX_A_REG /* eax contains snapshot for cmpxchgl */
1853 and $(~M_MLOCKED_MSK), LMTX_C_REG32 /* drop mutex */
1854 or $(M_ILOCKED_MSK), LMTX_C_REG32 /* pick up interlock */
1855
1856 PREEMPTION_DISABLE
1857 lock
1858 cmpxchg LMTX_C_REG32, M_STATE(LMTX_REG) /* atomic compare and exchange */
1859 jne Llmu_busy_disabled /* branch on failure to spin loop */
1860
1861 Llmu_unlock:
1862 xor LMTX_A_REG, LMTX_A_REG
1863 mov LMTX_A_REG, M_OWNER(LMTX_REG)
1864 mov LMTX_C_REG, LMTX_A_REG /* keep original state in %ecx for later evaluation */
1865 and $(~(M_ILOCKED_MSK | M_SPIN_MSK | M_PROMOTED_MSK)), LMTX_A_REG
1866
1867 test $(M_WAITERS_MSK), LMTX_A_REG32
1868 jz 2f
1869 dec LMTX_A_REG32 /* decrement waiter count */
1870 2:
1871 mov LMTX_A_REG32, M_STATE(LMTX_REG) /* since I own the interlock, I don't need an atomic update */
1872
1873 #if MACH_LDEBUG
1874 /* perform lock statistics after drop to prevent delay */
1875 mov %gs:CPU_ACTIVE_THREAD, LMTX_A_REG
1876 test LMTX_A_REG, LMTX_A_REG
1877 jz 1f
1878 decl TH_MUTEX_COUNT(LMTX_A_REG) /* lock statistic */
1879 1:
1880 #endif /* MACH_LDEBUG */
1881
1882 test $(M_PROMOTED_MSK | M_WAITERS_MSK), LMTX_C_REG32
1883 jz 3f
1884
1885 LMTX_CALLEXT2(lck_mtx_unlock_wakeup_x86, LMTX_C_REG)
1886 3:
1887 PREEMPTION_ENABLE
1888
1889 LMTX_CHK_EXTENDED
1890 jne 4f
1891
1892 leave
1893 #if CONFIG_DTRACE
1894 /* Dtrace: LS_LCK_MTX_UNLOCK_RELEASE */
1895 LOCKSTAT_LABEL(_lck_mtx_unlock_lockstat_patch_point)
1896 ret
1897 /* inherit lock pointer in LMTX_REG from above */
1898 LOCKSTAT_RECORD(LS_LCK_MTX_UNLOCK_RELEASE, LMTX_REG)
1899 #endif
1900 ret
1901 4:
1902 leave
1903 #if CONFIG_DTRACE
1904 /* Dtrace: LS_LCK_MTX_EXT_UNLOCK_RELEASE */
1905 LOCKSTAT_LABEL(_lck_mtx_ext_unlock_lockstat_patch_point)
1906 ret
1907 /* inherit lock pointer in LMTX_REG from above */
1908 LOCKSTAT_RECORD(LS_LCK_MTX_EXT_UNLOCK_RELEASE, LMTX_REG)
1909 #endif
1910 ret
1911
1912
1913 Llmu_busy_disabled:
1914 PREEMPTION_ENABLE
1915 Llmu_busy:
1916 PAUSE
1917 mov M_STATE(LMTX_REG), LMTX_C_REG32
1918 jmp Llmu_mutex
1919
1920 Llmu_ext:
1921 mov M_PTR(LMTX_REG), LMTX_REG
1922 mov M_OWNER(LMTX_REG), LMTX_A_REG
1923 mov %gs:CPU_ACTIVE_THREAD, LMTX_C_REG
1924 CHECK_UNLOCK(LMTX_C_REG, LMTX_A_REG)
1925 mov M_STATE(LMTX_REG), LMTX_C_REG32
1926 jmp Llmu_chktype
1927
1928
1929
1930 LEAF_ENTRY(lck_mtx_ilk_unlock)
1931 LOAD_LMTX_REG(L_ARG0) /* fetch lock pointer - no indirection here */
1932
1933 andl $(~M_ILOCKED_MSK), M_STATE(LMTX_REG)
1934
1935 PREEMPTION_ENABLE /* need to re-enable preemption */
1936
1937 LEAF_RET
1938
1939
1940
1941 LEAF_ENTRY(lck_mtx_lock_grab_mutex)
1942 LOAD_LMTX_REG(L_ARG0) /* fetch lock pointer - no indirection here */
1943
1944 mov M_STATE(LMTX_REG), LMTX_C_REG32
1945
1946 test $(M_ILOCKED_MSK | M_MLOCKED_MSK), LMTX_C_REG32 /* can't have the mutex yet */
1947 jnz 3f
1948
1949 mov LMTX_C_REG, LMTX_A_REG /* eax contains snapshot for cmpxchgl */
1950 or $(M_ILOCKED_MSK | M_MLOCKED_MSK), LMTX_C_REG32
1951
1952 PREEMPTION_DISABLE
1953 lock
1954 cmpxchg LMTX_C_REG32, M_STATE(LMTX_REG) /* atomic compare and exchange */
1955 jne 2f /* branch on failure to spin loop */
1956
1957 mov %gs:CPU_ACTIVE_THREAD, LMTX_A_REG
1958 mov LMTX_A_REG, M_OWNER(LMTX_REG) /* record owner of mutex */
1959 #if MACH_LDEBUG
1960 test LMTX_A_REG, LMTX_A_REG
1961 jz 1f
1962 incl TH_MUTEX_COUNT(LMTX_A_REG) /* lock statistic */
1963 1:
1964 #endif /* MACH_LDEBUG */
1965
1966 mov $1, LMTX_RET_REG /* return success */
1967 LEAF_RET
1968 2:
1969 PREEMPTION_ENABLE
1970 3:
1971 xor LMTX_RET_REG, LMTX_RET_REG /* return failure */
1972 LEAF_RET
1973
1974
1975
1976 LEAF_ENTRY(lck_mtx_lock_mark_destroyed)
1977 LOAD_LMTX_REG(L_ARG0)
1978 1:
1979 mov M_STATE(LMTX_REG), LMTX_C_REG32
1980 cmp $(MUTEX_IND), LMTX_C_REG32 /* Is this an indirect mutex? */
1981 jne 2f
1982
1983 movl $(MUTEX_DESTROYED), M_STATE(LMTX_REG) /* convert to destroyed state */
1984 jmp 3f
1985 2:
1986 test $(M_ILOCKED_MSK), LMTX_C_REG /* have to wait for interlock to clear */
1987 jnz 5f
1988
1989 PREEMPTION_DISABLE
1990 mov LMTX_C_REG, LMTX_A_REG /* eax contains snapshot for cmpxchgl */
1991 or $(M_ILOCKED_MSK), LMTX_C_REG32
1992 lock
1993 cmpxchg LMTX_C_REG32, M_STATE(LMTX_REG) /* atomic compare and exchange */
1994 jne 4f /* branch on failure to spin loop */
1995 movl $(MUTEX_DESTROYED), M_STATE(LMTX_REG) /* convert to destroyed state */
1996 PREEMPTION_ENABLE
1997 3:
1998 LEAF_RET /* return with M_ILOCKED set */
1999 4:
2000 PREEMPTION_ENABLE
2001 5:
2002 PAUSE
2003 jmp 1b
2004
2005 LEAF_ENTRY(preemption_underflow_panic)
2006 FRAME
2007 incl %gs:CPU_PREEMPTION_LEVEL
2008 ALIGN_STACK()
2009 LOAD_STRING_ARG0(16f)
2010 CALL_PANIC()
2011 hlt
2012 .data
2013 16: String "Preemption level underflow, possible cause unlocking an unlocked mutex or spinlock"
2014 .text
2015
2016
2017 LEAF_ENTRY(_disable_preemption)
2018 #if MACH_RT
2019 PREEMPTION_DISABLE
2020 #endif /* MACH_RT */
2021 LEAF_RET
2022
2023 LEAF_ENTRY(_enable_preemption)
2024 #if MACH_RT
2025 #if MACH_ASSERT
2026 cmpl $0,%gs:CPU_PREEMPTION_LEVEL
2027 jg 1f
2028 #if __i386__
2029 pushl %gs:CPU_PREEMPTION_LEVEL
2030 #else
2031 movl %gs:CPU_PREEMPTION_LEVEL,%esi
2032 #endif
2033 ALIGN_STACK()
2034 LOAD_STRING_ARG0(_enable_preemption_less_than_zero)
2035 CALL_PANIC()
2036 hlt
2037 .cstring
2038 _enable_preemption_less_than_zero:
2039 .asciz "_enable_preemption: preemption_level(%d) < 0!"
2040 .text
2041 1:
2042 #endif /* MACH_ASSERT */
2043 PREEMPTION_ENABLE
2044 #endif /* MACH_RT */
2045 LEAF_RET
2046
2047 LEAF_ENTRY(_enable_preemption_no_check)
2048 #if MACH_RT
2049 #if MACH_ASSERT
2050 cmpl $0,%gs:CPU_PREEMPTION_LEVEL
2051 jg 1f
2052 ALIGN_STACK()
2053 LOAD_STRING_ARG0(_enable_preemption_no_check_less_than_zero)
2054 CALL_PANIC()
2055 hlt
2056 .cstring
2057 _enable_preemption_no_check_less_than_zero:
2058 .asciz "_enable_preemption_no_check: preemption_level <= 0!"
2059 .text
2060 1:
2061 #endif /* MACH_ASSERT */
2062 _ENABLE_PREEMPTION_NO_CHECK
2063 #endif /* MACH_RT */
2064 LEAF_RET
2065
2066
2067 LEAF_ENTRY(_mp_disable_preemption)
2068 #if MACH_RT
2069 PREEMPTION_DISABLE
2070 #endif /* MACH_RT */
2071 LEAF_RET
2072
2073 LEAF_ENTRY(_mp_enable_preemption)
2074 #if MACH_RT
2075 #if MACH_ASSERT
2076 cmpl $0,%gs:CPU_PREEMPTION_LEVEL
2077 jg 1f
2078 #if __i386__
2079 pushl %gs:CPU_PREEMPTION_LEVEL
2080 #else
2081 movl %gs:CPU_PREEMPTION_LEVEL,%esi
2082 #endif
2083 ALIGN_PANIC()
2084 LOAD_STRING_ARG0(_mp_enable_preemption_less_than_zero)
2085 CALL_PANIC()
2086 hlt
2087 .cstring
2088 _mp_enable_preemption_less_than_zero:
2089 .asciz "_mp_enable_preemption: preemption_level (%d) <= 0!"
2090 .text
2091 1:
2092 #endif /* MACH_ASSERT */
2093 PREEMPTION_ENABLE
2094 #endif /* MACH_RT */
2095 LEAF_RET
2096
2097 LEAF_ENTRY(_mp_enable_preemption_no_check)
2098 #if MACH_RT
2099 #if MACH_ASSERT
2100 cmpl $0,%gs:CPU_PREEMPTION_LEVEL
2101 jg 1f
2102 ALIGN_STACK()
2103 LOAD_STRING_ARG0(_mp_enable_preemption_no_check_less_than_zero)
2104 CALL_PANIC()
2105 hlt
2106 .cstring
2107 _mp_enable_preemption_no_check_less_than_zero:
2108 .asciz "_mp_enable_preemption_no_check: preemption_level <= 0!"
2109 .text
2110 1:
2111 #endif /* MACH_ASSERT */
2112 _ENABLE_PREEMPTION_NO_CHECK
2113 #endif /* MACH_RT */
2114 LEAF_RET
2115
2116 #if __i386__
2117
2118 LEAF_ENTRY(i_bit_set)
2119 movl L_ARG0,%edx
2120 movl L_ARG1,%eax
2121 lock
2122 bts %edx,(%eax)
2123 LEAF_RET
2124
2125 LEAF_ENTRY(i_bit_clear)
2126 movl L_ARG0,%edx
2127 movl L_ARG1,%eax
2128 lock
2129 btr %edx,(%eax)
2130 LEAF_RET
2131
2132
2133 LEAF_ENTRY(bit_lock)
2134 movl L_ARG0,%ecx
2135 movl L_ARG1,%eax
2136 1:
2137 lock
2138 bts %ecx,(%eax)
2139 jb 1b
2140 LEAF_RET
2141
2142
2143 LEAF_ENTRY(bit_lock_try)
2144 movl L_ARG0,%ecx
2145 movl L_ARG1,%eax
2146 lock
2147 bts %ecx,(%eax)
2148 jb bit_lock_failed
2149 LEAF_RET /* %eax better not be null ! */
2150 bit_lock_failed:
2151 xorl %eax,%eax
2152 LEAF_RET
2153
2154 LEAF_ENTRY(bit_unlock)
2155 movl L_ARG0,%ecx
2156 movl L_ARG1,%eax
2157 lock
2158 btr %ecx,(%eax)
2159 LEAF_RET
2160
2161 /*
2162 * Atomic primitives, prototyped in kern/simple_lock.h
2163 */
2164 LEAF_ENTRY(hw_atomic_add)
2165 movl L_ARG0, %ecx /* Load address of operand */
2166 movl L_ARG1, %eax /* Load addend */
2167 movl %eax, %edx
2168 lock
2169 xaddl %eax, (%ecx) /* Atomic exchange and add */
2170 addl %edx, %eax /* Calculate result */
2171 LEAF_RET
2172
2173 LEAF_ENTRY(hw_atomic_sub)
2174 movl L_ARG0, %ecx /* Load address of operand */
2175 movl L_ARG1, %eax /* Load subtrahend */
2176 negl %eax
2177 movl %eax, %edx
2178 lock
2179 xaddl %eax, (%ecx) /* Atomic exchange and add */
2180 addl %edx, %eax /* Calculate result */
2181 LEAF_RET
2182
2183 LEAF_ENTRY(hw_atomic_or)
2184 movl L_ARG0, %ecx /* Load address of operand */
2185 movl (%ecx), %eax
2186 1:
2187 movl L_ARG1, %edx /* Load mask */
2188 orl %eax, %edx
2189 lock
2190 cmpxchgl %edx, (%ecx) /* Atomic CAS */
2191 jne 1b
2192 movl %edx, %eax /* Result */
2193 LEAF_RET
2194 /*
2195 * A variant of hw_atomic_or which doesn't return a value.
2196 * The implementation is thus comparatively more efficient.
2197 */
2198
2199 LEAF_ENTRY(hw_atomic_or_noret)
2200 movl L_ARG0, %ecx /* Load address of operand */
2201 movl L_ARG1, %edx /* Load mask */
2202 lock
2203 orl %edx, (%ecx) /* Atomic OR */
2204 LEAF_RET
2205
2206 LEAF_ENTRY(hw_atomic_and)
2207 movl L_ARG0, %ecx /* Load address of operand */
2208 movl (%ecx), %eax
2209 1:
2210 movl L_ARG1, %edx /* Load mask */
2211 andl %eax, %edx
2212 lock
2213 cmpxchgl %edx, (%ecx) /* Atomic CAS */
2214 jne 1b
2215 movl %edx, %eax /* Result */
2216 LEAF_RET
2217 /*
2218 * A variant of hw_atomic_and which doesn't return a value.
2219 * The implementation is thus comparatively more efficient.
2220 */
2221
2222 LEAF_ENTRY(hw_atomic_and_noret)
2223 movl L_ARG0, %ecx /* Load address of operand */
2224 movl L_ARG1, %edx /* Load mask */
2225 lock
2226 andl %edx, (%ecx) /* Atomic AND */
2227 LEAF_RET
2228
2229 #else /* !__i386__ */
2230
2231 LEAF_ENTRY(i_bit_set)
2232 lock
2233 bts %edi,(%rsi)
2234 LEAF_RET
2235
2236 LEAF_ENTRY(i_bit_clear)
2237 lock
2238 btr %edi,(%rsi)
2239 LEAF_RET
2240
2241
2242 LEAF_ENTRY(bit_lock)
2243 1:
2244 lock
2245 bts %edi,(%rsi)
2246 jb 1b
2247 LEAF_RET
2248
2249
2250 LEAF_ENTRY(bit_lock_try)
2251 lock
2252 bts %edi,(%rsi)
2253 jb bit_lock_failed
2254 movl $1, %eax
2255 LEAF_RET
2256 bit_lock_failed:
2257 xorl %eax,%eax
2258 LEAF_RET
2259
2260 LEAF_ENTRY(bit_unlock)
2261 lock
2262 btr %edi,(%rsi)
2263 LEAF_RET
2264
2265
2266 /*
2267 * Atomic primitives, prototyped in kern/simple_lock.h
2268 */
2269 LEAF_ENTRY(hw_atomic_add)
2270 movl %esi, %eax /* Load addend */
2271 lock
2272 xaddl %eax, (%rdi) /* Atomic exchange and add */
2273 addl %esi, %eax /* Calculate result */
2274 LEAF_RET
2275
2276 LEAF_ENTRY(hw_atomic_sub)
2277 negl %esi
2278 movl %esi, %eax
2279 lock
2280 xaddl %eax, (%rdi) /* Atomic exchange and add */
2281 addl %esi, %eax /* Calculate result */
2282 LEAF_RET
2283
2284 LEAF_ENTRY(hw_atomic_or)
2285 movl (%rdi), %eax
2286 1:
2287 movl %esi, %edx /* Load mask */
2288 orl %eax, %edx
2289 lock
2290 cmpxchgl %edx, (%rdi) /* Atomic CAS */
2291 jne 1b
2292 movl %edx, %eax /* Result */
2293 LEAF_RET
2294 /*
2295 * A variant of hw_atomic_or which doesn't return a value.
2296 * The implementation is thus comparatively more efficient.
2297 */
2298
2299 LEAF_ENTRY(hw_atomic_or_noret)
2300 lock
2301 orl %esi, (%rdi) /* Atomic OR */
2302 LEAF_RET
2303
2304
2305 LEAF_ENTRY(hw_atomic_and)
2306 movl (%rdi), %eax
2307 1:
2308 movl %esi, %edx /* Load mask */
2309 andl %eax, %edx
2310 lock
2311 cmpxchgl %edx, (%rdi) /* Atomic CAS */
2312 jne 1b
2313 movl %edx, %eax /* Result */
2314 LEAF_RET
2315 /*
2316 * A variant of hw_atomic_and which doesn't return a value.
2317 * The implementation is thus comparatively more efficient.
2318 */
2319
2320 LEAF_ENTRY(hw_atomic_and_noret)
2321 lock
2322 andl %esi, (%rdi) /* Atomic OR */
2323 LEAF_RET
2324
2325 #endif /* !__i386 __ */
mirror of XNU