2 * Copyright (c) 2000-2007 Apple Inc. All rights reserved.
4 * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
6 * This file contains Original Code and/or Modifications of Original Code
7 * as defined in and that are subject to the Apple Public Source License
8 * Version 2.0 (the 'License'). You may not use this file except in
9 * compliance with the License. The rights granted to you under the License
10 * may not be used to create, or enable the creation or redistribution of,
11 * unlawful or unlicensed copies of an Apple operating system, or to
12 * circumvent, violate, or enable the circumvention or violation of, any
13 * terms of an Apple operating system software license agreement.
15 * Please obtain a copy of the License at
16 * http://www.opensource.apple.com/apsl/ and read it before using this file.
18 * The Original Code and all software distributed under the License are
19 * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
20 * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
21 * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
22 * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
23 * Please see the License for the specific language governing rights and
24 * limitations under the License.
26 * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
32 * Mach Operating System
33 * Copyright (c) 1991,1990,1989,1988,1987 Carnegie Mellon University
34 * All Rights Reserved.
36 * Permission to use, copy, modify and distribute this software and its
37 * documentation is hereby granted, provided that both the copyright
38 * notice and this permission notice appear in all copies of the
39 * software, derivative works or modified versions, and any portions
40 * thereof, and that both notices appear in supporting documentation.
42 * CARNEGIE MELLON ALLOWS FREE USE OF THIS SOFTWARE IN ITS "AS IS"
43 * CONDITION. CARNEGIE MELLON DISCLAIMS ANY LIABILITY OF ANY KIND FOR
44 * ANY DAMAGES WHATSOEVER RESULTING FROM THE USE OF THIS SOFTWARE.
46 * Carnegie Mellon requests users of this software to return to
48 * Software Distribution Coordinator or Software.Distribution@CS.CMU.EDU
49 * School of Computer Science
50 * Carnegie Mellon University
51 * Pittsburgh PA 15213-3890
53 * any improvements or extensions that they make and grant Carnegie Mellon
54 * the rights to redistribute these changes.
57 #include <mach_ldebug.h>
60 #include <mach/kern_return.h>
61 #include <mach/mach_host_server.h>
62 #include <mach_debug/lockgroup_info.h>
64 #include <kern/locks.h>
65 #include <kern/misc_protos.h>
66 #include <kern/kalloc.h>
67 #include <kern/thread.h>
68 #include <kern/processor.h>
69 #include <kern/sched_prim.h>
70 #include <kern/debug.h>
74 #include <sys/kdebug.h>
78 * We need only enough declarations from the BSD-side to be able to
79 * test if our probe is active, and to call __dtrace_probe(). Setting
80 * NEED_DTRACE_DEFS gets a local copy of those definitions pulled in.
82 #define NEED_DTRACE_DEFS
83 #include <../bsd/sys/lockstat.h>
86 #define LCK_MTX_SLEEP_CODE 0
87 #define LCK_MTX_SLEEP_DEADLINE_CODE 1
88 #define LCK_MTX_LCK_WAIT_CODE 2
89 #define LCK_MTX_UNLCK_WAKEUP_CODE 3
92 static queue_head_t lck_grp_queue
;
93 static unsigned int lck_grp_cnt
;
95 decl_lck_mtx_data(static,lck_grp_lock
)
96 static lck_mtx_ext_t lck_grp_lock_ext
;
98 lck_grp_attr_t LockDefaultGroupAttr
;
99 lck_grp_t LockCompatGroup
;
100 lck_attr_t LockDefaultLckAttr
;
103 * Routine: lck_mod_init
110 queue_init(&lck_grp_queue
);
113 * Need to bootstrap the LockCompatGroup instead of calling lck_grp_init() here. This avoids
114 * grabbing the lck_grp_lock before it is initialized.
117 bzero(&LockCompatGroup
, sizeof(lck_grp_t
));
118 (void) strncpy(LockCompatGroup
.lck_grp_name
, "Compatibility APIs", LCK_GRP_MAX_NAME
);
120 if (LcksOpts
& enaLkStat
)
121 LockCompatGroup
.lck_grp_attr
= LCK_GRP_ATTR_STAT
;
123 LockCompatGroup
.lck_grp_attr
= LCK_ATTR_NONE
;
125 LockCompatGroup
.lck_grp_refcnt
= 1;
127 enqueue_tail(&lck_grp_queue
, (queue_entry_t
)&LockCompatGroup
);
130 lck_grp_attr_setdefault(&LockDefaultGroupAttr
);
131 lck_attr_setdefault(&LockDefaultLckAttr
);
133 lck_mtx_init_ext(&lck_grp_lock
, &lck_grp_lock_ext
, &LockCompatGroup
, &LockDefaultLckAttr
);
138 * Routine: lck_grp_attr_alloc_init
142 lck_grp_attr_alloc_init(
145 lck_grp_attr_t
*attr
;
147 if ((attr
= (lck_grp_attr_t
*)kalloc(sizeof(lck_grp_attr_t
))) != 0)
148 lck_grp_attr_setdefault(attr
);
155 * Routine: lck_grp_attr_setdefault
159 lck_grp_attr_setdefault(
160 lck_grp_attr_t
*attr
)
162 if (LcksOpts
& enaLkStat
)
163 attr
->grp_attr_val
= LCK_GRP_ATTR_STAT
;
165 attr
->grp_attr_val
= 0;
170 * Routine: lck_grp_attr_setstat
174 lck_grp_attr_setstat(
175 lck_grp_attr_t
*attr
)
177 (void)hw_atomic_or(&attr
->grp_attr_val
, LCK_GRP_ATTR_STAT
);
182 * Routine: lck_grp_attr_free
187 lck_grp_attr_t
*attr
)
189 kfree(attr
, sizeof(lck_grp_attr_t
));
194 * Routine: lck_grp_alloc_init
199 const char* grp_name
,
200 lck_grp_attr_t
*attr
)
204 if ((grp
= (lck_grp_t
*)kalloc(sizeof(lck_grp_t
))) != 0)
205 lck_grp_init(grp
, grp_name
, attr
);
212 * Routine: lck_grp_init
218 const char* grp_name
,
219 lck_grp_attr_t
*attr
)
221 bzero((void *)grp
, sizeof(lck_grp_t
));
223 (void) strncpy(grp
->lck_grp_name
, grp_name
, LCK_GRP_MAX_NAME
);
225 if (attr
!= LCK_GRP_ATTR_NULL
)
226 grp
->lck_grp_attr
= attr
->grp_attr_val
;
227 else if (LcksOpts
& enaLkStat
)
228 grp
->lck_grp_attr
= LCK_GRP_ATTR_STAT
;
230 grp
->lck_grp_attr
= LCK_ATTR_NONE
;
232 grp
->lck_grp_refcnt
= 1;
234 lck_mtx_lock(&lck_grp_lock
);
235 enqueue_tail(&lck_grp_queue
, (queue_entry_t
)grp
);
237 lck_mtx_unlock(&lck_grp_lock
);
243 * Routine: lck_grp_free
250 lck_mtx_lock(&lck_grp_lock
);
252 (void)remque((queue_entry_t
)grp
);
253 lck_mtx_unlock(&lck_grp_lock
);
254 lck_grp_deallocate(grp
);
259 * Routine: lck_grp_reference
266 (void)hw_atomic_add(&grp
->lck_grp_refcnt
, 1);
271 * Routine: lck_grp_deallocate
278 if (hw_atomic_sub(&grp
->lck_grp_refcnt
, 1) == 0)
279 kfree(grp
, sizeof(lck_grp_t
));
283 * Routine: lck_grp_lckcnt_incr
291 unsigned int *lckcnt
;
295 lckcnt
= &grp
->lck_grp_spincnt
;
298 lckcnt
= &grp
->lck_grp_mtxcnt
;
301 lckcnt
= &grp
->lck_grp_rwcnt
;
304 return panic("lck_grp_lckcnt_incr(): invalid lock type: %d\n", lck_type
);
307 (void)hw_atomic_add(lckcnt
, 1);
311 * Routine: lck_grp_lckcnt_decr
319 unsigned int *lckcnt
;
323 lckcnt
= &grp
->lck_grp_spincnt
;
326 lckcnt
= &grp
->lck_grp_mtxcnt
;
329 lckcnt
= &grp
->lck_grp_rwcnt
;
332 return panic("lck_grp_lckcnt_decr(): invalid lock type: %d\n", lck_type
);
335 (void)hw_atomic_sub(lckcnt
, 1);
339 * Routine: lck_attr_alloc_init
348 if ((attr
= (lck_attr_t
*)kalloc(sizeof(lck_attr_t
))) != 0)
349 lck_attr_setdefault(attr
);
356 * Routine: lck_attr_setdefault
364 if (LcksOpts
& enaLkDeb
)
365 attr
->lck_attr_val
= LCK_ATTR_DEBUG
;
367 attr
->lck_attr_val
= LCK_ATTR_NONE
;
369 attr
->lck_attr_val
= LCK_ATTR_DEBUG
;
375 * Routine: lck_attr_setdebug
381 (void)hw_atomic_or(&attr
->lck_attr_val
, LCK_ATTR_DEBUG
);
385 * Routine: lck_attr_setdebug
391 (void)hw_atomic_and(&attr
->lck_attr_val
, ~LCK_ATTR_DEBUG
);
396 * Routine: lck_attr_rw_shared_priority
399 lck_attr_rw_shared_priority(
402 (void)hw_atomic_or(&attr
->lck_attr_val
, LCK_ATTR_RW_SHARED_PRIORITY
);
407 * Routine: lck_attr_free
413 kfree(attr
, sizeof(lck_attr_t
));
418 * Routine: lck_spin_sleep
423 lck_sleep_action_t lck_sleep_action
,
425 wait_interrupt_t interruptible
)
429 if ((lck_sleep_action
& ~LCK_SLEEP_MASK
) != 0)
430 panic("Invalid lock sleep action %x\n", lck_sleep_action
);
432 res
= assert_wait(event
, interruptible
);
433 if (res
== THREAD_WAITING
) {
434 lck_spin_unlock(lck
);
435 res
= thread_block(THREAD_CONTINUE_NULL
);
436 if (!(lck_sleep_action
& LCK_SLEEP_UNLOCK
))
440 if (lck_sleep_action
& LCK_SLEEP_UNLOCK
)
441 lck_spin_unlock(lck
);
448 * Routine: lck_spin_sleep_deadline
451 lck_spin_sleep_deadline(
453 lck_sleep_action_t lck_sleep_action
,
455 wait_interrupt_t interruptible
,
460 if ((lck_sleep_action
& ~LCK_SLEEP_MASK
) != 0)
461 panic("Invalid lock sleep action %x\n", lck_sleep_action
);
463 res
= assert_wait_deadline(event
, interruptible
, deadline
);
464 if (res
== THREAD_WAITING
) {
465 lck_spin_unlock(lck
);
466 res
= thread_block(THREAD_CONTINUE_NULL
);
467 if (!(lck_sleep_action
& LCK_SLEEP_UNLOCK
))
471 if (lck_sleep_action
& LCK_SLEEP_UNLOCK
)
472 lck_spin_unlock(lck
);
479 * Routine: lck_mtx_sleep
484 lck_sleep_action_t lck_sleep_action
,
486 wait_interrupt_t interruptible
)
490 KERNEL_DEBUG(MACHDBG_CODE(DBG_MACH_LOCKS
, LCK_MTX_SLEEP_CODE
) | DBG_FUNC_START
,
491 (int)lck
, (int)lck_sleep_action
, (int)event
, (int)interruptible
, 0);
493 if ((lck_sleep_action
& ~LCK_SLEEP_MASK
) != 0)
494 panic("Invalid lock sleep action %x\n", lck_sleep_action
);
496 res
= assert_wait(event
, interruptible
);
497 if (res
== THREAD_WAITING
) {
499 res
= thread_block(THREAD_CONTINUE_NULL
);
500 if (!(lck_sleep_action
& LCK_SLEEP_UNLOCK
)) {
501 if ((lck_sleep_action
& LCK_SLEEP_SPIN
))
502 lck_mtx_lock_spin(lck
);
508 if (lck_sleep_action
& LCK_SLEEP_UNLOCK
)
511 KERNEL_DEBUG(MACHDBG_CODE(DBG_MACH_LOCKS
, LCK_MTX_SLEEP_CODE
) | DBG_FUNC_END
, (int)res
, 0, 0, 0, 0);
518 * Routine: lck_mtx_sleep_deadline
521 lck_mtx_sleep_deadline(
523 lck_sleep_action_t lck_sleep_action
,
525 wait_interrupt_t interruptible
,
530 KERNEL_DEBUG(MACHDBG_CODE(DBG_MACH_LOCKS
, LCK_MTX_SLEEP_DEADLINE_CODE
) | DBG_FUNC_START
,
531 (int)lck
, (int)lck_sleep_action
, (int)event
, (int)interruptible
, 0);
533 if ((lck_sleep_action
& ~LCK_SLEEP_MASK
) != 0)
534 panic("Invalid lock sleep action %x\n", lck_sleep_action
);
536 res
= assert_wait_deadline(event
, interruptible
, deadline
);
537 if (res
== THREAD_WAITING
) {
539 res
= thread_block(THREAD_CONTINUE_NULL
);
540 if (!(lck_sleep_action
& LCK_SLEEP_UNLOCK
))
544 if (lck_sleep_action
& LCK_SLEEP_UNLOCK
)
547 KERNEL_DEBUG(MACHDBG_CODE(DBG_MACH_LOCKS
, LCK_MTX_SLEEP_DEADLINE_CODE
) | DBG_FUNC_END
, (int)res
, 0, 0, 0, 0);
553 * Routine: lck_mtx_lock_wait
555 * Invoked in order to wait on contention.
557 * Called with the interlock locked and
558 * returns it unlocked.
565 thread_t self
= current_thread();
568 spl_t s
= splsched();
570 uint64_t sleep_start
= 0;
572 if (lockstat_probemap
[LS_LCK_MTX_LOCK_BLOCK
] || lockstat_probemap
[LS_LCK_MTX_EXT_LOCK_BLOCK
]) {
573 sleep_start
= mach_absolute_time();
577 if (lck
->lck_mtx_tag
!= LCK_MTX_TAG_INDIRECT
)
580 mutex
= &lck
->lck_mtx_ptr
->lck_mtx
;
582 KERNEL_DEBUG(MACHDBG_CODE(DBG_MACH_LOCKS
, LCK_MTX_LCK_WAIT_CODE
) | DBG_FUNC_START
, (int)lck
, (int)holder
, 0, 0, 0);
584 priority
= self
->sched_pri
;
585 if (priority
< self
->priority
)
586 priority
= self
->priority
;
587 if (priority
< BASEPRI_DEFAULT
)
588 priority
= BASEPRI_DEFAULT
;
591 if (mutex
->lck_mtx_pri
== 0)
592 holder
->promotions
++;
593 holder
->sched_mode
|= TH_MODE_PROMOTED
;
594 if ( mutex
->lck_mtx_pri
< priority
&&
595 holder
->sched_pri
< priority
) {
596 KERNEL_DEBUG_CONSTANT(
597 MACHDBG_CODE(DBG_MACH_SCHED
,MACH_PROMOTE
) | DBG_FUNC_NONE
,
598 holder
->sched_pri
, priority
, holder
, lck
, 0);
600 set_sched_pri(holder
, priority
);
602 thread_unlock(holder
);
605 if (mutex
->lck_mtx_pri
< priority
)
606 mutex
->lck_mtx_pri
= priority
;
607 if (self
->pending_promoter
[self
->pending_promoter_index
] == NULL
) {
608 self
->pending_promoter
[self
->pending_promoter_index
] = mutex
;
609 mutex
->lck_mtx_waiters
++;
612 if (self
->pending_promoter
[self
->pending_promoter_index
] != mutex
) {
613 self
->pending_promoter
[++self
->pending_promoter_index
] = mutex
;
614 mutex
->lck_mtx_waiters
++;
617 assert_wait((event_t
)(((unsigned int*)lck
)+((sizeof(lck_mtx_t
)-1)/sizeof(unsigned int))), THREAD_UNINT
);
618 lck_mtx_ilk_unlock(mutex
);
620 thread_block(THREAD_CONTINUE_NULL
);
622 KERNEL_DEBUG(MACHDBG_CODE(DBG_MACH_LOCKS
, LCK_MTX_LCK_WAIT_CODE
) | DBG_FUNC_END
, 0, 0, 0, 0, 0);
625 * Record the Dtrace lockstat probe for blocking, block time
626 * measured from when we were entered.
629 if (lck
->lck_mtx_tag
!= LCK_MTX_TAG_INDIRECT
) {
630 LOCKSTAT_RECORD(LS_LCK_MTX_LOCK_BLOCK
, lck
,
631 mach_absolute_time() - sleep_start
);
633 LOCKSTAT_RECORD(LS_LCK_MTX_EXT_LOCK_BLOCK
, lck
,
634 mach_absolute_time() - sleep_start
);
641 * Routine: lck_mtx_lock_acquire
643 * Invoked on acquiring the mutex when there is
646 * Returns the current number of waiters.
648 * Called with the interlock locked.
651 lck_mtx_lock_acquire(
654 thread_t thread
= current_thread();
657 if (lck
->lck_mtx_tag
!= LCK_MTX_TAG_INDIRECT
)
660 mutex
= &lck
->lck_mtx_ptr
->lck_mtx
;
662 if (thread
->pending_promoter
[thread
->pending_promoter_index
] == mutex
) {
663 thread
->pending_promoter
[thread
->pending_promoter_index
] = NULL
;
664 if (thread
->pending_promoter_index
> 0)
665 thread
->pending_promoter_index
--;
666 mutex
->lck_mtx_waiters
--;
669 if (mutex
->lck_mtx_waiters
> 0) {
670 integer_t priority
= mutex
->lck_mtx_pri
;
671 spl_t s
= splsched();
674 thread
->promotions
++;
675 thread
->sched_mode
|= TH_MODE_PROMOTED
;
676 if (thread
->sched_pri
< priority
) {
677 KERNEL_DEBUG_CONSTANT(
678 MACHDBG_CODE(DBG_MACH_SCHED
,MACH_PROMOTE
) | DBG_FUNC_NONE
,
679 thread
->sched_pri
, priority
, 0, lck
, 0);
681 set_sched_pri(thread
, priority
);
683 thread_unlock(thread
);
687 mutex
->lck_mtx_pri
= 0;
689 return (mutex
->lck_mtx_waiters
);
693 * Routine: lck_mtx_unlock_wakeup
695 * Invoked on unlock when there is contention.
697 * Called with the interlock locked.
700 lck_mtx_unlock_wakeup (
704 thread_t thread
= current_thread();
707 if (lck
->lck_mtx_tag
!= LCK_MTX_TAG_INDIRECT
)
710 mutex
= &lck
->lck_mtx_ptr
->lck_mtx
;
713 KERNEL_DEBUG(MACHDBG_CODE(DBG_MACH_LOCKS
, LCK_MTX_UNLCK_WAKEUP_CODE
) | DBG_FUNC_START
, (int)lck
, (int)holder
, 0, 0, 0);
715 if (thread
!= holder
)
716 panic("lck_mtx_unlock_wakeup: mutex %p holder %p\n", mutex
, holder
);
718 if (thread
->promotions
> 0) {
719 spl_t s
= splsched();
722 if ( --thread
->promotions
== 0 &&
723 (thread
->sched_mode
& TH_MODE_PROMOTED
) ) {
724 thread
->sched_mode
&= ~TH_MODE_PROMOTED
;
725 if (thread
->sched_mode
& TH_MODE_ISDEPRESSED
) {
726 KERNEL_DEBUG_CONSTANT(
727 MACHDBG_CODE(DBG_MACH_SCHED
,MACH_DEMOTE
) | DBG_FUNC_NONE
,
728 thread
->sched_pri
, DEPRESSPRI
, 0, lck
, 0);
730 set_sched_pri(thread
, DEPRESSPRI
);
733 if (thread
->priority
< thread
->sched_pri
) {
734 KERNEL_DEBUG_CONSTANT(
735 MACHDBG_CODE(DBG_MACH_SCHED
,MACH_DEMOTE
) |
737 thread
->sched_pri
, thread
->priority
,
741 compute_priority(thread
, FALSE
);
744 thread_unlock(thread
);
747 assert(mutex
->lck_mtx_waiters
> 0);
748 thread_wakeup_one((event_t
)(((unsigned int*)lck
)+(sizeof(lck_mtx_t
)-1)/sizeof(unsigned int)));
750 KERNEL_DEBUG(MACHDBG_CODE(DBG_MACH_LOCKS
, LCK_MTX_UNLCK_WAKEUP_CODE
) | DBG_FUNC_END
, 0, 0, 0, 0, 0);
754 lck_mtx_unlockspin_wakeup (
757 assert(lck
->lck_mtx_waiters
> 0);
758 thread_wakeup_one((event_t
)(((unsigned int*)lck
)+(sizeof(lck_mtx_t
)-1)/sizeof(unsigned int)));
760 KERNEL_DEBUG(MACHDBG_CODE(DBG_MACH_LOCKS
, LCK_MTX_UNLCK_WAKEUP_CODE
) | DBG_FUNC_NONE
, (int)lck
, 0, 0, 1, 0);
763 * When there are waiters, we skip the hot-patch spot in the
764 * fastpath, so we record it here.
766 LOCKSTAT_RECORD(LS_LCK_MTX_UNLOCK_RELEASE
, lck
, 0);
772 * Routine: mutex_pause
774 * Called by former callers of simple_lock_pause().
776 #define MAX_COLLISION_COUNTS 32
777 #define MAX_COLLISION 8
779 unsigned int max_collision_count
[MAX_COLLISION_COUNTS
];
781 uint32_t collision_backoffs
[MAX_COLLISION
] = {
782 10, 50, 100, 200, 400, 600, 800, 1000
787 mutex_pause(uint32_t collisions
)
789 wait_result_t wait_result
;
792 if (collisions
>= MAX_COLLISION_COUNTS
)
793 collisions
= MAX_COLLISION_COUNTS
- 1;
794 max_collision_count
[collisions
]++;
796 if (collisions
>= MAX_COLLISION
)
797 collisions
= MAX_COLLISION
- 1;
798 back_off
= collision_backoffs
[collisions
];
800 wait_result
= assert_wait_timeout((event_t
)mutex_pause
, THREAD_UNINT
, back_off
, NSEC_PER_USEC
);
801 assert(wait_result
== THREAD_WAITING
);
803 wait_result
= thread_block(THREAD_CONTINUE_NULL
);
804 assert(wait_result
== THREAD_TIMED_OUT
);
808 unsigned int mutex_yield_wait
= 0;
809 unsigned int mutex_yield_no_wait
= 0;
818 lck_mtx_assert(lck
, LCK_MTX_ASSERT_OWNED
);
821 if (lck
->lck_mtx_tag
== LCK_MTX_TAG_INDIRECT
)
822 waiters
= lck
->lck_mtx_ptr
->lck_mtx
.lck_mtx_waiters
;
824 waiters
= lck
->lck_mtx_waiters
;
827 mutex_yield_no_wait
++;
838 * Routine: lck_rw_sleep
843 lck_sleep_action_t lck_sleep_action
,
845 wait_interrupt_t interruptible
)
848 lck_rw_type_t lck_rw_type
;
850 if ((lck_sleep_action
& ~LCK_SLEEP_MASK
) != 0)
851 panic("Invalid lock sleep action %x\n", lck_sleep_action
);
853 res
= assert_wait(event
, interruptible
);
854 if (res
== THREAD_WAITING
) {
855 lck_rw_type
= lck_rw_done(lck
);
856 res
= thread_block(THREAD_CONTINUE_NULL
);
857 if (!(lck_sleep_action
& LCK_SLEEP_UNLOCK
)) {
858 if (!(lck_sleep_action
& (LCK_SLEEP_SHARED
|LCK_SLEEP_EXCLUSIVE
)))
859 lck_rw_lock(lck
, lck_rw_type
);
860 else if (lck_sleep_action
& LCK_SLEEP_EXCLUSIVE
)
861 lck_rw_lock_exclusive(lck
);
863 lck_rw_lock_shared(lck
);
867 if (lck_sleep_action
& LCK_SLEEP_UNLOCK
)
868 (void)lck_rw_done(lck
);
875 * Routine: lck_rw_sleep_deadline
878 lck_rw_sleep_deadline(
880 lck_sleep_action_t lck_sleep_action
,
882 wait_interrupt_t interruptible
,
886 lck_rw_type_t lck_rw_type
;
888 if ((lck_sleep_action
& ~LCK_SLEEP_MASK
) != 0)
889 panic("Invalid lock sleep action %x\n", lck_sleep_action
);
891 res
= assert_wait_deadline(event
, interruptible
, deadline
);
892 if (res
== THREAD_WAITING
) {
893 lck_rw_type
= lck_rw_done(lck
);
894 res
= thread_block(THREAD_CONTINUE_NULL
);
895 if (!(lck_sleep_action
& LCK_SLEEP_UNLOCK
)) {
896 if (!(lck_sleep_action
& (LCK_SLEEP_SHARED
|LCK_SLEEP_EXCLUSIVE
)))
897 lck_rw_lock(lck
, lck_rw_type
);
898 else if (lck_sleep_action
& LCK_SLEEP_EXCLUSIVE
)
899 lck_rw_lock_exclusive(lck
);
901 lck_rw_lock_shared(lck
);
905 if (lck_sleep_action
& LCK_SLEEP_UNLOCK
)
906 (void)lck_rw_done(lck
);
914 lockgroup_info_array_t
*lockgroup_infop
,
915 mach_msg_type_number_t
*lockgroup_infoCntp
)
917 lockgroup_info_t
*lockgroup_info_base
;
918 lockgroup_info_t
*lockgroup_info
;
919 vm_offset_t lockgroup_info_addr
;
920 vm_size_t lockgroup_info_size
;
927 if (host
== HOST_NULL
)
928 return KERN_INVALID_HOST
;
930 lck_mtx_lock(&lck_grp_lock
);
932 lockgroup_info_size
= round_page(lck_grp_cnt
* sizeof *lockgroup_info
);
933 kr
= kmem_alloc_pageable(ipc_kernel_map
,
934 &lockgroup_info_addr
, lockgroup_info_size
);
935 if (kr
!= KERN_SUCCESS
) {
936 lck_mtx_unlock(&lck_grp_lock
);
940 lockgroup_info_base
= (lockgroup_info_t
*) lockgroup_info_addr
;
941 lck_grp
= (lck_grp_t
*)queue_first(&lck_grp_queue
);
942 lockgroup_info
= lockgroup_info_base
;
944 for (i
= 0; i
< lck_grp_cnt
; i
++) {
946 lockgroup_info
->lock_spin_cnt
= lck_grp
->lck_grp_spincnt
;
947 lockgroup_info
->lock_spin_util_cnt
= lck_grp
->lck_grp_stat
.lck_grp_spin_stat
.lck_grp_spin_util_cnt
;
948 lockgroup_info
->lock_spin_held_cnt
= lck_grp
->lck_grp_stat
.lck_grp_spin_stat
.lck_grp_spin_held_cnt
;
949 lockgroup_info
->lock_spin_miss_cnt
= lck_grp
->lck_grp_stat
.lck_grp_spin_stat
.lck_grp_spin_miss_cnt
;
950 lockgroup_info
->lock_spin_held_max
= lck_grp
->lck_grp_stat
.lck_grp_spin_stat
.lck_grp_spin_held_max
;
951 lockgroup_info
->lock_spin_held_cum
= lck_grp
->lck_grp_stat
.lck_grp_spin_stat
.lck_grp_spin_held_cum
;
953 lockgroup_info
->lock_mtx_cnt
= lck_grp
->lck_grp_mtxcnt
;
954 lockgroup_info
->lock_mtx_util_cnt
= lck_grp
->lck_grp_stat
.lck_grp_mtx_stat
.lck_grp_mtx_util_cnt
;
955 lockgroup_info
->lock_mtx_held_cnt
= lck_grp
->lck_grp_stat
.lck_grp_mtx_stat
.lck_grp_mtx_held_cnt
;
956 lockgroup_info
->lock_mtx_miss_cnt
= lck_grp
->lck_grp_stat
.lck_grp_mtx_stat
.lck_grp_mtx_miss_cnt
;
957 lockgroup_info
->lock_mtx_wait_cnt
= lck_grp
->lck_grp_stat
.lck_grp_mtx_stat
.lck_grp_mtx_wait_cnt
;
958 lockgroup_info
->lock_mtx_held_max
= lck_grp
->lck_grp_stat
.lck_grp_mtx_stat
.lck_grp_mtx_held_max
;
959 lockgroup_info
->lock_mtx_held_cum
= lck_grp
->lck_grp_stat
.lck_grp_mtx_stat
.lck_grp_mtx_held_cum
;
960 lockgroup_info
->lock_mtx_wait_max
= lck_grp
->lck_grp_stat
.lck_grp_mtx_stat
.lck_grp_mtx_wait_max
;
961 lockgroup_info
->lock_mtx_wait_cum
= lck_grp
->lck_grp_stat
.lck_grp_mtx_stat
.lck_grp_mtx_wait_cum
;
963 lockgroup_info
->lock_rw_cnt
= lck_grp
->lck_grp_rwcnt
;
964 lockgroup_info
->lock_rw_util_cnt
= lck_grp
->lck_grp_stat
.lck_grp_rw_stat
.lck_grp_rw_util_cnt
;
965 lockgroup_info
->lock_rw_held_cnt
= lck_grp
->lck_grp_stat
.lck_grp_rw_stat
.lck_grp_rw_held_cnt
;
966 lockgroup_info
->lock_rw_miss_cnt
= lck_grp
->lck_grp_stat
.lck_grp_rw_stat
.lck_grp_rw_miss_cnt
;
967 lockgroup_info
->lock_rw_wait_cnt
= lck_grp
->lck_grp_stat
.lck_grp_rw_stat
.lck_grp_rw_wait_cnt
;
968 lockgroup_info
->lock_rw_held_max
= lck_grp
->lck_grp_stat
.lck_grp_rw_stat
.lck_grp_rw_held_max
;
969 lockgroup_info
->lock_rw_held_cum
= lck_grp
->lck_grp_stat
.lck_grp_rw_stat
.lck_grp_rw_held_cum
;
970 lockgroup_info
->lock_rw_wait_max
= lck_grp
->lck_grp_stat
.lck_grp_rw_stat
.lck_grp_rw_wait_max
;
971 lockgroup_info
->lock_rw_wait_cum
= lck_grp
->lck_grp_stat
.lck_grp_rw_stat
.lck_grp_rw_wait_cum
;
973 (void) strncpy(lockgroup_info
->lockgroup_name
,lck_grp
->lck_grp_name
, LOCKGROUP_MAX_NAME
);
975 lck_grp
= (lck_grp_t
*)(queue_next((queue_entry_t
)(lck_grp
)));
979 *lockgroup_infoCntp
= lck_grp_cnt
;
980 lck_mtx_unlock(&lck_grp_lock
);
982 used
= (*lockgroup_infoCntp
) * sizeof *lockgroup_info
;
984 if (used
!= lockgroup_info_size
)
985 bzero((char *) lockgroup_info
, lockgroup_info_size
- used
);
987 kr
= vm_map_copyin(ipc_kernel_map
, (vm_map_address_t
)lockgroup_info_addr
,
988 (vm_map_size_t
)lockgroup_info_size
, TRUE
, ©
);
989 assert(kr
== KERN_SUCCESS
);
991 *lockgroup_infop
= (lockgroup_info_t
*) copy
;
993 return(KERN_SUCCESS
);
997 * Compatibility module
1000 extern lck_rw_t
*lock_alloc_EXT( boolean_t can_sleep
, unsigned short tag0
, unsigned short tag1
);
1001 extern void lock_done_EXT(lck_rw_t
*lock
);
1002 extern void lock_free_EXT(lck_rw_t
*lock
);
1003 extern void lock_init_EXT(lck_rw_t
*lock
, boolean_t can_sleep
, unsigned short tag0
, unsigned short tag1
);
1004 extern void lock_read_EXT(lck_rw_t
*lock
);
1005 extern boolean_t
lock_read_to_write_EXT(lck_rw_t
*lock
);
1006 extern void lock_write_EXT(lck_rw_t
*lock
);
1007 extern void lock_write_to_read_EXT(lck_rw_t
*lock
);
1008 extern wait_result_t
thread_sleep_lock_write_EXT(
1009 event_t event
, lck_rw_t
*lock
, wait_interrupt_t interruptible
);
1011 extern void usimple_lock_EXT(lck_spin_t
*lock
);
1012 extern void usimple_lock_init_EXT(lck_spin_t
*lock
, unsigned short tag
);
1013 extern unsigned int usimple_lock_try_EXT(lck_spin_t
*lock
);
1014 extern void usimple_unlock_EXT(lck_spin_t
*lock
);
1015 extern wait_result_t
thread_sleep_usimple_lock_EXT(event_t event
, lck_spin_t
*lock
, wait_interrupt_t interruptible
);
1018 lck_mtx_t
* mutex_alloc_EXT(__unused
unsigned short tag
);
1019 void mutex_free_EXT(lck_mtx_t
*mutex
);
1020 void mutex_init_EXT(lck_mtx_t
*mutex
, __unused
unsigned short tag
);
1021 wait_result_t
thread_sleep_mutex_EXT(event_t event
, lck_mtx_t
*mutex
, wait_interrupt_t interruptible
);
1022 wait_result_t
thread_sleep_mutex_deadline_EXT(event_t event
, lck_mtx_t
*mutex
, uint64_t deadline
, wait_interrupt_t interruptible
);
1026 __unused boolean_t can_sleep
,
1027 __unused
unsigned short tag0
,
1028 __unused
unsigned short tag1
)
1030 return( lck_rw_alloc_init( &LockCompatGroup
, LCK_ATTR_NULL
));
1037 (void) lck_rw_done(lock
);
1044 lck_rw_free(lock
, &LockCompatGroup
);
1050 __unused boolean_t can_sleep
,
1051 __unused
unsigned short tag0
,
1052 __unused
unsigned short tag1
)
1054 lck_rw_init(lock
, &LockCompatGroup
, LCK_ATTR_NULL
);
1061 lck_rw_lock_shared( lock
);
1065 lock_read_to_write_EXT(
1068 return( lck_rw_lock_shared_to_exclusive(lock
));
1075 lck_rw_lock_exclusive(lock
);
1079 lock_write_to_read_EXT(
1082 lck_rw_lock_exclusive_to_shared(lock
);
1086 thread_sleep_lock_write_EXT(
1089 wait_interrupt_t interruptible
)
1091 return( lck_rw_sleep(lock
, LCK_SLEEP_EXCLUSIVE
, event
, interruptible
));
1098 lck_spin_lock(lock
);
1102 usimple_lock_init_EXT(
1104 __unused
unsigned short tag
)
1106 lck_spin_init(lock
, &LockCompatGroup
, LCK_ATTR_NULL
);
1110 usimple_lock_try_EXT(
1113 return(lck_spin_try_lock(lock
));
1120 lck_spin_unlock(lock
);
1124 thread_sleep_usimple_lock_EXT(
1127 wait_interrupt_t interruptible
)
1129 return( lck_spin_sleep(lock
, LCK_SLEEP_DEFAULT
, event
, interruptible
));
1133 __unused
unsigned short tag
)
1135 return(lck_mtx_alloc_init(&LockCompatGroup
, LCK_ATTR_NULL
));
1142 lck_mtx_free(mutex
, &LockCompatGroup
);
1148 __unused
unsigned short tag
)
1150 lck_mtx_init(mutex
, &LockCompatGroup
, LCK_ATTR_NULL
);
1154 thread_sleep_mutex_EXT(
1157 wait_interrupt_t interruptible
)
1159 return( lck_mtx_sleep(mutex
, LCK_SLEEP_DEFAULT
, event
, interruptible
));
1163 thread_sleep_mutex_deadline_EXT(
1167 wait_interrupt_t interruptible
)
1169 return( lck_mtx_sleep_deadline(mutex
, LCK_SLEEP_DEFAULT
, event
, interruptible
, deadline
));