]> git.saurik.com Git - apple/xnu.git/blame - osfmk/kern/locks.c
projects / apple / xnu.git / blame
? search:
summary | shortlog | log | commit | commitdiff | tree
blob | blame (incremental) | history | HEAD
xnu-1228.0.2.tar.gz
[apple/xnu.git] / osfmk / kern / locks.c
CommitLineData
91447636 1/*
2d21ac55 2 * Copyright (c) 2000-2007 Apple Inc. All rights reserved.
91447636 3 *
2d21ac55 4 * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
91447636 5 *
2d21ac55
A		 6 * This file contains Original Code and/or Modifications of Original Code
7 * as defined in and that are subject to the Apple Public Source License
8 * Version 2.0 (the 'License'). You may not use this file except in
9 * compliance with the License. The rights granted to you under the License
10 * may not be used to create, or enable the creation or redistribution of,
11 * unlawful or unlicensed copies of an Apple operating system, or to
12 * circumvent, violate, or enable the circumvention or violation of, any
13 * terms of an Apple operating system software license agreement.
8f6c56a5 14 *
2d21ac55
A		 15 * Please obtain a copy of the License at
16 * http://www.opensource.apple.com/apsl/ and read it before using this file.
17 *
18 * The Original Code and all software distributed under the License are
19 * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
8f6c56a5
A		 20 * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
21 * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
2d21ac55
A		 22 * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
23 * Please see the License for the specific language governing rights and
24 * limitations under the License.
8f6c56a5 25 *
2d21ac55 26 * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
91447636
A		 27 */
28/*
29 * @OSF_COPYRIGHT@
30 */
31/*
32 * Mach Operating System
33 * Copyright (c) 1991,1990,1989,1988,1987 Carnegie Mellon University
34 * All Rights Reserved.
35 *
36 * Permission to use, copy, modify and distribute this software and its
37 * documentation is hereby granted, provided that both the copyright
38 * notice and this permission notice appear in all copies of the
39 * software, derivative works or modified versions, and any portions
40 * thereof, and that both notices appear in supporting documentation.
41 *
42 * CARNEGIE MELLON ALLOWS FREE USE OF THIS SOFTWARE IN ITS "AS IS"
43 * CONDITION. CARNEGIE MELLON DISCLAIMS ANY LIABILITY OF ANY KIND FOR
44 * ANY DAMAGES WHATSOEVER RESULTING FROM THE USE OF THIS SOFTWARE.
45 *
46 * Carnegie Mellon requests users of this software to return to
47 *
48 * Software Distribution Coordinator or Software.Distribution@CS.CMU.EDU
49 * School of Computer Science
50 * Carnegie Mellon University
51 * Pittsburgh PA 15213-3890
52 *
53 * any improvements or extensions that they make and grant Carnegie Mellon
54 * the rights to redistribute these changes.
55 */
56#include <mach_kdb.h>
57#include <mach_ldebug.h>
58#include <debug.h>
59
60#include <mach/kern_return.h>
61#include <mach/mach_host_server.h>
62#include <mach_debug/lockgroup_info.h>
63
64#include <kern/locks.h>
65#include <kern/misc_protos.h>
66#include <kern/kalloc.h>
67#include <kern/thread.h>
68#include <kern/processor.h>
69#include <kern/sched_prim.h>
70#include <kern/debug.h>
71#include <string.h>
72
73
74#include <sys/kdebug.h>
75
2d21ac55
A		 76#if CONFIG_DTRACE
77/*
78 * We need only enough declarations from the BSD-side to be able to
79 * test if our probe is active, and to call __dtrace_probe(). Setting
80 * NEED_DTRACE_DEFS gets a local copy of those definitions pulled in.
81 */
82#define NEED_DTRACE_DEFS
83#include <../bsd/sys/lockstat.h>
84#endif
85
91447636
A		 86#define LCK_MTX_SLEEP_CODE 0
87#define LCK_MTX_SLEEP_DEADLINE_CODE 1
88#define LCK_MTX_LCK_WAIT_CODE 2
89#define LCK_MTX_UNLCK_WAKEUP_CODE 3
90
91
92static queue_head_t lck_grp_queue;
93static unsigned int lck_grp_cnt;
94
95decl_mutex_data(static,lck_grp_lock)
96
97lck_grp_attr_t LockDefaultGroupAttr;
98lck_grp_t LockCompatGroup;
99lck_attr_t LockDefaultLckAttr;
100
101/*
102 * Routine: lck_mod_init
103 */
104
105void
106lck_mod_init(
107 void)
108{
109 queue_init(&lck_grp_queue);
110 mutex_init(&lck_grp_lock, 0);
111 lck_grp_cnt = 0;
112 lck_grp_attr_setdefault( &LockDefaultGroupAttr);
113 lck_grp_init( &LockCompatGroup, "Compatibility APIs", LCK_GRP_ATTR_NULL);
114 lck_attr_setdefault(&LockDefaultLckAttr);
115}
116
117/*
118 * Routine: lck_grp_attr_alloc_init
119 */
120
121lck_grp_attr_t *
122lck_grp_attr_alloc_init(
123 void)
124{
125 lck_grp_attr_t *attr;
126
127 if ((attr = (lck_grp_attr_t *)kalloc(sizeof(lck_grp_attr_t))) != 0)
128 lck_grp_attr_setdefault(attr);
129
130 return(attr);
131}
132
133
134/*
135 * Routine: lck_grp_attr_setdefault
136 */
137
138void
139lck_grp_attr_setdefault(
140 lck_grp_attr_t *attr)
141{
142 if (LcksOpts & enaLkStat)
143 attr->grp_attr_val = LCK_GRP_ATTR_STAT;
144 else
145 attr->grp_attr_val = 0;
146}
147
148
149/*
150 * Routine: lck_grp_attr_setstat
151 */
152
153void
154lck_grp_attr_setstat(
155 lck_grp_attr_t *attr)
156{
2d21ac55 157 (void)hw_atomic_or(&attr->grp_attr_val, LCK_GRP_ATTR_STAT);
91447636
A		 158}
159
160
161/*
162 * Routine: lck_grp_attr_free
163 */
164
165void
166lck_grp_attr_free(
167 lck_grp_attr_t *attr)
168{
169 kfree(attr, sizeof(lck_grp_attr_t));
170}
171
172
173/*
174 * Routine: lck_grp_alloc_init
175 */
176
177lck_grp_t *
178lck_grp_alloc_init(
179 const char* grp_name,
180 lck_grp_attr_t *attr)
181{
182 lck_grp_t *grp;
183
184 if ((grp = (lck_grp_t *)kalloc(sizeof(lck_grp_t))) != 0)
185 lck_grp_init(grp, grp_name, attr);
186
187 return(grp);
188}
189
190
191/*
192 * Routine: lck_grp_init
193 */
194
195void
196lck_grp_init(
197 lck_grp_t *grp,
198 const char* grp_name,
199 lck_grp_attr_t *attr)
200{
201 bzero((void *)grp, sizeof(lck_grp_t));
202
203 (void) strncpy(grp->lck_grp_name, grp_name, LCK_GRP_MAX_NAME);
204
205 if (attr != LCK_GRP_ATTR_NULL)
206 grp->lck_grp_attr = attr->grp_attr_val;
207 else if (LcksOpts & enaLkStat)
208 grp->lck_grp_attr = LCK_GRP_ATTR_STAT;
209 else
210 grp->lck_grp_attr = LCK_ATTR_NONE;
211
212 grp->lck_grp_refcnt = 1;
213
214 mutex_lock(&lck_grp_lock);
215 enqueue_tail(&lck_grp_queue, (queue_entry_t)grp);
216 lck_grp_cnt++;
217 mutex_unlock(&lck_grp_lock);
218
219}
220
221
222/*
223 * Routine: lck_grp_free
224 */
225
226void
227lck_grp_free(
228 lck_grp_t *grp)
229{
230 mutex_lock(&lck_grp_lock);
231 lck_grp_cnt--;
232 (void)remque((queue_entry_t)grp);
233 mutex_unlock(&lck_grp_lock);
234 lck_grp_deallocate(grp);
235}
236
237
238/*
239 * Routine: lck_grp_reference
240 */
241
242void
243lck_grp_reference(
244 lck_grp_t *grp)
245{
2d21ac55 246 (void)hw_atomic_add(&grp->lck_grp_refcnt, 1);
91447636
A		 247}
248
249
250/*
251 * Routine: lck_grp_deallocate
252 */
253
254void
255lck_grp_deallocate(
256 lck_grp_t *grp)
257{
2d21ac55 258 if (hw_atomic_sub(&grp->lck_grp_refcnt, 1) == 0)
91447636
A		 259 kfree(grp, sizeof(lck_grp_t));
260}
261
262/*
263 * Routine: lck_grp_lckcnt_incr
264 */
265
266void
267lck_grp_lckcnt_incr(
268 lck_grp_t *grp,
269 lck_type_t lck_type)
270{
271 unsigned int *lckcnt;
272
273 switch (lck_type) {
274 case LCK_TYPE_SPIN:
275 lckcnt = &grp->lck_grp_spincnt;
276 break;
277 case LCK_TYPE_MTX:
278 lckcnt = &grp->lck_grp_mtxcnt;
279 break;
280 case LCK_TYPE_RW:
281 lckcnt = &grp->lck_grp_rwcnt;
282 break;
283 default:
284 return panic("lck_grp_lckcnt_incr(): invalid lock type: %d\n", lck_type);
285 }
286
2d21ac55 287 (void)hw_atomic_add(lckcnt, 1);
91447636
A		 288}
289
290/*
291 * Routine: lck_grp_lckcnt_decr
292 */
293
294void
295lck_grp_lckcnt_decr(
296 lck_grp_t *grp,
297 lck_type_t lck_type)
298{
299 unsigned int *lckcnt;
300
301 switch (lck_type) {
302 case LCK_TYPE_SPIN:
303 lckcnt = &grp->lck_grp_spincnt;
304 break;
305 case LCK_TYPE_MTX:
306 lckcnt = &grp->lck_grp_mtxcnt;
307 break;
308 case LCK_TYPE_RW:
309 lckcnt = &grp->lck_grp_rwcnt;
310 break;
311 default:
312 return panic("lck_grp_lckcnt_decr(): invalid lock type: %d\n", lck_type);
313 }
314
2d21ac55 315 (void)hw_atomic_sub(lckcnt, 1);
91447636
A		 316}
317
318/*
319 * Routine: lck_attr_alloc_init
320 */
321
322lck_attr_t *
323lck_attr_alloc_init(
324 void)
325{
326 lck_attr_t *attr;
327
328 if ((attr = (lck_attr_t *)kalloc(sizeof(lck_attr_t))) != 0)
329 lck_attr_setdefault(attr);
330
331 return(attr);
332}
333
334
335/*
336 * Routine: lck_attr_setdefault
337 */
338
339void
340lck_attr_setdefault(
341 lck_attr_t *attr)
342{
343#if !DEBUG
344 if (LcksOpts & enaLkDeb)
345 attr->lck_attr_val = LCK_ATTR_DEBUG;
346 else
347 attr->lck_attr_val = LCK_ATTR_NONE;
348#else
349 attr->lck_attr_val = LCK_ATTR_DEBUG;
350#endif
351
352}
353
354
355/*
356 * Routine: lck_attr_setdebug
357 */
358void
359lck_attr_setdebug(
360 lck_attr_t *attr)
361{
2d21ac55
A		 362 (void)hw_atomic_or(&attr->lck_attr_val, LCK_ATTR_DEBUG);
363}
364
365/*
366 * Routine: lck_attr_setdebug
367 */
368void
369lck_attr_cleardebug(
370 lck_attr_t *attr)
371{
372 (void)hw_atomic_and(&attr->lck_attr_val, ~LCK_ATTR_DEBUG);
91447636
A		 373}
374
375
0c530ab8
A		 376/*
377 * Routine: lck_attr_rw_shared_priority
378 */
379void
380lck_attr_rw_shared_priority(
381 lck_attr_t *attr)
382{
2d21ac55 383 (void)hw_atomic_or(&attr->lck_attr_val, LCK_ATTR_RW_SHARED_PRIORITY);
0c530ab8
A		 384}
385
386
91447636
A		 387/*
388 * Routine: lck_attr_free
389 */
390void
391lck_attr_free(
392 lck_attr_t *attr)
393{
394 kfree(attr, sizeof(lck_attr_t));
395}
396
397
398/*
399 * Routine: lck_spin_sleep
400 */
401wait_result_t
402lck_spin_sleep(
403 lck_spin_t *lck,
404 lck_sleep_action_t lck_sleep_action,
405 event_t event,
406 wait_interrupt_t interruptible)
407{
408 wait_result_t res;
409
410 if ((lck_sleep_action & ~LCK_SLEEP_MASK) != 0)
411 panic("Invalid lock sleep action %x\n", lck_sleep_action);
412
413 res = assert_wait(event, interruptible);
414 if (res == THREAD_WAITING) {
415 lck_spin_unlock(lck);
416 res = thread_block(THREAD_CONTINUE_NULL);
417 if (!(lck_sleep_action & LCK_SLEEP_UNLOCK))
418 lck_spin_lock(lck);
419 }
420 else
421 if (lck_sleep_action & LCK_SLEEP_UNLOCK)
422 lck_spin_unlock(lck);
423
424 return res;
425}
426
427
428/*
429 * Routine: lck_spin_sleep_deadline
430 */
431wait_result_t
432lck_spin_sleep_deadline(
433 lck_spin_t *lck,
434 lck_sleep_action_t lck_sleep_action,
435 event_t event,
436 wait_interrupt_t interruptible,
437 uint64_t deadline)
438{
439 wait_result_t res;
440
441 if ((lck_sleep_action & ~LCK_SLEEP_MASK) != 0)
442 panic("Invalid lock sleep action %x\n", lck_sleep_action);
443
444 res = assert_wait_deadline(event, interruptible, deadline);
445 if (res == THREAD_WAITING) {
446 lck_spin_unlock(lck);
447 res = thread_block(THREAD_CONTINUE_NULL);
448 if (!(lck_sleep_action & LCK_SLEEP_UNLOCK))
449 lck_spin_lock(lck);
450 }
451 else
452 if (lck_sleep_action & LCK_SLEEP_UNLOCK)
453 lck_spin_unlock(lck);
454
455 return res;
456}
457
458
459/*
460 * Routine: lck_mtx_sleep
461 */
462wait_result_t
463lck_mtx_sleep(
464 lck_mtx_t *lck,
465 lck_sleep_action_t lck_sleep_action,
466 event_t event,
467 wait_interrupt_t interruptible)
468{
469 wait_result_t res;
470
471 KERNEL_DEBUG(MACHDBG_CODE(DBG_MACH_LOCKS, LCK_MTX_SLEEP_CODE) | DBG_FUNC_START,
472 (int)lck, (int)lck_sleep_action, (int)event, (int)interruptible, 0);
473
474 if ((lck_sleep_action & ~LCK_SLEEP_MASK) != 0)
475 panic("Invalid lock sleep action %x\n", lck_sleep_action);
476
477 res = assert_wait(event, interruptible);
478 if (res == THREAD_WAITING) {
479 lck_mtx_unlock(lck);
480 res = thread_block(THREAD_CONTINUE_NULL);
481 if (!(lck_sleep_action & LCK_SLEEP_UNLOCK))
482 lck_mtx_lock(lck);
483 }
484 else
485 if (lck_sleep_action & LCK_SLEEP_UNLOCK)
486 lck_mtx_unlock(lck);
487
488 KERNEL_DEBUG(MACHDBG_CODE(DBG_MACH_LOCKS, LCK_MTX_SLEEP_CODE) | DBG_FUNC_END, (int)res, 0, 0, 0, 0);
489
490 return res;
491}
492
493
494/*
495 * Routine: lck_mtx_sleep_deadline
496 */
497wait_result_t
498lck_mtx_sleep_deadline(
499 lck_mtx_t *lck,
500 lck_sleep_action_t lck_sleep_action,
501 event_t event,
502 wait_interrupt_t interruptible,
503 uint64_t deadline)
504{
505 wait_result_t res;
506
507 KERNEL_DEBUG(MACHDBG_CODE(DBG_MACH_LOCKS, LCK_MTX_SLEEP_DEADLINE_CODE) | DBG_FUNC_START,
508 (int)lck, (int)lck_sleep_action, (int)event, (int)interruptible, 0);
509
510 if ((lck_sleep_action & ~LCK_SLEEP_MASK) != 0)
511 panic("Invalid lock sleep action %x\n", lck_sleep_action);
512
513 res = assert_wait_deadline(event, interruptible, deadline);
514 if (res == THREAD_WAITING) {
515 lck_mtx_unlock(lck);
516 res = thread_block(THREAD_CONTINUE_NULL);
517 if (!(lck_sleep_action & LCK_SLEEP_UNLOCK))
518 lck_mtx_lock(lck);
519 }
520 else
521 if (lck_sleep_action & LCK_SLEEP_UNLOCK)
522 lck_mtx_unlock(lck);
523
524 KERNEL_DEBUG(MACHDBG_CODE(DBG_MACH_LOCKS, LCK_MTX_SLEEP_DEADLINE_CODE) | DBG_FUNC_END, (int)res, 0, 0, 0, 0);
525
526 return res;
527}
528
529/*
530 * Routine: lck_mtx_lock_wait
531 *
532 * Invoked in order to wait on contention.
533 *
534 * Called with the interlock locked and
535 * returns it unlocked.
536 */
537void
538lck_mtx_lock_wait (
539 lck_mtx_t *lck,
540 thread_t holder)
541{
542 thread_t self = current_thread();
543 lck_mtx_t *mutex;
544 integer_t priority;
545 spl_t s = splsched();
2d21ac55
A		 546#if CONFIG_DTRACE
547 uint64_t sleep_start = 0;
548
549 if (lockstat_probemap[LS_LCK_MTX_LOCK_BLOCK] || lockstat_probemap[LS_LCK_MTX_EXT_LOCK_BLOCK]) {
550 sleep_start = mach_absolute_time();
551 }
552#endif
91447636
A		 553
554 if (lck->lck_mtx_tag != LCK_MTX_TAG_INDIRECT)
555 mutex = lck;
556 else
557 mutex = &lck->lck_mtx_ptr->lck_mtx;
558
559 KERNEL_DEBUG(MACHDBG_CODE(DBG_MACH_LOCKS, LCK_MTX_LCK_WAIT_CODE) | DBG_FUNC_START, (int)lck, (int)holder, 0, 0, 0);
560
561 priority = self->sched_pri;
562 if (priority < self->priority)
563 priority = self->priority;
564 if (priority > MINPRI_KERNEL)
565 priority = MINPRI_KERNEL;
566 else
567 if (priority < BASEPRI_DEFAULT)
568 priority = BASEPRI_DEFAULT;
569
570 thread_lock(holder);
571 if (mutex->lck_mtx_pri == 0)
572 holder->promotions++;
573 if (holder->priority < MINPRI_KERNEL) {
574 holder->sched_mode |= TH_MODE_PROMOTED;
575 if ( mutex->lck_mtx_pri < priority &&
576 holder->sched_pri < priority ) {
577 KERNEL_DEBUG_CONSTANT(
578 MACHDBG_CODE(DBG_MACH_SCHED,MACH_PROMOTE) | DBG_FUNC_NONE,
579 holder->sched_pri, priority, (int)holder, (int)lck, 0);
580
581 set_sched_pri(holder, priority);
582 }
583 }
584 thread_unlock(holder);
585 splx(s);
586
587 if (mutex->lck_mtx_pri < priority)
588 mutex->lck_mtx_pri = priority;
589 if (self->pending_promoter[self->pending_promoter_index] == NULL) {
590 self->pending_promoter[self->pending_promoter_index] = mutex;
591 mutex->lck_mtx_waiters++;
592 }
593 else
594 if (self->pending_promoter[self->pending_promoter_index] != mutex) {
595 self->pending_promoter[++self->pending_promoter_index] = mutex;
596 mutex->lck_mtx_waiters++;
597 }
598
599 assert_wait((event_t)(((unsigned int*)lck)+((sizeof(lck_mtx_t)-1)/sizeof(unsigned int))), THREAD_UNINT);
600 lck_mtx_ilk_unlock(mutex);
601
602 thread_block(THREAD_CONTINUE_NULL);
603
604 KERNEL_DEBUG(MACHDBG_CODE(DBG_MACH_LOCKS, LCK_MTX_LCK_WAIT_CODE) | DBG_FUNC_END, 0, 0, 0, 0, 0);
2d21ac55
A		 605#if CONFIG_DTRACE
606 /*
607 * Record the Dtrace lockstat probe for blocking, block time
608 * measured from when we were entered.
609 */
610 if (sleep_start) {
611 if (lck->lck_mtx_tag != LCK_MTX_TAG_INDIRECT) {
612 LOCKSTAT_RECORD(LS_LCK_MTX_LOCK_BLOCK, lck,
613 mach_absolute_time() - sleep_start);
614 } else {
615 LOCKSTAT_RECORD(LS_LCK_MTX_EXT_LOCK_BLOCK, lck,
616 mach_absolute_time() - sleep_start);
617 }
618 }
619#endif
91447636
A		 620}
621
622/*
623 * Routine: lck_mtx_lock_acquire
624 *
625 * Invoked on acquiring the mutex when there is
626 * contention.
627 *
628 * Returns the current number of waiters.
629 *
630 * Called with the interlock locked.
631 */
632int
633lck_mtx_lock_acquire(
634 lck_mtx_t *lck)
635{
636 thread_t thread = current_thread();
637 lck_mtx_t *mutex;
638
639 if (lck->lck_mtx_tag != LCK_MTX_TAG_INDIRECT)
640 mutex = lck;
641 else
642 mutex = &lck->lck_mtx_ptr->lck_mtx;
643
644 if (thread->pending_promoter[thread->pending_promoter_index] == mutex) {
645 thread->pending_promoter[thread->pending_promoter_index] = NULL;
646 if (thread->pending_promoter_index > 0)
647 thread->pending_promoter_index--;
648 mutex->lck_mtx_waiters--;
649 }
650
651 if (mutex->lck_mtx_waiters > 0) {
652 integer_t priority = mutex->lck_mtx_pri;
653 spl_t s = splsched();
654
655 thread_lock(thread);
656 thread->promotions++;
657 if (thread->priority < MINPRI_KERNEL) {
658 thread->sched_mode |= TH_MODE_PROMOTED;
659 if (thread->sched_pri < priority) {
660 KERNEL_DEBUG_CONSTANT(
661 MACHDBG_CODE(DBG_MACH_SCHED,MACH_PROMOTE) | DBG_FUNC_NONE,
662 thread->sched_pri, priority, 0, (int)lck, 0);
663
664 set_sched_pri(thread, priority);
665 }
666 }
667 thread_unlock(thread);
668 splx(s);
669 }
670 else
671 mutex->lck_mtx_pri = 0;
672
673 return (mutex->lck_mtx_waiters);
674}
675
676/*
677 * Routine: lck_mtx_unlock_wakeup
678 *
679 * Invoked on unlock when there is contention.
680 *
681 * Called with the interlock locked.
682 */
683void
684lck_mtx_unlock_wakeup (
685 lck_mtx_t *lck,
686 thread_t holder)
687{
688 thread_t thread = current_thread();
689 lck_mtx_t *mutex;
690
691 if (lck->lck_mtx_tag != LCK_MTX_TAG_INDIRECT)
692 mutex = lck;
693 else
694 mutex = &lck->lck_mtx_ptr->lck_mtx;
695
696
697 KERNEL_DEBUG(MACHDBG_CODE(DBG_MACH_LOCKS, LCK_MTX_UNLCK_WAKEUP_CODE) | DBG_FUNC_START, (int)lck, (int)holder, 0, 0, 0);
698
699 if (thread != holder)
2d21ac55 700 panic("lck_mtx_unlock_wakeup: mutex %p holder %p\n", mutex, holder);
91447636
A		 701
702 if (thread->promotions > 0) {
703 spl_t s = splsched();
704
705 thread_lock(thread);
706 if ( --thread->promotions == 0 &&
707 (thread->sched_mode & TH_MODE_PROMOTED) ) {
708 thread->sched_mode &= ~TH_MODE_PROMOTED;
709 if (thread->sched_mode & TH_MODE_ISDEPRESSED) {
710 KERNEL_DEBUG_CONSTANT(
711 MACHDBG_CODE(DBG_MACH_SCHED,MACH_DEMOTE) | DBG_FUNC_NONE,
712 thread->sched_pri, DEPRESSPRI, 0, (int)lck, 0);
713
714 set_sched_pri(thread, DEPRESSPRI);
715 }
716 else {
717 if (thread->priority < thread->sched_pri) {
718 KERNEL_DEBUG_CONSTANT(
719 MACHDBG_CODE(DBG_MACH_SCHED,MACH_DEMOTE) |
720 DBG_FUNC_NONE,
721 thread->sched_pri, thread->priority,
722 0, (int)lck, 0);
723 }
724
725 compute_priority(thread, FALSE);
726 }
727 }
728 thread_unlock(thread);
729 splx(s);
730 }
731 assert(mutex->lck_mtx_waiters > 0);
732 thread_wakeup_one((event_t)(((unsigned int*)lck)+(sizeof(lck_mtx_t)-1)/sizeof(unsigned int)));
733
734 KERNEL_DEBUG(MACHDBG_CODE(DBG_MACH_LOCKS, LCK_MTX_UNLCK_WAKEUP_CODE) | DBG_FUNC_END, 0, 0, 0, 0, 0);
735}
736
2d21ac55
A		 737void
738lck_mtx_unlockspin_wakeup (
739 lck_mtx_t *lck)
740{
741 assert(lck->lck_mtx_waiters > 0);
742 thread_wakeup_one((event_t)(((unsigned int*)lck)+(sizeof(lck_mtx_t)-1)/sizeof(unsigned int)));
743
744 KERNEL_DEBUG(MACHDBG_CODE(DBG_MACH_LOCKS, LCK_MTX_UNLCK_WAKEUP_CODE) | DBG_FUNC_NONE, (int)lck, 0, 0, 1, 0);
745#if CONFIG_DTRACE
746 /*
747 * When there are waiters, we skip the hot-patch spot in the
748 * fastpath, so we record it here.
749 */
750 LOCKSTAT_RECORD(LS_LCK_MTX_UNLOCK_RELEASE, lck, 0);
751#endif
752}
753
754
91447636
A		 755/*
756 * Routine: mutex_pause
757 *
758 * Called by former callers of simple_lock_pause().
759 */
2d21ac55
A		 760#define MAX_COLLISION_COUNTS 32
761#define MAX_COLLISION 8
762
763unsigned int max_collision_count[MAX_COLLISION_COUNTS];
764
765uint32_t collision_backoffs[MAX_COLLISION] = {
766 10, 50, 100, 200, 400, 600, 800, 1000
767};
768
91447636
A		 769
770void
2d21ac55 771mutex_pause(uint32_t collisions)
91447636
A		 772{
773 wait_result_t wait_result;
2d21ac55 774 uint32_t back_off;
91447636 775
2d21ac55
A		 776 if (collisions >= MAX_COLLISION_COUNTS)
777 collisions = MAX_COLLISION_COUNTS - 1;
778 max_collision_count[collisions]++;
779
780 if (collisions >= MAX_COLLISION)
781 collisions = MAX_COLLISION - 1;
782 back_off = collision_backoffs[collisions];
783
784 wait_result = assert_wait_timeout((event_t)mutex_pause, THREAD_UNINT, back_off, NSEC_PER_USEC);
91447636
A		 785 assert(wait_result == THREAD_WAITING);
786
787 wait_result = thread_block(THREAD_CONTINUE_NULL);
788 assert(wait_result == THREAD_TIMED_OUT);
789}
790
2d21ac55
A		 791
792unsigned int mutex_yield_wait = 0;
793unsigned int mutex_yield_no_wait = 0;
794
795void
796mutex_yield(
797 mutex_t *mutex)
798{
799 lck_mtx_t *lck;
800
801#if DEBUG
802 _mutex_assert(mutex, MA_OWNED);
803#endif /* DEBUG */
804
805 lck = (lck_mtx_t *) mutex;
806 if (lck->lck_mtx_tag == LCK_MTX_TAG_INDIRECT)
807 lck = &lck->lck_mtx_ptr->lck_mtx;
808
809 if (! lck->lck_mtx_waiters) {
810 mutex_yield_no_wait++;
811 } else {
812 mutex_yield_wait++;
813 mutex_unlock(mutex);
814 mutex_pause(0);
815 mutex_lock(mutex);
816 }
817}
818
819
91447636
A		 820/*
821 * Routine: lck_rw_sleep
822 */
823wait_result_t
824lck_rw_sleep(
825 lck_rw_t *lck,
826 lck_sleep_action_t lck_sleep_action,
827 event_t event,
828 wait_interrupt_t interruptible)
829{
830 wait_result_t res;
831 lck_rw_type_t lck_rw_type;
832
833 if ((lck_sleep_action & ~LCK_SLEEP_MASK) != 0)
834 panic("Invalid lock sleep action %x\n", lck_sleep_action);
835
836 res = assert_wait(event, interruptible);
837 if (res == THREAD_WAITING) {
838 lck_rw_type = lck_rw_done(lck);
839 res = thread_block(THREAD_CONTINUE_NULL);
840 if (!(lck_sleep_action & LCK_SLEEP_UNLOCK)) {
841 if (!(lck_sleep_action & (LCK_SLEEP_SHARED|LCK_SLEEP_EXCLUSIVE)))
842 lck_rw_lock(lck, lck_rw_type);
843 else if (lck_sleep_action & LCK_SLEEP_EXCLUSIVE)
844 lck_rw_lock_exclusive(lck);
845 else
846 lck_rw_lock_shared(lck);
847 }
848 }
849 else
850 if (lck_sleep_action & LCK_SLEEP_UNLOCK)
851 (void)lck_rw_done(lck);
852
853 return res;
854}
855
856
857/*
858 * Routine: lck_rw_sleep_deadline
859 */
860wait_result_t
861lck_rw_sleep_deadline(
862 lck_rw_t *lck,
863 lck_sleep_action_t lck_sleep_action,
864 event_t event,
865 wait_interrupt_t interruptible,
866 uint64_t deadline)
867{
868 wait_result_t res;
869 lck_rw_type_t lck_rw_type;
870
871 if ((lck_sleep_action & ~LCK_SLEEP_MASK) != 0)
872 panic("Invalid lock sleep action %x\n", lck_sleep_action);
873
874 res = assert_wait_deadline(event, interruptible, deadline);
875 if (res == THREAD_WAITING) {
876 lck_rw_type = lck_rw_done(lck);
877 res = thread_block(THREAD_CONTINUE_NULL);
878 if (!(lck_sleep_action & LCK_SLEEP_UNLOCK)) {
879 if (!(lck_sleep_action & (LCK_SLEEP_SHARED|LCK_SLEEP_EXCLUSIVE)))
880 lck_rw_lock(lck, lck_rw_type);
881 else if (lck_sleep_action & LCK_SLEEP_EXCLUSIVE)
882 lck_rw_lock_exclusive(lck);
883 else
884 lck_rw_lock_shared(lck);
885 }
886 }
887 else
888 if (lck_sleep_action & LCK_SLEEP_UNLOCK)
889 (void)lck_rw_done(lck);
890
891 return res;
892}
893
894kern_return_t
895host_lockgroup_info(
896 host_t host,
897 lockgroup_info_array_t *lockgroup_infop,
898 mach_msg_type_number_t *lockgroup_infoCntp)
899{
900 lockgroup_info_t *lockgroup_info_base;
901 lockgroup_info_t *lockgroup_info;
902 vm_offset_t lockgroup_info_addr;
903 vm_size_t lockgroup_info_size;
904 lck_grp_t *lck_grp;
905 unsigned int i;
906 vm_size_t used;
907 vm_map_copy_t copy;
908 kern_return_t kr;
909
910 if (host == HOST_NULL)
911 return KERN_INVALID_HOST;
912
913 mutex_lock(&lck_grp_lock);
914
915 lockgroup_info_size = round_page(lck_grp_cnt * sizeof *lockgroup_info);
916 kr = kmem_alloc_pageable(ipc_kernel_map,
917 &lockgroup_info_addr, lockgroup_info_size);
918 if (kr != KERN_SUCCESS) {
919 mutex_unlock(&lck_grp_lock);
920 return(kr);
921 }
922
923 lockgroup_info_base = (lockgroup_info_t *) lockgroup_info_addr;
924 lck_grp = (lck_grp_t *)queue_first(&lck_grp_queue);
925 lockgroup_info = lockgroup_info_base;
926
927 for (i = 0; i < lck_grp_cnt; i++) {
928
929 lockgroup_info->lock_spin_cnt = lck_grp->lck_grp_spincnt;
930 lockgroup_info->lock_spin_util_cnt = lck_grp->lck_grp_stat.lck_grp_spin_stat.lck_grp_spin_util_cnt;
931 lockgroup_info->lock_spin_held_cnt = lck_grp->lck_grp_stat.lck_grp_spin_stat.lck_grp_spin_held_cnt;
932 lockgroup_info->lock_spin_miss_cnt = lck_grp->lck_grp_stat.lck_grp_spin_stat.lck_grp_spin_miss_cnt;
933 lockgroup_info->lock_spin_held_max = lck_grp->lck_grp_stat.lck_grp_spin_stat.lck_grp_spin_held_max;
934 lockgroup_info->lock_spin_held_cum = lck_grp->lck_grp_stat.lck_grp_spin_stat.lck_grp_spin_held_cum;
935
936 lockgroup_info->lock_mtx_cnt = lck_grp->lck_grp_mtxcnt;
937 lockgroup_info->lock_mtx_util_cnt = lck_grp->lck_grp_stat.lck_grp_mtx_stat.lck_grp_mtx_util_cnt;
938 lockgroup_info->lock_mtx_held_cnt = lck_grp->lck_grp_stat.lck_grp_mtx_stat.lck_grp_mtx_held_cnt;
939 lockgroup_info->lock_mtx_miss_cnt = lck_grp->lck_grp_stat.lck_grp_mtx_stat.lck_grp_mtx_miss_cnt;
940 lockgroup_info->lock_mtx_wait_cnt = lck_grp->lck_grp_stat.lck_grp_mtx_stat.lck_grp_mtx_wait_cnt;
941 lockgroup_info->lock_mtx_held_max = lck_grp->lck_grp_stat.lck_grp_mtx_stat.lck_grp_mtx_held_max;
942 lockgroup_info->lock_mtx_held_cum = lck_grp->lck_grp_stat.lck_grp_mtx_stat.lck_grp_mtx_held_cum;
943 lockgroup_info->lock_mtx_wait_max = lck_grp->lck_grp_stat.lck_grp_mtx_stat.lck_grp_mtx_wait_max;
944 lockgroup_info->lock_mtx_wait_cum = lck_grp->lck_grp_stat.lck_grp_mtx_stat.lck_grp_mtx_wait_cum;
945
946 lockgroup_info->lock_rw_cnt = lck_grp->lck_grp_rwcnt;
947 lockgroup_info->lock_rw_util_cnt = lck_grp->lck_grp_stat.lck_grp_rw_stat.lck_grp_rw_util_cnt;
948 lockgroup_info->lock_rw_held_cnt = lck_grp->lck_grp_stat.lck_grp_rw_stat.lck_grp_rw_held_cnt;
949 lockgroup_info->lock_rw_miss_cnt = lck_grp->lck_grp_stat.lck_grp_rw_stat.lck_grp_rw_miss_cnt;
950 lockgroup_info->lock_rw_wait_cnt = lck_grp->lck_grp_stat.lck_grp_rw_stat.lck_grp_rw_wait_cnt;
951 lockgroup_info->lock_rw_held_max = lck_grp->lck_grp_stat.lck_grp_rw_stat.lck_grp_rw_held_max;
952 lockgroup_info->lock_rw_held_cum = lck_grp->lck_grp_stat.lck_grp_rw_stat.lck_grp_rw_held_cum;
953 lockgroup_info->lock_rw_wait_max = lck_grp->lck_grp_stat.lck_grp_rw_stat.lck_grp_rw_wait_max;
954 lockgroup_info->lock_rw_wait_cum = lck_grp->lck_grp_stat.lck_grp_rw_stat.lck_grp_rw_wait_cum;
955
956 (void) strncpy(lockgroup_info->lockgroup_name,lck_grp->lck_grp_name, LOCKGROUP_MAX_NAME);
957
958 lck_grp = (lck_grp_t *)(queue_next((queue_entry_t)(lck_grp)));
959 lockgroup_info++;
960 }
961
962 *lockgroup_infoCntp = lck_grp_cnt;
963 mutex_unlock(&lck_grp_lock);
964
965 used = (*lockgroup_infoCntp) * sizeof *lockgroup_info;
966
967 if (used != lockgroup_info_size)
968 bzero((char *) lockgroup_info, lockgroup_info_size - used);
969
970 kr = vm_map_copyin(ipc_kernel_map, (vm_map_address_t)lockgroup_info_addr,
971 (vm_map_size_t)lockgroup_info_size, TRUE, &copy);
972 assert(kr == KERN_SUCCESS);
973
974 *lockgroup_infop = (lockgroup_info_t *) copy;
975
976 return(KERN_SUCCESS);
977}
978
979/*
980 * Compatibility module
981 */
982
983extern lck_rw_t *lock_alloc_EXT( boolean_t can_sleep, unsigned short tag0, unsigned short tag1);
984extern void lock_done_EXT(lck_rw_t *lock);
985extern void lock_free_EXT(lck_rw_t *lock);
986extern void lock_init_EXT(lck_rw_t *lock, boolean_t can_sleep, unsigned short tag0, unsigned short tag1);
987extern void lock_read_EXT(lck_rw_t *lock);
988extern boolean_t lock_read_to_write_EXT(lck_rw_t *lock);
989extern void lock_write_EXT(lck_rw_t *lock);
990extern void lock_write_to_read_EXT(lck_rw_t *lock);
991extern wait_result_t thread_sleep_lock_write_EXT(
992 event_t event, lck_rw_t *lock, wait_interrupt_t interruptible);
993
994extern lck_mtx_t *mutex_alloc_EXT(unsigned short tag);
995extern void mutex_free_EXT(lck_mtx_t *mutex);
996extern void mutex_init_EXT(lck_mtx_t *mutex, unsigned short tag);
997extern void mutex_lock_EXT(lck_mtx_t *mutex);
998extern boolean_t mutex_try_EXT(lck_mtx_t *mutex);
999extern void mutex_unlock_EXT(lck_mtx_t *mutex);
1000extern wait_result_t thread_sleep_mutex_EXT(
1001 event_t event, lck_mtx_t *mutex, wait_interrupt_t interruptible);
1002extern wait_result_t thread_sleep_mutex_deadline_EXT(
1003 event_t event, lck_mtx_t *mutex, uint64_t deadline, wait_interrupt_t interruptible);
1004
1005extern void usimple_lock_EXT(lck_spin_t *lock);
1006extern void usimple_lock_init_EXT(lck_spin_t *lock, unsigned short tag);
1007extern unsigned int usimple_lock_try_EXT(lck_spin_t *lock);
1008extern void usimple_unlock_EXT(lck_spin_t *lock);
1009extern wait_result_t thread_sleep_usimple_lock_EXT(event_t event, lck_spin_t *lock, wait_interrupt_t interruptible);
1010
1011lck_rw_t *
1012lock_alloc_EXT(
1013 __unused boolean_t can_sleep,
1014 __unused unsigned short tag0,
1015 __unused unsigned short tag1)
1016{
1017 return( lck_rw_alloc_init( &LockCompatGroup, LCK_ATTR_NULL));
1018}
1019
1020void
1021lock_done_EXT(
1022 lck_rw_t *lock)
1023{
1024 (void) lck_rw_done(lock);
1025}
1026
1027void
1028lock_free_EXT(
1029 lck_rw_t *lock)
1030{
1031 lck_rw_free(lock, &LockCompatGroup);
1032}
1033
1034void
1035lock_init_EXT(
1036 lck_rw_t *lock,
1037 __unused boolean_t can_sleep,
1038 __unused unsigned short tag0,
1039 __unused unsigned short tag1)
1040{
1041 lck_rw_init(lock, &LockCompatGroup, LCK_ATTR_NULL);
1042}
1043
1044void
1045lock_read_EXT(
1046 lck_rw_t *lock)
1047{
1048 lck_rw_lock_shared( lock);
1049}
1050
1051boolean_t
1052lock_read_to_write_EXT(
1053 lck_rw_t *lock)
1054{
1055 return( lck_rw_lock_shared_to_exclusive(lock));
1056}
1057
1058void
1059lock_write_EXT(
1060 lck_rw_t *lock)
1061{
1062 lck_rw_lock_exclusive(lock);
1063}
1064
1065void
1066lock_write_to_read_EXT(
1067 lck_rw_t *lock)
1068{
1069 lck_rw_lock_exclusive_to_shared(lock);
1070}
1071
1072wait_result_t
1073thread_sleep_lock_write_EXT(
1074 event_t event,
1075 lck_rw_t *lock,
1076 wait_interrupt_t interruptible)
1077{
1078 return( lck_rw_sleep(lock, LCK_SLEEP_EXCLUSIVE, event, interruptible));
1079}
1080
1081lck_mtx_t *
1082mutex_alloc_EXT(
1083 __unused unsigned short tag)
1084{
1085 return(lck_mtx_alloc_init(&LockCompatGroup, LCK_ATTR_NULL));
1086}
1087
1088void
1089mutex_free_EXT(
1090 lck_mtx_t *mutex)
1091{
1092 lck_mtx_free(mutex, &LockCompatGroup);
1093}
1094
1095void
1096mutex_init_EXT(
1097 lck_mtx_t *mutex,
1098 __unused unsigned short tag)
1099{
1100 lck_mtx_init(mutex, &LockCompatGroup, LCK_ATTR_NULL);
1101}
1102
1103void
1104mutex_lock_EXT(
1105 lck_mtx_t *mutex)
1106{
1107 lck_mtx_lock(mutex);
1108}
1109
1110boolean_t
1111mutex_try_EXT(
1112 lck_mtx_t *mutex)
1113{
1114 return(lck_mtx_try_lock(mutex));
1115}
1116
1117void
1118mutex_unlock_EXT(
1119 lck_mtx_t *mutex)
1120{
1121 lck_mtx_unlock(mutex);
1122}
1123
1124wait_result_t
1125thread_sleep_mutex_EXT(
1126 event_t event,
1127 lck_mtx_t *mutex,
1128 wait_interrupt_t interruptible)
1129{
1130 return( lck_mtx_sleep(mutex, LCK_SLEEP_DEFAULT, event, interruptible));
1131}
1132
1133wait_result_t
1134thread_sleep_mutex_deadline_EXT(
1135 event_t event,
1136 lck_mtx_t *mutex,
1137 uint64_t deadline,
1138 wait_interrupt_t interruptible)
1139{
1140 return( lck_mtx_sleep_deadline(mutex, LCK_SLEEP_DEFAULT, event, interruptible, deadline));
1141}
1142
1143void
1144usimple_lock_EXT(
1145 lck_spin_t *lock)
1146{
1147 lck_spin_lock(lock);
1148}
1149
1150void
1151usimple_lock_init_EXT(
1152 lck_spin_t *lock,
1153 __unused unsigned short tag)
1154{
1155 lck_spin_init(lock, &LockCompatGroup, LCK_ATTR_NULL);
1156}
1157
1158unsigned int
1159usimple_lock_try_EXT(
1160 lck_spin_t *lock)
1161{
0c530ab8 1162 return(lck_spin_try_lock(lock));
91447636
A		 1163}
1164
1165void
1166usimple_unlock_EXT(
1167 lck_spin_t *lock)
1168{
1169 lck_spin_unlock(lock);
1170}
1171
1172wait_result_t
1173thread_sleep_usimple_lock_EXT(
1174 event_t event,
1175 lck_spin_t *lock,
1176 wait_interrupt_t interruptible)
1177{
1178 return( lck_spin_sleep(lock, LCK_SLEEP_DEFAULT, event, interruptible));
1179}
mirror of XNU