]> git.saurik.com Git - apple/xnu.git/blame - osfmk/kern/locks.c
projects / apple / xnu.git / blame
? search:
summary | shortlog | log | commit | commitdiff | tree
blob | blame (incremental) | history | HEAD
xnu-1228.15.4.tar.gz
[apple/xnu.git] / osfmk / kern / locks.c
CommitLineData
91447636 1/*
2d21ac55 2 * Copyright (c) 2000-2007 Apple Inc. All rights reserved.
91447636 3 *
2d21ac55 4 * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
91447636 5 *
2d21ac55
A		 6 * This file contains Original Code and/or Modifications of Original Code
7 * as defined in and that are subject to the Apple Public Source License
8 * Version 2.0 (the 'License'). You may not use this file except in
9 * compliance with the License. The rights granted to you under the License
10 * may not be used to create, or enable the creation or redistribution of,
11 * unlawful or unlicensed copies of an Apple operating system, or to
12 * circumvent, violate, or enable the circumvention or violation of, any
13 * terms of an Apple operating system software license agreement.
8f6c56a5 14 *
2d21ac55
A		 15 * Please obtain a copy of the License at
16 * http://www.opensource.apple.com/apsl/ and read it before using this file.
17 *
18 * The Original Code and all software distributed under the License are
19 * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
8f6c56a5
A		 20 * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
21 * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
2d21ac55
A		 22 * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
23 * Please see the License for the specific language governing rights and
24 * limitations under the License.
8f6c56a5 25 *
2d21ac55 26 * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
91447636
A		 27 */
28/*
29 * @OSF_COPYRIGHT@
30 */
31/*
32 * Mach Operating System
33 * Copyright (c) 1991,1990,1989,1988,1987 Carnegie Mellon University
34 * All Rights Reserved.
35 *
36 * Permission to use, copy, modify and distribute this software and its
37 * documentation is hereby granted, provided that both the copyright
38 * notice and this permission notice appear in all copies of the
39 * software, derivative works or modified versions, and any portions
40 * thereof, and that both notices appear in supporting documentation.
41 *
42 * CARNEGIE MELLON ALLOWS FREE USE OF THIS SOFTWARE IN ITS "AS IS"
43 * CONDITION. CARNEGIE MELLON DISCLAIMS ANY LIABILITY OF ANY KIND FOR
44 * ANY DAMAGES WHATSOEVER RESULTING FROM THE USE OF THIS SOFTWARE.
45 *
46 * Carnegie Mellon requests users of this software to return to
47 *
48 * Software Distribution Coordinator or Software.Distribution@CS.CMU.EDU
49 * School of Computer Science
50 * Carnegie Mellon University
51 * Pittsburgh PA 15213-3890
52 *
53 * any improvements or extensions that they make and grant Carnegie Mellon
54 * the rights to redistribute these changes.
55 */
56#include <mach_kdb.h>
57#include <mach_ldebug.h>
58#include <debug.h>
59
60#include <mach/kern_return.h>
61#include <mach/mach_host_server.h>
62#include <mach_debug/lockgroup_info.h>
63
64#include <kern/locks.h>
65#include <kern/misc_protos.h>
66#include <kern/kalloc.h>
67#include <kern/thread.h>
68#include <kern/processor.h>
69#include <kern/sched_prim.h>
70#include <kern/debug.h>
71#include <string.h>
72
73
74#include <sys/kdebug.h>
75
2d21ac55
A		 76#if CONFIG_DTRACE
77/*
78 * We need only enough declarations from the BSD-side to be able to
79 * test if our probe is active, and to call __dtrace_probe(). Setting
80 * NEED_DTRACE_DEFS gets a local copy of those definitions pulled in.
81 */
82#define NEED_DTRACE_DEFS
83#include <../bsd/sys/lockstat.h>
84#endif
85
91447636
A		 86#define LCK_MTX_SLEEP_CODE 0
87#define LCK_MTX_SLEEP_DEADLINE_CODE 1
88#define LCK_MTX_LCK_WAIT_CODE 2
89#define LCK_MTX_UNLCK_WAKEUP_CODE 3
90
91
92static queue_head_t lck_grp_queue;
93static unsigned int lck_grp_cnt;
94
95decl_mutex_data(static,lck_grp_lock)
96
97lck_grp_attr_t LockDefaultGroupAttr;
98lck_grp_t LockCompatGroup;
99lck_attr_t LockDefaultLckAttr;
100
101/*
102 * Routine: lck_mod_init
103 */
104
105void
106lck_mod_init(
107 void)
108{
109 queue_init(&lck_grp_queue);
110 mutex_init(&lck_grp_lock, 0);
111 lck_grp_cnt = 0;
112 lck_grp_attr_setdefault( &LockDefaultGroupAttr);
113 lck_grp_init( &LockCompatGroup, "Compatibility APIs", LCK_GRP_ATTR_NULL);
114 lck_attr_setdefault(&LockDefaultLckAttr);
115}
116
117/*
118 * Routine: lck_grp_attr_alloc_init
119 */
120
121lck_grp_attr_t *
122lck_grp_attr_alloc_init(
123 void)
124{
125 lck_grp_attr_t *attr;
126
127 if ((attr = (lck_grp_attr_t *)kalloc(sizeof(lck_grp_attr_t))) != 0)
128 lck_grp_attr_setdefault(attr);
129
130 return(attr);
131}
132
133
134/*
135 * Routine: lck_grp_attr_setdefault
136 */
137
138void
139lck_grp_attr_setdefault(
140 lck_grp_attr_t *attr)
141{
142 if (LcksOpts & enaLkStat)
143 attr->grp_attr_val = LCK_GRP_ATTR_STAT;
144 else
145 attr->grp_attr_val = 0;
146}
147
148
149/*
150 * Routine: lck_grp_attr_setstat
151 */
152
153void
154lck_grp_attr_setstat(
155 lck_grp_attr_t *attr)
156{
2d21ac55 157 (void)hw_atomic_or(&attr->grp_attr_val, LCK_GRP_ATTR_STAT);
91447636
A		 158}
159
160
161/*
162 * Routine: lck_grp_attr_free
163 */
164
165void
166lck_grp_attr_free(
167 lck_grp_attr_t *attr)
168{
169 kfree(attr, sizeof(lck_grp_attr_t));
170}
171
172
173/*
174 * Routine: lck_grp_alloc_init
175 */
176
177lck_grp_t *
178lck_grp_alloc_init(
179 const char* grp_name,
180 lck_grp_attr_t *attr)
181{
182 lck_grp_t *grp;
183
184 if ((grp = (lck_grp_t *)kalloc(sizeof(lck_grp_t))) != 0)
185 lck_grp_init(grp, grp_name, attr);
186
187 return(grp);
188}
189
190
191/*
192 * Routine: lck_grp_init
193 */
194
195void
196lck_grp_init(
197 lck_grp_t *grp,
198 const char* grp_name,
199 lck_grp_attr_t *attr)
200{
201 bzero((void *)grp, sizeof(lck_grp_t));
202
203 (void) strncpy(grp->lck_grp_name, grp_name, LCK_GRP_MAX_NAME);
204
205 if (attr != LCK_GRP_ATTR_NULL)
206 grp->lck_grp_attr = attr->grp_attr_val;
207 else if (LcksOpts & enaLkStat)
208 grp->lck_grp_attr = LCK_GRP_ATTR_STAT;
209 else
210 grp->lck_grp_attr = LCK_ATTR_NONE;
211
212 grp->lck_grp_refcnt = 1;
213
214 mutex_lock(&lck_grp_lock);
215 enqueue_tail(&lck_grp_queue, (queue_entry_t)grp);
216 lck_grp_cnt++;
217 mutex_unlock(&lck_grp_lock);
218
219}
220
221
222/*
223 * Routine: lck_grp_free
224 */
225
226void
227lck_grp_free(
228 lck_grp_t *grp)
229{
230 mutex_lock(&lck_grp_lock);
231 lck_grp_cnt--;
232 (void)remque((queue_entry_t)grp);
233 mutex_unlock(&lck_grp_lock);
234 lck_grp_deallocate(grp);
235}
236
237
238/*
239 * Routine: lck_grp_reference
240 */
241
242void
243lck_grp_reference(
244 lck_grp_t *grp)
245{
2d21ac55 246 (void)hw_atomic_add(&grp->lck_grp_refcnt, 1);
91447636
A		 247}
248
249
250/*
251 * Routine: lck_grp_deallocate
252 */
253
254void
255lck_grp_deallocate(
256 lck_grp_t *grp)
257{
2d21ac55 258 if (hw_atomic_sub(&grp->lck_grp_refcnt, 1) == 0)
91447636
A		 259 kfree(grp, sizeof(lck_grp_t));
260}
261
262/*
263 * Routine: lck_grp_lckcnt_incr
264 */
265
266void
267lck_grp_lckcnt_incr(
268 lck_grp_t *grp,
269 lck_type_t lck_type)
270{
271 unsigned int *lckcnt;
272
273 switch (lck_type) {
274 case LCK_TYPE_SPIN:
275 lckcnt = &grp->lck_grp_spincnt;
276 break;
277 case LCK_TYPE_MTX:
278 lckcnt = &grp->lck_grp_mtxcnt;
279 break;
280 case LCK_TYPE_RW:
281 lckcnt = &grp->lck_grp_rwcnt;
282 break;
283 default:
284 return panic("lck_grp_lckcnt_incr(): invalid lock type: %d\n", lck_type);
285 }
286
2d21ac55 287 (void)hw_atomic_add(lckcnt, 1);
91447636
A		 288}
289
290/*
291 * Routine: lck_grp_lckcnt_decr
292 */
293
294void
295lck_grp_lckcnt_decr(
296 lck_grp_t *grp,
297 lck_type_t lck_type)
298{
299 unsigned int *lckcnt;
300
301 switch (lck_type) {
302 case LCK_TYPE_SPIN:
303 lckcnt = &grp->lck_grp_spincnt;
304 break;
305 case LCK_TYPE_MTX:
306 lckcnt = &grp->lck_grp_mtxcnt;
307 break;
308 case LCK_TYPE_RW:
309 lckcnt = &grp->lck_grp_rwcnt;
310 break;
311 default:
312 return panic("lck_grp_lckcnt_decr(): invalid lock type: %d\n", lck_type);
313 }
314
2d21ac55 315 (void)hw_atomic_sub(lckcnt, 1);
91447636
A		 316}
317
318/*
319 * Routine: lck_attr_alloc_init
320 */
321
322lck_attr_t *
323lck_attr_alloc_init(
324 void)
325{
326 lck_attr_t *attr;
327
328 if ((attr = (lck_attr_t *)kalloc(sizeof(lck_attr_t))) != 0)
329 lck_attr_setdefault(attr);
330
331 return(attr);
332}
333
334
335/*
336 * Routine: lck_attr_setdefault
337 */
338
339void
340lck_attr_setdefault(
341 lck_attr_t *attr)
342{
343#if !DEBUG
593a1d5f
A		 344 if (LcksOpts & enaLkDeb)
345 attr->lck_attr_val = LCK_ATTR_DEBUG;
346 else
347 attr->lck_attr_val = LCK_ATTR_NONE;
91447636 348#else
593a1d5f
A		 349 attr->lck_attr_val = LCK_ATTR_DEBUG;
350#endif /* !DEBUG */
91447636
A		 351}
352
353
354/*
355 * Routine: lck_attr_setdebug
356 */
357void
358lck_attr_setdebug(
359 lck_attr_t *attr)
360{
2d21ac55
A		 361 (void)hw_atomic_or(&attr->lck_attr_val, LCK_ATTR_DEBUG);
362}
363
364/*
365 * Routine: lck_attr_setdebug
366 */
367void
368lck_attr_cleardebug(
369 lck_attr_t *attr)
370{
371 (void)hw_atomic_and(&attr->lck_attr_val, ~LCK_ATTR_DEBUG);
91447636
A		 372}
373
374
0c530ab8
A		 375/*
376 * Routine: lck_attr_rw_shared_priority
377 */
378void
379lck_attr_rw_shared_priority(
380 lck_attr_t *attr)
381{
2d21ac55 382 (void)hw_atomic_or(&attr->lck_attr_val, LCK_ATTR_RW_SHARED_PRIORITY);
0c530ab8
A		 383}
384
385
91447636
A		 386/*
387 * Routine: lck_attr_free
388 */
389void
390lck_attr_free(
391 lck_attr_t *attr)
392{
393 kfree(attr, sizeof(lck_attr_t));
394}
395
396
397/*
398 * Routine: lck_spin_sleep
399 */
400wait_result_t
401lck_spin_sleep(
402 lck_spin_t *lck,
403 lck_sleep_action_t lck_sleep_action,
404 event_t event,
405 wait_interrupt_t interruptible)
406{
407 wait_result_t res;
408
409 if ((lck_sleep_action & ~LCK_SLEEP_MASK) != 0)
410 panic("Invalid lock sleep action %x\n", lck_sleep_action);
411
412 res = assert_wait(event, interruptible);
413 if (res == THREAD_WAITING) {
414 lck_spin_unlock(lck);
415 res = thread_block(THREAD_CONTINUE_NULL);
416 if (!(lck_sleep_action & LCK_SLEEP_UNLOCK))
417 lck_spin_lock(lck);
418 }
419 else
420 if (lck_sleep_action & LCK_SLEEP_UNLOCK)
421 lck_spin_unlock(lck);
422
423 return res;
424}
425
426
427/*
428 * Routine: lck_spin_sleep_deadline
429 */
430wait_result_t
431lck_spin_sleep_deadline(
432 lck_spin_t *lck,
433 lck_sleep_action_t lck_sleep_action,
434 event_t event,
435 wait_interrupt_t interruptible,
436 uint64_t deadline)
437{
438 wait_result_t res;
439
440 if ((lck_sleep_action & ~LCK_SLEEP_MASK) != 0)
441 panic("Invalid lock sleep action %x\n", lck_sleep_action);
442
443 res = assert_wait_deadline(event, interruptible, deadline);
444 if (res == THREAD_WAITING) {
445 lck_spin_unlock(lck);
446 res = thread_block(THREAD_CONTINUE_NULL);
447 if (!(lck_sleep_action & LCK_SLEEP_UNLOCK))
448 lck_spin_lock(lck);
449 }
450 else
451 if (lck_sleep_action & LCK_SLEEP_UNLOCK)
452 lck_spin_unlock(lck);
453
454 return res;
455}
456
457
458/*
459 * Routine: lck_mtx_sleep
460 */
461wait_result_t
462lck_mtx_sleep(
463 lck_mtx_t *lck,
464 lck_sleep_action_t lck_sleep_action,
465 event_t event,
466 wait_interrupt_t interruptible)
467{
468 wait_result_t res;
469
470 KERNEL_DEBUG(MACHDBG_CODE(DBG_MACH_LOCKS, LCK_MTX_SLEEP_CODE) | DBG_FUNC_START,
471 (int)lck, (int)lck_sleep_action, (int)event, (int)interruptible, 0);
472
473 if ((lck_sleep_action & ~LCK_SLEEP_MASK) != 0)
474 panic("Invalid lock sleep action %x\n", lck_sleep_action);
475
476 res = assert_wait(event, interruptible);
477 if (res == THREAD_WAITING) {
478 lck_mtx_unlock(lck);
479 res = thread_block(THREAD_CONTINUE_NULL);
480 if (!(lck_sleep_action & LCK_SLEEP_UNLOCK))
481 lck_mtx_lock(lck);
482 }
483 else
484 if (lck_sleep_action & LCK_SLEEP_UNLOCK)
485 lck_mtx_unlock(lck);
486
487 KERNEL_DEBUG(MACHDBG_CODE(DBG_MACH_LOCKS, LCK_MTX_SLEEP_CODE) | DBG_FUNC_END, (int)res, 0, 0, 0, 0);
488
489 return res;
490}
491
492
493/*
494 * Routine: lck_mtx_sleep_deadline
495 */
496wait_result_t
497lck_mtx_sleep_deadline(
498 lck_mtx_t *lck,
499 lck_sleep_action_t lck_sleep_action,
500 event_t event,
501 wait_interrupt_t interruptible,
502 uint64_t deadline)
503{
504 wait_result_t res;
505
506 KERNEL_DEBUG(MACHDBG_CODE(DBG_MACH_LOCKS, LCK_MTX_SLEEP_DEADLINE_CODE) | DBG_FUNC_START,
507 (int)lck, (int)lck_sleep_action, (int)event, (int)interruptible, 0);
508
509 if ((lck_sleep_action & ~LCK_SLEEP_MASK) != 0)
510 panic("Invalid lock sleep action %x\n", lck_sleep_action);
511
512 res = assert_wait_deadline(event, interruptible, deadline);
513 if (res == THREAD_WAITING) {
514 lck_mtx_unlock(lck);
515 res = thread_block(THREAD_CONTINUE_NULL);
516 if (!(lck_sleep_action & LCK_SLEEP_UNLOCK))
517 lck_mtx_lock(lck);
518 }
519 else
520 if (lck_sleep_action & LCK_SLEEP_UNLOCK)
521 lck_mtx_unlock(lck);
522
523 KERNEL_DEBUG(MACHDBG_CODE(DBG_MACH_LOCKS, LCK_MTX_SLEEP_DEADLINE_CODE) | DBG_FUNC_END, (int)res, 0, 0, 0, 0);
524
525 return res;
526}
527
528/*
529 * Routine: lck_mtx_lock_wait
530 *
531 * Invoked in order to wait on contention.
532 *
533 * Called with the interlock locked and
534 * returns it unlocked.
535 */
536void
537lck_mtx_lock_wait (
538 lck_mtx_t *lck,
539 thread_t holder)
540{
541 thread_t self = current_thread();
542 lck_mtx_t *mutex;
543 integer_t priority;
544 spl_t s = splsched();
2d21ac55
A		 545#if CONFIG_DTRACE
546 uint64_t sleep_start = 0;
547
548 if (lockstat_probemap[LS_LCK_MTX_LOCK_BLOCK] || lockstat_probemap[LS_LCK_MTX_EXT_LOCK_BLOCK]) {
549 sleep_start = mach_absolute_time();
550 }
551#endif
91447636
A		 552
553 if (lck->lck_mtx_tag != LCK_MTX_TAG_INDIRECT)
554 mutex = lck;
555 else
556 mutex = &lck->lck_mtx_ptr->lck_mtx;
557
558 KERNEL_DEBUG(MACHDBG_CODE(DBG_MACH_LOCKS, LCK_MTX_LCK_WAIT_CODE) | DBG_FUNC_START, (int)lck, (int)holder, 0, 0, 0);
559
560 priority = self->sched_pri;
561 if (priority < self->priority)
562 priority = self->priority;
91447636
A		 563 if (priority < BASEPRI_DEFAULT)
564 priority = BASEPRI_DEFAULT;
565
566 thread_lock(holder);
567 if (mutex->lck_mtx_pri == 0)
568 holder->promotions++;
4a3eedf9
A		 569 holder->sched_mode |= TH_MODE_PROMOTED;
570 if ( mutex->lck_mtx_pri < priority &&
91447636 571 holder->sched_pri < priority ) {
4a3eedf9
A		 572 KERNEL_DEBUG_CONSTANT(
573 MACHDBG_CODE(DBG_MACH_SCHED,MACH_PROMOTE) | DBG_FUNC_NONE,
91447636
A		 574 holder->sched_pri, priority, (int)holder, (int)lck, 0);
575
4a3eedf9 576 set_sched_pri(holder, priority);
91447636
A		 577 }
578 thread_unlock(holder);
579 splx(s);
580
581 if (mutex->lck_mtx_pri < priority)
582 mutex->lck_mtx_pri = priority;
583 if (self->pending_promoter[self->pending_promoter_index] == NULL) {
584 self->pending_promoter[self->pending_promoter_index] = mutex;
585 mutex->lck_mtx_waiters++;
586 }
587 else
588 if (self->pending_promoter[self->pending_promoter_index] != mutex) {
589 self->pending_promoter[++self->pending_promoter_index] = mutex;
590 mutex->lck_mtx_waiters++;
591 }
592
593 assert_wait((event_t)(((unsigned int*)lck)+((sizeof(lck_mtx_t)-1)/sizeof(unsigned int))), THREAD_UNINT);
594 lck_mtx_ilk_unlock(mutex);
595
596 thread_block(THREAD_CONTINUE_NULL);
597
598 KERNEL_DEBUG(MACHDBG_CODE(DBG_MACH_LOCKS, LCK_MTX_LCK_WAIT_CODE) | DBG_FUNC_END, 0, 0, 0, 0, 0);
2d21ac55
A		 599#if CONFIG_DTRACE
600 /*
601 * Record the Dtrace lockstat probe for blocking, block time
602 * measured from when we were entered.
603 */
604 if (sleep_start) {
605 if (lck->lck_mtx_tag != LCK_MTX_TAG_INDIRECT) {
606 LOCKSTAT_RECORD(LS_LCK_MTX_LOCK_BLOCK, lck,
607 mach_absolute_time() - sleep_start);
608 } else {
609 LOCKSTAT_RECORD(LS_LCK_MTX_EXT_LOCK_BLOCK, lck,
610 mach_absolute_time() - sleep_start);
611 }
612 }
613#endif
91447636
A		 614}
615
616/*
617 * Routine: lck_mtx_lock_acquire
618 *
619 * Invoked on acquiring the mutex when there is
620 * contention.
621 *
622 * Returns the current number of waiters.
623 *
624 * Called with the interlock locked.
625 */
626int
627lck_mtx_lock_acquire(
628 lck_mtx_t *lck)
629{
630 thread_t thread = current_thread();
631 lck_mtx_t *mutex;
632
633 if (lck->lck_mtx_tag != LCK_MTX_TAG_INDIRECT)
634 mutex = lck;
635 else
636 mutex = &lck->lck_mtx_ptr->lck_mtx;
637
638 if (thread->pending_promoter[thread->pending_promoter_index] == mutex) {
639 thread->pending_promoter[thread->pending_promoter_index] = NULL;
640 if (thread->pending_promoter_index > 0)
641 thread->pending_promoter_index--;
642 mutex->lck_mtx_waiters--;
643 }
644
645 if (mutex->lck_mtx_waiters > 0) {
646 integer_t priority = mutex->lck_mtx_pri;
647 spl_t s = splsched();
648
649 thread_lock(thread);
650 thread->promotions++;
4a3eedf9
A		 651 thread->sched_mode |= TH_MODE_PROMOTED;
652 if (thread->sched_pri < priority) {
653 KERNEL_DEBUG_CONSTANT(
654 MACHDBG_CODE(DBG_MACH_SCHED,MACH_PROMOTE) | DBG_FUNC_NONE,
91447636
A		 655 thread->sched_pri, priority, 0, (int)lck, 0);
656
4a3eedf9 657 set_sched_pri(thread, priority);
91447636
A		 658 }
659 thread_unlock(thread);
660 splx(s);
661 }
662 else
663 mutex->lck_mtx_pri = 0;
664
665 return (mutex->lck_mtx_waiters);
666}
667
668/*
669 * Routine: lck_mtx_unlock_wakeup
670 *
671 * Invoked on unlock when there is contention.
672 *
673 * Called with the interlock locked.
674 */
675void
676lck_mtx_unlock_wakeup (
677 lck_mtx_t *lck,
678 thread_t holder)
679{
680 thread_t thread = current_thread();
681 lck_mtx_t *mutex;
682
683 if (lck->lck_mtx_tag != LCK_MTX_TAG_INDIRECT)
684 mutex = lck;
685 else
686 mutex = &lck->lck_mtx_ptr->lck_mtx;
687
688
689 KERNEL_DEBUG(MACHDBG_CODE(DBG_MACH_LOCKS, LCK_MTX_UNLCK_WAKEUP_CODE) | DBG_FUNC_START, (int)lck, (int)holder, 0, 0, 0);
690
691 if (thread != holder)
2d21ac55 692 panic("lck_mtx_unlock_wakeup: mutex %p holder %p\n", mutex, holder);
91447636
A		 693
694 if (thread->promotions > 0) {
695 spl_t s = splsched();
696
697 thread_lock(thread);
698 if ( --thread->promotions == 0 &&
699 (thread->sched_mode & TH_MODE_PROMOTED) ) {
700 thread->sched_mode &= ~TH_MODE_PROMOTED;
701 if (thread->sched_mode & TH_MODE_ISDEPRESSED) {
702 KERNEL_DEBUG_CONSTANT(
703 MACHDBG_CODE(DBG_MACH_SCHED,MACH_DEMOTE) | DBG_FUNC_NONE,
704 thread->sched_pri, DEPRESSPRI, 0, (int)lck, 0);
705
706 set_sched_pri(thread, DEPRESSPRI);
707 }
708 else {
709 if (thread->priority < thread->sched_pri) {
710 KERNEL_DEBUG_CONSTANT(
711 MACHDBG_CODE(DBG_MACH_SCHED,MACH_DEMOTE) |
712 DBG_FUNC_NONE,
713 thread->sched_pri, thread->priority,
714 0, (int)lck, 0);
715 }
716
717 compute_priority(thread, FALSE);
718 }
719 }
720 thread_unlock(thread);
721 splx(s);
722 }
723 assert(mutex->lck_mtx_waiters > 0);
724 thread_wakeup_one((event_t)(((unsigned int*)lck)+(sizeof(lck_mtx_t)-1)/sizeof(unsigned int)));
725
726 KERNEL_DEBUG(MACHDBG_CODE(DBG_MACH_LOCKS, LCK_MTX_UNLCK_WAKEUP_CODE) | DBG_FUNC_END, 0, 0, 0, 0, 0);
727}
728
2d21ac55
A		 729void
730lck_mtx_unlockspin_wakeup (
731 lck_mtx_t *lck)
732{
733 assert(lck->lck_mtx_waiters > 0);
734 thread_wakeup_one((event_t)(((unsigned int*)lck)+(sizeof(lck_mtx_t)-1)/sizeof(unsigned int)));
735
736 KERNEL_DEBUG(MACHDBG_CODE(DBG_MACH_LOCKS, LCK_MTX_UNLCK_WAKEUP_CODE) | DBG_FUNC_NONE, (int)lck, 0, 0, 1, 0);
737#if CONFIG_DTRACE
738 /*
739 * When there are waiters, we skip the hot-patch spot in the
740 * fastpath, so we record it here.
741 */
742 LOCKSTAT_RECORD(LS_LCK_MTX_UNLOCK_RELEASE, lck, 0);
743#endif
744}
745
746
91447636
A		 747/*
748 * Routine: mutex_pause
749 *
750 * Called by former callers of simple_lock_pause().
751 */
2d21ac55
A		 752#define MAX_COLLISION_COUNTS 32
753#define MAX_COLLISION 8
754
755unsigned int max_collision_count[MAX_COLLISION_COUNTS];
756
757uint32_t collision_backoffs[MAX_COLLISION] = {
758 10, 50, 100, 200, 400, 600, 800, 1000
759};
760
91447636
A		 761
762void
2d21ac55 763mutex_pause(uint32_t collisions)
91447636
A		 764{
765 wait_result_t wait_result;
2d21ac55 766 uint32_t back_off;
91447636 767
2d21ac55
A		 768 if (collisions >= MAX_COLLISION_COUNTS)
769 collisions = MAX_COLLISION_COUNTS - 1;
770 max_collision_count[collisions]++;
771
772 if (collisions >= MAX_COLLISION)
773 collisions = MAX_COLLISION - 1;
774 back_off = collision_backoffs[collisions];
775
776 wait_result = assert_wait_timeout((event_t)mutex_pause, THREAD_UNINT, back_off, NSEC_PER_USEC);
91447636
A		 777 assert(wait_result == THREAD_WAITING);
778
779 wait_result = thread_block(THREAD_CONTINUE_NULL);
780 assert(wait_result == THREAD_TIMED_OUT);
781}
782
2d21ac55
A		 783
784unsigned int mutex_yield_wait = 0;
785unsigned int mutex_yield_no_wait = 0;
786
787void
788mutex_yield(
789 mutex_t *mutex)
790{
791 lck_mtx_t *lck;
792
793#if DEBUG
794 _mutex_assert(mutex, MA_OWNED);
795#endif /* DEBUG */
796
797 lck = (lck_mtx_t *) mutex;
798 if (lck->lck_mtx_tag == LCK_MTX_TAG_INDIRECT)
799 lck = &lck->lck_mtx_ptr->lck_mtx;
800
801 if (! lck->lck_mtx_waiters) {
802 mutex_yield_no_wait++;
803 } else {
804 mutex_yield_wait++;
805 mutex_unlock(mutex);
806 mutex_pause(0);
807 mutex_lock(mutex);
808 }
809}
810
811
91447636
A		 812/*
813 * Routine: lck_rw_sleep
814 */
815wait_result_t
816lck_rw_sleep(
817 lck_rw_t *lck,
818 lck_sleep_action_t lck_sleep_action,
819 event_t event,
820 wait_interrupt_t interruptible)
821{
822 wait_result_t res;
823 lck_rw_type_t lck_rw_type;
824
825 if ((lck_sleep_action & ~LCK_SLEEP_MASK) != 0)
826 panic("Invalid lock sleep action %x\n", lck_sleep_action);
827
828 res = assert_wait(event, interruptible);
829 if (res == THREAD_WAITING) {
830 lck_rw_type = lck_rw_done(lck);
831 res = thread_block(THREAD_CONTINUE_NULL);
832 if (!(lck_sleep_action & LCK_SLEEP_UNLOCK)) {
833 if (!(lck_sleep_action & (LCK_SLEEP_SHARED|LCK_SLEEP_EXCLUSIVE)))
834 lck_rw_lock(lck, lck_rw_type);
835 else if (lck_sleep_action & LCK_SLEEP_EXCLUSIVE)
836 lck_rw_lock_exclusive(lck);
837 else
838 lck_rw_lock_shared(lck);
839 }
840 }
841 else
842 if (lck_sleep_action & LCK_SLEEP_UNLOCK)
843 (void)lck_rw_done(lck);
844
845 return res;
846}
847
848
849/*
850 * Routine: lck_rw_sleep_deadline
851 */
852wait_result_t
853lck_rw_sleep_deadline(
854 lck_rw_t *lck,
855 lck_sleep_action_t lck_sleep_action,
856 event_t event,
857 wait_interrupt_t interruptible,
858 uint64_t deadline)
859{
860 wait_result_t res;
861 lck_rw_type_t lck_rw_type;
862
863 if ((lck_sleep_action & ~LCK_SLEEP_MASK) != 0)
864 panic("Invalid lock sleep action %x\n", lck_sleep_action);
865
866 res = assert_wait_deadline(event, interruptible, deadline);
867 if (res == THREAD_WAITING) {
868 lck_rw_type = lck_rw_done(lck);
869 res = thread_block(THREAD_CONTINUE_NULL);
870 if (!(lck_sleep_action & LCK_SLEEP_UNLOCK)) {
871 if (!(lck_sleep_action & (LCK_SLEEP_SHARED|LCK_SLEEP_EXCLUSIVE)))
872 lck_rw_lock(lck, lck_rw_type);
873 else if (lck_sleep_action & LCK_SLEEP_EXCLUSIVE)
874 lck_rw_lock_exclusive(lck);
875 else
876 lck_rw_lock_shared(lck);
877 }
878 }
879 else
880 if (lck_sleep_action & LCK_SLEEP_UNLOCK)
881 (void)lck_rw_done(lck);
882
883 return res;
884}
885
886kern_return_t
887host_lockgroup_info(
888 host_t host,
889 lockgroup_info_array_t *lockgroup_infop,
890 mach_msg_type_number_t *lockgroup_infoCntp)
891{
892 lockgroup_info_t *lockgroup_info_base;
893 lockgroup_info_t *lockgroup_info;
894 vm_offset_t lockgroup_info_addr;
895 vm_size_t lockgroup_info_size;
896 lck_grp_t *lck_grp;
897 unsigned int i;
898 vm_size_t used;
899 vm_map_copy_t copy;
900 kern_return_t kr;
901
902 if (host == HOST_NULL)
903 return KERN_INVALID_HOST;
904
905 mutex_lock(&lck_grp_lock);
906
907 lockgroup_info_size = round_page(lck_grp_cnt * sizeof *lockgroup_info);
908 kr = kmem_alloc_pageable(ipc_kernel_map,
909 &lockgroup_info_addr, lockgroup_info_size);
910 if (kr != KERN_SUCCESS) {
911 mutex_unlock(&lck_grp_lock);
912 return(kr);
913 }
914
915 lockgroup_info_base = (lockgroup_info_t *) lockgroup_info_addr;
916 lck_grp = (lck_grp_t *)queue_first(&lck_grp_queue);
917 lockgroup_info = lockgroup_info_base;
918
919 for (i = 0; i < lck_grp_cnt; i++) {
920
921 lockgroup_info->lock_spin_cnt = lck_grp->lck_grp_spincnt;
922 lockgroup_info->lock_spin_util_cnt = lck_grp->lck_grp_stat.lck_grp_spin_stat.lck_grp_spin_util_cnt;
923 lockgroup_info->lock_spin_held_cnt = lck_grp->lck_grp_stat.lck_grp_spin_stat.lck_grp_spin_held_cnt;
924 lockgroup_info->lock_spin_miss_cnt = lck_grp->lck_grp_stat.lck_grp_spin_stat.lck_grp_spin_miss_cnt;
925 lockgroup_info->lock_spin_held_max = lck_grp->lck_grp_stat.lck_grp_spin_stat.lck_grp_spin_held_max;
926 lockgroup_info->lock_spin_held_cum = lck_grp->lck_grp_stat.lck_grp_spin_stat.lck_grp_spin_held_cum;
927
928 lockgroup_info->lock_mtx_cnt = lck_grp->lck_grp_mtxcnt;
929 lockgroup_info->lock_mtx_util_cnt = lck_grp->lck_grp_stat.lck_grp_mtx_stat.lck_grp_mtx_util_cnt;
930 lockgroup_info->lock_mtx_held_cnt = lck_grp->lck_grp_stat.lck_grp_mtx_stat.lck_grp_mtx_held_cnt;
931 lockgroup_info->lock_mtx_miss_cnt = lck_grp->lck_grp_stat.lck_grp_mtx_stat.lck_grp_mtx_miss_cnt;
932 lockgroup_info->lock_mtx_wait_cnt = lck_grp->lck_grp_stat.lck_grp_mtx_stat.lck_grp_mtx_wait_cnt;
933 lockgroup_info->lock_mtx_held_max = lck_grp->lck_grp_stat.lck_grp_mtx_stat.lck_grp_mtx_held_max;
934 lockgroup_info->lock_mtx_held_cum = lck_grp->lck_grp_stat.lck_grp_mtx_stat.lck_grp_mtx_held_cum;
935 lockgroup_info->lock_mtx_wait_max = lck_grp->lck_grp_stat.lck_grp_mtx_stat.lck_grp_mtx_wait_max;
936 lockgroup_info->lock_mtx_wait_cum = lck_grp->lck_grp_stat.lck_grp_mtx_stat.lck_grp_mtx_wait_cum;
937
938 lockgroup_info->lock_rw_cnt = lck_grp->lck_grp_rwcnt;
939 lockgroup_info->lock_rw_util_cnt = lck_grp->lck_grp_stat.lck_grp_rw_stat.lck_grp_rw_util_cnt;
940 lockgroup_info->lock_rw_held_cnt = lck_grp->lck_grp_stat.lck_grp_rw_stat.lck_grp_rw_held_cnt;
941 lockgroup_info->lock_rw_miss_cnt = lck_grp->lck_grp_stat.lck_grp_rw_stat.lck_grp_rw_miss_cnt;
942 lockgroup_info->lock_rw_wait_cnt = lck_grp->lck_grp_stat.lck_grp_rw_stat.lck_grp_rw_wait_cnt;
943 lockgroup_info->lock_rw_held_max = lck_grp->lck_grp_stat.lck_grp_rw_stat.lck_grp_rw_held_max;
944 lockgroup_info->lock_rw_held_cum = lck_grp->lck_grp_stat.lck_grp_rw_stat.lck_grp_rw_held_cum;
945 lockgroup_info->lock_rw_wait_max = lck_grp->lck_grp_stat.lck_grp_rw_stat.lck_grp_rw_wait_max;
946 lockgroup_info->lock_rw_wait_cum = lck_grp->lck_grp_stat.lck_grp_rw_stat.lck_grp_rw_wait_cum;
947
948 (void) strncpy(lockgroup_info->lockgroup_name,lck_grp->lck_grp_name, LOCKGROUP_MAX_NAME);
949
950 lck_grp = (lck_grp_t *)(queue_next((queue_entry_t)(lck_grp)));
951 lockgroup_info++;
952 }
953
954 *lockgroup_infoCntp = lck_grp_cnt;
955 mutex_unlock(&lck_grp_lock);
956
957 used = (*lockgroup_infoCntp) * sizeof *lockgroup_info;
958
959 if (used != lockgroup_info_size)
960 bzero((char *) lockgroup_info, lockgroup_info_size - used);
961
962 kr = vm_map_copyin(ipc_kernel_map, (vm_map_address_t)lockgroup_info_addr,
963 (vm_map_size_t)lockgroup_info_size, TRUE, &copy);
964 assert(kr == KERN_SUCCESS);
965
966 *lockgroup_infop = (lockgroup_info_t *) copy;
967
968 return(KERN_SUCCESS);
969}
970
971/*
972 * Compatibility module
973 */
974
975extern lck_rw_t *lock_alloc_EXT( boolean_t can_sleep, unsigned short tag0, unsigned short tag1);
976extern void lock_done_EXT(lck_rw_t *lock);
977extern void lock_free_EXT(lck_rw_t *lock);
978extern void lock_init_EXT(lck_rw_t *lock, boolean_t can_sleep, unsigned short tag0, unsigned short tag1);
979extern void lock_read_EXT(lck_rw_t *lock);
980extern boolean_t lock_read_to_write_EXT(lck_rw_t *lock);
981extern void lock_write_EXT(lck_rw_t *lock);
982extern void lock_write_to_read_EXT(lck_rw_t *lock);
983extern wait_result_t thread_sleep_lock_write_EXT(
984 event_t event, lck_rw_t *lock, wait_interrupt_t interruptible);
985
986extern lck_mtx_t *mutex_alloc_EXT(unsigned short tag);
987extern void mutex_free_EXT(lck_mtx_t *mutex);
988extern void mutex_init_EXT(lck_mtx_t *mutex, unsigned short tag);
989extern void mutex_lock_EXT(lck_mtx_t *mutex);
990extern boolean_t mutex_try_EXT(lck_mtx_t *mutex);
991extern void mutex_unlock_EXT(lck_mtx_t *mutex);
992extern wait_result_t thread_sleep_mutex_EXT(
993 event_t event, lck_mtx_t *mutex, wait_interrupt_t interruptible);
994extern wait_result_t thread_sleep_mutex_deadline_EXT(
995 event_t event, lck_mtx_t *mutex, uint64_t deadline, wait_interrupt_t interruptible);
996
997extern void usimple_lock_EXT(lck_spin_t *lock);
998extern void usimple_lock_init_EXT(lck_spin_t *lock, unsigned short tag);
999extern unsigned int usimple_lock_try_EXT(lck_spin_t *lock);
1000extern void usimple_unlock_EXT(lck_spin_t *lock);
1001extern wait_result_t thread_sleep_usimple_lock_EXT(event_t event, lck_spin_t *lock, wait_interrupt_t interruptible);
1002
1003lck_rw_t *
1004lock_alloc_EXT(
1005 __unused boolean_t can_sleep,
1006 __unused unsigned short tag0,
1007 __unused unsigned short tag1)
1008{
1009 return( lck_rw_alloc_init( &LockCompatGroup, LCK_ATTR_NULL));
1010}
1011
1012void
1013lock_done_EXT(
1014 lck_rw_t *lock)
1015{
1016 (void) lck_rw_done(lock);
1017}
1018
1019void
1020lock_free_EXT(
1021 lck_rw_t *lock)
1022{
1023 lck_rw_free(lock, &LockCompatGroup);
1024}
1025
1026void
1027lock_init_EXT(
1028 lck_rw_t *lock,
1029 __unused boolean_t can_sleep,
1030 __unused unsigned short tag0,
1031 __unused unsigned short tag1)
1032{
1033 lck_rw_init(lock, &LockCompatGroup, LCK_ATTR_NULL);
1034}
1035
1036void
1037lock_read_EXT(
1038 lck_rw_t *lock)
1039{
1040 lck_rw_lock_shared( lock);
1041}
1042
1043boolean_t
1044lock_read_to_write_EXT(
1045 lck_rw_t *lock)
1046{
1047 return( lck_rw_lock_shared_to_exclusive(lock));
1048}
1049
1050void
1051lock_write_EXT(
1052 lck_rw_t *lock)
1053{
1054 lck_rw_lock_exclusive(lock);
1055}
1056
1057void
1058lock_write_to_read_EXT(
1059 lck_rw_t *lock)
1060{
1061 lck_rw_lock_exclusive_to_shared(lock);
1062}
1063
1064wait_result_t
1065thread_sleep_lock_write_EXT(
1066 event_t event,
1067 lck_rw_t *lock,
1068 wait_interrupt_t interruptible)
1069{
1070 return( lck_rw_sleep(lock, LCK_SLEEP_EXCLUSIVE, event, interruptible));
1071}
1072
1073lck_mtx_t *
1074mutex_alloc_EXT(
1075 __unused unsigned short tag)
1076{
1077 return(lck_mtx_alloc_init(&LockCompatGroup, LCK_ATTR_NULL));
1078}
1079
1080void
1081mutex_free_EXT(
1082 lck_mtx_t *mutex)
1083{
1084 lck_mtx_free(mutex, &LockCompatGroup);
1085}
1086
1087void
1088mutex_init_EXT(
1089 lck_mtx_t *mutex,
1090 __unused unsigned short tag)
1091{
1092 lck_mtx_init(mutex, &LockCompatGroup, LCK_ATTR_NULL);
1093}
1094
1095void
1096mutex_lock_EXT(
1097 lck_mtx_t *mutex)
1098{
1099 lck_mtx_lock(mutex);
1100}
1101
1102boolean_t
1103mutex_try_EXT(
1104 lck_mtx_t *mutex)
1105{
1106 return(lck_mtx_try_lock(mutex));
1107}
1108
1109void
1110mutex_unlock_EXT(
1111 lck_mtx_t *mutex)
1112{
1113 lck_mtx_unlock(mutex);
1114}
1115
1116wait_result_t
1117thread_sleep_mutex_EXT(
1118 event_t event,
1119 lck_mtx_t *mutex,
1120 wait_interrupt_t interruptible)
1121{
1122 return( lck_mtx_sleep(mutex, LCK_SLEEP_DEFAULT, event, interruptible));
1123}
1124
1125wait_result_t
1126thread_sleep_mutex_deadline_EXT(
1127 event_t event,
1128 lck_mtx_t *mutex,
1129 uint64_t deadline,
1130 wait_interrupt_t interruptible)
1131{
1132 return( lck_mtx_sleep_deadline(mutex, LCK_SLEEP_DEFAULT, event, interruptible, deadline));
1133}
1134
1135void
1136usimple_lock_EXT(
1137 lck_spin_t *lock)
1138{
1139 lck_spin_lock(lock);
1140}
1141
1142void
1143usimple_lock_init_EXT(
1144 lck_spin_t *lock,
1145 __unused unsigned short tag)
1146{
1147 lck_spin_init(lock, &LockCompatGroup, LCK_ATTR_NULL);
1148}
1149
1150unsigned int
1151usimple_lock_try_EXT(
1152 lck_spin_t *lock)
1153{
0c530ab8 1154 return(lck_spin_try_lock(lock));
91447636
A		 1155}
1156
1157void
1158usimple_unlock_EXT(
1159 lck_spin_t *lock)
1160{
1161 lck_spin_unlock(lock);
1162}
1163
1164wait_result_t
1165thread_sleep_usimple_lock_EXT(
1166 event_t event,
1167 lck_spin_t *lock,
1168 wait_interrupt_t interruptible)
1169{
1170 return( lck_spin_sleep(lock, LCK_SLEEP_DEFAULT, event, interruptible));
1171}
mirror of XNU