osfmk/kern/locks.c

   1 /*
   2  * Copyright (c) 2000-2007 Apple Inc. All rights reserved.
   3  *
   4  * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
   5  *
   6  * This file contains Original Code and/or Modifications of Original Code
   7  * as defined in and that are subject to the Apple Public Source License
   8  * Version 2.0 (the 'License'). You may not use this file except in
   9  * compliance with the License. The rights granted to you under the License
  10  * may not be used to create, or enable the creation or redistribution of,
  11  * unlawful or unlicensed copies of an Apple operating system, or to
  12  * circumvent, violate, or enable the circumvention or violation of, any
  13  * terms of an Apple operating system software license agreement.
  14  *
  15  * Please obtain a copy of the License at
  16  * http://www.opensource.apple.com/apsl/ and read it before using this file.
  17  *
  18  * The Original Code and all software distributed under the License are
  19  * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  20  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  21  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
  22  * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
  23  * Please see the License for the specific language governing rights and
  24  * limitations under the License.
  25  *
  26  * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  27  */
  28 /*
  29  * @OSF_COPYRIGHT@
  30  */
  31 /*
  32  * Mach Operating System
  33  * Copyright (c) 1991,1990,1989,1988,1987 Carnegie Mellon University
  34  * All Rights Reserved.
  35  *
  36  * Permission to use, copy, modify and distribute this software and its
  37  * documentation is hereby granted, provided that both the copyright
  38  * notice and this permission notice appear in all copies of the
  39  * software, derivative works or modified versions, and any portions
  40  * thereof, and that both notices appear in supporting documentation.
  41  *
  42  * CARNEGIE MELLON ALLOWS FREE USE OF THIS SOFTWARE IN ITS "AS IS"
  43  * CONDITION.  CARNEGIE MELLON DISCLAIMS ANY LIABILITY OF ANY KIND FOR
  44  * ANY DAMAGES WHATSOEVER RESULTING FROM THE USE OF THIS SOFTWARE.
  45  *
  46  * Carnegie Mellon requests users of this software to return to
  47  *
  48  *  Software Distribution Coordinator  or  Software.Distribution@CS.CMU.EDU
  49  *  School of Computer Science
  50  *  Carnegie Mellon University
  51  *  Pittsburgh PA 15213-3890
  52  *
  53  * any improvements or extensions that they make and grant Carnegie Mellon
  54  * the rights to redistribute these changes.
  55  */
  56 #include <mach_kdb.h>
  57 #include <mach_ldebug.h>
  58 #include <debug.h>
  59
  60 #include <mach/kern_return.h>
  61 #include <mach/mach_host_server.h>
  62 #include <mach_debug/lockgroup_info.h>
  63
  64 #include <kern/locks.h>
  65 #include <kern/misc_protos.h>
  66 #include <kern/kalloc.h>
  67 #include <kern/thread.h>
  68 #include <kern/processor.h>
  69 #include <kern/sched_prim.h>
  70 #include <kern/debug.h>
  71 #include <string.h>
  72
  73
  74 #include <sys/kdebug.h>
  75
  76 #if     CONFIG_DTRACE
  77 /*
  78  * We need only enough declarations from the BSD-side to be able to
  79  * test if our probe is active, and to call __dtrace_probe().  Setting
  80  * NEED_DTRACE_DEFS gets a local copy of those definitions pulled in.
  81  */
  82 #define NEED_DTRACE_DEFS
  83 #include <../bsd/sys/lockstat.h>
  84 #endif
  85
  86 #define LCK_MTX_SLEEP_CODE              0
  87 #define LCK_MTX_SLEEP_DEADLINE_CODE     1
  88 #define LCK_MTX_LCK_WAIT_CODE           2
  89 #define LCK_MTX_UNLCK_WAKEUP_CODE       3
  90
  91
  92 static queue_head_t     lck_grp_queue;
  93 static unsigned int     lck_grp_cnt;
  94
  95 decl_mutex_data(static,lck_grp_lock)
  96
  97 lck_grp_attr_t  LockDefaultGroupAttr;
  98 lck_grp_t       LockCompatGroup;
  99 lck_attr_t      LockDefaultLckAttr;
 100
 101 /*
 102  * Routine:     lck_mod_init
 103  */
 104
 105 void
 106 lck_mod_init(
 107         void)
 108 {
 109         queue_init(&lck_grp_queue);
 110         mutex_init(&lck_grp_lock, 0);
 111         lck_grp_cnt = 0;
 112         lck_grp_attr_setdefault( &LockDefaultGroupAttr);
 113         lck_grp_init( &LockCompatGroup, "Compatibility APIs", LCK_GRP_ATTR_NULL);
 114         lck_attr_setdefault(&LockDefaultLckAttr);
 115 }
 116
 117 /*
 118  * Routine:     lck_grp_attr_alloc_init
 119  */
 120
 121 lck_grp_attr_t  *
 122 lck_grp_attr_alloc_init(
 123         void)
 124 {
 125         lck_grp_attr_t  *attr;
 126
 127         if ((attr = (lck_grp_attr_t *)kalloc(sizeof(lck_grp_attr_t))) != 0)
 128                 lck_grp_attr_setdefault(attr);
 129
 130         return(attr);
 131 }
 132
 133
 134 /*
 135  * Routine:     lck_grp_attr_setdefault
 136  */
 137
 138 void
 139 lck_grp_attr_setdefault(
 140         lck_grp_attr_t  *attr)
 141 {
 142         if (LcksOpts & enaLkStat)
 143                 attr->grp_attr_val = LCK_GRP_ATTR_STAT;
 144         else
 145                 attr->grp_attr_val = 0;
 146 }
 147
 148
 149 /*
 150  * Routine:     lck_grp_attr_setstat
 151  */
 152
 153 void
 154 lck_grp_attr_setstat(
 155         lck_grp_attr_t  *attr)
 156 {
 157         (void)hw_atomic_or(&attr->grp_attr_val, LCK_GRP_ATTR_STAT);
 158 }
 159
 160
 161 /*
 162  * Routine:     lck_grp_attr_free
 163  */
 164
 165 void
 166 lck_grp_attr_free(
 167         lck_grp_attr_t  *attr)
 168 {
 169         kfree(attr, sizeof(lck_grp_attr_t));
 170 }
 171
 172
 173 /*
 174  * Routine:     lck_grp_alloc_init
 175  */
 176
 177 lck_grp_t *
 178 lck_grp_alloc_init(
 179         const char*     grp_name,
 180         lck_grp_attr_t  *attr)
 181 {
 182         lck_grp_t       *grp;
 183
 184         if ((grp = (lck_grp_t *)kalloc(sizeof(lck_grp_t))) != 0)
 185                 lck_grp_init(grp, grp_name, attr);
 186
 187         return(grp);
 188 }
 189
 190
 191 /*
 192  * Routine:     lck_grp_init
 193  */
 194
 195 void
 196 lck_grp_init(
 197         lck_grp_t               *grp,
 198         const char*             grp_name,
 199         lck_grp_attr_t  *attr)
 200 {
 201         bzero((void *)grp, sizeof(lck_grp_t));
 202
 203         (void) strncpy(grp->lck_grp_name, grp_name, LCK_GRP_MAX_NAME);
 204
 205         if (attr != LCK_GRP_ATTR_NULL)
 206                 grp->lck_grp_attr = attr->grp_attr_val;
 207         else if (LcksOpts & enaLkStat)
 208                 grp->lck_grp_attr = LCK_GRP_ATTR_STAT;
 209         else
 210                 grp->lck_grp_attr = LCK_ATTR_NONE;
 211
 212         grp->lck_grp_refcnt = 1;
 213
 214         mutex_lock(&lck_grp_lock);
 215         enqueue_tail(&lck_grp_queue, (queue_entry_t)grp);
 216         lck_grp_cnt++;
 217         mutex_unlock(&lck_grp_lock);
 218
 219 }
 220
 221
 222 /*
 223  * Routine:     lck_grp_free
 224  */
 225
 226 void
 227 lck_grp_free(
 228         lck_grp_t       *grp)
 229 {
 230         mutex_lock(&lck_grp_lock);
 231         lck_grp_cnt--;
 232         (void)remque((queue_entry_t)grp);
 233         mutex_unlock(&lck_grp_lock);
 234         lck_grp_deallocate(grp);
 235 }
 236
 237
 238 /*
 239  * Routine:     lck_grp_reference
 240  */
 241
 242 void
 243 lck_grp_reference(
 244         lck_grp_t       *grp)
 245 {
 246         (void)hw_atomic_add(&grp->lck_grp_refcnt, 1);
 247 }
 248
 249
 250 /*
 251  * Routine:     lck_grp_deallocate
 252  */
 253
 254 void
 255 lck_grp_deallocate(
 256         lck_grp_t       *grp)
 257 {
 258         if (hw_atomic_sub(&grp->lck_grp_refcnt, 1) == 0)
 259                 kfree(grp, sizeof(lck_grp_t));
 260 }
 261
 262 /*
 263  * Routine:     lck_grp_lckcnt_incr
 264  */
 265
 266 void
 267 lck_grp_lckcnt_incr(
 268         lck_grp_t       *grp,
 269         lck_type_t      lck_type)
 270 {
 271         unsigned int    *lckcnt;
 272
 273         switch (lck_type) {
 274         case LCK_TYPE_SPIN:
 275                 lckcnt = &grp->lck_grp_spincnt;
 276                 break;
 277         case LCK_TYPE_MTX:
 278                 lckcnt = &grp->lck_grp_mtxcnt;
 279                 break;
 280         case LCK_TYPE_RW:
 281                 lckcnt = &grp->lck_grp_rwcnt;
 282                 break;
 283         default:
 284                 return panic("lck_grp_lckcnt_incr(): invalid lock type: %d\n", lck_type);
 285         }
 286
 287         (void)hw_atomic_add(lckcnt, 1);
 288 }
 289
 290 /*
 291  * Routine:     lck_grp_lckcnt_decr
 292  */
 293
 294 void
 295 lck_grp_lckcnt_decr(
 296         lck_grp_t       *grp,
 297         lck_type_t      lck_type)
 298 {
 299         unsigned int    *lckcnt;
 300
 301         switch (lck_type) {
 302         case LCK_TYPE_SPIN:
 303                 lckcnt = &grp->lck_grp_spincnt;
 304                 break;
 305         case LCK_TYPE_MTX:
 306                 lckcnt = &grp->lck_grp_mtxcnt;
 307                 break;
 308         case LCK_TYPE_RW:
 309                 lckcnt = &grp->lck_grp_rwcnt;
 310                 break;
 311         default:
 312                 return panic("lck_grp_lckcnt_decr(): invalid lock type: %d\n", lck_type);
 313         }
 314
 315         (void)hw_atomic_sub(lckcnt, 1);
 316 }
 317
 318 /*
 319  * Routine:     lck_attr_alloc_init
 320  */
 321
 322 lck_attr_t *
 323 lck_attr_alloc_init(
 324         void)
 325 {
 326         lck_attr_t      *attr;
 327
 328         if ((attr = (lck_attr_t *)kalloc(sizeof(lck_attr_t))) != 0)
 329                 lck_attr_setdefault(attr);
 330
 331         return(attr);
 332 }
 333
 334
 335 /*
 336  * Routine:     lck_attr_setdefault
 337  */
 338
 339 void
 340 lck_attr_setdefault(
 341         lck_attr_t      *attr)
 342 {
 343 #if     !DEBUG
 344         if (LcksOpts & enaLkDeb)
 345                 attr->lck_attr_val =  LCK_ATTR_DEBUG;
 346         else
 347                 attr->lck_attr_val =  LCK_ATTR_NONE;
 348 #else
 349         attr->lck_attr_val =  LCK_ATTR_DEBUG;
 350 #endif  /* !DEBUG */
 351 }
 352
 353
 354 /*
 355  * Routine:     lck_attr_setdebug
 356  */
 357 void
 358 lck_attr_setdebug(
 359         lck_attr_t      *attr)
 360 {
 361         (void)hw_atomic_or(&attr->lck_attr_val, LCK_ATTR_DEBUG);
 362 }
 363
 364 /*
 365  * Routine:     lck_attr_setdebug
 366  */
 367 void
 368 lck_attr_cleardebug(
 369         lck_attr_t      *attr)
 370 {
 371         (void)hw_atomic_and(&attr->lck_attr_val, ~LCK_ATTR_DEBUG);
 372 }
 373
 374
 375 /*
 376  * Routine:     lck_attr_rw_shared_priority
 377  */
 378 void
 379 lck_attr_rw_shared_priority(
 380         lck_attr_t      *attr)
 381 {
 382         (void)hw_atomic_or(&attr->lck_attr_val, LCK_ATTR_RW_SHARED_PRIORITY);
 383 }
 384
 385
 386 /*
 387  * Routine:     lck_attr_free
 388  */
 389 void
 390 lck_attr_free(
 391         lck_attr_t      *attr)
 392 {
 393         kfree(attr, sizeof(lck_attr_t));
 394 }
 395
 396
 397 /*
 398  * Routine:     lck_spin_sleep
 399  */
 400 wait_result_t
 401 lck_spin_sleep(
 402         lck_spin_t              *lck,
 403         lck_sleep_action_t      lck_sleep_action,
 404         event_t                 event,
 405         wait_interrupt_t        interruptible)
 406 {
 407         wait_result_t   res;
 408
 409         if ((lck_sleep_action & ~LCK_SLEEP_MASK) != 0)
 410                 panic("Invalid lock sleep action %x\n", lck_sleep_action);
 411
 412         res = assert_wait(event, interruptible);
 413         if (res == THREAD_WAITING) {
 414                 lck_spin_unlock(lck);
 415                 res = thread_block(THREAD_CONTINUE_NULL);
 416                 if (!(lck_sleep_action & LCK_SLEEP_UNLOCK))
 417                         lck_spin_lock(lck);
 418         }
 419         else
 420         if (lck_sleep_action & LCK_SLEEP_UNLOCK)
 421                 lck_spin_unlock(lck);
 422
 423         return res;
 424 }
 425
 426
 427 /*
 428  * Routine:     lck_spin_sleep_deadline
 429  */
 430 wait_result_t
 431 lck_spin_sleep_deadline(
 432         lck_spin_t              *lck,
 433         lck_sleep_action_t      lck_sleep_action,
 434         event_t                 event,
 435         wait_interrupt_t        interruptible,
 436         uint64_t                deadline)
 437 {
 438         wait_result_t   res;
 439
 440         if ((lck_sleep_action & ~LCK_SLEEP_MASK) != 0)
 441                 panic("Invalid lock sleep action %x\n", lck_sleep_action);
 442
 443         res = assert_wait_deadline(event, interruptible, deadline);
 444         if (res == THREAD_WAITING) {
 445                 lck_spin_unlock(lck);
 446                 res = thread_block(THREAD_CONTINUE_NULL);
 447                 if (!(lck_sleep_action & LCK_SLEEP_UNLOCK))
 448                         lck_spin_lock(lck);
 449         }
 450         else
 451         if (lck_sleep_action & LCK_SLEEP_UNLOCK)
 452                 lck_spin_unlock(lck);
 453
 454         return res;
 455 }
 456
 457
 458 /*
 459  * Routine:     lck_mtx_sleep
 460  */
 461 wait_result_t
 462 lck_mtx_sleep(
 463         lck_mtx_t               *lck,
 464         lck_sleep_action_t      lck_sleep_action,
 465         event_t                 event,
 466         wait_interrupt_t        interruptible)
 467 {
 468         wait_result_t   res;
 469
 470         KERNEL_DEBUG(MACHDBG_CODE(DBG_MACH_LOCKS, LCK_MTX_SLEEP_CODE) | DBG_FUNC_START,
 471                      (int)lck, (int)lck_sleep_action, (int)event, (int)interruptible, 0);
 472
 473         if ((lck_sleep_action & ~LCK_SLEEP_MASK) != 0)
 474                 panic("Invalid lock sleep action %x\n", lck_sleep_action);
 475
 476         res = assert_wait(event, interruptible);
 477         if (res == THREAD_WAITING) {
 478                 lck_mtx_unlock(lck);
 479                 res = thread_block(THREAD_CONTINUE_NULL);
 480                 if (!(lck_sleep_action & LCK_SLEEP_UNLOCK))
 481                         lck_mtx_lock(lck);
 482         }
 483         else
 484         if (lck_sleep_action & LCK_SLEEP_UNLOCK)
 485                 lck_mtx_unlock(lck);
 486
 487         KERNEL_DEBUG(MACHDBG_CODE(DBG_MACH_LOCKS, LCK_MTX_SLEEP_CODE) | DBG_FUNC_END, (int)res, 0, 0, 0, 0);
 488
 489         return res;
 490 }
 491
 492
 493 /*
 494  * Routine:     lck_mtx_sleep_deadline
 495  */
 496 wait_result_t
 497 lck_mtx_sleep_deadline(
 498         lck_mtx_t               *lck,
 499         lck_sleep_action_t      lck_sleep_action,
 500         event_t                 event,
 501         wait_interrupt_t        interruptible,
 502         uint64_t                deadline)
 503 {
 504         wait_result_t   res;
 505
 506         KERNEL_DEBUG(MACHDBG_CODE(DBG_MACH_LOCKS, LCK_MTX_SLEEP_DEADLINE_CODE) | DBG_FUNC_START,
 507                      (int)lck, (int)lck_sleep_action, (int)event, (int)interruptible, 0);
 508
 509         if ((lck_sleep_action & ~LCK_SLEEP_MASK) != 0)
 510                 panic("Invalid lock sleep action %x\n", lck_sleep_action);
 511
 512         res = assert_wait_deadline(event, interruptible, deadline);
 513         if (res == THREAD_WAITING) {
 514                 lck_mtx_unlock(lck);
 515                 res = thread_block(THREAD_CONTINUE_NULL);
 516                 if (!(lck_sleep_action & LCK_SLEEP_UNLOCK))
 517                         lck_mtx_lock(lck);
 518         }
 519         else
 520         if (lck_sleep_action & LCK_SLEEP_UNLOCK)
 521                 lck_mtx_unlock(lck);
 522
 523         KERNEL_DEBUG(MACHDBG_CODE(DBG_MACH_LOCKS, LCK_MTX_SLEEP_DEADLINE_CODE) | DBG_FUNC_END, (int)res, 0, 0, 0, 0);
 524
 525         return res;
 526 }
 527
 528 /*
 529  * Routine:     lck_mtx_lock_wait
 530  *
 531  * Invoked in order to wait on contention.
 532  *
 533  * Called with the interlock locked and
 534  * returns it unlocked.
 535  */
 536 void
 537 lck_mtx_lock_wait (
 538         lck_mtx_t                       *lck,
 539         thread_t                        holder)
 540 {
 541         thread_t                self = current_thread();
 542         lck_mtx_t               *mutex;
 543         integer_t               priority;
 544         spl_t                   s = splsched();
 545 #if     CONFIG_DTRACE
 546         uint64_t                sleep_start = 0;
 547
 548         if (lockstat_probemap[LS_LCK_MTX_LOCK_BLOCK] || lockstat_probemap[LS_LCK_MTX_EXT_LOCK_BLOCK]) {
 549                 sleep_start = mach_absolute_time();
 550         }
 551 #endif
 552
 553         if (lck->lck_mtx_tag != LCK_MTX_TAG_INDIRECT)
 554                 mutex = lck;
 555         else
 556                 mutex = &lck->lck_mtx_ptr->lck_mtx;
 557
 558         KERNEL_DEBUG(MACHDBG_CODE(DBG_MACH_LOCKS, LCK_MTX_LCK_WAIT_CODE) | DBG_FUNC_START, (int)lck, (int)holder, 0, 0, 0);
 559
 560         priority = self->sched_pri;
 561         if (priority < self->priority)
 562                 priority = self->priority;
 563         if (priority < BASEPRI_DEFAULT)
 564                 priority = BASEPRI_DEFAULT;
 565
 566         thread_lock(holder);
 567         if (mutex->lck_mtx_pri == 0)
 568                 holder->promotions++;
 569         holder->sched_mode |= TH_MODE_PROMOTED;
 570         if (            mutex->lck_mtx_pri < priority   &&
 571                                 holder->sched_pri < priority            ) {
 572                 KERNEL_DEBUG_CONSTANT(
 573                         MACHDBG_CODE(DBG_MACH_SCHED,MACH_PROMOTE) | DBG_FUNC_NONE,
 574                                         holder->sched_pri, priority, (int)holder, (int)lck, 0);
 575
 576                 set_sched_pri(holder, priority);
 577         }
 578         thread_unlock(holder);
 579         splx(s);
 580
 581         if (mutex->lck_mtx_pri < priority)
 582                 mutex->lck_mtx_pri = priority;
 583         if (self->pending_promoter[self->pending_promoter_index] == NULL) {
 584                 self->pending_promoter[self->pending_promoter_index] = mutex;
 585                 mutex->lck_mtx_waiters++;
 586         }
 587         else
 588         if (self->pending_promoter[self->pending_promoter_index] != mutex) {
 589                 self->pending_promoter[++self->pending_promoter_index] = mutex;
 590                 mutex->lck_mtx_waiters++;
 591         }
 592
 593         assert_wait((event_t)(((unsigned int*)lck)+((sizeof(lck_mtx_t)-1)/sizeof(unsigned int))), THREAD_UNINT);
 594         lck_mtx_ilk_unlock(mutex);
 595
 596         thread_block(THREAD_CONTINUE_NULL);
 597
 598         KERNEL_DEBUG(MACHDBG_CODE(DBG_MACH_LOCKS, LCK_MTX_LCK_WAIT_CODE) | DBG_FUNC_END, 0, 0, 0, 0, 0);
 599 #if     CONFIG_DTRACE
 600         /*
 601          * Record the Dtrace lockstat probe for blocking, block time
 602          * measured from when we were entered.
 603          */
 604         if (sleep_start) {
 605                 if (lck->lck_mtx_tag != LCK_MTX_TAG_INDIRECT) {
 606                         LOCKSTAT_RECORD(LS_LCK_MTX_LOCK_BLOCK, lck,
 607                             mach_absolute_time() - sleep_start);
 608                 } else {
 609                         LOCKSTAT_RECORD(LS_LCK_MTX_EXT_LOCK_BLOCK, lck,
 610                             mach_absolute_time() - sleep_start);
 611                 }
 612         }
 613 #endif
 614 }
 615
 616 /*
 617  * Routine:     lck_mtx_lock_acquire
 618  *
 619  * Invoked on acquiring the mutex when there is
 620  * contention.
 621  *
 622  * Returns the current number of waiters.
 623  *
 624  * Called with the interlock locked.
 625  */
 626 int
 627 lck_mtx_lock_acquire(
 628         lck_mtx_t               *lck)
 629 {
 630         thread_t                thread = current_thread();
 631         lck_mtx_t               *mutex;
 632
 633         if (lck->lck_mtx_tag != LCK_MTX_TAG_INDIRECT)
 634                 mutex = lck;
 635         else
 636                 mutex = &lck->lck_mtx_ptr->lck_mtx;
 637
 638         if (thread->pending_promoter[thread->pending_promoter_index] == mutex) {
 639                 thread->pending_promoter[thread->pending_promoter_index] = NULL;
 640                 if (thread->pending_promoter_index > 0)
 641                         thread->pending_promoter_index--;
 642                 mutex->lck_mtx_waiters--;
 643         }
 644
 645         if (mutex->lck_mtx_waiters > 0) {
 646                 integer_t               priority = mutex->lck_mtx_pri;
 647                 spl_t                   s = splsched();
 648
 649                 thread_lock(thread);
 650                 thread->promotions++;
 651                 thread->sched_mode |= TH_MODE_PROMOTED;
 652                 if (thread->sched_pri < priority) {
 653                         KERNEL_DEBUG_CONSTANT(
 654                                 MACHDBG_CODE(DBG_MACH_SCHED,MACH_PROMOTE) | DBG_FUNC_NONE,
 655                                                 thread->sched_pri, priority, 0, (int)lck, 0);
 656
 657                         set_sched_pri(thread, priority);
 658                 }
 659                 thread_unlock(thread);
 660                 splx(s);
 661         }
 662         else
 663                 mutex->lck_mtx_pri = 0;
 664
 665         return (mutex->lck_mtx_waiters);
 666 }
 667
 668 /*
 669  * Routine:     lck_mtx_unlock_wakeup
 670  *
 671  * Invoked on unlock when there is contention.
 672  *
 673  * Called with the interlock locked.
 674  */
 675 void
 676 lck_mtx_unlock_wakeup (
 677         lck_mtx_t                       *lck,
 678         thread_t                        holder)
 679 {
 680         thread_t                thread = current_thread();
 681         lck_mtx_t               *mutex;
 682
 683         if (lck->lck_mtx_tag != LCK_MTX_TAG_INDIRECT)
 684                 mutex = lck;
 685         else
 686                 mutex = &lck->lck_mtx_ptr->lck_mtx;
 687
 688
 689         KERNEL_DEBUG(MACHDBG_CODE(DBG_MACH_LOCKS, LCK_MTX_UNLCK_WAKEUP_CODE) | DBG_FUNC_START, (int)lck, (int)holder, 0, 0, 0);
 690
 691         if (thread != holder)
 692                 panic("lck_mtx_unlock_wakeup: mutex %p holder %p\n", mutex, holder);
 693
 694         if (thread->promotions > 0) {
 695                 spl_t           s = splsched();
 696
 697                 thread_lock(thread);
 698                 if (    --thread->promotions == 0                               &&
 699                                 (thread->sched_mode & TH_MODE_PROMOTED)         ) {
 700                         thread->sched_mode &= ~TH_MODE_PROMOTED;
 701                         if (thread->sched_mode & TH_MODE_ISDEPRESSED) {
 702                                 KERNEL_DEBUG_CONSTANT(
 703                                         MACHDBG_CODE(DBG_MACH_SCHED,MACH_DEMOTE) | DBG_FUNC_NONE,
 704                                                   thread->sched_pri, DEPRESSPRI, 0, (int)lck, 0);
 705
 706                                 set_sched_pri(thread, DEPRESSPRI);
 707                         }
 708                         else {
 709                                 if (thread->priority < thread->sched_pri) {
 710                                         KERNEL_DEBUG_CONSTANT(
 711                                                 MACHDBG_CODE(DBG_MACH_SCHED,MACH_DEMOTE) |
 712                                                                                                                         DBG_FUNC_NONE,
 713                                                         thread->sched_pri, thread->priority,
 714                                                                         0, (int)lck, 0);
 715                                 }
 716
 717                                 compute_priority(thread, FALSE);
 718                         }
 719                 }
 720                 thread_unlock(thread);
 721                 splx(s);
 722         }
 723         assert(mutex->lck_mtx_waiters > 0);
 724         thread_wakeup_one((event_t)(((unsigned int*)lck)+(sizeof(lck_mtx_t)-1)/sizeof(unsigned int)));
 725
 726         KERNEL_DEBUG(MACHDBG_CODE(DBG_MACH_LOCKS, LCK_MTX_UNLCK_WAKEUP_CODE) | DBG_FUNC_END, 0, 0, 0, 0, 0);
 727 }
 728
 729 void
 730 lck_mtx_unlockspin_wakeup (
 731         lck_mtx_t                       *lck)
 732 {
 733         assert(lck->lck_mtx_waiters > 0);
 734         thread_wakeup_one((event_t)(((unsigned int*)lck)+(sizeof(lck_mtx_t)-1)/sizeof(unsigned int)));
 735
 736         KERNEL_DEBUG(MACHDBG_CODE(DBG_MACH_LOCKS, LCK_MTX_UNLCK_WAKEUP_CODE) | DBG_FUNC_NONE, (int)lck, 0, 0, 1, 0);
 737 #if CONFIG_DTRACE
 738         /*
 739          * When there are waiters, we skip the hot-patch spot in the
 740          * fastpath, so we record it here.
 741          */
 742         LOCKSTAT_RECORD(LS_LCK_MTX_UNLOCK_RELEASE, lck, 0);
 743 #endif
 744 }
 745
 746
 747 /*
 748  * Routine:     mutex_pause
 749  *
 750  * Called by former callers of simple_lock_pause().
 751  */
 752 #define MAX_COLLISION_COUNTS    32
 753 #define MAX_COLLISION   8
 754
 755 unsigned int max_collision_count[MAX_COLLISION_COUNTS];
 756
 757 uint32_t collision_backoffs[MAX_COLLISION] = {
 758         10, 50, 100, 200, 400, 600, 800, 1000
 759 };
 760
 761
 762 void
 763 mutex_pause(uint32_t collisions)
 764 {
 765         wait_result_t wait_result;
 766         uint32_t        back_off;
 767
 768         if (collisions >= MAX_COLLISION_COUNTS)
 769                 collisions = MAX_COLLISION_COUNTS - 1;
 770         max_collision_count[collisions]++;
 771
 772         if (collisions >= MAX_COLLISION)
 773                 collisions = MAX_COLLISION - 1;
 774         back_off = collision_backoffs[collisions];
 775
 776         wait_result = assert_wait_timeout((event_t)mutex_pause, THREAD_UNINT, back_off, NSEC_PER_USEC);
 777         assert(wait_result == THREAD_WAITING);
 778
 779         wait_result = thread_block(THREAD_CONTINUE_NULL);
 780         assert(wait_result == THREAD_TIMED_OUT);
 781 }
 782
 783
 784 unsigned int mutex_yield_wait = 0;
 785 unsigned int mutex_yield_no_wait = 0;
 786
 787 void
 788 mutex_yield(
 789             mutex_t     *mutex)
 790 {
 791         lck_mtx_t       *lck;
 792
 793 #if DEBUG
 794         _mutex_assert(mutex, MA_OWNED);
 795 #endif /* DEBUG */
 796
 797         lck = (lck_mtx_t *) mutex;
 798         if (lck->lck_mtx_tag == LCK_MTX_TAG_INDIRECT)
 799                 lck = &lck->lck_mtx_ptr->lck_mtx;
 800
 801         if (! lck->lck_mtx_waiters) {
 802                 mutex_yield_no_wait++;
 803         } else {
 804                 mutex_yield_wait++;
 805                 mutex_unlock(mutex);
 806                 mutex_pause(0);
 807                 mutex_lock(mutex);
 808         }
 809 }
 810
 811
 812 /*
 813  * Routine:     lck_rw_sleep
 814  */
 815 wait_result_t
 816 lck_rw_sleep(
 817         lck_rw_t                *lck,
 818         lck_sleep_action_t      lck_sleep_action,
 819         event_t                 event,
 820         wait_interrupt_t        interruptible)
 821 {
 822         wait_result_t   res;
 823         lck_rw_type_t   lck_rw_type;
 824
 825         if ((lck_sleep_action & ~LCK_SLEEP_MASK) != 0)
 826                 panic("Invalid lock sleep action %x\n", lck_sleep_action);
 827
 828         res = assert_wait(event, interruptible);
 829         if (res == THREAD_WAITING) {
 830                 lck_rw_type = lck_rw_done(lck);
 831                 res = thread_block(THREAD_CONTINUE_NULL);
 832                 if (!(lck_sleep_action & LCK_SLEEP_UNLOCK)) {
 833                         if (!(lck_sleep_action & (LCK_SLEEP_SHARED|LCK_SLEEP_EXCLUSIVE)))
 834                                 lck_rw_lock(lck, lck_rw_type);
 835                         else if (lck_sleep_action & LCK_SLEEP_EXCLUSIVE)
 836                                 lck_rw_lock_exclusive(lck);
 837                         else
 838                                 lck_rw_lock_shared(lck);
 839                 }
 840         }
 841         else
 842         if (lck_sleep_action & LCK_SLEEP_UNLOCK)
 843                 (void)lck_rw_done(lck);
 844
 845         return res;
 846 }
 847
 848
 849 /*
 850  * Routine:     lck_rw_sleep_deadline
 851  */
 852 wait_result_t
 853 lck_rw_sleep_deadline(
 854         lck_rw_t                *lck,
 855         lck_sleep_action_t      lck_sleep_action,
 856         event_t                 event,
 857         wait_interrupt_t        interruptible,
 858         uint64_t                deadline)
 859 {
 860         wait_result_t   res;
 861         lck_rw_type_t   lck_rw_type;
 862
 863         if ((lck_sleep_action & ~LCK_SLEEP_MASK) != 0)
 864                 panic("Invalid lock sleep action %x\n", lck_sleep_action);
 865
 866         res = assert_wait_deadline(event, interruptible, deadline);
 867         if (res == THREAD_WAITING) {
 868                 lck_rw_type = lck_rw_done(lck);
 869                 res = thread_block(THREAD_CONTINUE_NULL);
 870                 if (!(lck_sleep_action & LCK_SLEEP_UNLOCK)) {
 871                         if (!(lck_sleep_action & (LCK_SLEEP_SHARED|LCK_SLEEP_EXCLUSIVE)))
 872                                 lck_rw_lock(lck, lck_rw_type);
 873                         else if (lck_sleep_action & LCK_SLEEP_EXCLUSIVE)
 874                                 lck_rw_lock_exclusive(lck);
 875                         else
 876                                 lck_rw_lock_shared(lck);
 877                 }
 878         }
 879         else
 880         if (lck_sleep_action & LCK_SLEEP_UNLOCK)
 881                 (void)lck_rw_done(lck);
 882
 883         return res;
 884 }
 885
 886 kern_return_t
 887 host_lockgroup_info(
 888         host_t                                  host,
 889         lockgroup_info_array_t  *lockgroup_infop,
 890         mach_msg_type_number_t  *lockgroup_infoCntp)
 891 {
 892         lockgroup_info_t        *lockgroup_info_base;
 893         lockgroup_info_t        *lockgroup_info;
 894         vm_offset_t                     lockgroup_info_addr;
 895         vm_size_t                       lockgroup_info_size;
 896         lck_grp_t                       *lck_grp;
 897         unsigned int            i;
 898         vm_size_t                       used;
 899         vm_map_copy_t           copy;
 900         kern_return_t           kr;
 901
 902         if (host == HOST_NULL)
 903                 return KERN_INVALID_HOST;
 904
 905         mutex_lock(&lck_grp_lock);
 906
 907         lockgroup_info_size = round_page(lck_grp_cnt * sizeof *lockgroup_info);
 908         kr = kmem_alloc_pageable(ipc_kernel_map,
 909                                                  &lockgroup_info_addr, lockgroup_info_size);
 910         if (kr != KERN_SUCCESS) {
 911                 mutex_unlock(&lck_grp_lock);
 912                 return(kr);
 913         }
 914
 915         lockgroup_info_base = (lockgroup_info_t *) lockgroup_info_addr;
 916         lck_grp = (lck_grp_t *)queue_first(&lck_grp_queue);
 917         lockgroup_info = lockgroup_info_base;
 918
 919         for (i = 0; i < lck_grp_cnt; i++) {
 920
 921                 lockgroup_info->lock_spin_cnt = lck_grp->lck_grp_spincnt;
 922                 lockgroup_info->lock_spin_util_cnt = lck_grp->lck_grp_stat.lck_grp_spin_stat.lck_grp_spin_util_cnt;
 923                 lockgroup_info->lock_spin_held_cnt = lck_grp->lck_grp_stat.lck_grp_spin_stat.lck_grp_spin_held_cnt;
 924                 lockgroup_info->lock_spin_miss_cnt = lck_grp->lck_grp_stat.lck_grp_spin_stat.lck_grp_spin_miss_cnt;
 925                 lockgroup_info->lock_spin_held_max = lck_grp->lck_grp_stat.lck_grp_spin_stat.lck_grp_spin_held_max;
 926                 lockgroup_info->lock_spin_held_cum = lck_grp->lck_grp_stat.lck_grp_spin_stat.lck_grp_spin_held_cum;
 927
 928                 lockgroup_info->lock_mtx_cnt = lck_grp->lck_grp_mtxcnt;
 929                 lockgroup_info->lock_mtx_util_cnt = lck_grp->lck_grp_stat.lck_grp_mtx_stat.lck_grp_mtx_util_cnt;
 930                 lockgroup_info->lock_mtx_held_cnt = lck_grp->lck_grp_stat.lck_grp_mtx_stat.lck_grp_mtx_held_cnt;
 931                 lockgroup_info->lock_mtx_miss_cnt = lck_grp->lck_grp_stat.lck_grp_mtx_stat.lck_grp_mtx_miss_cnt;
 932                 lockgroup_info->lock_mtx_wait_cnt = lck_grp->lck_grp_stat.lck_grp_mtx_stat.lck_grp_mtx_wait_cnt;
 933                 lockgroup_info->lock_mtx_held_max = lck_grp->lck_grp_stat.lck_grp_mtx_stat.lck_grp_mtx_held_max;
 934                 lockgroup_info->lock_mtx_held_cum = lck_grp->lck_grp_stat.lck_grp_mtx_stat.lck_grp_mtx_held_cum;
 935                 lockgroup_info->lock_mtx_wait_max = lck_grp->lck_grp_stat.lck_grp_mtx_stat.lck_grp_mtx_wait_max;
 936                 lockgroup_info->lock_mtx_wait_cum = lck_grp->lck_grp_stat.lck_grp_mtx_stat.lck_grp_mtx_wait_cum;
 937
 938                 lockgroup_info->lock_rw_cnt = lck_grp->lck_grp_rwcnt;
 939                 lockgroup_info->lock_rw_util_cnt = lck_grp->lck_grp_stat.lck_grp_rw_stat.lck_grp_rw_util_cnt;
 940                 lockgroup_info->lock_rw_held_cnt = lck_grp->lck_grp_stat.lck_grp_rw_stat.lck_grp_rw_held_cnt;
 941                 lockgroup_info->lock_rw_miss_cnt = lck_grp->lck_grp_stat.lck_grp_rw_stat.lck_grp_rw_miss_cnt;
 942                 lockgroup_info->lock_rw_wait_cnt = lck_grp->lck_grp_stat.lck_grp_rw_stat.lck_grp_rw_wait_cnt;
 943                 lockgroup_info->lock_rw_held_max = lck_grp->lck_grp_stat.lck_grp_rw_stat.lck_grp_rw_held_max;
 944                 lockgroup_info->lock_rw_held_cum = lck_grp->lck_grp_stat.lck_grp_rw_stat.lck_grp_rw_held_cum;
 945                 lockgroup_info->lock_rw_wait_max = lck_grp->lck_grp_stat.lck_grp_rw_stat.lck_grp_rw_wait_max;
 946                 lockgroup_info->lock_rw_wait_cum = lck_grp->lck_grp_stat.lck_grp_rw_stat.lck_grp_rw_wait_cum;
 947
 948                 (void) strncpy(lockgroup_info->lockgroup_name,lck_grp->lck_grp_name, LOCKGROUP_MAX_NAME);
 949
 950                 lck_grp = (lck_grp_t *)(queue_next((queue_entry_t)(lck_grp)));
 951                 lockgroup_info++;
 952         }
 953
 954         *lockgroup_infoCntp = lck_grp_cnt;
 955         mutex_unlock(&lck_grp_lock);
 956
 957         used = (*lockgroup_infoCntp) * sizeof *lockgroup_info;
 958
 959         if (used != lockgroup_info_size)
 960                 bzero((char *) lockgroup_info, lockgroup_info_size - used);
 961
 962         kr = vm_map_copyin(ipc_kernel_map, (vm_map_address_t)lockgroup_info_addr,
 963                            (vm_map_size_t)lockgroup_info_size, TRUE, &copy);
 964         assert(kr == KERN_SUCCESS);
 965
 966         *lockgroup_infop = (lockgroup_info_t *) copy;
 967
 968         return(KERN_SUCCESS);
 969 }
 970
 971 /*
 972  * Compatibility module
 973  */
 974
 975 extern lck_rw_t         *lock_alloc_EXT( boolean_t can_sleep, unsigned short  tag0, unsigned short  tag1);
 976 extern void             lock_done_EXT(lck_rw_t *lock);
 977 extern void             lock_free_EXT(lck_rw_t *lock);
 978 extern void             lock_init_EXT(lck_rw_t *lock, boolean_t can_sleep, unsigned short tag0, unsigned short tag1);
 979 extern void             lock_read_EXT(lck_rw_t *lock);
 980 extern boolean_t        lock_read_to_write_EXT(lck_rw_t *lock);
 981 extern void             lock_write_EXT(lck_rw_t *lock);
 982 extern void             lock_write_to_read_EXT(lck_rw_t *lock);
 983 extern wait_result_t    thread_sleep_lock_write_EXT(
 984                                 event_t event, lck_rw_t *lock, wait_interrupt_t interruptible);
 985
 986 extern lck_mtx_t        *mutex_alloc_EXT(unsigned short tag);
 987 extern void             mutex_free_EXT(lck_mtx_t *mutex);
 988 extern void             mutex_init_EXT(lck_mtx_t *mutex, unsigned short tag);
 989 extern void             mutex_lock_EXT(lck_mtx_t *mutex);
 990 extern boolean_t        mutex_try_EXT(lck_mtx_t *mutex);
 991 extern void             mutex_unlock_EXT(lck_mtx_t *mutex);
 992 extern wait_result_t    thread_sleep_mutex_EXT(
 993                                 event_t event, lck_mtx_t *mutex, wait_interrupt_t interruptible);
 994 extern wait_result_t    thread_sleep_mutex_deadline_EXT(
 995                                 event_t event, lck_mtx_t *mutex, uint64_t deadline, wait_interrupt_t interruptible);
 996
 997 extern void             usimple_lock_EXT(lck_spin_t *lock);
 998 extern void             usimple_lock_init_EXT(lck_spin_t *lock, unsigned short tag);
 999 extern unsigned int     usimple_lock_try_EXT(lck_spin_t *lock);
1000 extern void             usimple_unlock_EXT(lck_spin_t *lock);
1001 extern wait_result_t    thread_sleep_usimple_lock_EXT(event_t event, lck_spin_t *lock, wait_interrupt_t interruptible);
1002
1003 lck_rw_t *
1004 lock_alloc_EXT(
1005         __unused boolean_t       can_sleep,
1006         __unused unsigned short  tag0,
1007         __unused unsigned short  tag1)
1008 {
1009         return( lck_rw_alloc_init( &LockCompatGroup, LCK_ATTR_NULL));
1010 }
1011
1012 void
1013 lock_done_EXT(
1014         lck_rw_t        *lock)
1015 {
1016         (void) lck_rw_done(lock);
1017 }
1018
1019 void
1020 lock_free_EXT(
1021         lck_rw_t        *lock)
1022 {
1023         lck_rw_free(lock, &LockCompatGroup);
1024 }
1025
1026 void
1027 lock_init_EXT(
1028         lck_rw_t        *lock,
1029         __unused boolean_t      can_sleep,
1030         __unused unsigned short tag0,
1031         __unused unsigned short tag1)
1032 {
1033         lck_rw_init(lock, &LockCompatGroup, LCK_ATTR_NULL);
1034 }
1035
1036 void
1037 lock_read_EXT(
1038         lck_rw_t        *lock)
1039 {
1040         lck_rw_lock_shared( lock);
1041 }
1042
1043 boolean_t
1044 lock_read_to_write_EXT(
1045         lck_rw_t        *lock)
1046 {
1047         return( lck_rw_lock_shared_to_exclusive(lock));
1048 }
1049
1050 void
1051 lock_write_EXT(
1052         lck_rw_t        *lock)
1053 {
1054         lck_rw_lock_exclusive(lock);
1055 }
1056
1057 void
1058 lock_write_to_read_EXT(
1059         lck_rw_t        *lock)
1060 {
1061         lck_rw_lock_exclusive_to_shared(lock);
1062 }
1063
1064 wait_result_t
1065 thread_sleep_lock_write_EXT(
1066         event_t                 event,
1067         lck_rw_t                *lock,
1068         wait_interrupt_t        interruptible)
1069 {
1070         return( lck_rw_sleep(lock, LCK_SLEEP_EXCLUSIVE, event, interruptible));
1071 }
1072
1073 lck_mtx_t *
1074 mutex_alloc_EXT(
1075         __unused unsigned short         tag)
1076 {
1077         return(lck_mtx_alloc_init(&LockCompatGroup, LCK_ATTR_NULL));
1078 }
1079
1080 void
1081 mutex_free_EXT(
1082         lck_mtx_t               *mutex)
1083 {
1084         lck_mtx_free(mutex, &LockCompatGroup);
1085 }
1086
1087 void
1088 mutex_init_EXT(
1089         lck_mtx_t               *mutex,
1090         __unused unsigned short tag)
1091 {
1092         lck_mtx_init(mutex, &LockCompatGroup, LCK_ATTR_NULL);
1093 }
1094
1095 void
1096 mutex_lock_EXT(
1097         lck_mtx_t               *mutex)
1098 {
1099         lck_mtx_lock(mutex);
1100 }
1101
1102 boolean_t
1103 mutex_try_EXT(
1104         lck_mtx_t               *mutex)
1105 {
1106         return(lck_mtx_try_lock(mutex));
1107 }
1108
1109 void
1110 mutex_unlock_EXT(
1111         lck_mtx_t               *mutex)
1112 {
1113         lck_mtx_unlock(mutex);
1114 }
1115
1116 wait_result_t
1117 thread_sleep_mutex_EXT(
1118         event_t                 event,
1119         lck_mtx_t               *mutex,
1120         wait_interrupt_t        interruptible)
1121 {
1122         return( lck_mtx_sleep(mutex, LCK_SLEEP_DEFAULT, event, interruptible));
1123 }
1124
1125 wait_result_t
1126 thread_sleep_mutex_deadline_EXT(
1127         event_t                 event,
1128         lck_mtx_t               *mutex,
1129         uint64_t                deadline,
1130         wait_interrupt_t        interruptible)
1131 {
1132         return( lck_mtx_sleep_deadline(mutex, LCK_SLEEP_DEFAULT, event, interruptible, deadline));
1133 }
1134
1135 void
1136 usimple_lock_EXT(
1137         lck_spin_t              *lock)
1138 {
1139         lck_spin_lock(lock);
1140 }
1141
1142 void
1143 usimple_lock_init_EXT(
1144         lck_spin_t              *lock,
1145         __unused unsigned short tag)
1146 {
1147         lck_spin_init(lock, &LockCompatGroup, LCK_ATTR_NULL);
1148 }
1149
1150 unsigned int
1151 usimple_lock_try_EXT(
1152         lck_spin_t              *lock)
1153 {
1154         return(lck_spin_try_lock(lock));
1155 }
1156
1157 void
1158 usimple_unlock_EXT(
1159         lck_spin_t              *lock)
1160 {
1161         lck_spin_unlock(lock);
1162 }
1163
1164 wait_result_t
1165 thread_sleep_usimple_lock_EXT(
1166         event_t                 event,
1167         lck_spin_t              *lock,
1168         wait_interrupt_t        interruptible)
1169 {
1170         return( lck_spin_sleep(lock, LCK_SLEEP_DEFAULT, event, interruptible));
1171 }