osfmk/kern/sync_lock.c

   1 /*
   2  * Copyright (c) 2000 Apple Computer, Inc. All rights reserved.
   3  *
   4  * @APPLE_LICENSE_HEADER_START@
   5  *
   6  * The contents of this file constitute Original Code as defined in and
   7  * are subject to the Apple Public Source License Version 1.1 (the
   8  * "License").  You may not use this file except in compliance with the
   9  * License.  Please obtain a copy of the License at
  10  * http://www.apple.com/publicsource and read it before using this file.
  11  *
  12  * This Original Code and all software distributed under the License are
  13  * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  14  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  15  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
  16  * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
  17  * License for the specific language governing rights and limitations
  18  * under the License.
  19  *
  20  * @APPLE_LICENSE_HEADER_END@
  21  */
  22 /*
  23  * @OSF_COPYRIGHT@
  24  *
  25  */
  26 /*
  27  *      File:   kern/sync_lock.c
  28  *      Author: Joseph CaraDonna
  29  *
  30  *      Contains RT distributed lock synchronization services.
  31  */
  32
  33 #include <kern/etap_macros.h>
  34 #include <kern/misc_protos.h>
  35 #include <kern/sync_lock.h>
  36 #include <kern/sched_prim.h>
  37 #include <kern/ipc_kobject.h>
  38 #include <kern/ipc_sync.h>
  39 #include <kern/etap_macros.h>
  40 #include <kern/thread.h>
  41 #include <kern/task.h>
  42
  43 #include <ipc/ipc_port.h>
  44 #include <ipc/ipc_space.h>
  45
  46 /*
  47  *      Ulock ownership MACROS
  48  *
  49  *      Assumes: ulock internal lock is held
  50  */
  51
  52 #define ulock_ownership_set(ul, th)                             \
  53         MACRO_BEGIN                                             \
  54         thread_act_t _th_act;                                   \
  55         _th_act = (th)->top_act;                                \
  56         act_lock(_th_act);                                      \
  57         enqueue (&_th_act->held_ulocks, (queue_entry_t) (ul));  \
  58         act_unlock(_th_act);                                    \
  59         (ul)->holder = _th_act;                                 \
  60         MACRO_END
  61
  62 #define ulock_ownership_clear(ul)                               \
  63         MACRO_BEGIN                                             \
  64         thread_act_t _th_act;                                   \
  65         _th_act = (ul)->holder;                                 \
  66         if (_th_act->active) {                                  \
  67                 act_lock(_th_act);                              \
  68                 remqueue(&_th_act->held_ulocks,                 \
  69                          (queue_entry_t) (ul));                 \
  70                 act_unlock(_th_act);                            \
  71         } else {                                                \
  72                 remqueue(&_th_act->held_ulocks,                 \
  73                          (queue_entry_t) (ul));                 \
  74         }                                                       \
  75         (ul)->holder = THR_ACT_NULL;                            \
  76         MACRO_END
  77
  78 /*
  79  *      Lock set ownership MACROS
  80  */
  81
  82 #define lock_set_ownership_set(ls, t)                           \
  83         MACRO_BEGIN                                             \
  84         task_lock((t));                                         \
  85         enqueue_head(&(t)->lock_set_list, (queue_entry_t) (ls));\
  86         (t)->lock_sets_owned++;                                 \
  87         task_unlock((t));                                       \
  88         (ls)->owner = (t);                                      \
  89         MACRO_END
  90
  91 #define lock_set_ownership_clear(ls, t)                         \
  92         MACRO_BEGIN                                             \
  93         task_lock((t));                                         \
  94         remqueue(&(t)->lock_set_list, (queue_entry_t) (ls));    \
  95         (t)->lock_sets_owned--;                                 \
  96         task_unlock((t));                                       \
  97         MACRO_END
  98
  99 unsigned int lock_set_event;
 100 #define LOCK_SET_EVENT ((event_t)&lock_set_event)
 101
 102 unsigned int lock_set_handoff;
 103 #define LOCK_SET_HANDOFF ((event_t)&lock_set_handoff)
 104
 105 /*
 106  *      ROUTINE:        lock_set_init           [private]
 107  *
 108  *      Initialize the lock_set subsystem.
 109  *
 110  *      For now, we don't have anything to do here.
 111  */
 112 void
 113 lock_set_init(void)
 114 {
 115         return;
 116 }
 117
 118
 119 /*
 120  *      ROUTINE:        lock_set_create         [exported]
 121  *
 122  *      Creates a lock set.
 123  *      The port representing the lock set is returned as a parameter.
 124  */
 125 kern_return_t
 126 lock_set_create (
 127         task_t          task,
 128         lock_set_t      *new_lock_set,
 129         int             n_ulocks,
 130         int             policy)
 131 {
 132         lock_set_t      lock_set = LOCK_SET_NULL;
 133         ulock_t         ulock;
 134         int             size;
 135         int             x;
 136
 137         *new_lock_set = LOCK_SET_NULL;
 138
 139         if (task == TASK_NULL || n_ulocks <= 0 || policy > SYNC_POLICY_MAX)
 140                 return KERN_INVALID_ARGUMENT;
 141
 142         size = sizeof(struct lock_set) + (sizeof(struct ulock) * (n_ulocks-1));
 143         lock_set = (lock_set_t) kalloc (size);
 144
 145         if (lock_set == LOCK_SET_NULL)
 146                 return KERN_RESOURCE_SHORTAGE;
 147
 148
 149         lock_set_lock_init(lock_set);
 150         lock_set->n_ulocks = n_ulocks;
 151         lock_set->ref_count = 1;
 152
 153         /*
 154          *  Create and initialize the lock set port
 155          */
 156         lock_set->port = ipc_port_alloc_kernel();
 157         if (lock_set->port == IP_NULL) {
 158                 /* This will deallocate the lock set */
 159                 lock_set_dereference(lock_set);
 160                 return KERN_RESOURCE_SHORTAGE;
 161         }
 162
 163         ipc_kobject_set (lock_set->port,
 164                         (ipc_kobject_t) lock_set,
 165                         IKOT_LOCK_SET);
 166
 167         /*
 168          *  Initialize each ulock in the lock set
 169          */
 170
 171         for (x=0; x < n_ulocks; x++) {
 172                 ulock = (ulock_t) &lock_set->ulock_list[x];
 173                 ulock_lock_init(ulock);
 174                 ulock->lock_set  = lock_set;
 175                 ulock->holder    = THR_ACT_NULL;
 176                 ulock->blocked   = FALSE;
 177                 ulock->unstable  = FALSE;
 178                 ulock->ho_wait   = FALSE;
 179                 wait_queue_init(&ulock->wait_queue, policy);
 180         }
 181
 182         lock_set_ownership_set(lock_set, task);
 183
 184         lock_set->active = TRUE;
 185         *new_lock_set = lock_set;
 186
 187         return KERN_SUCCESS;
 188 }
 189
 190 /*
 191  *      ROUTINE:        lock_set_destroy        [exported]
 192  *
 193  *      Destroys a lock set.  This call will only succeed if the
 194  *      specified task is the SAME task name specified at the lock set's
 195  *      creation.
 196  *
 197  *      NOTES:
 198  *      - All threads currently blocked on the lock set's ulocks are awoken.
 199  *      - These threads will return with the KERN_LOCK_SET_DESTROYED error.
 200  */
 201 kern_return_t
 202 lock_set_destroy (task_t task, lock_set_t lock_set)
 203 {
 204         thread_t        thread;
 205         ulock_t         ulock;
 206         int             i;
 207
 208         if (task == TASK_NULL || lock_set == LOCK_SET_NULL)
 209                 return KERN_INVALID_ARGUMENT;
 210
 211         if (lock_set->owner != task)
 212                 return KERN_INVALID_RIGHT;
 213
 214         lock_set_lock(lock_set);
 215         if (!lock_set->active) {
 216                 lock_set_unlock(lock_set);
 217                 return KERN_LOCK_SET_DESTROYED;
 218         }
 219
 220         /*
 221          *  Deactivate lock set
 222          */
 223         lock_set->active = FALSE;
 224
 225         /*
 226          *  If a ulock is currently held in the target lock set:
 227          *
 228          *  1) Wakeup all threads blocked on the ulock (if any).  Threads
 229          *     may be blocked waiting normally, or waiting for a handoff.
 230          *     Blocked threads will return with KERN_LOCK_SET_DESTROYED.
 231          *
 232          *  2) ulock ownership is cleared.
 233          *     The thread currently holding the ulock is revoked of its
 234          *     ownership.
 235          */
 236         for (i = 0; i < lock_set->n_ulocks; i++) {
 237                 ulock = &lock_set->ulock_list[i];
 238
 239                 ulock_lock(ulock);
 240
 241                 if (ulock->accept_wait) {
 242                         ulock->accept_wait = FALSE;
 243                         wait_queue_wakeup_one(&ulock->wait_queue,
 244                                               LOCK_SET_HANDOFF,
 245                                               THREAD_RESTART);
 246                 }
 247
 248                 if (ulock->holder) {
 249                         if (ulock->blocked) {
 250                                 ulock->blocked = FALSE;
 251                                 wait_queue_wakeup_all(&ulock->wait_queue,
 252                                                       LOCK_SET_EVENT,
 253                                                       THREAD_RESTART);
 254                         }
 255                         if (ulock->ho_wait) {
 256                                 ulock->ho_wait = FALSE;
 257                                 wait_queue_wakeup_one(&ulock->wait_queue,
 258                                                       LOCK_SET_HANDOFF,
 259                                                       THREAD_RESTART);
 260                         }
 261                         ulock_ownership_clear(ulock);
 262                 }
 263
 264                 ulock_unlock(ulock);
 265         }
 266
 267         lock_set_unlock(lock_set);
 268         lock_set_ownership_clear(lock_set, task);
 269
 270         /*
 271          *  Deallocate
 272          *
 273          *  Drop the lock set reference, which inturn destroys the
 274          *  lock set structure if the reference count goes to zero.
 275          */
 276
 277         ipc_port_dealloc_kernel(lock_set->port);
 278         lock_set_dereference(lock_set);
 279
 280         return KERN_SUCCESS;
 281 }
 282
 283 kern_return_t
 284 lock_acquire (lock_set_t lock_set, int lock_id)
 285 {
 286         ulock_t   ulock;
 287
 288         if (lock_set == LOCK_SET_NULL)
 289                 return KERN_INVALID_ARGUMENT;
 290
 291         if (lock_id < 0 || lock_id >= lock_set->n_ulocks)
 292                 return KERN_INVALID_ARGUMENT;
 293
 294  retry:
 295         lock_set_lock(lock_set);
 296         if (!lock_set->active) {
 297                 lock_set_unlock(lock_set);
 298                 return KERN_LOCK_SET_DESTROYED;
 299         }
 300
 301         ulock = (ulock_t) &lock_set->ulock_list[lock_id];
 302         ulock_lock(ulock);
 303         lock_set_unlock(lock_set);
 304
 305         /*
 306          *  Block the current thread if the lock is already held.
 307          */
 308
 309         if (ulock->holder != THR_ACT_NULL) {
 310                 int wait_result;
 311
 312                 lock_set_unlock(lock_set);
 313
 314                 if (ulock->holder == current_act()) {
 315                         ulock_unlock(ulock);
 316                         return KERN_LOCK_OWNED_SELF;
 317                 }
 318
 319                 ulock->blocked = TRUE;
 320                 (void)wait_queue_assert_wait(&ulock->wait_queue,
 321                                        LOCK_SET_EVENT,
 322                                        THREAD_ABORTSAFE);
 323                 ulock_unlock(ulock);
 324
 325                 /*
 326                  *  Block - Wait for lock to become available.
 327                  */
 328
 329                 wait_result = thread_block((void (*)(void))0);
 330
 331                 /*
 332                  *  Check the result status:
 333                  *
 334                  *  Check to see why thread was woken up.  In all cases, we
 335                  *  already have been removed from the queue.
 336                  */
 337                 switch (wait_result) {
 338                 case THREAD_AWAKENED:
 339                         /* lock transitioned from old locker to us */
 340                         /* he already made us owner */
 341                         return (ulock->unstable) ? KERN_LOCK_UNSTABLE :
 342                                                    KERN_SUCCESS;
 343
 344                 case THREAD_INTERRUPTED:
 345                         return KERN_ABORTED;
 346
 347                 case THREAD_RESTART:
 348                         goto retry;  /* probably a dead lock_set */
 349
 350                 default:
 351                         panic("lock_acquire\n");
 352                 }
 353         }
 354
 355         /*
 356          *  Assign lock ownership
 357          */
 358         ulock_ownership_set(ulock, current_thread());
 359         ulock_unlock(ulock);
 360
 361         return (ulock->unstable) ? KERN_LOCK_UNSTABLE : KERN_SUCCESS;
 362 }
 363
 364 kern_return_t
 365 lock_release (lock_set_t lock_set, int lock_id)
 366 {
 367         ulock_t  ulock;
 368
 369         if (lock_set == LOCK_SET_NULL)
 370                 return KERN_INVALID_ARGUMENT;
 371
 372         if (lock_id < 0 || lock_id >= lock_set->n_ulocks)
 373                 return KERN_INVALID_ARGUMENT;
 374
 375         ulock = (ulock_t) &lock_set->ulock_list[lock_id];
 376
 377         return (lock_release_internal(ulock, current_act()));
 378 }
 379
 380 kern_return_t
 381 lock_try (lock_set_t lock_set, int lock_id)
 382 {
 383         ulock_t   ulock;
 384
 385
 386         if (lock_set == LOCK_SET_NULL)
 387                 return KERN_INVALID_ARGUMENT;
 388
 389         if (lock_id < 0 || lock_id >= lock_set->n_ulocks)
 390                 return KERN_INVALID_ARGUMENT;
 391
 392
 393         lock_set_lock(lock_set);
 394         if (!lock_set->active) {
 395                 lock_set_unlock(lock_set);
 396                 return KERN_LOCK_SET_DESTROYED;
 397         }
 398
 399         ulock = (ulock_t) &lock_set->ulock_list[lock_id];
 400         ulock_lock(ulock);
 401         lock_set_unlock(lock_set);
 402
 403         /*
 404          *  If the lock is already owned, we return without blocking.
 405          *
 406          *  An ownership status is returned to inform the caller as to
 407          *  whether it already holds the lock or another thread does.
 408          */
 409
 410         if (ulock->holder != THR_ACT_NULL) {
 411                 lock_set_unlock(lock_set);
 412
 413                 if (ulock->holder == current_act()) {
 414                         ulock_unlock(ulock);
 415                         return KERN_LOCK_OWNED_SELF;
 416                 }
 417
 418                 ulock_unlock(ulock);
 419                 return KERN_LOCK_OWNED;
 420         }
 421
 422         /*
 423          *  Add the ulock to the lock set's held_ulocks list.
 424          */
 425
 426         ulock_ownership_set(ulock, current_thread());
 427         ulock_unlock(ulock);
 428
 429         return (ulock->unstable) ? KERN_LOCK_UNSTABLE : KERN_SUCCESS;
 430 }
 431
 432 kern_return_t
 433 lock_make_stable (lock_set_t lock_set, int lock_id)
 434 {
 435         ulock_t  ulock;
 436
 437
 438         if (lock_set == LOCK_SET_NULL)
 439                 return KERN_INVALID_ARGUMENT;
 440
 441         if (lock_id < 0 || lock_id >= lock_set->n_ulocks)
 442                 return KERN_INVALID_ARGUMENT;
 443
 444
 445         lock_set_lock(lock_set);
 446         if (!lock_set->active) {
 447                 lock_set_unlock(lock_set);
 448                 return KERN_LOCK_SET_DESTROYED;
 449         }
 450
 451         ulock = (ulock_t) &lock_set->ulock_list[lock_id];
 452         ulock_lock(ulock);
 453         lock_set_unlock(lock_set);
 454
 455         if (ulock->holder != current_act()) {
 456                 ulock_unlock(ulock);
 457                 return KERN_INVALID_RIGHT;
 458         }
 459
 460         ulock->unstable = FALSE;
 461         ulock_unlock(ulock);
 462
 463         return KERN_SUCCESS;
 464 }
 465
 466 /*
 467  *      ROUTINE:        lock_make_unstable      [internal]
 468  *
 469  *      Marks the lock as unstable.
 470  *
 471  *      NOTES:
 472  *      - All future acquisitions of the lock will return with a
 473  *        KERN_LOCK_UNSTABLE status, until the lock is made stable again.
 474  */
 475 kern_return_t
 476 lock_make_unstable (ulock_t ulock, thread_act_t thr_act)
 477 {
 478         lock_set_t      lock_set;
 479
 480
 481         lock_set = ulock->lock_set;
 482         lock_set_lock(lock_set);
 483         if (!lock_set->active) {
 484                 lock_set_unlock(lock_set);
 485                 return KERN_LOCK_SET_DESTROYED;
 486         }
 487
 488         ulock_lock(ulock);
 489         lock_set_unlock(lock_set);
 490
 491         if (ulock->holder != thr_act) {
 492                 ulock_unlock(ulock);
 493                 return KERN_INVALID_RIGHT;
 494         }
 495
 496         ulock->unstable = TRUE;
 497         ulock_unlock(ulock);
 498
 499         return KERN_SUCCESS;
 500 }
 501
 502 /*
 503  *      ROUTINE:        lock_release_internal   [internal]
 504  *
 505  *      Releases the ulock.
 506  *      If any threads are blocked waiting for the ulock, one is woken-up.
 507  *
 508  */
 509 kern_return_t
 510 lock_release_internal (ulock_t ulock, thread_act_t thr_act)
 511 {
 512         lock_set_t      lock_set;
 513         int             result;
 514
 515
 516         if ((lock_set = ulock->lock_set) == LOCK_SET_NULL)
 517                 return KERN_INVALID_ARGUMENT;
 518
 519         lock_set_lock(lock_set);
 520         if (!lock_set->active) {
 521                 lock_set_unlock(lock_set);
 522                 return KERN_LOCK_SET_DESTROYED;
 523         }
 524         ulock_lock(ulock);
 525         lock_set_unlock(lock_set);
 526
 527         if (ulock->holder != thr_act) {
 528                 ulock_unlock(ulock);
 529                 lock_set_unlock(lock_set);
 530                 return KERN_INVALID_RIGHT;
 531         }
 532
 533         /*
 534          *  If we have a hint that threads might be waiting,
 535          *  try to transfer the lock ownership to a waiting thread
 536          *  and wake it up.
 537          */
 538         if (ulock->blocked) {
 539                 wait_queue_t    wq = &ulock->wait_queue;
 540                 thread_t        thread;
 541                 spl_t           s;
 542
 543                 s = splsched();
 544                 wait_queue_lock(wq);
 545                 thread = wait_queue_wakeup_identity_locked(wq,
 546                                                            LOCK_SET_EVENT,
 547                                                            THREAD_AWAKENED,
 548                                                            TRUE);
 549                 /* wait_queue now unlocked, thread locked */
 550
 551                 if (thread != THREAD_NULL) {
 552                         /*
 553                          * JMM - These ownership transfer macros have a
 554                          * locking/race problem.  To keep the thread from
 555                          * changing states on us (nullifying the ownership
 556                          * assignment) we need to keep the thread locked
 557                          * during the assignment.  But we can't because the
 558                          * macros take an activation lock, which is a mutex.
 559                          * Since this code was already broken before I got
 560                          * here, I will leave it for now.
 561                          */
 562                         thread_unlock(thread);
 563                         splx(s);
 564
 565                         /*
 566                          *  Transfer ulock ownership
 567                          *  from the current thread to the acquisition thread.
 568                          */
 569                         ulock_ownership_clear(ulock);
 570                         ulock_ownership_set(ulock, thread);
 571                         ulock_unlock(ulock);
 572
 573                         return KERN_SUCCESS;
 574                 } else {
 575                         ulock->blocked = FALSE;
 576                         splx(s);
 577                 }
 578         }
 579
 580         /*
 581          *  Disown ulock
 582          */
 583         ulock_ownership_clear(ulock);
 584         ulock_unlock(ulock);
 585
 586         return KERN_SUCCESS;
 587 }
 588
 589 kern_return_t
 590 lock_handoff (lock_set_t lock_set, int lock_id)
 591 {
 592         ulock_t   ulock;
 593         int       wait_result;
 594
 595
 596         if (lock_set == LOCK_SET_NULL)
 597                 return KERN_INVALID_ARGUMENT;
 598
 599         if (lock_id < 0 || lock_id >= lock_set->n_ulocks)
 600                 return KERN_INVALID_ARGUMENT;
 601
 602  retry:
 603         lock_set_lock(lock_set);
 604
 605         if (!lock_set->active) {
 606                 lock_set_unlock(lock_set);
 607                 return KERN_LOCK_SET_DESTROYED;
 608         }
 609
 610         ulock = (ulock_t) &lock_set->ulock_list[lock_id];
 611         ulock_lock(ulock);
 612         lock_set_unlock(lock_set);
 613
 614         if (ulock->holder != current_act()) {
 615                 ulock_unlock(ulock);
 616                 lock_set_unlock(lock_set);
 617                 return KERN_INVALID_RIGHT;
 618         }
 619
 620         /*
 621          *  If the accepting thread (the receiver) is already waiting
 622          *  to accept the lock from the handoff thread (the sender),
 623          *  then perform the hand-off now.
 624          */
 625
 626         if (ulock->accept_wait) {
 627                 wait_queue_t    wq = &ulock->wait_queue;
 628                 thread_t        thread;
 629                 spl_t           s;
 630
 631                 /*
 632                  *  See who the lucky devil is, if he is still there waiting.
 633                  */
 634                 s = splsched();
 635                 wait_queue_lock(wq);
 636                 thread = wait_queue_wakeup_identity_locked(
 637                                            wq,
 638                                            LOCK_SET_HANDOFF,
 639                                            THREAD_AWAKENED,
 640                                            TRUE);
 641                 /* wait queue unlocked, thread locked */
 642
 643                 /*
 644                  *  Transfer lock ownership
 645                  */
 646                 if (thread != THREAD_NULL) {
 647                         /*
 648                          * JMM - These ownership transfer macros have a
 649                          * locking/race problem.  To keep the thread from
 650                          * changing states on us (nullifying the ownership
 651                          * assignment) we need to keep the thread locked
 652                          * during the assignment.  But we can't because the
 653                          * macros take an activation lock, which is a mutex.
 654                          * Since this code was already broken before I got
 655                          * here, I will leave it for now.
 656                          */
 657                         thread_unlock(thread);
 658                         splx(s);
 659
 660                         ulock_ownership_clear(ulock);
 661                         ulock_ownership_set(ulock, thread);
 662                         ulock->accept_wait = FALSE;
 663                         ulock_unlock(ulock);
 664                         return KERN_SUCCESS;
 665                 } else {
 666
 667                         /*
 668                          * OOPS.  The accepting thread must have been aborted.
 669                          * and is racing back to clear the flag that says is
 670                          * waiting for an accept.  He will clear it when we
 671                          * release the lock, so just fall thru and wait for
 672                          * the next accept thread (that's the way it is
 673                          * specified).
 674                          */
 675                         splx(s);
 676                 }
 677         }
 678
 679         /*
 680          * Indicate that there is a hand-off thread waiting, and then wait
 681          * for an accepting thread.
 682          */
 683         ulock->ho_wait = TRUE;
 684         (void)wait_queue_assert_wait(&ulock->wait_queue,
 685                                LOCK_SET_HANDOFF,
 686                                THREAD_ABORTSAFE);
 687         ulock_unlock(ulock);
 688
 689         ETAP_SET_REASON(current_thread(), BLOCKED_ON_LOCK_HANDOFF);
 690         wait_result = thread_block((void (*)(void))0);
 691
 692         /*
 693          *  If the thread was woken-up via some action other than
 694          *  lock_handoff_accept or lock_set_destroy (i.e. thread_terminate),
 695          *  then we need to clear the ulock's handoff state.
 696          */
 697         switch (wait_result) {
 698
 699         case THREAD_AWAKENED:
 700                 return KERN_SUCCESS;
 701
 702         case THREAD_INTERRUPTED:
 703                 ulock_lock(ulock);
 704                 assert(ulock->holder == current_act());
 705                 ulock->ho_wait = FALSE;
 706                 ulock_unlock(ulock);
 707                 return KERN_ABORTED;
 708
 709         case THREAD_RESTART:
 710                 goto retry;
 711
 712         default:
 713                 panic("lock_handoff");
 714         }
 715 }
 716
 717 kern_return_t
 718 lock_handoff_accept (lock_set_t lock_set, int lock_id)
 719 {
 720         ulock_t   ulock;
 721         int       wait_result;
 722
 723
 724         if (lock_set == LOCK_SET_NULL)
 725                 return KERN_INVALID_ARGUMENT;
 726
 727         if (lock_id < 0 || lock_id >= lock_set->n_ulocks)
 728                 return KERN_INVALID_ARGUMENT;
 729
 730  retry:
 731         lock_set_lock(lock_set);
 732         if (!lock_set->active) {
 733                 lock_set_unlock(lock_set);
 734                 return KERN_LOCK_SET_DESTROYED;
 735         }
 736
 737         ulock = (ulock_t) &lock_set->ulock_list[lock_id];
 738         ulock_lock(ulock);
 739         lock_set_unlock(lock_set);
 740
 741         /*
 742          * If there is another accepting thread that beat us, just
 743          * return with an error.
 744          */
 745         if (ulock->accept_wait) {
 746                 ulock_unlock(ulock);
 747                 return KERN_ALREADY_WAITING;
 748         }
 749
 750         if (ulock->holder == current_act()) {
 751                 ulock_unlock(ulock);
 752                 return KERN_LOCK_OWNED_SELF;
 753         }
 754
 755         /*
 756          *  If the handoff thread (the sender) is already waiting to
 757          *  hand-off the lock to the accepting thread (the receiver),
 758          *  then perform the hand-off now.
 759          */
 760         if (ulock->ho_wait) {
 761                 wait_queue_t    wq = &ulock->wait_queue;
 762                 thread_t        thread;
 763
 764                 /*
 765                  *  See who the lucky devil is, if he is still there waiting.
 766                  */
 767                 assert(ulock->holder != THR_ACT_NULL);
 768                 thread = ulock->holder->thread;
 769
 770                 if (wait_queue_wakeup_thread(wq,
 771                                             LOCK_SET_HANDOFF,
 772                                             thread,
 773                                             THREAD_AWAKENED) == KERN_SUCCESS) {
 774                         /*
 775                          * Holder thread was still waiting to give it
 776                          * away.  Take over ownership.
 777                          */
 778                         ulock_ownership_clear(ulock);
 779                         ulock_ownership_set(ulock, current_thread());
 780                         ulock->ho_wait = FALSE;
 781                         ulock_unlock(ulock);
 782                         return (ulock->unstable) ? KERN_LOCK_UNSTABLE :
 783                                                    KERN_SUCCESS;
 784                 }
 785
 786                 /*
 787                  * OOPS.  The owner was aborted out of the handoff.
 788                  * He will clear his own flag when he gets back.
 789                  * in the meantime, we will wait as if we didn't
 790                  * even see his flag (by falling thru).
 791                  */
 792         }
 793
 794         ulock->accept_wait = TRUE;
 795         (void)wait_queue_assert_wait(&ulock->wait_queue,
 796                                LOCK_SET_HANDOFF,
 797                                THREAD_ABORTSAFE);
 798         ulock_unlock(ulock);
 799
 800         ETAP_SET_REASON(current_thread(), BLOCKED_ON_LOCK_HANDOFF);
 801         wait_result = thread_block((void (*)(void))0);
 802
 803         /*
 804          *  If the thread was woken-up via some action other than
 805          *  lock_handoff_accept or lock_set_destroy (i.e. thread_terminate),
 806          *  then we need to clear the ulock's handoff state.
 807          */
 808         switch (wait_result) {
 809
 810         case THREAD_AWAKENED:
 811                 return KERN_SUCCESS;
 812
 813         case THREAD_INTERRUPTED:
 814                 ulock_lock(ulock);
 815                 ulock->accept_wait = FALSE;
 816                 ulock_unlock(ulock);
 817                 return KERN_ABORTED;
 818
 819         case THREAD_RESTART:
 820                 goto retry;
 821
 822         default:
 823                 panic("lock_handoff_accept");
 824         }
 825 }
 826
 827 /*
 828  *      Routine:        lock_set_reference
 829  *
 830  *      Take out a reference on a lock set.  This keeps the data structure
 831  *      in existence (but the lock set may be deactivated).
 832  */
 833 void
 834 lock_set_reference(lock_set_t lock_set)
 835 {
 836         lock_set_lock(lock_set);
 837         lock_set->ref_count++;
 838         lock_set_unlock(lock_set);
 839 }
 840
 841 /*
 842  *      Routine:        lock_set_dereference
 843  *
 844  *      Release a reference on a lock set.  If this is the last reference,
 845  *      the lock set data structure is deallocated.
 846  */
 847 void
 848 lock_set_dereference(lock_set_t lock_set)
 849 {
 850         int     ref_count;
 851         int     size;
 852
 853         lock_set_lock(lock_set);
 854         ref_count = --(lock_set->ref_count);
 855         lock_set_unlock(lock_set);
 856
 857         if (ref_count == 0) {
 858                 size =  sizeof(struct lock_set) +
 859                         (sizeof(struct ulock) * (lock_set->n_ulocks - 1));
 860                 kfree((vm_offset_t) lock_set, size);
 861         }
 862 }