osfmk/kern/sync_sema.c

   1 /*
   2  * Copyright (c) 2000-2009 Apple Inc. All rights reserved.
   3  *
   4  * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
   5  *
   6  * This file contains Original Code and/or Modifications of Original Code
   7  * as defined in and that are subject to the Apple Public Source License
   8  * Version 2.0 (the 'License'). You may not use this file except in
   9  * compliance with the License. The rights granted to you under the License
  10  * may not be used to create, or enable the creation or redistribution of,
  11  * unlawful or unlicensed copies of an Apple operating system, or to
  12  * circumvent, violate, or enable the circumvention or violation of, any
  13  * terms of an Apple operating system software license agreement.
  14  *
  15  * Please obtain a copy of the License at
  16  * http://www.opensource.apple.com/apsl/ and read it before using this file.
  17  *
  18  * The Original Code and all software distributed under the License are
  19  * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  20  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  21  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
  22  * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
  23  * Please see the License for the specific language governing rights and
  24  * limitations under the License.
  25  *
  26  * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  27  */
  28 /*
  29  * @OSF_COPYRIGHT@
  30  *
  31  */
  32 /*
  33  *      File:   kern/sync_sema.c
  34  *      Author: Joseph CaraDonna
  35  *
  36  *      Contains RT distributed semaphore synchronization services.
  37  */
  38
  39 #include <mach/mach_types.h>
  40 #include <mach/mach_traps.h>
  41 #include <mach/kern_return.h>
  42 #include <mach/semaphore.h>
  43 #include <mach/sync_policy.h>
  44 #include <mach/task.h>
  45
  46 #include <kern/misc_protos.h>
  47 #include <kern/sync_sema.h>
  48 #include <kern/spl.h>
  49 #include <kern/ipc_kobject.h>
  50 #include <kern/ipc_sync.h>
  51 #include <kern/ipc_tt.h>
  52 #include <kern/thread.h>
  53 #include <kern/clock.h>
  54 #include <ipc/ipc_port.h>
  55 #include <ipc/ipc_space.h>
  56 #include <kern/host.h>
  57 #include <kern/wait_queue.h>
  58 #include <kern/zalloc.h>
  59 #include <kern/mach_param.h>
  60
  61 #include <libkern/OSAtomic.h>
  62
  63 static unsigned int semaphore_event;
  64 #define SEMAPHORE_EVENT CAST_EVENT64_T(&semaphore_event)
  65
  66 zone_t semaphore_zone;
  67 unsigned int semaphore_max;
  68
  69 /* Forward declarations */
  70
  71
  72 kern_return_t
  73 semaphore_wait_trap_internal(
  74                                 mach_port_name_t name,
  75                                 void (*caller_cont)(kern_return_t));
  76
  77 kern_return_t
  78 semaphore_wait_signal_trap_internal(
  79                                 mach_port_name_t wait_name,
  80                                 mach_port_name_t signal_name,
  81                                 void (*caller_cont)(kern_return_t));
  82
  83 kern_return_t
  84 semaphore_timedwait_trap_internal(
  85                                 mach_port_name_t name,
  86                                 unsigned int sec,
  87                                 clock_res_t nsec,
  88                                 void (*caller_cont)(kern_return_t));
  89
  90 kern_return_t
  91 semaphore_timedwait_signal_trap_internal(
  92                                 mach_port_name_t wait_name,
  93                                 mach_port_name_t signal_name,
  94                                 unsigned int sec,
  95                                 clock_res_t nsec,
  96                                 void (*caller_cont)(kern_return_t));
  97
  98 kern_return_t
  99 semaphore_signal_internal_trap(mach_port_name_t sema_name);
 100
 101 kern_return_t
 102 semaphore_signal_internal(
 103                         semaphore_t             semaphore,
 104                         thread_t                        thread,
 105                         int                             options);
 106
 107 kern_return_t
 108 semaphore_convert_wait_result(
 109                         int                             wait_result);
 110
 111 void
 112 semaphore_wait_continue(void);
 113
 114 static kern_return_t
 115 semaphore_wait_internal(
 116                         semaphore_t             wait_semaphore,
 117                         semaphore_t             signal_semaphore,
 118                         uint64_t                deadline,
 119                         int                             option,
 120                         void (*caller_cont)(kern_return_t));
 121
 122 static __inline__ uint64_t
 123 semaphore_deadline(
 124         unsigned int            sec,
 125         clock_res_t                     nsec)
 126 {
 127         uint64_t        abstime;
 128
 129         nanoseconds_to_absolutetime((uint64_t)sec *     NSEC_PER_SEC + nsec, &abstime);
 130         clock_absolutetime_interval_to_deadline(abstime, &abstime);
 131
 132         return (abstime);
 133 }
 134
 135 /*
 136  *      ROUTINE:        semaphore_init          [private]
 137  *
 138  *      Initialize the semaphore mechanisms.
 139  *      Right now, we only need to initialize the semaphore zone.
 140  */
 141 void
 142 semaphore_init(void)
 143 {
 144   semaphore_zone = zinit(sizeof(struct semaphore),
 145                         semaphore_max * sizeof(struct semaphore),
 146                         sizeof(struct semaphore),
 147                         "semaphores");
 148   zone_change(semaphore_zone, Z_NOENCRYPT, TRUE);
 149 }
 150
 151 /*
 152  *      Routine:        semaphore_create
 153  *
 154  *      Creates a semaphore.
 155  *      The port representing the semaphore is returned as a parameter.
 156  */
 157 kern_return_t
 158 semaphore_create(
 159         task_t                  task,
 160         semaphore_t             *new_semaphore,
 161         int                             policy,
 162         int                             value)
 163 {
 164         semaphore_t              s = SEMAPHORE_NULL;
 165         kern_return_t           kret;
 166
 167
 168         *new_semaphore = SEMAPHORE_NULL;
 169         if (task == TASK_NULL || value < 0 || policy > SYNC_POLICY_MAX)
 170                 return KERN_INVALID_ARGUMENT;
 171
 172         s = (semaphore_t) zalloc (semaphore_zone);
 173
 174         if (s == SEMAPHORE_NULL)
 175                 return KERN_RESOURCE_SHORTAGE;
 176
 177         kret = wait_queue_init(&s->wait_queue, policy); /* also inits lock */
 178         if (kret != KERN_SUCCESS) {
 179                 zfree(semaphore_zone, s);
 180                 return kret;
 181         }
 182
 183         s->count = value;
 184
 185         /*
 186          * One reference for caller, one for port, and one for owner
 187          * task (if not the kernel itself).
 188          */
 189         s->ref_count = (task == kernel_task) ? 2 : 3;
 190
 191         /*
 192          *  Create and initialize the semaphore port
 193          */
 194         s->port = ipc_port_alloc_kernel();
 195         if (s->port == IP_NULL) {
 196                 zfree(semaphore_zone, s);
 197                 return KERN_RESOURCE_SHORTAGE;
 198         }
 199
 200         ipc_kobject_set (s->port, (ipc_kobject_t) s, IKOT_SEMAPHORE);
 201
 202         /*
 203          *  Associate the new semaphore with the task by adding
 204          *  the new semaphore to the task's semaphore list.
 205          *
 206          *  Associate the task with the new semaphore by having the
 207          *  semaphores task pointer point to the owning task's structure.
 208          */
 209         task_lock(task);
 210         enqueue_head(&task->semaphore_list, (queue_entry_t) s);
 211         task->semaphores_owned++;
 212         s->owner = task;
 213         s->active = TRUE;
 214         task_unlock(task);
 215
 216         *new_semaphore = s;
 217
 218         return KERN_SUCCESS;
 219 }
 220
 221 /*
 222  *      Routine:        semaphore_destroy
 223  *
 224  *      Destroys a semaphore.  This call will only succeed if the
 225  *      specified task is the SAME task name specified at the semaphore's
 226  *      creation.
 227  *
 228  *      All threads currently blocked on the semaphore are awoken.  These
 229  *      threads will return with the KERN_TERMINATED error.
 230  */
 231 kern_return_t
 232 semaphore_destroy(
 233         task_t                  task,
 234         semaphore_t             semaphore)
 235 {
 236         int                             old_count;
 237         spl_t                   spl_level;
 238
 239
 240         if (task == TASK_NULL || semaphore == SEMAPHORE_NULL)
 241                 return KERN_INVALID_ARGUMENT;
 242
 243         /*
 244          *  Disown semaphore
 245          */
 246         task_lock(task);
 247         if (semaphore->owner != task) {
 248                 task_unlock(task);
 249                 return KERN_INVALID_ARGUMENT;
 250         }
 251         remqueue((queue_entry_t) semaphore);
 252         semaphore->owner = TASK_NULL;
 253         task->semaphores_owned--;
 254         task_unlock(task);
 255
 256         spl_level = splsched();
 257         semaphore_lock(semaphore);
 258
 259         /*
 260          *  Deactivate semaphore
 261          */
 262         assert(semaphore->active);
 263         semaphore->active = FALSE;
 264
 265         /*
 266          *  Wakeup blocked threads
 267          */
 268         old_count = semaphore->count;
 269         semaphore->count = 0;
 270
 271         if (old_count < 0) {
 272                 wait_queue_wakeup64_all_locked(&semaphore->wait_queue,
 273                                              SEMAPHORE_EVENT,
 274                                              THREAD_RESTART,
 275                                              TRUE);             /* unlock? */
 276         } else {
 277                 semaphore_unlock(semaphore);
 278         }
 279         splx(spl_level);
 280
 281         /*
 282          *  Deallocate
 283          *
 284          *  Drop the task's semaphore reference, which in turn deallocates
 285          *  the semaphore structure if the reference count goes to zero.
 286          */
 287         semaphore_dereference(semaphore);
 288         return KERN_SUCCESS;
 289 }
 290
 291 /*
 292  *      Routine:        semaphore_signal_internal
 293  *
 294  *              Signals the semaphore as direct.
 295  *      Assumptions:
 296  *              Semaphore is locked.
 297  */
 298 kern_return_t
 299 semaphore_signal_internal(
 300         semaphore_t             semaphore,
 301         thread_t                thread,
 302         int                             options)
 303 {
 304         kern_return_t kr;
 305         spl_t  spl_level;
 306
 307         spl_level = splsched();
 308         semaphore_lock(semaphore);
 309
 310         if (!semaphore->active) {
 311                 semaphore_unlock(semaphore);
 312                 splx(spl_level);
 313                 return KERN_TERMINATED;
 314         }
 315
 316         if (thread != THREAD_NULL) {
 317                 if (semaphore->count < 0) {
 318                         kr = wait_queue_wakeup64_thread_locked(
 319                                         &semaphore->wait_queue,
 320                                         SEMAPHORE_EVENT,
 321                                         thread,
 322                                         THREAD_AWAKENED,
 323                                         TRUE);  /* unlock? */
 324                 } else {
 325                         semaphore_unlock(semaphore);
 326                         kr = KERN_NOT_WAITING;
 327                 }
 328                 splx(spl_level);
 329                 return kr;
 330         }
 331
 332         if (options & SEMAPHORE_SIGNAL_ALL) {
 333                 int old_count = semaphore->count;
 334
 335                 if (old_count < 0) {
 336                         semaphore->count = 0;  /* always reset */
 337                         kr = wait_queue_wakeup64_all_locked(
 338                                         &semaphore->wait_queue,
 339                                         SEMAPHORE_EVENT,
 340                                         THREAD_AWAKENED,
 341                                         TRUE);          /* unlock? */
 342                 } else {
 343                         if (options & SEMAPHORE_SIGNAL_PREPOST)
 344                                 semaphore->count++;
 345                         semaphore_unlock(semaphore);
 346                         kr = KERN_SUCCESS;
 347                 }
 348                 splx(spl_level);
 349                 return kr;
 350         }
 351
 352         if (semaphore->count < 0) {
 353                 if (wait_queue_wakeup64_one_locked(
 354                                         &semaphore->wait_queue,
 355                                         SEMAPHORE_EVENT,
 356                                         THREAD_AWAKENED,
 357                                         FALSE) == KERN_SUCCESS) {
 358                         semaphore_unlock(semaphore);
 359                         splx(spl_level);
 360                         return KERN_SUCCESS;
 361                 } else
 362                         semaphore->count = 0;  /* all waiters gone */
 363         }
 364
 365         if (options & SEMAPHORE_SIGNAL_PREPOST) {
 366                 semaphore->count++;
 367         }
 368
 369         semaphore_unlock(semaphore);
 370         splx(spl_level);
 371         return KERN_NOT_WAITING;
 372 }
 373
 374 /*
 375  *      Routine:        semaphore_signal_thread
 376  *
 377  *      If the specified thread is blocked on the semaphore, it is
 378  *      woken up.  If a NULL thread was supplied, then any one
 379  *      thread is woken up.  Otherwise the caller gets KERN_NOT_WAITING
 380  *      and the semaphore is unchanged.
 381  */
 382 kern_return_t
 383 semaphore_signal_thread(
 384         semaphore_t     semaphore,
 385         thread_t        thread)
 386 {
 387         kern_return_t           ret;
 388
 389         if (semaphore == SEMAPHORE_NULL)
 390                 return KERN_INVALID_ARGUMENT;
 391
 392         ret = semaphore_signal_internal(semaphore,
 393                                         thread,
 394                                         SEMAPHORE_OPTION_NONE);
 395         return ret;
 396 }
 397
 398 /*
 399  *      Routine:        semaphore_signal_thread_trap
 400  *
 401  *      Trap interface to the semaphore_signal_thread function.
 402  */
 403 kern_return_t
 404 semaphore_signal_thread_trap(
 405         struct semaphore_signal_thread_trap_args *args)
 406 {
 407         mach_port_name_t sema_name = args->signal_name;
 408         mach_port_name_t thread_name = args->thread_name;
 409         semaphore_t     semaphore;
 410         thread_t        thread;
 411         kern_return_t   kr;
 412
 413         /*
 414          * MACH_PORT_NULL is not an error. It means that we want to
 415          * select any one thread that is already waiting, but not to
 416          * pre-post the semaphore.
 417          */
 418         if (thread_name != MACH_PORT_NULL) {
 419                 thread = port_name_to_thread(thread_name);
 420                 if (thread == THREAD_NULL)
 421                         return KERN_INVALID_ARGUMENT;
 422         } else
 423                 thread = THREAD_NULL;
 424
 425         kr = port_name_to_semaphore(sema_name, &semaphore);
 426         if (kr == KERN_SUCCESS) {
 427                 kr = semaphore_signal_internal(semaphore,
 428                                 thread,
 429                                 SEMAPHORE_OPTION_NONE);
 430                 semaphore_dereference(semaphore);
 431         }
 432         if (thread != THREAD_NULL) {
 433                 thread_deallocate(thread);
 434         }
 435         return kr;
 436 }
 437
 438
 439
 440 /*
 441  *      Routine:        semaphore_signal
 442  *
 443  *              Traditional (in-kernel client and MIG interface) semaphore
 444  *              signal routine.  Most users will access the trap version.
 445  *
 446  *              This interface in not defined to return info about whether
 447  *              this call found a thread waiting or not.  The internal
 448  *              routines (and future external routines) do.  We have to
 449  *              convert those into plain KERN_SUCCESS returns.
 450  */
 451 kern_return_t
 452 semaphore_signal(
 453         semaphore_t             semaphore)
 454 {
 455         kern_return_t           kr;
 456
 457         if (semaphore == SEMAPHORE_NULL)
 458                 return KERN_INVALID_ARGUMENT;
 459
 460         kr = semaphore_signal_internal(semaphore,
 461                                        THREAD_NULL,
 462                                        SEMAPHORE_SIGNAL_PREPOST);
 463         if (kr == KERN_NOT_WAITING)
 464                 return KERN_SUCCESS;
 465         return kr;
 466 }
 467
 468 /*
 469  *      Routine:        semaphore_signal_trap
 470  *
 471  *      Trap interface to the semaphore_signal function.
 472  */
 473 kern_return_t
 474 semaphore_signal_trap(
 475         struct semaphore_signal_trap_args *args)
 476 {
 477         mach_port_name_t sema_name = args->signal_name;
 478
 479         return (semaphore_signal_internal_trap(sema_name));
 480 }
 481
 482 kern_return_t
 483 semaphore_signal_internal_trap(mach_port_name_t sema_name)
 484 {
 485         semaphore_t     semaphore;
 486         kern_return_t kr;
 487
 488         kr = port_name_to_semaphore(sema_name, &semaphore);
 489         if (kr == KERN_SUCCESS) {
 490                 kr = semaphore_signal_internal(semaphore,
 491                                 THREAD_NULL,
 492                                 SEMAPHORE_SIGNAL_PREPOST);
 493                 semaphore_dereference(semaphore);
 494                 if (kr == KERN_NOT_WAITING)
 495                         kr = KERN_SUCCESS;
 496         }
 497         return kr;
 498 }
 499
 500 /*
 501  *      Routine:        semaphore_signal_all
 502  *
 503  *      Awakens ALL threads currently blocked on the semaphore.
 504  *      The semaphore count returns to zero.
 505  */
 506 kern_return_t
 507 semaphore_signal_all(
 508         semaphore_t             semaphore)
 509 {
 510         kern_return_t kr;
 511
 512         if (semaphore == SEMAPHORE_NULL)
 513                 return KERN_INVALID_ARGUMENT;
 514
 515         kr = semaphore_signal_internal(semaphore,
 516                                        THREAD_NULL,
 517                                        SEMAPHORE_SIGNAL_ALL);
 518         if (kr == KERN_NOT_WAITING)
 519                 return KERN_SUCCESS;
 520         return kr;
 521 }
 522
 523 /*
 524  *      Routine:        semaphore_signal_all_trap
 525  *
 526  *      Trap interface to the semaphore_signal_all function.
 527  */
 528 kern_return_t
 529 semaphore_signal_all_trap(
 530         struct semaphore_signal_all_trap_args *args)
 531 {
 532         mach_port_name_t sema_name = args->signal_name;
 533         semaphore_t     semaphore;
 534         kern_return_t kr;
 535
 536         kr = port_name_to_semaphore(sema_name, &semaphore);
 537         if (kr == KERN_SUCCESS) {
 538                 kr = semaphore_signal_internal(semaphore,
 539                                 THREAD_NULL,
 540                                 SEMAPHORE_SIGNAL_ALL);
 541                 semaphore_dereference(semaphore);
 542                 if (kr == KERN_NOT_WAITING)
 543                         kr = KERN_SUCCESS;
 544         }
 545         return kr;
 546 }
 547
 548 /*
 549  *      Routine:        semaphore_convert_wait_result
 550  *
 551  *      Generate the return code after a semaphore wait/block.  It
 552  *      takes the wait result as an input and coverts that to an
 553  *      appropriate result.
 554  */
 555 kern_return_t
 556 semaphore_convert_wait_result(int wait_result)
 557 {
 558         switch (wait_result) {
 559         case THREAD_AWAKENED:
 560                 return KERN_SUCCESS;
 561
 562         case THREAD_TIMED_OUT:
 563                 return KERN_OPERATION_TIMED_OUT;
 564
 565         case THREAD_INTERRUPTED:
 566                 return KERN_ABORTED;
 567
 568         case THREAD_RESTART:
 569                 return KERN_TERMINATED;
 570
 571         default:
 572                 panic("semaphore_block\n");
 573                 return KERN_FAILURE;
 574         }
 575 }
 576
 577 /*
 578  *      Routine:        semaphore_wait_continue
 579  *
 580  *      Common continuation routine after waiting on a semphore.
 581  *      It returns directly to user space.
 582  */
 583 void
 584 semaphore_wait_continue(void)
 585 {
 586         thread_t self = current_thread();
 587         int wait_result = self->wait_result;
 588         void (*caller_cont)(kern_return_t) = self->sth_continuation;
 589
 590         assert(self->sth_waitsemaphore != SEMAPHORE_NULL);
 591         semaphore_dereference(self->sth_waitsemaphore);
 592         if (self->sth_signalsemaphore != SEMAPHORE_NULL)
 593                 semaphore_dereference(self->sth_signalsemaphore);
 594
 595         assert(caller_cont != (void (*)(kern_return_t))0);
 596         (*caller_cont)(semaphore_convert_wait_result(wait_result));
 597 }
 598
 599 /*
 600  *      Routine:        semaphore_wait_internal
 601  *
 602  *              Decrements the semaphore count by one.  If the count is
 603  *              negative after the decrement, the calling thread blocks
 604  *              (possibly at a continuation and/or with a timeout).
 605  *
 606  *      Assumptions:
 607  *              The reference
 608  *              A reference is held on the signal semaphore.
 609  */
 610 static kern_return_t
 611 semaphore_wait_internal(
 612         semaphore_t             wait_semaphore,
 613         semaphore_t             signal_semaphore,
 614         uint64_t                deadline,
 615         int                             option,
 616         void                    (*caller_cont)(kern_return_t))
 617 {
 618         int                                     wait_result;
 619         spl_t                           spl_level;
 620         kern_return_t           kr = KERN_ALREADY_WAITING;
 621
 622         spl_level = splsched();
 623         semaphore_lock(wait_semaphore);
 624
 625         if (!wait_semaphore->active) {
 626                 kr = KERN_TERMINATED;
 627         } else if (wait_semaphore->count > 0) {
 628                 wait_semaphore->count--;
 629                 kr = KERN_SUCCESS;
 630         } else if (option & SEMAPHORE_TIMEOUT_NOBLOCK) {
 631                 kr = KERN_OPERATION_TIMED_OUT;
 632         } else {
 633                 thread_t        self = current_thread();
 634
 635                 wait_semaphore->count = -1;  /* we don't keep an actual count */
 636                 thread_lock(self);
 637                 (void)wait_queue_assert_wait64_locked(
 638                                         &wait_semaphore->wait_queue,
 639                                         SEMAPHORE_EVENT,
 640                                         THREAD_ABORTSAFE, deadline,
 641                                         self);
 642                 thread_unlock(self);
 643         }
 644         semaphore_unlock(wait_semaphore);
 645         splx(spl_level);
 646
 647         /*
 648          * wait_semaphore is unlocked so we are free to go ahead and
 649          * signal the signal_semaphore (if one was provided).
 650          */
 651         if (signal_semaphore != SEMAPHORE_NULL) {
 652                 kern_return_t signal_kr;
 653
 654                 /*
 655                  * lock the signal semaphore reference we got and signal it.
 656                  * This will NOT block (we cannot block after having asserted
 657                  * our intention to wait above).
 658                  */
 659                 signal_kr = semaphore_signal_internal(signal_semaphore,
 660                                                       THREAD_NULL,
 661                                                       SEMAPHORE_SIGNAL_PREPOST);
 662
 663                 if (signal_kr == KERN_NOT_WAITING)
 664                         signal_kr = KERN_SUCCESS;
 665                 else if (signal_kr == KERN_TERMINATED) {
 666                         /*
 667                          * Uh!Oh!  The semaphore we were to signal died.
 668                          * We have to get ourselves out of the wait in
 669                          * case we get stuck here forever (it is assumed
 670                          * that the semaphore we were posting is gating
 671                          * the decision by someone else to post the
 672                          * semaphore we are waiting on).  People will
 673                          * discover the other dead semaphore soon enough.
 674                          * If we got out of the wait cleanly (someone
 675                          * already posted a wakeup to us) then return that
 676                          * (most important) result.  Otherwise,
 677                          * return the KERN_TERMINATED status.
 678                          */
 679                         thread_t self = current_thread();
 680
 681                         clear_wait(self, THREAD_INTERRUPTED);
 682                         kr = semaphore_convert_wait_result(self->wait_result);
 683                         if (kr == KERN_ABORTED)
 684                                 kr = KERN_TERMINATED;
 685                 }
 686         }
 687
 688         /*
 689          * If we had an error, or we didn't really need to wait we can
 690          * return now that we have signalled the signal semaphore.
 691          */
 692         if (kr != KERN_ALREADY_WAITING)
 693                 return kr;
 694
 695         /*
 696          * Now, we can block.  If the caller supplied a continuation
 697          * pointer of his own for after the block, block with the
 698          * appropriate semaphore continuation.  Thiswill gather the
 699          * semaphore results, release references on the semaphore(s),
 700          * and then call the caller's continuation.
 701          */
 702         if (caller_cont) {
 703                 thread_t self = current_thread();
 704
 705                 self->sth_continuation = caller_cont;
 706                 self->sth_waitsemaphore = wait_semaphore;
 707                 self->sth_signalsemaphore = signal_semaphore;
 708                 wait_result = thread_block((thread_continue_t)semaphore_wait_continue);
 709         }
 710         else {
 711                 wait_result = thread_block(THREAD_CONTINUE_NULL);
 712         }
 713
 714         return (semaphore_convert_wait_result(wait_result));
 715 }
 716
 717
 718 /*
 719  *      Routine:        semaphore_wait
 720  *
 721  *      Traditional (non-continuation) interface presented to
 722  *      in-kernel clients to wait on a semaphore.
 723  */
 724 kern_return_t
 725 semaphore_wait(
 726         semaphore_t             semaphore)
 727 {
 728
 729         if (semaphore == SEMAPHORE_NULL)
 730                 return KERN_INVALID_ARGUMENT;
 731
 732         return(semaphore_wait_internal(semaphore,
 733                                            SEMAPHORE_NULL,
 734                                            0ULL, SEMAPHORE_OPTION_NONE,
 735                                        (void (*)(kern_return_t))0));
 736 }
 737
 738 kern_return_t
 739 semaphore_wait_noblock(
 740         semaphore_t             semaphore)
 741 {
 742
 743         if (semaphore == SEMAPHORE_NULL)
 744                 return KERN_INVALID_ARGUMENT;
 745
 746         return(semaphore_wait_internal(semaphore,
 747                                            SEMAPHORE_NULL,
 748                                            0ULL, SEMAPHORE_TIMEOUT_NOBLOCK,
 749                                        (void (*)(kern_return_t))0));
 750 }
 751
 752 kern_return_t
 753 semaphore_wait_deadline(
 754         semaphore_t             semaphore,
 755         uint64_t                deadline)
 756 {
 757
 758         if (semaphore == SEMAPHORE_NULL)
 759                 return KERN_INVALID_ARGUMENT;
 760
 761         return(semaphore_wait_internal(semaphore,
 762                                            SEMAPHORE_NULL,
 763                                            deadline, SEMAPHORE_OPTION_NONE,
 764                                        (void (*)(kern_return_t))0));
 765 }
 766
 767 /*
 768  *      Trap:   semaphore_wait_trap
 769  *
 770  *      Trap version of semaphore wait.  Called on behalf of user-level
 771  *      clients.
 772  */
 773
 774 kern_return_t
 775 semaphore_wait_trap(
 776         struct semaphore_wait_trap_args *args)
 777 {
 778         return(semaphore_wait_trap_internal(args->wait_name, thread_syscall_return));
 779 }
 780
 781
 782
 783 kern_return_t
 784 semaphore_wait_trap_internal(
 785         mach_port_name_t name,
 786         void (*caller_cont)(kern_return_t))
 787 {
 788         semaphore_t     semaphore;
 789         kern_return_t kr;
 790
 791         kr = port_name_to_semaphore(name, &semaphore);
 792         if (kr == KERN_SUCCESS) {
 793                 kr = semaphore_wait_internal(semaphore,
 794                                 SEMAPHORE_NULL,
 795                                 0ULL, SEMAPHORE_OPTION_NONE,
 796                                 caller_cont);
 797                 semaphore_dereference(semaphore);
 798         }
 799         return kr;
 800 }
 801
 802 /*
 803  *      Routine:        semaphore_timedwait
 804  *
 805  *      Traditional (non-continuation) interface presented to
 806  *      in-kernel clients to wait on a semaphore with a timeout.
 807  *
 808  *      A timeout of {0,0} is considered non-blocking.
 809  */
 810 kern_return_t
 811 semaphore_timedwait(
 812         semaphore_t             semaphore,
 813         mach_timespec_t         wait_time)
 814 {
 815         int                             option = SEMAPHORE_OPTION_NONE;
 816         uint64_t                deadline = 0;
 817
 818         if (semaphore == SEMAPHORE_NULL)
 819                 return KERN_INVALID_ARGUMENT;
 820
 821         if(BAD_MACH_TIMESPEC(&wait_time))
 822                 return KERN_INVALID_VALUE;
 823
 824         if (wait_time.tv_sec == 0 && wait_time.tv_nsec == 0)
 825                 option = SEMAPHORE_TIMEOUT_NOBLOCK;
 826         else
 827                 deadline = semaphore_deadline(wait_time.tv_sec, wait_time.tv_nsec);
 828
 829         return (semaphore_wait_internal(semaphore,
 830                                         SEMAPHORE_NULL,
 831                                         deadline, option,
 832                                         (void(*)(kern_return_t))0));
 833
 834 }
 835
 836 /*
 837  *      Trap:   semaphore_timedwait_trap
 838  *
 839  *      Trap version of a semaphore_timedwait.  The timeout parameter
 840  *      is passed in two distinct parts and re-assembled on this side
 841  *      of the trap interface (to accomodate calling conventions that
 842  *      pass structures as pointers instead of inline in registers without
 843  *      having to add a copyin).
 844  *
 845  *      A timeout of {0,0} is considered non-blocking.
 846  */
 847 kern_return_t
 848 semaphore_timedwait_trap(
 849         struct semaphore_timedwait_trap_args *args)
 850 {
 851
 852         return(semaphore_timedwait_trap_internal(args->wait_name, args->sec, args->nsec, thread_syscall_return));
 853 }
 854
 855
 856 kern_return_t
 857 semaphore_timedwait_trap_internal(
 858         mach_port_name_t name,
 859         unsigned int            sec,
 860         clock_res_t             nsec,
 861         void (*caller_cont)(kern_return_t))
 862 {
 863         semaphore_t semaphore;
 864         mach_timespec_t wait_time;
 865         kern_return_t kr;
 866
 867         wait_time.tv_sec = sec;
 868         wait_time.tv_nsec = nsec;
 869         if(BAD_MACH_TIMESPEC(&wait_time))
 870                 return KERN_INVALID_VALUE;
 871
 872         kr = port_name_to_semaphore(name, &semaphore);
 873         if (kr == KERN_SUCCESS) {
 874                 int                             option = SEMAPHORE_OPTION_NONE;
 875                 uint64_t                deadline = 0;
 876
 877                 if (sec == 0 && nsec == 0)
 878                         option = SEMAPHORE_TIMEOUT_NOBLOCK;
 879                 else
 880                         deadline = semaphore_deadline(sec, nsec);
 881
 882                 kr = semaphore_wait_internal(semaphore,
 883                                 SEMAPHORE_NULL,
 884                                 deadline, option,
 885                                 caller_cont);
 886                 semaphore_dereference(semaphore);
 887         }
 888         return kr;
 889 }
 890
 891 /*
 892  *      Routine:        semaphore_wait_signal
 893  *
 894  *      Atomically register a wait on a semaphore and THEN signal
 895  *      another.  This is the in-kernel entry point that does not
 896  *      block at a continuation and does not free a signal_semaphore
 897  *      reference.
 898  */
 899 kern_return_t
 900 semaphore_wait_signal(
 901         semaphore_t             wait_semaphore,
 902         semaphore_t             signal_semaphore)
 903 {
 904         if (wait_semaphore == SEMAPHORE_NULL)
 905                 return KERN_INVALID_ARGUMENT;
 906
 907         return(semaphore_wait_internal(wait_semaphore,
 908                                        signal_semaphore,
 909                                            0ULL, SEMAPHORE_OPTION_NONE,
 910                                        (void(*)(kern_return_t))0));
 911 }
 912
 913 /*
 914  *      Trap:   semaphore_wait_signal_trap
 915  *
 916  *      Atomically register a wait on a semaphore and THEN signal
 917  *      another.  This is the trap version from user space.
 918  */
 919 kern_return_t
 920 semaphore_wait_signal_trap(
 921         struct semaphore_wait_signal_trap_args *args)
 922 {
 923         return(semaphore_wait_signal_trap_internal(args->wait_name, args->signal_name, thread_syscall_return));
 924 }
 925
 926 kern_return_t
 927 semaphore_wait_signal_trap_internal(
 928         mach_port_name_t wait_name,
 929         mach_port_name_t signal_name,
 930         void (*caller_cont)(kern_return_t))
 931 {
 932         semaphore_t wait_semaphore;
 933         semaphore_t signal_semaphore;
 934         kern_return_t kr;
 935
 936         kr = port_name_to_semaphore(signal_name, &signal_semaphore);
 937         if (kr == KERN_SUCCESS) {
 938                 kr = port_name_to_semaphore(wait_name, &wait_semaphore);
 939                 if (kr == KERN_SUCCESS) {
 940                         kr = semaphore_wait_internal(wait_semaphore,
 941                                         signal_semaphore,
 942                                         0ULL, SEMAPHORE_OPTION_NONE,
 943                                         caller_cont);
 944                         semaphore_dereference(wait_semaphore);
 945                 }
 946                 semaphore_dereference(signal_semaphore);
 947         }
 948         return kr;
 949 }
 950
 951
 952 /*
 953  *      Routine:        semaphore_timedwait_signal
 954  *
 955  *      Atomically register a wait on a semaphore and THEN signal
 956  *      another.  This is the in-kernel entry point that does not
 957  *      block at a continuation.
 958  *
 959  *      A timeout of {0,0} is considered non-blocking.
 960  */
 961 kern_return_t
 962 semaphore_timedwait_signal(
 963         semaphore_t             wait_semaphore,
 964         semaphore_t             signal_semaphore,
 965         mach_timespec_t         wait_time)
 966 {
 967         int                             option = SEMAPHORE_OPTION_NONE;
 968         uint64_t                deadline = 0;
 969
 970         if (wait_semaphore == SEMAPHORE_NULL)
 971                 return KERN_INVALID_ARGUMENT;
 972
 973         if(BAD_MACH_TIMESPEC(&wait_time))
 974                 return KERN_INVALID_VALUE;
 975
 976         if (wait_time.tv_sec == 0 && wait_time.tv_nsec == 0)
 977                 option = SEMAPHORE_TIMEOUT_NOBLOCK;
 978         else
 979                 deadline = semaphore_deadline(wait_time.tv_sec, wait_time.tv_nsec);
 980
 981         return(semaphore_wait_internal(wait_semaphore,
 982                                        signal_semaphore,
 983                                            deadline, option,
 984                                        (void(*)(kern_return_t))0));
 985 }
 986
 987 /*
 988  *      Trap:   semaphore_timedwait_signal_trap
 989  *
 990  *      Atomically register a timed wait on a semaphore and THEN signal
 991  *      another.  This is the trap version from user space.
 992  */
 993 kern_return_t
 994 semaphore_timedwait_signal_trap(
 995         struct semaphore_timedwait_signal_trap_args *args)
 996 {
 997         return(semaphore_timedwait_signal_trap_internal(args->wait_name, args->signal_name, args->sec, args->nsec, thread_syscall_return));
 998 }
 999
1000 kern_return_t
1001 semaphore_timedwait_signal_trap_internal(
1002         mach_port_name_t wait_name,
1003         mach_port_name_t signal_name,
1004         unsigned int sec,
1005         clock_res_t nsec,
1006         void (*caller_cont)(kern_return_t))
1007 {
1008         semaphore_t wait_semaphore;
1009         semaphore_t signal_semaphore;
1010         mach_timespec_t wait_time;
1011         kern_return_t kr;
1012
1013         wait_time.tv_sec = sec;
1014         wait_time.tv_nsec = nsec;
1015         if(BAD_MACH_TIMESPEC(&wait_time))
1016                 return KERN_INVALID_VALUE;
1017
1018         kr = port_name_to_semaphore(signal_name, &signal_semaphore);
1019         if (kr == KERN_SUCCESS) {
1020                 kr = port_name_to_semaphore(wait_name, &wait_semaphore);
1021                 if (kr == KERN_SUCCESS) {
1022                         int                             option = SEMAPHORE_OPTION_NONE;
1023                         uint64_t                deadline = 0;
1024
1025                         if (sec == 0 && nsec == 0)
1026                                 option = SEMAPHORE_TIMEOUT_NOBLOCK;
1027                         else
1028                                 deadline = semaphore_deadline(sec, nsec);
1029
1030                         kr = semaphore_wait_internal(wait_semaphore,
1031                                         signal_semaphore,
1032                                         deadline, option,
1033                                         caller_cont);
1034                         semaphore_dereference(wait_semaphore);
1035                 }
1036                 semaphore_dereference(signal_semaphore);
1037         }
1038         return kr;
1039 }
1040
1041
1042 /*
1043  *      Routine:        semaphore_reference
1044  *
1045  *      Take out a reference on a semaphore.  This keeps the data structure
1046  *      in existence (but the semaphore may be deactivated).
1047  */
1048 void
1049 semaphore_reference(
1050         semaphore_t             semaphore)
1051 {
1052         (void)hw_atomic_add(&semaphore->ref_count, 1);
1053 }
1054
1055 /*
1056  *      Routine:        semaphore_dereference
1057  *
1058  *      Release a reference on a semaphore.  If this is the last reference,
1059  *      the semaphore data structure is deallocated.
1060  */
1061 void
1062 semaphore_dereference(
1063         semaphore_t             semaphore)
1064 {
1065         int                     ref_count;
1066
1067         if (semaphore != NULL) {
1068                 ref_count = hw_atomic_sub(&semaphore->ref_count, 1);
1069
1070                 if (ref_count == 1) {
1071                         ipc_port_t port = semaphore->port;
1072
1073                         if (IP_VALID(port) &&
1074                             OSCompareAndSwapPtr(port, IP_NULL, &semaphore->port)) {
1075                                 /*
1076                                  * We get to disassociate the port from the sema and
1077                                  * drop the port's reference on the sema.
1078                                  */
1079                                 ipc_port_dealloc_kernel(port);
1080                                 ref_count = hw_atomic_sub(&semaphore->ref_count, 1);
1081                         }
1082                 }
1083                 if (ref_count == 0) {
1084                         assert(wait_queue_empty(&semaphore->wait_queue));
1085                         zfree(semaphore_zone, semaphore);
1086                 }
1087         }
1088 }