osfmk/kern/sync_sema.c

   1 /*
   2  * Copyright (c) 2000-2009 Apple Inc. All rights reserved.
   3  *
   4  * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
   5  *
   6  * This file contains Original Code and/or Modifications of Original Code
   7  * as defined in and that are subject to the Apple Public Source License
   8  * Version 2.0 (the 'License'). You may not use this file except in
   9  * compliance with the License. The rights granted to you under the License
  10  * may not be used to create, or enable the creation or redistribution of,
  11  * unlawful or unlicensed copies of an Apple operating system, or to
  12  * circumvent, violate, or enable the circumvention or violation of, any
  13  * terms of an Apple operating system software license agreement.
  14  *
  15  * Please obtain a copy of the License at
  16  * http://www.opensource.apple.com/apsl/ and read it before using this file.
  17  *
  18  * The Original Code and all software distributed under the License are
  19  * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  20  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  21  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
  22  * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
  23  * Please see the License for the specific language governing rights and
  24  * limitations under the License.
  25  *
  26  * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  27  */
  28 /*
  29  * @OSF_COPYRIGHT@
  30  *
  31  */
  32 /*
  33  *      File:   kern/sync_sema.c
  34  *      Author: Joseph CaraDonna
  35  *
  36  *      Contains RT distributed semaphore synchronization services.
  37  */
  38
  39 #include <mach/mach_types.h>
  40 #include <mach/mach_traps.h>
  41 #include <mach/kern_return.h>
  42 #include <mach/semaphore.h>
  43 #include <mach/sync_policy.h>
  44 #include <mach/task.h>
  45
  46 #include <kern/misc_protos.h>
  47 #include <kern/sync_sema.h>
  48 #include <kern/spl.h>
  49 #include <kern/ipc_kobject.h>
  50 #include <kern/ipc_sync.h>
  51 #include <kern/ipc_tt.h>
  52 #include <kern/thread.h>
  53 #include <kern/clock.h>
  54 #include <ipc/ipc_port.h>
  55 #include <ipc/ipc_space.h>
  56 #include <kern/host.h>
  57 #include <kern/wait_queue.h>
  58 #include <kern/zalloc.h>
  59 #include <kern/mach_param.h>
  60
  61 #include <libkern/OSAtomic.h>
  62
  63 static unsigned int semaphore_event;
  64 #define SEMAPHORE_EVENT CAST_EVENT64_T(&semaphore_event)
  65
  66 zone_t semaphore_zone;
  67 unsigned int semaphore_max;
  68
  69 /* Forward declarations */
  70
  71
  72 kern_return_t
  73 semaphore_wait_trap_internal(
  74                                 mach_port_name_t name,
  75                                 void (*caller_cont)(kern_return_t));
  76
  77 kern_return_t
  78 semaphore_wait_signal_trap_internal(
  79                                 mach_port_name_t wait_name,
  80                                 mach_port_name_t signal_name,
  81                                 void (*caller_cont)(kern_return_t));
  82
  83 kern_return_t
  84 semaphore_timedwait_trap_internal(
  85                                 mach_port_name_t name,
  86                                 unsigned int sec,
  87                                 clock_res_t nsec,
  88                                 void (*caller_cont)(kern_return_t));
  89
  90 kern_return_t
  91 semaphore_timedwait_signal_trap_internal(
  92                                 mach_port_name_t wait_name,
  93                                 mach_port_name_t signal_name,
  94                                 unsigned int sec,
  95                                 clock_res_t nsec,
  96                                 void (*caller_cont)(kern_return_t));
  97
  98 kern_return_t
  99 semaphore_signal_internal_trap(mach_port_name_t sema_name);
 100
 101 kern_return_t
 102 semaphore_signal_internal(
 103                         semaphore_t             semaphore,
 104                         thread_t                        thread,
 105                         int                             options);
 106
 107 kern_return_t
 108 semaphore_convert_wait_result(
 109                         int                             wait_result);
 110
 111 void
 112 semaphore_wait_continue(void);
 113
 114 static kern_return_t
 115 semaphore_wait_internal(
 116                         semaphore_t             wait_semaphore,
 117                         semaphore_t             signal_semaphore,
 118                         uint64_t                deadline,
 119                         int                             option,
 120                         void (*caller_cont)(kern_return_t));
 121
 122 static __inline__ uint64_t
 123 semaphore_deadline(
 124         unsigned int            sec,
 125         clock_res_t                     nsec)
 126 {
 127         uint64_t        abstime;
 128
 129         nanoseconds_to_absolutetime((uint64_t)sec *     NSEC_PER_SEC + nsec, &abstime);
 130         clock_absolutetime_interval_to_deadline(abstime, &abstime);
 131
 132         return (abstime);
 133 }
 134
 135 /*
 136  *      ROUTINE:        semaphore_init          [private]
 137  *
 138  *      Initialize the semaphore mechanisms.
 139  *      Right now, we only need to initialize the semaphore zone.
 140  */
 141 void
 142 semaphore_init(void)
 143 {
 144   semaphore_zone = zinit(sizeof(struct semaphore),
 145                         semaphore_max * sizeof(struct semaphore),
 146                         sizeof(struct semaphore),
 147                         "semaphores");
 148   zone_change(semaphore_zone, Z_NOENCRYPT, TRUE);
 149 }
 150
 151 /*
 152  *      Routine:        semaphore_create
 153  *
 154  *      Creates a semaphore.
 155  *      The port representing the semaphore is returned as a parameter.
 156  */
 157 kern_return_t
 158 semaphore_create(
 159         task_t                  task,
 160         semaphore_t             *new_semaphore,
 161         int                             policy,
 162         int                             value)
 163 {
 164         semaphore_t              s = SEMAPHORE_NULL;
 165         kern_return_t           kret;
 166
 167
 168         *new_semaphore = SEMAPHORE_NULL;
 169         if (task == TASK_NULL || value < 0 || policy > SYNC_POLICY_MAX)
 170                 return KERN_INVALID_ARGUMENT;
 171
 172         s = (semaphore_t) zalloc (semaphore_zone);
 173
 174         if (s == SEMAPHORE_NULL)
 175                 return KERN_RESOURCE_SHORTAGE;
 176
 177         kret = wait_queue_init(&s->wait_queue, policy); /* also inits lock */
 178         if (kret != KERN_SUCCESS) {
 179                 zfree(semaphore_zone, s);
 180                 return kret;
 181         }
 182
 183         s->count = value;
 184
 185         /*
 186          * One reference for caller, one for port, and one for owner
 187          * task (if not the kernel itself).
 188          */
 189         s->ref_count = (task == kernel_task) ? 2 : 3;
 190
 191         /*
 192          *  Create and initialize the semaphore port
 193          */
 194         s->port = ipc_port_alloc_kernel();
 195         if (s->port == IP_NULL) {
 196                 zfree(semaphore_zone, s);
 197                 return KERN_RESOURCE_SHORTAGE;
 198         }
 199
 200         ipc_kobject_set (s->port, (ipc_kobject_t) s, IKOT_SEMAPHORE);
 201
 202         /*
 203          *  Associate the new semaphore with the task by adding
 204          *  the new semaphore to the task's semaphore list.
 205          *
 206          *  Associate the task with the new semaphore by having the
 207          *  semaphores task pointer point to the owning task's structure.
 208          */
 209         task_lock(task);
 210         enqueue_head(&task->semaphore_list, (queue_entry_t) s);
 211         task->semaphores_owned++;
 212         s->owner = task;
 213         s->active = TRUE;
 214         task_unlock(task);
 215
 216         *new_semaphore = s;
 217
 218         return KERN_SUCCESS;
 219 }
 220
 221 /*
 222  *      Routine:        semaphore_destroy
 223  *
 224  *      Destroys a semaphore.  This call will only succeed if the
 225  *      specified task is the SAME task name specified at the semaphore's
 226  *      creation.
 227  *
 228  *      All threads currently blocked on the semaphore are awoken.  These
 229  *      threads will return with the KERN_TERMINATED error.
 230  */
 231 kern_return_t
 232 semaphore_destroy(
 233         task_t                  task,
 234         semaphore_t             semaphore)
 235 {
 236         int                             old_count;
 237         spl_t                   spl_level;
 238
 239
 240         if (task == TASK_NULL || semaphore == SEMAPHORE_NULL)
 241                 return KERN_INVALID_ARGUMENT;
 242
 243         /*
 244          *  Disown semaphore
 245          */
 246         task_lock(task);
 247         if (semaphore->owner != task) {
 248                 task_unlock(task);
 249                 return KERN_INVALID_ARGUMENT;
 250         }
 251         remqueue((queue_entry_t) semaphore);
 252         semaphore->owner = TASK_NULL;
 253         task->semaphores_owned--;
 254         task_unlock(task);
 255
 256         spl_level = splsched();
 257         semaphore_lock(semaphore);
 258
 259         /*
 260          *  Deactivate semaphore
 261          */
 262         assert(semaphore->active);
 263         semaphore->active = FALSE;
 264
 265         /*
 266          *  Wakeup blocked threads
 267          */
 268         old_count = semaphore->count;
 269         semaphore->count = 0;
 270
 271         if (old_count < 0) {
 272                 wait_queue_wakeup64_all_locked(&semaphore->wait_queue,
 273                                              SEMAPHORE_EVENT,
 274                                              THREAD_RESTART,
 275                                              TRUE);             /* unlock? */
 276         } else {
 277                 semaphore_unlock(semaphore);
 278         }
 279         splx(spl_level);
 280
 281         /*
 282          *  Deallocate
 283          *
 284          *  Drop the task's semaphore reference, which in turn deallocates
 285          *  the semaphore structure if the reference count goes to zero.
 286          */
 287         semaphore_dereference(semaphore);
 288         return KERN_SUCCESS;
 289 }
 290
 291 /*
 292  *      Routine:        semaphore_signal_internal
 293  *
 294  *              Signals the semaphore as direct.
 295  *      Assumptions:
 296  *              Semaphore is locked.
 297  */
 298 kern_return_t
 299 semaphore_signal_internal(
 300         semaphore_t             semaphore,
 301         thread_t                thread,
 302         int                             options)
 303 {
 304         kern_return_t kr;
 305         spl_t  spl_level;
 306
 307         spl_level = splsched();
 308         semaphore_lock(semaphore);
 309
 310         if (!semaphore->active) {
 311                 semaphore_unlock(semaphore);
 312                 splx(spl_level);
 313                 return KERN_TERMINATED;
 314         }
 315
 316         if (thread != THREAD_NULL) {
 317                 if (semaphore->count < 0) {
 318                         kr = wait_queue_wakeup64_thread_locked(
 319                                         &semaphore->wait_queue,
 320                                         SEMAPHORE_EVENT,
 321                                         thread,
 322                                         THREAD_AWAKENED,
 323                                         TRUE);  /* unlock? */
 324                 } else {
 325                         semaphore_unlock(semaphore);
 326                         kr = KERN_NOT_WAITING;
 327                 }
 328                 splx(spl_level);
 329                 return kr;
 330         }
 331
 332         if (options & SEMAPHORE_SIGNAL_ALL) {
 333                 int old_count = semaphore->count;
 334
 335                 if (old_count < 0) {
 336                         semaphore->count = 0;  /* always reset */
 337                         kr = wait_queue_wakeup64_all_locked(
 338                                         &semaphore->wait_queue,
 339                                         SEMAPHORE_EVENT,
 340                                         THREAD_AWAKENED,
 341                                         TRUE);          /* unlock? */
 342                 } else {
 343                         if (options & SEMAPHORE_SIGNAL_PREPOST)
 344                                 semaphore->count++;
 345                         semaphore_unlock(semaphore);
 346                         kr = KERN_SUCCESS;
 347                 }
 348                 splx(spl_level);
 349                 return kr;
 350         }
 351
 352         if (semaphore->count < 0) {
 353                 if (wait_queue_wakeup64_one_locked(
 354                                         &semaphore->wait_queue,
 355                                         SEMAPHORE_EVENT,
 356                                         THREAD_AWAKENED,
 357                                         FALSE) == KERN_SUCCESS) {
 358                         semaphore_unlock(semaphore);
 359                         splx(spl_level);
 360                         return KERN_SUCCESS;
 361                 } else
 362                         semaphore->count = 0;  /* all waiters gone */
 363         }
 364
 365         if (options & SEMAPHORE_SIGNAL_PREPOST) {
 366                 semaphore->count++;
 367         }
 368
 369         semaphore_unlock(semaphore);
 370         splx(spl_level);
 371         return KERN_NOT_WAITING;
 372 }
 373
 374 /*
 375  *      Routine:        semaphore_signal_thread
 376  *
 377  *      If the specified thread is blocked on the semaphore, it is
 378  *      woken up.  If a NULL thread was supplied, then any one
 379  *      thread is woken up.  Otherwise the caller gets KERN_NOT_WAITING
 380  *      and the semaphore is unchanged.
 381  */
 382 kern_return_t
 383 semaphore_signal_thread(
 384         semaphore_t     semaphore,
 385         thread_t        thread)
 386 {
 387         kern_return_t           ret;
 388
 389         if (semaphore == SEMAPHORE_NULL)
 390                 return KERN_INVALID_ARGUMENT;
 391
 392         ret = semaphore_signal_internal(semaphore,
 393                                         thread,
 394                                         SEMAPHORE_OPTION_NONE);
 395         return ret;
 396 }
 397
 398 /*
 399  *      Routine:        semaphore_signal_thread_trap
 400  *
 401  *      Trap interface to the semaphore_signal_thread function.
 402  */
 403 kern_return_t
 404 semaphore_signal_thread_trap(
 405         struct semaphore_signal_thread_trap_args *args)
 406 {
 407         mach_port_name_t sema_name = args->signal_name;
 408         mach_port_name_t thread_name = args->thread_name;
 409         semaphore_t     semaphore;
 410         thread_t        thread;
 411         kern_return_t   kr;
 412
 413         /*
 414          * MACH_PORT_NULL is not an error. It means that we want to
 415          * select any one thread that is already waiting, but not to
 416          * pre-post the semaphore.
 417          */
 418         if (thread_name != MACH_PORT_NULL) {
 419                 thread = port_name_to_thread(thread_name);
 420                 if (thread == THREAD_NULL)
 421                         return KERN_INVALID_ARGUMENT;
 422         } else
 423                 thread = THREAD_NULL;
 424
 425         kr = port_name_to_semaphore(sema_name, &semaphore);
 426         if (kr == KERN_SUCCESS) {
 427                 kr = semaphore_signal_internal(semaphore,
 428                                 thread,
 429                                 SEMAPHORE_OPTION_NONE);
 430                 semaphore_dereference(semaphore);
 431         }
 432         if (thread != THREAD_NULL) {
 433                 thread_deallocate(thread);
 434         }
 435         return kr;
 436 }
 437
 438
 439
 440 /*
 441  *      Routine:        semaphore_signal
 442  *
 443  *              Traditional (in-kernel client and MIG interface) semaphore
 444  *              signal routine.  Most users will access the trap version.
 445  *
 446  *              This interface in not defined to return info about whether
 447  *              this call found a thread waiting or not.  The internal
 448  *              routines (and future external routines) do.  We have to
 449  *              convert those into plain KERN_SUCCESS returns.
 450  */
 451 kern_return_t
 452 semaphore_signal(
 453         semaphore_t             semaphore)
 454 {
 455         kern_return_t           kr;
 456
 457         if (semaphore == SEMAPHORE_NULL)
 458                 return KERN_INVALID_ARGUMENT;
 459
 460         kr = semaphore_signal_internal(semaphore,
 461                                        THREAD_NULL,
 462                                        SEMAPHORE_SIGNAL_PREPOST);
 463         if (kr == KERN_NOT_WAITING)
 464                 return KERN_SUCCESS;
 465         return kr;
 466 }
 467
 468 /*
 469  *      Routine:        semaphore_signal_trap
 470  *
 471  *      Trap interface to the semaphore_signal function.
 472  */
 473 kern_return_t
 474 semaphore_signal_trap(
 475         struct semaphore_signal_trap_args *args)
 476 {
 477         mach_port_name_t sema_name = args->signal_name;
 478
 479         return (semaphore_signal_internal_trap(sema_name));
 480 }
 481
 482 kern_return_t
 483 semaphore_signal_internal_trap(mach_port_name_t sema_name)
 484 {
 485         semaphore_t     semaphore;
 486         kern_return_t kr;
 487
 488         kr = port_name_to_semaphore(sema_name, &semaphore);
 489         if (kr == KERN_SUCCESS) {
 490                 kr = semaphore_signal_internal(semaphore,
 491                                 THREAD_NULL,
 492                                 SEMAPHORE_SIGNAL_PREPOST);
 493                 semaphore_dereference(semaphore);
 494                 if (kr == KERN_NOT_WAITING)
 495                         kr = KERN_SUCCESS;
 496         }
 497         return kr;
 498 }
 499
 500 /*
 501  *      Routine:        semaphore_signal_all
 502  *
 503  *      Awakens ALL threads currently blocked on the semaphore.
 504  *      The semaphore count returns to zero.
 505  */
 506 kern_return_t
 507 semaphore_signal_all(
 508         semaphore_t             semaphore)
 509 {
 510         kern_return_t kr;
 511
 512         if (semaphore == SEMAPHORE_NULL)
 513                 return KERN_INVALID_ARGUMENT;
 514
 515         kr = semaphore_signal_internal(semaphore,
 516                                        THREAD_NULL,
 517                                        SEMAPHORE_SIGNAL_ALL);
 518         if (kr == KERN_NOT_WAITING)
 519                 return KERN_SUCCESS;
 520         return kr;
 521 }
 522
 523 /*
 524  *      Routine:        semaphore_signal_all_trap
 525  *
 526  *      Trap interface to the semaphore_signal_all function.
 527  */
 528 kern_return_t
 529 semaphore_signal_all_trap(
 530         struct semaphore_signal_all_trap_args *args)
 531 {
 532         mach_port_name_t sema_name = args->signal_name;
 533         semaphore_t     semaphore;
 534         kern_return_t kr;
 535
 536         kr = port_name_to_semaphore(sema_name, &semaphore);
 537         if (kr == KERN_SUCCESS) {
 538                 kr = semaphore_signal_internal(semaphore,
 539                                 THREAD_NULL,
 540                                 SEMAPHORE_SIGNAL_ALL);
 541                 semaphore_dereference(semaphore);
 542                 if (kr == KERN_NOT_WAITING)
 543                         kr = KERN_SUCCESS;
 544         }
 545         return kr;
 546 }
 547
 548 /*
 549  *      Routine:        semaphore_convert_wait_result
 550  *
 551  *      Generate the return code after a semaphore wait/block.  It
 552  *      takes the wait result as an input and coverts that to an
 553  *      appropriate result.
 554  */
 555 kern_return_t
 556 semaphore_convert_wait_result(int wait_result)
 557 {
 558         switch (wait_result) {
 559         case THREAD_AWAKENED:
 560                 return KERN_SUCCESS;
 561
 562         case THREAD_TIMED_OUT:
 563                 return KERN_OPERATION_TIMED_OUT;
 564
 565         case THREAD_INTERRUPTED:
 566                 return KERN_ABORTED;
 567
 568         case THREAD_RESTART:
 569                 return KERN_TERMINATED;
 570
 571         default:
 572                 panic("semaphore_block\n");
 573                 return KERN_FAILURE;
 574         }
 575 }
 576
 577 /*
 578  *      Routine:        semaphore_wait_continue
 579  *
 580  *      Common continuation routine after waiting on a semphore.
 581  *      It returns directly to user space.
 582  */
 583 void
 584 semaphore_wait_continue(void)
 585 {
 586         thread_t self = current_thread();
 587         int wait_result = self->wait_result;
 588         void (*caller_cont)(kern_return_t) = self->sth_continuation;
 589
 590         assert(self->sth_waitsemaphore != SEMAPHORE_NULL);
 591         semaphore_dereference(self->sth_waitsemaphore);
 592         if (self->sth_signalsemaphore != SEMAPHORE_NULL)
 593                 semaphore_dereference(self->sth_signalsemaphore);
 594
 595         assert(caller_cont != (void (*)(kern_return_t))0);
 596         (*caller_cont)(semaphore_convert_wait_result(wait_result));
 597 }
 598
 599 /*
 600  *      Routine:        semaphore_wait_internal
 601  *
 602  *              Decrements the semaphore count by one.  If the count is
 603  *              negative after the decrement, the calling thread blocks
 604  *              (possibly at a continuation and/or with a timeout).
 605  *
 606  *      Assumptions:
 607  *              The reference
 608  *              A reference is held on the signal semaphore.
 609  */
 610 static kern_return_t
 611 semaphore_wait_internal(
 612         semaphore_t             wait_semaphore,
 613         semaphore_t             signal_semaphore,
 614         uint64_t                deadline,
 615         int                             option,
 616         void                    (*caller_cont)(kern_return_t))
 617 {
 618         int                                     wait_result;
 619         spl_t                           spl_level;
 620         kern_return_t           kr = KERN_ALREADY_WAITING;
 621
 622         spl_level = splsched();
 623         semaphore_lock(wait_semaphore);
 624
 625         if (!wait_semaphore->active) {
 626                 kr = KERN_TERMINATED;
 627         } else if (wait_semaphore->count > 0) {
 628                 wait_semaphore->count--;
 629                 kr = KERN_SUCCESS;
 630         } else if (option & SEMAPHORE_TIMEOUT_NOBLOCK) {
 631                 kr = KERN_OPERATION_TIMED_OUT;
 632         } else {
 633                 thread_t        self = current_thread();
 634
 635                 wait_semaphore->count = -1;  /* we don't keep an actual count */
 636                 thread_lock(self);
 637                 (void)wait_queue_assert_wait64_locked(
 638                                         &wait_semaphore->wait_queue,
 639                                         SEMAPHORE_EVENT,
 640                                         THREAD_ABORTSAFE,
 641                                         TIMEOUT_URGENCY_USER_NORMAL,
 642                                         deadline, 0,
 643                                         self);
 644                 thread_unlock(self);
 645         }
 646         semaphore_unlock(wait_semaphore);
 647         splx(spl_level);
 648
 649         /*
 650          * wait_semaphore is unlocked so we are free to go ahead and
 651          * signal the signal_semaphore (if one was provided).
 652          */
 653         if (signal_semaphore != SEMAPHORE_NULL) {
 654                 kern_return_t signal_kr;
 655
 656                 /*
 657                  * lock the signal semaphore reference we got and signal it.
 658                  * This will NOT block (we cannot block after having asserted
 659                  * our intention to wait above).
 660                  */
 661                 signal_kr = semaphore_signal_internal(signal_semaphore,
 662                                                       THREAD_NULL,
 663                                                       SEMAPHORE_SIGNAL_PREPOST);
 664
 665                 if (signal_kr == KERN_NOT_WAITING)
 666                         signal_kr = KERN_SUCCESS;
 667                 else if (signal_kr == KERN_TERMINATED) {
 668                         /*
 669                          * Uh!Oh!  The semaphore we were to signal died.
 670                          * We have to get ourselves out of the wait in
 671                          * case we get stuck here forever (it is assumed
 672                          * that the semaphore we were posting is gating
 673                          * the decision by someone else to post the
 674                          * semaphore we are waiting on).  People will
 675                          * discover the other dead semaphore soon enough.
 676                          * If we got out of the wait cleanly (someone
 677                          * already posted a wakeup to us) then return that
 678                          * (most important) result.  Otherwise,
 679                          * return the KERN_TERMINATED status.
 680                          */
 681                         thread_t self = current_thread();
 682
 683                         clear_wait(self, THREAD_INTERRUPTED);
 684                         kr = semaphore_convert_wait_result(self->wait_result);
 685                         if (kr == KERN_ABORTED)
 686                                 kr = KERN_TERMINATED;
 687                 }
 688         }
 689
 690         /*
 691          * If we had an error, or we didn't really need to wait we can
 692          * return now that we have signalled the signal semaphore.
 693          */
 694         if (kr != KERN_ALREADY_WAITING)
 695                 return kr;
 696
 697         /*
 698          * Now, we can block.  If the caller supplied a continuation
 699          * pointer of his own for after the block, block with the
 700          * appropriate semaphore continuation.  Thiswill gather the
 701          * semaphore results, release references on the semaphore(s),
 702          * and then call the caller's continuation.
 703          */
 704         if (caller_cont) {
 705                 thread_t self = current_thread();
 706
 707                 self->sth_continuation = caller_cont;
 708                 self->sth_waitsemaphore = wait_semaphore;
 709                 self->sth_signalsemaphore = signal_semaphore;
 710                 wait_result = thread_block((thread_continue_t)semaphore_wait_continue);
 711         }
 712         else {
 713                 wait_result = thread_block(THREAD_CONTINUE_NULL);
 714         }
 715
 716         return (semaphore_convert_wait_result(wait_result));
 717 }
 718
 719
 720 /*
 721  *      Routine:        semaphore_wait
 722  *
 723  *      Traditional (non-continuation) interface presented to
 724  *      in-kernel clients to wait on a semaphore.
 725  */
 726 kern_return_t
 727 semaphore_wait(
 728         semaphore_t             semaphore)
 729 {
 730
 731         if (semaphore == SEMAPHORE_NULL)
 732                 return KERN_INVALID_ARGUMENT;
 733
 734         return(semaphore_wait_internal(semaphore,
 735                                            SEMAPHORE_NULL,
 736                                            0ULL, SEMAPHORE_OPTION_NONE,
 737                                        (void (*)(kern_return_t))0));
 738 }
 739
 740 kern_return_t
 741 semaphore_wait_noblock(
 742         semaphore_t             semaphore)
 743 {
 744
 745         if (semaphore == SEMAPHORE_NULL)
 746                 return KERN_INVALID_ARGUMENT;
 747
 748         return(semaphore_wait_internal(semaphore,
 749                                            SEMAPHORE_NULL,
 750                                            0ULL, SEMAPHORE_TIMEOUT_NOBLOCK,
 751                                        (void (*)(kern_return_t))0));
 752 }
 753
 754 kern_return_t
 755 semaphore_wait_deadline(
 756         semaphore_t             semaphore,
 757         uint64_t                deadline)
 758 {
 759
 760         if (semaphore == SEMAPHORE_NULL)
 761                 return KERN_INVALID_ARGUMENT;
 762
 763         return(semaphore_wait_internal(semaphore,
 764                                            SEMAPHORE_NULL,
 765                                            deadline, SEMAPHORE_OPTION_NONE,
 766                                        (void (*)(kern_return_t))0));
 767 }
 768
 769 /*
 770  *      Trap:   semaphore_wait_trap
 771  *
 772  *      Trap version of semaphore wait.  Called on behalf of user-level
 773  *      clients.
 774  */
 775
 776 kern_return_t
 777 semaphore_wait_trap(
 778         struct semaphore_wait_trap_args *args)
 779 {
 780         return(semaphore_wait_trap_internal(args->wait_name, thread_syscall_return));
 781 }
 782
 783
 784
 785 kern_return_t
 786 semaphore_wait_trap_internal(
 787         mach_port_name_t name,
 788         void (*caller_cont)(kern_return_t))
 789 {
 790         semaphore_t     semaphore;
 791         kern_return_t kr;
 792
 793         kr = port_name_to_semaphore(name, &semaphore);
 794         if (kr == KERN_SUCCESS) {
 795                 kr = semaphore_wait_internal(semaphore,
 796                                 SEMAPHORE_NULL,
 797                                 0ULL, SEMAPHORE_OPTION_NONE,
 798                                 caller_cont);
 799                 semaphore_dereference(semaphore);
 800         }
 801         return kr;
 802 }
 803
 804 /*
 805  *      Routine:        semaphore_timedwait
 806  *
 807  *      Traditional (non-continuation) interface presented to
 808  *      in-kernel clients to wait on a semaphore with a timeout.
 809  *
 810  *      A timeout of {0,0} is considered non-blocking.
 811  */
 812 kern_return_t
 813 semaphore_timedwait(
 814         semaphore_t             semaphore,
 815         mach_timespec_t         wait_time)
 816 {
 817         int                             option = SEMAPHORE_OPTION_NONE;
 818         uint64_t                deadline = 0;
 819
 820         if (semaphore == SEMAPHORE_NULL)
 821                 return KERN_INVALID_ARGUMENT;
 822
 823         if(BAD_MACH_TIMESPEC(&wait_time))
 824                 return KERN_INVALID_VALUE;
 825
 826         if (wait_time.tv_sec == 0 && wait_time.tv_nsec == 0)
 827                 option = SEMAPHORE_TIMEOUT_NOBLOCK;
 828         else
 829                 deadline = semaphore_deadline(wait_time.tv_sec, wait_time.tv_nsec);
 830
 831         return (semaphore_wait_internal(semaphore,
 832                                         SEMAPHORE_NULL,
 833                                         deadline, option,
 834                                         (void(*)(kern_return_t))0));
 835
 836 }
 837
 838 /*
 839  *      Trap:   semaphore_timedwait_trap
 840  *
 841  *      Trap version of a semaphore_timedwait.  The timeout parameter
 842  *      is passed in two distinct parts and re-assembled on this side
 843  *      of the trap interface (to accomodate calling conventions that
 844  *      pass structures as pointers instead of inline in registers without
 845  *      having to add a copyin).
 846  *
 847  *      A timeout of {0,0} is considered non-blocking.
 848  */
 849 kern_return_t
 850 semaphore_timedwait_trap(
 851         struct semaphore_timedwait_trap_args *args)
 852 {
 853
 854         return(semaphore_timedwait_trap_internal(args->wait_name, args->sec, args->nsec, thread_syscall_return));
 855 }
 856
 857
 858 kern_return_t
 859 semaphore_timedwait_trap_internal(
 860         mach_port_name_t name,
 861         unsigned int            sec,
 862         clock_res_t             nsec,
 863         void (*caller_cont)(kern_return_t))
 864 {
 865         semaphore_t semaphore;
 866         mach_timespec_t wait_time;
 867         kern_return_t kr;
 868
 869         wait_time.tv_sec = sec;
 870         wait_time.tv_nsec = nsec;
 871         if(BAD_MACH_TIMESPEC(&wait_time))
 872                 return KERN_INVALID_VALUE;
 873
 874         kr = port_name_to_semaphore(name, &semaphore);
 875         if (kr == KERN_SUCCESS) {
 876                 int                             option = SEMAPHORE_OPTION_NONE;
 877                 uint64_t                deadline = 0;
 878
 879                 if (sec == 0 && nsec == 0)
 880                         option = SEMAPHORE_TIMEOUT_NOBLOCK;
 881                 else
 882                         deadline = semaphore_deadline(sec, nsec);
 883
 884                 kr = semaphore_wait_internal(semaphore,
 885                                 SEMAPHORE_NULL,
 886                                 deadline, option,
 887                                 caller_cont);
 888                 semaphore_dereference(semaphore);
 889         }
 890         return kr;
 891 }
 892
 893 /*
 894  *      Routine:        semaphore_wait_signal
 895  *
 896  *      Atomically register a wait on a semaphore and THEN signal
 897  *      another.  This is the in-kernel entry point that does not
 898  *      block at a continuation and does not free a signal_semaphore
 899  *      reference.
 900  */
 901 kern_return_t
 902 semaphore_wait_signal(
 903         semaphore_t             wait_semaphore,
 904         semaphore_t             signal_semaphore)
 905 {
 906         if (wait_semaphore == SEMAPHORE_NULL)
 907                 return KERN_INVALID_ARGUMENT;
 908
 909         return(semaphore_wait_internal(wait_semaphore,
 910                                        signal_semaphore,
 911                                            0ULL, SEMAPHORE_OPTION_NONE,
 912                                        (void(*)(kern_return_t))0));
 913 }
 914
 915 /*
 916  *      Trap:   semaphore_wait_signal_trap
 917  *
 918  *      Atomically register a wait on a semaphore and THEN signal
 919  *      another.  This is the trap version from user space.
 920  */
 921 kern_return_t
 922 semaphore_wait_signal_trap(
 923         struct semaphore_wait_signal_trap_args *args)
 924 {
 925         return(semaphore_wait_signal_trap_internal(args->wait_name, args->signal_name, thread_syscall_return));
 926 }
 927
 928 kern_return_t
 929 semaphore_wait_signal_trap_internal(
 930         mach_port_name_t wait_name,
 931         mach_port_name_t signal_name,
 932         void (*caller_cont)(kern_return_t))
 933 {
 934         semaphore_t wait_semaphore;
 935         semaphore_t signal_semaphore;
 936         kern_return_t kr;
 937
 938         kr = port_name_to_semaphore(signal_name, &signal_semaphore);
 939         if (kr == KERN_SUCCESS) {
 940                 kr = port_name_to_semaphore(wait_name, &wait_semaphore);
 941                 if (kr == KERN_SUCCESS) {
 942                         kr = semaphore_wait_internal(wait_semaphore,
 943                                         signal_semaphore,
 944                                         0ULL, SEMAPHORE_OPTION_NONE,
 945                                         caller_cont);
 946                         semaphore_dereference(wait_semaphore);
 947                 }
 948                 semaphore_dereference(signal_semaphore);
 949         }
 950         return kr;
 951 }
 952
 953
 954 /*
 955  *      Routine:        semaphore_timedwait_signal
 956  *
 957  *      Atomically register a wait on a semaphore and THEN signal
 958  *      another.  This is the in-kernel entry point that does not
 959  *      block at a continuation.
 960  *
 961  *      A timeout of {0,0} is considered non-blocking.
 962  */
 963 kern_return_t
 964 semaphore_timedwait_signal(
 965         semaphore_t             wait_semaphore,
 966         semaphore_t             signal_semaphore,
 967         mach_timespec_t         wait_time)
 968 {
 969         int                             option = SEMAPHORE_OPTION_NONE;
 970         uint64_t                deadline = 0;
 971
 972         if (wait_semaphore == SEMAPHORE_NULL)
 973                 return KERN_INVALID_ARGUMENT;
 974
 975         if(BAD_MACH_TIMESPEC(&wait_time))
 976                 return KERN_INVALID_VALUE;
 977
 978         if (wait_time.tv_sec == 0 && wait_time.tv_nsec == 0)
 979                 option = SEMAPHORE_TIMEOUT_NOBLOCK;
 980         else
 981                 deadline = semaphore_deadline(wait_time.tv_sec, wait_time.tv_nsec);
 982
 983         return(semaphore_wait_internal(wait_semaphore,
 984                                        signal_semaphore,
 985                                            deadline, option,
 986                                        (void(*)(kern_return_t))0));
 987 }
 988
 989 /*
 990  *      Trap:   semaphore_timedwait_signal_trap
 991  *
 992  *      Atomically register a timed wait on a semaphore and THEN signal
 993  *      another.  This is the trap version from user space.
 994  */
 995 kern_return_t
 996 semaphore_timedwait_signal_trap(
 997         struct semaphore_timedwait_signal_trap_args *args)
 998 {
 999         return(semaphore_timedwait_signal_trap_internal(args->wait_name, args->signal_name, args->sec, args->nsec, thread_syscall_return));
1000 }
1001
1002 kern_return_t
1003 semaphore_timedwait_signal_trap_internal(
1004         mach_port_name_t wait_name,
1005         mach_port_name_t signal_name,
1006         unsigned int sec,
1007         clock_res_t nsec,
1008         void (*caller_cont)(kern_return_t))
1009 {
1010         semaphore_t wait_semaphore;
1011         semaphore_t signal_semaphore;
1012         mach_timespec_t wait_time;
1013         kern_return_t kr;
1014
1015         wait_time.tv_sec = sec;
1016         wait_time.tv_nsec = nsec;
1017         if(BAD_MACH_TIMESPEC(&wait_time))
1018                 return KERN_INVALID_VALUE;
1019
1020         kr = port_name_to_semaphore(signal_name, &signal_semaphore);
1021         if (kr == KERN_SUCCESS) {
1022                 kr = port_name_to_semaphore(wait_name, &wait_semaphore);
1023                 if (kr == KERN_SUCCESS) {
1024                         int                             option = SEMAPHORE_OPTION_NONE;
1025                         uint64_t                deadline = 0;
1026
1027                         if (sec == 0 && nsec == 0)
1028                                 option = SEMAPHORE_TIMEOUT_NOBLOCK;
1029                         else
1030                                 deadline = semaphore_deadline(sec, nsec);
1031
1032                         kr = semaphore_wait_internal(wait_semaphore,
1033                                         signal_semaphore,
1034                                         deadline, option,
1035                                         caller_cont);
1036                         semaphore_dereference(wait_semaphore);
1037                 }
1038                 semaphore_dereference(signal_semaphore);
1039         }
1040         return kr;
1041 }
1042
1043
1044 /*
1045  *      Routine:        semaphore_reference
1046  *
1047  *      Take out a reference on a semaphore.  This keeps the data structure
1048  *      in existence (but the semaphore may be deactivated).
1049  */
1050 void
1051 semaphore_reference(
1052         semaphore_t             semaphore)
1053 {
1054         (void)hw_atomic_add(&semaphore->ref_count, 1);
1055 }
1056
1057 /*
1058  *      Routine:        semaphore_dereference
1059  *
1060  *      Release a reference on a semaphore.  If this is the last reference,
1061  *      the semaphore data structure is deallocated.
1062  */
1063 void
1064 semaphore_dereference(
1065         semaphore_t             semaphore)
1066 {
1067         int                     ref_count;
1068
1069         if (semaphore != NULL) {
1070                 ref_count = hw_atomic_sub(&semaphore->ref_count, 1);
1071
1072                 if (ref_count == 1) {
1073                         ipc_port_t port = semaphore->port;
1074
1075                         if (IP_VALID(port) &&
1076                             OSCompareAndSwapPtr(port, IP_NULL, &semaphore->port)) {
1077                                 /*
1078                                  * We get to disassociate the port from the sema and
1079                                  * drop the port's reference on the sema.
1080                                  */
1081                                 ipc_port_dealloc_kernel(port);
1082                                 ref_count = hw_atomic_sub(&semaphore->ref_count, 1);
1083                         }
1084                 }
1085                 if (ref_count == 0) {
1086                         assert(wait_queue_empty(&semaphore->wait_queue));
1087                         zfree(semaphore_zone, semaphore);
1088                 }
1089         }
1090 }