osfmk/kern/sync_sema.c

   1 /*
   2  * Copyright (c) 2000-2009 Apple Inc. All rights reserved.
   3  *
   4  * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
   5  *
   6  * This file contains Original Code and/or Modifications of Original Code
   7  * as defined in and that are subject to the Apple Public Source License
   8  * Version 2.0 (the 'License'). You may not use this file except in
   9  * compliance with the License. The rights granted to you under the License
  10  * may not be used to create, or enable the creation or redistribution of,
  11  * unlawful or unlicensed copies of an Apple operating system, or to
  12  * circumvent, violate, or enable the circumvention or violation of, any
  13  * terms of an Apple operating system software license agreement.
  14  *
  15  * Please obtain a copy of the License at
  16  * http://www.opensource.apple.com/apsl/ and read it before using this file.
  17  *
  18  * The Original Code and all software distributed under the License are
  19  * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  20  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  21  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
  22  * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
  23  * Please see the License for the specific language governing rights and
  24  * limitations under the License.
  25  *
  26  * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  27  */
  28 /*
  29  * @OSF_COPYRIGHT@
  30  *
  31  */
  32 /*
  33  *      File:   kern/sync_sema.c
  34  *      Author: Joseph CaraDonna
  35  *
  36  *      Contains RT distributed semaphore synchronization services.
  37  */
  38
  39 #include <mach/mach_types.h>
  40 #include <mach/mach_traps.h>
  41 #include <mach/kern_return.h>
  42 #include <mach/semaphore.h>
  43 #include <mach/sync_policy.h>
  44 #include <mach/task.h>
  45
  46 #include <kern/misc_protos.h>
  47 #include <kern/sync_sema.h>
  48 #include <kern/spl.h>
  49 #include <kern/ipc_kobject.h>
  50 #include <kern/ipc_sync.h>
  51 #include <kern/ipc_tt.h>
  52 #include <kern/thread.h>
  53 #include <kern/clock.h>
  54 #include <ipc/ipc_port.h>
  55 #include <ipc/ipc_space.h>
  56 #include <kern/host.h>
  57 #include <kern/waitq.h>
  58 #include <kern/zalloc.h>
  59 #include <kern/mach_param.h>
  60
  61 #include <libkern/OSAtomic.h>
  62
  63 static unsigned int semaphore_event;
  64 #define SEMAPHORE_EVENT CAST_EVENT64_T(&semaphore_event)
  65
  66 zone_t semaphore_zone;
  67 unsigned int semaphore_max;
  68
  69 /* Forward declarations */
  70
  71
  72 kern_return_t
  73 semaphore_wait_trap_internal(
  74                                 mach_port_name_t name,
  75                                 void (*caller_cont)(kern_return_t));
  76
  77 kern_return_t
  78 semaphore_wait_signal_trap_internal(
  79                                 mach_port_name_t wait_name,
  80                                 mach_port_name_t signal_name,
  81                                 void (*caller_cont)(kern_return_t));
  82
  83 kern_return_t
  84 semaphore_timedwait_trap_internal(
  85                                 mach_port_name_t name,
  86                                 unsigned int sec,
  87                                 clock_res_t nsec,
  88                                 void (*caller_cont)(kern_return_t));
  89
  90 kern_return_t
  91 semaphore_timedwait_signal_trap_internal(
  92                                 mach_port_name_t wait_name,
  93                                 mach_port_name_t signal_name,
  94                                 unsigned int sec,
  95                                 clock_res_t nsec,
  96                                 void (*caller_cont)(kern_return_t));
  97
  98 kern_return_t
  99 semaphore_signal_internal_trap(mach_port_name_t sema_name);
 100
 101 kern_return_t
 102 semaphore_signal_internal(
 103                         semaphore_t             semaphore,
 104                         thread_t                        thread,
 105                         int                             options);
 106
 107 kern_return_t
 108 semaphore_convert_wait_result(
 109                         int                             wait_result);
 110
 111 void
 112 semaphore_wait_continue(void);
 113
 114 static kern_return_t
 115 semaphore_wait_internal(
 116                         semaphore_t             wait_semaphore,
 117                         semaphore_t             signal_semaphore,
 118                         uint64_t                deadline,
 119                         int                             option,
 120                         void (*caller_cont)(kern_return_t));
 121
 122 static __inline__ uint64_t
 123 semaphore_deadline(
 124         unsigned int            sec,
 125         clock_res_t                     nsec)
 126 {
 127         uint64_t        abstime;
 128
 129         nanoseconds_to_absolutetime((uint64_t)sec *     NSEC_PER_SEC + nsec, &abstime);
 130         clock_absolutetime_interval_to_deadline(abstime, &abstime);
 131
 132         return (abstime);
 133 }
 134
 135 /*
 136  *      ROUTINE:        semaphore_init          [private]
 137  *
 138  *      Initialize the semaphore mechanisms.
 139  *      Right now, we only need to initialize the semaphore zone.
 140  */
 141 void
 142 semaphore_init(void)
 143 {
 144   semaphore_zone = zinit(sizeof(struct semaphore),
 145                         semaphore_max * sizeof(struct semaphore),
 146                         sizeof(struct semaphore),
 147                         "semaphores");
 148   zone_change(semaphore_zone, Z_NOENCRYPT, TRUE);
 149 }
 150
 151 /*
 152  *      Routine:        semaphore_create
 153  *
 154  *      Creates a semaphore.
 155  *      The port representing the semaphore is returned as a parameter.
 156  */
 157 kern_return_t
 158 semaphore_create(
 159         task_t                  task,
 160         semaphore_t             *new_semaphore,
 161         int                             policy,
 162         int                             value)
 163 {
 164         semaphore_t              s = SEMAPHORE_NULL;
 165         kern_return_t           kret;
 166
 167
 168         *new_semaphore = SEMAPHORE_NULL;
 169         if (task == TASK_NULL || value < 0 || policy > SYNC_POLICY_MAX)
 170                 return KERN_INVALID_ARGUMENT;
 171
 172         s = (semaphore_t) zalloc (semaphore_zone);
 173
 174         if (s == SEMAPHORE_NULL)
 175                 return KERN_RESOURCE_SHORTAGE;
 176
 177         kret = waitq_init(&s->waitq, policy | SYNC_POLICY_DISABLE_IRQ); /* also inits lock */
 178         if (kret != KERN_SUCCESS) {
 179                 zfree(semaphore_zone, s);
 180                 return kret;
 181         }
 182
 183         /*
 184          * Initialize the semaphore values.
 185          */
 186         s->port = IP_NULL;
 187         s->ref_count = 1;
 188         s->count = value;
 189         s->active = TRUE;
 190         s->owner = task;
 191
 192         /*
 193          *  Associate the new semaphore with the task by adding
 194          *  the new semaphore to the task's semaphore list.
 195          */
 196         task_lock(task);
 197         enqueue_head(&task->semaphore_list, (queue_entry_t) s);
 198         task->semaphores_owned++;
 199         task_unlock(task);
 200
 201         *new_semaphore = s;
 202
 203         return KERN_SUCCESS;
 204 }
 205
 206 /*
 207  *      Routine:        semaphore_destroy_internal
 208  *
 209  *      This call will only succeed if the specified task is the SAME task
 210  *      specified at the semaphore's creation.
 211  *
 212  *      All threads currently blocked on the semaphore are awoken.  These
 213  *      threads will return with the KERN_TERMINATED error.
 214  */
 215 kern_return_t
 216 semaphore_destroy_internal(
 217         task_t                  task,
 218         semaphore_t             semaphore)
 219 {
 220         int                     old_count;
 221         spl_t                   spl_level;
 222
 223         /*
 224          *  Disown semaphore
 225          */
 226         task_lock(task);
 227         if (semaphore->owner != task) {
 228                 task_unlock(task);
 229                 return KERN_INVALID_ARGUMENT;
 230         }
 231         spl_level = splsched();
 232         semaphore_lock(semaphore);
 233
 234         remqueue((queue_entry_t) semaphore);
 235         semaphore->owner = TASK_NULL;
 236         task->semaphores_owned--;
 237
 238         task_unlock(task);
 239
 240         /*
 241          *  Deactivate semaphore
 242          */
 243         assert(semaphore->active);
 244         semaphore->active = FALSE;
 245
 246         /*
 247          *  Wakeup blocked threads
 248          */
 249         old_count = semaphore->count;
 250         semaphore->count = 0;
 251
 252         if (old_count < 0) {
 253                 waitq_wakeup64_all_locked(&semaphore->waitq,
 254                                           SEMAPHORE_EVENT,
 255                                           THREAD_RESTART, NULL,
 256                                           WAITQ_ALL_PRIORITIES,
 257                                           WAITQ_UNLOCK);
 258                 /* waitq/semaphore is unlocked */
 259         } else {
 260                 semaphore_unlock(semaphore);
 261         }
 262         splx(spl_level);
 263
 264         return KERN_SUCCESS;
 265 }
 266
 267 /*
 268  *      Routine:        semaphore_destroy
 269  *
 270  *      Destroys a semaphore and consume the caller's reference on the
 271  *      semaphore.
 272  */
 273 kern_return_t
 274 semaphore_destroy(
 275         task_t                  task,
 276         semaphore_t             semaphore)
 277 {
 278         kern_return_t kr;
 279
 280         if (semaphore == SEMAPHORE_NULL)
 281                 return KERN_INVALID_ARGUMENT;
 282
 283         if (task == TASK_NULL) {
 284                 kr = KERN_INVALID_ARGUMENT;
 285         } else {
 286                 kr = semaphore_destroy_internal(task, semaphore);
 287         }
 288         semaphore_dereference(semaphore);
 289         return kr;
 290 }
 291
 292 /*
 293  *      Routine:        semaphore_signal_internal
 294  *
 295  *              Signals the semaphore as direct.
 296  *      Assumptions:
 297  *              Semaphore is locked.
 298  */
 299 kern_return_t
 300 semaphore_signal_internal(
 301         semaphore_t             semaphore,
 302         thread_t                thread,
 303         int                             options)
 304 {
 305         kern_return_t kr;
 306         spl_t  spl_level;
 307
 308         spl_level = splsched();
 309         semaphore_lock(semaphore);
 310
 311         if (!semaphore->active) {
 312                 semaphore_unlock(semaphore);
 313                 splx(spl_level);
 314                 return KERN_TERMINATED;
 315         }
 316
 317         if (thread != THREAD_NULL) {
 318                 if (semaphore->count < 0) {
 319                         kr = waitq_wakeup64_thread_locked(
 320                                         &semaphore->waitq,
 321                                         SEMAPHORE_EVENT,
 322                                         thread,
 323                                         THREAD_AWAKENED,
 324                                         WAITQ_UNLOCK);
 325                         /* waitq/semaphore is unlocked */
 326                 } else {
 327                         kr = KERN_NOT_WAITING;
 328                         semaphore_unlock(semaphore);
 329                 }
 330                 splx(spl_level);
 331                 return kr;
 332         }
 333
 334         if (options & SEMAPHORE_SIGNAL_ALL) {
 335                 int old_count = semaphore->count;
 336
 337                 kr = KERN_NOT_WAITING;
 338                 if (old_count < 0) {
 339                         semaphore->count = 0;  /* always reset */
 340                         kr = waitq_wakeup64_all_locked(
 341                                         &semaphore->waitq,
 342                                         SEMAPHORE_EVENT,
 343                                         THREAD_AWAKENED, NULL,
 344                                         WAITQ_ALL_PRIORITIES,
 345                                         WAITQ_UNLOCK);
 346                         /* waitq / semaphore is unlocked */
 347                 } else {
 348                         if (options & SEMAPHORE_SIGNAL_PREPOST)
 349                                 semaphore->count++;
 350                         kr = KERN_SUCCESS;
 351                         semaphore_unlock(semaphore);
 352                 }
 353                 splx(spl_level);
 354                 return kr;
 355         }
 356
 357         if (semaphore->count < 0) {
 358                 kr = waitq_wakeup64_one_locked(
 359                                         &semaphore->waitq,
 360                                         SEMAPHORE_EVENT,
 361                                         THREAD_AWAKENED, NULL,
 362                                         WAITQ_ALL_PRIORITIES,
 363                                         WAITQ_KEEP_LOCKED);
 364                 if (kr == KERN_SUCCESS) {
 365                         semaphore_unlock(semaphore);
 366                         splx(spl_level);
 367                         return KERN_SUCCESS;
 368                 } else {
 369                         semaphore->count = 0;  /* all waiters gone */
 370                 }
 371         }
 372
 373         if (options & SEMAPHORE_SIGNAL_PREPOST) {
 374                 semaphore->count++;
 375         }
 376
 377         semaphore_unlock(semaphore);
 378         splx(spl_level);
 379         return KERN_NOT_WAITING;
 380 }
 381
 382 /*
 383  *      Routine:        semaphore_signal_thread
 384  *
 385  *      If the specified thread is blocked on the semaphore, it is
 386  *      woken up.  If a NULL thread was supplied, then any one
 387  *      thread is woken up.  Otherwise the caller gets KERN_NOT_WAITING
 388  *      and the semaphore is unchanged.
 389  */
 390 kern_return_t
 391 semaphore_signal_thread(
 392         semaphore_t     semaphore,
 393         thread_t        thread)
 394 {
 395         kern_return_t           ret;
 396
 397         if (semaphore == SEMAPHORE_NULL)
 398                 return KERN_INVALID_ARGUMENT;
 399
 400         ret = semaphore_signal_internal(semaphore,
 401                                         thread,
 402                                         SEMAPHORE_OPTION_NONE);
 403         return ret;
 404 }
 405
 406 /*
 407  *      Routine:        semaphore_signal_thread_trap
 408  *
 409  *      Trap interface to the semaphore_signal_thread function.
 410  */
 411 kern_return_t
 412 semaphore_signal_thread_trap(
 413         struct semaphore_signal_thread_trap_args *args)
 414 {
 415         mach_port_name_t sema_name = args->signal_name;
 416         mach_port_name_t thread_name = args->thread_name;
 417         semaphore_t     semaphore;
 418         thread_t        thread;
 419         kern_return_t   kr;
 420
 421         /*
 422          * MACH_PORT_NULL is not an error. It means that we want to
 423          * select any one thread that is already waiting, but not to
 424          * pre-post the semaphore.
 425          */
 426         if (thread_name != MACH_PORT_NULL) {
 427                 thread = port_name_to_thread(thread_name);
 428                 if (thread == THREAD_NULL)
 429                         return KERN_INVALID_ARGUMENT;
 430         } else
 431                 thread = THREAD_NULL;
 432
 433         kr = port_name_to_semaphore(sema_name, &semaphore);
 434         if (kr == KERN_SUCCESS) {
 435                 kr = semaphore_signal_internal(semaphore,
 436                                 thread,
 437                                 SEMAPHORE_OPTION_NONE);
 438                 semaphore_dereference(semaphore);
 439         }
 440         if (thread != THREAD_NULL) {
 441                 thread_deallocate(thread);
 442         }
 443         return kr;
 444 }
 445
 446
 447
 448 /*
 449  *      Routine:        semaphore_signal
 450  *
 451  *              Traditional (in-kernel client and MIG interface) semaphore
 452  *              signal routine.  Most users will access the trap version.
 453  *
 454  *              This interface in not defined to return info about whether
 455  *              this call found a thread waiting or not.  The internal
 456  *              routines (and future external routines) do.  We have to
 457  *              convert those into plain KERN_SUCCESS returns.
 458  */
 459 kern_return_t
 460 semaphore_signal(
 461         semaphore_t             semaphore)
 462 {
 463         kern_return_t           kr;
 464
 465         if (semaphore == SEMAPHORE_NULL)
 466                 return KERN_INVALID_ARGUMENT;
 467
 468         kr = semaphore_signal_internal(semaphore,
 469                                        THREAD_NULL,
 470                                        SEMAPHORE_SIGNAL_PREPOST);
 471         if (kr == KERN_NOT_WAITING)
 472                 return KERN_SUCCESS;
 473         return kr;
 474 }
 475
 476 /*
 477  *      Routine:        semaphore_signal_trap
 478  *
 479  *      Trap interface to the semaphore_signal function.
 480  */
 481 kern_return_t
 482 semaphore_signal_trap(
 483         struct semaphore_signal_trap_args *args)
 484 {
 485         mach_port_name_t sema_name = args->signal_name;
 486
 487         return (semaphore_signal_internal_trap(sema_name));
 488 }
 489
 490 kern_return_t
 491 semaphore_signal_internal_trap(mach_port_name_t sema_name)
 492 {
 493         semaphore_t     semaphore;
 494         kern_return_t kr;
 495
 496         kr = port_name_to_semaphore(sema_name, &semaphore);
 497         if (kr == KERN_SUCCESS) {
 498                 kr = semaphore_signal_internal(semaphore,
 499                                 THREAD_NULL,
 500                                 SEMAPHORE_SIGNAL_PREPOST);
 501                 semaphore_dereference(semaphore);
 502                 if (kr == KERN_NOT_WAITING)
 503                         kr = KERN_SUCCESS;
 504         }
 505         return kr;
 506 }
 507
 508 /*
 509  *      Routine:        semaphore_signal_all
 510  *
 511  *      Awakens ALL threads currently blocked on the semaphore.
 512  *      The semaphore count returns to zero.
 513  */
 514 kern_return_t
 515 semaphore_signal_all(
 516         semaphore_t             semaphore)
 517 {
 518         kern_return_t kr;
 519
 520         if (semaphore == SEMAPHORE_NULL)
 521                 return KERN_INVALID_ARGUMENT;
 522
 523         kr = semaphore_signal_internal(semaphore,
 524                                        THREAD_NULL,
 525                                        SEMAPHORE_SIGNAL_ALL);
 526         if (kr == KERN_NOT_WAITING)
 527                 return KERN_SUCCESS;
 528         return kr;
 529 }
 530
 531 /*
 532  *      Routine:        semaphore_signal_all_trap
 533  *
 534  *      Trap interface to the semaphore_signal_all function.
 535  */
 536 kern_return_t
 537 semaphore_signal_all_trap(
 538         struct semaphore_signal_all_trap_args *args)
 539 {
 540         mach_port_name_t sema_name = args->signal_name;
 541         semaphore_t     semaphore;
 542         kern_return_t kr;
 543
 544         kr = port_name_to_semaphore(sema_name, &semaphore);
 545         if (kr == KERN_SUCCESS) {
 546                 kr = semaphore_signal_internal(semaphore,
 547                                 THREAD_NULL,
 548                                 SEMAPHORE_SIGNAL_ALL);
 549                 semaphore_dereference(semaphore);
 550                 if (kr == KERN_NOT_WAITING)
 551                         kr = KERN_SUCCESS;
 552         }
 553         return kr;
 554 }
 555
 556 /*
 557  *      Routine:        semaphore_convert_wait_result
 558  *
 559  *      Generate the return code after a semaphore wait/block.  It
 560  *      takes the wait result as an input and coverts that to an
 561  *      appropriate result.
 562  */
 563 kern_return_t
 564 semaphore_convert_wait_result(int wait_result)
 565 {
 566         switch (wait_result) {
 567         case THREAD_AWAKENED:
 568                 return KERN_SUCCESS;
 569
 570         case THREAD_TIMED_OUT:
 571                 return KERN_OPERATION_TIMED_OUT;
 572
 573         case THREAD_INTERRUPTED:
 574                 return KERN_ABORTED;
 575
 576         case THREAD_RESTART:
 577                 return KERN_TERMINATED;
 578
 579         default:
 580                 panic("semaphore_block\n");
 581                 return KERN_FAILURE;
 582         }
 583 }
 584
 585 /*
 586  *      Routine:        semaphore_wait_continue
 587  *
 588  *      Common continuation routine after waiting on a semphore.
 589  *      It returns directly to user space.
 590  */
 591 void
 592 semaphore_wait_continue(void)
 593 {
 594         thread_t self = current_thread();
 595         int wait_result = self->wait_result;
 596         void (*caller_cont)(kern_return_t) = self->sth_continuation;
 597
 598         assert(self->sth_waitsemaphore != SEMAPHORE_NULL);
 599         semaphore_dereference(self->sth_waitsemaphore);
 600         if (self->sth_signalsemaphore != SEMAPHORE_NULL)
 601                 semaphore_dereference(self->sth_signalsemaphore);
 602
 603         assert(caller_cont != (void (*)(kern_return_t))0);
 604         (*caller_cont)(semaphore_convert_wait_result(wait_result));
 605 }
 606
 607 /*
 608  *      Routine:        semaphore_wait_internal
 609  *
 610  *              Decrements the semaphore count by one.  If the count is
 611  *              negative after the decrement, the calling thread blocks
 612  *              (possibly at a continuation and/or with a timeout).
 613  *
 614  *      Assumptions:
 615  *              The reference
 616  *              A reference is held on the signal semaphore.
 617  */
 618 static kern_return_t
 619 semaphore_wait_internal(
 620         semaphore_t             wait_semaphore,
 621         semaphore_t             signal_semaphore,
 622         uint64_t                deadline,
 623         int                             option,
 624         void                    (*caller_cont)(kern_return_t))
 625 {
 626         int                                     wait_result;
 627         spl_t                           spl_level;
 628         kern_return_t           kr = KERN_ALREADY_WAITING;
 629
 630         spl_level = splsched();
 631         semaphore_lock(wait_semaphore);
 632
 633         if (!wait_semaphore->active) {
 634                 kr = KERN_TERMINATED;
 635         } else if (wait_semaphore->count > 0) {
 636                 wait_semaphore->count--;
 637                 kr = KERN_SUCCESS;
 638         } else if (option & SEMAPHORE_TIMEOUT_NOBLOCK) {
 639                 kr = KERN_OPERATION_TIMED_OUT;
 640         } else {
 641                 thread_t        self = current_thread();
 642
 643                 wait_semaphore->count = -1;  /* we don't keep an actual count */
 644                 thread_lock(self);
 645                 (void)waitq_assert_wait64_locked(
 646                                         &wait_semaphore->waitq,
 647                                         SEMAPHORE_EVENT,
 648                                         THREAD_ABORTSAFE,
 649                                         TIMEOUT_URGENCY_USER_NORMAL,
 650                                         deadline, TIMEOUT_NO_LEEWAY,
 651                                         self);
 652                 thread_unlock(self);
 653         }
 654         semaphore_unlock(wait_semaphore);
 655         splx(spl_level);
 656
 657         /*
 658          * wait_semaphore is unlocked so we are free to go ahead and
 659          * signal the signal_semaphore (if one was provided).
 660          */
 661         if (signal_semaphore != SEMAPHORE_NULL) {
 662                 kern_return_t signal_kr;
 663
 664                 /*
 665                  * lock the signal semaphore reference we got and signal it.
 666                  * This will NOT block (we cannot block after having asserted
 667                  * our intention to wait above).
 668                  */
 669                 signal_kr = semaphore_signal_internal(signal_semaphore,
 670                                                       THREAD_NULL,
 671                                                       SEMAPHORE_SIGNAL_PREPOST);
 672
 673                 if (signal_kr == KERN_NOT_WAITING)
 674                         signal_kr = KERN_SUCCESS;
 675                 else if (signal_kr == KERN_TERMINATED) {
 676                         /*
 677                          * Uh!Oh!  The semaphore we were to signal died.
 678                          * We have to get ourselves out of the wait in
 679                          * case we get stuck here forever (it is assumed
 680                          * that the semaphore we were posting is gating
 681                          * the decision by someone else to post the
 682                          * semaphore we are waiting on).  People will
 683                          * discover the other dead semaphore soon enough.
 684                          * If we got out of the wait cleanly (someone
 685                          * already posted a wakeup to us) then return that
 686                          * (most important) result.  Otherwise,
 687                          * return the KERN_TERMINATED status.
 688                          */
 689                         thread_t self = current_thread();
 690
 691                         clear_wait(self, THREAD_INTERRUPTED);
 692                         kr = semaphore_convert_wait_result(self->wait_result);
 693                         if (kr == KERN_ABORTED)
 694                                 kr = KERN_TERMINATED;
 695                 }
 696         }
 697
 698         /*
 699          * If we had an error, or we didn't really need to wait we can
 700          * return now that we have signalled the signal semaphore.
 701          */
 702         if (kr != KERN_ALREADY_WAITING)
 703                 return kr;
 704
 705         /*
 706          * Now, we can block.  If the caller supplied a continuation
 707          * pointer of his own for after the block, block with the
 708          * appropriate semaphore continuation.  Thiswill gather the
 709          * semaphore results, release references on the semaphore(s),
 710          * and then call the caller's continuation.
 711          */
 712         if (caller_cont) {
 713                 thread_t self = current_thread();
 714
 715                 self->sth_continuation = caller_cont;
 716                 self->sth_waitsemaphore = wait_semaphore;
 717                 self->sth_signalsemaphore = signal_semaphore;
 718                 wait_result = thread_block((thread_continue_t)semaphore_wait_continue);
 719         }
 720         else {
 721                 wait_result = thread_block(THREAD_CONTINUE_NULL);
 722         }
 723
 724         return (semaphore_convert_wait_result(wait_result));
 725 }
 726
 727
 728 /*
 729  *      Routine:        semaphore_wait
 730  *
 731  *      Traditional (non-continuation) interface presented to
 732  *      in-kernel clients to wait on a semaphore.
 733  */
 734 kern_return_t
 735 semaphore_wait(
 736         semaphore_t             semaphore)
 737 {
 738
 739         if (semaphore == SEMAPHORE_NULL)
 740                 return KERN_INVALID_ARGUMENT;
 741
 742         return(semaphore_wait_internal(semaphore,
 743                                            SEMAPHORE_NULL,
 744                                            0ULL, SEMAPHORE_OPTION_NONE,
 745                                        (void (*)(kern_return_t))0));
 746 }
 747
 748 kern_return_t
 749 semaphore_wait_noblock(
 750         semaphore_t             semaphore)
 751 {
 752
 753         if (semaphore == SEMAPHORE_NULL)
 754                 return KERN_INVALID_ARGUMENT;
 755
 756         return(semaphore_wait_internal(semaphore,
 757                                            SEMAPHORE_NULL,
 758                                            0ULL, SEMAPHORE_TIMEOUT_NOBLOCK,
 759                                        (void (*)(kern_return_t))0));
 760 }
 761
 762 kern_return_t
 763 semaphore_wait_deadline(
 764         semaphore_t             semaphore,
 765         uint64_t                deadline)
 766 {
 767
 768         if (semaphore == SEMAPHORE_NULL)
 769                 return KERN_INVALID_ARGUMENT;
 770
 771         return(semaphore_wait_internal(semaphore,
 772                                            SEMAPHORE_NULL,
 773                                            deadline, SEMAPHORE_OPTION_NONE,
 774                                        (void (*)(kern_return_t))0));
 775 }
 776
 777 /*
 778  *      Trap:   semaphore_wait_trap
 779  *
 780  *      Trap version of semaphore wait.  Called on behalf of user-level
 781  *      clients.
 782  */
 783
 784 kern_return_t
 785 semaphore_wait_trap(
 786         struct semaphore_wait_trap_args *args)
 787 {
 788         return(semaphore_wait_trap_internal(args->wait_name, thread_syscall_return));
 789 }
 790
 791
 792
 793 kern_return_t
 794 semaphore_wait_trap_internal(
 795         mach_port_name_t name,
 796         void (*caller_cont)(kern_return_t))
 797 {
 798         semaphore_t     semaphore;
 799         kern_return_t kr;
 800
 801         kr = port_name_to_semaphore(name, &semaphore);
 802         if (kr == KERN_SUCCESS) {
 803                 kr = semaphore_wait_internal(semaphore,
 804                                 SEMAPHORE_NULL,
 805                                 0ULL, SEMAPHORE_OPTION_NONE,
 806                                 caller_cont);
 807                 semaphore_dereference(semaphore);
 808         }
 809         return kr;
 810 }
 811
 812 /*
 813  *      Routine:        semaphore_timedwait
 814  *
 815  *      Traditional (non-continuation) interface presented to
 816  *      in-kernel clients to wait on a semaphore with a timeout.
 817  *
 818  *      A timeout of {0,0} is considered non-blocking.
 819  */
 820 kern_return_t
 821 semaphore_timedwait(
 822         semaphore_t             semaphore,
 823         mach_timespec_t         wait_time)
 824 {
 825         int                             option = SEMAPHORE_OPTION_NONE;
 826         uint64_t                deadline = 0;
 827
 828         if (semaphore == SEMAPHORE_NULL)
 829                 return KERN_INVALID_ARGUMENT;
 830
 831         if(BAD_MACH_TIMESPEC(&wait_time))
 832                 return KERN_INVALID_VALUE;
 833
 834         if (wait_time.tv_sec == 0 && wait_time.tv_nsec == 0)
 835                 option = SEMAPHORE_TIMEOUT_NOBLOCK;
 836         else
 837                 deadline = semaphore_deadline(wait_time.tv_sec, wait_time.tv_nsec);
 838
 839         return (semaphore_wait_internal(semaphore,
 840                                         SEMAPHORE_NULL,
 841                                         deadline, option,
 842                                         (void(*)(kern_return_t))0));
 843
 844 }
 845
 846 /*
 847  *      Trap:   semaphore_timedwait_trap
 848  *
 849  *      Trap version of a semaphore_timedwait.  The timeout parameter
 850  *      is passed in two distinct parts and re-assembled on this side
 851  *      of the trap interface (to accomodate calling conventions that
 852  *      pass structures as pointers instead of inline in registers without
 853  *      having to add a copyin).
 854  *
 855  *      A timeout of {0,0} is considered non-blocking.
 856  */
 857 kern_return_t
 858 semaphore_timedwait_trap(
 859         struct semaphore_timedwait_trap_args *args)
 860 {
 861
 862         return(semaphore_timedwait_trap_internal(args->wait_name, args->sec, args->nsec, thread_syscall_return));
 863 }
 864
 865
 866 kern_return_t
 867 semaphore_timedwait_trap_internal(
 868         mach_port_name_t name,
 869         unsigned int            sec,
 870         clock_res_t             nsec,
 871         void (*caller_cont)(kern_return_t))
 872 {
 873         semaphore_t semaphore;
 874         mach_timespec_t wait_time;
 875         kern_return_t kr;
 876
 877         wait_time.tv_sec = sec;
 878         wait_time.tv_nsec = nsec;
 879         if(BAD_MACH_TIMESPEC(&wait_time))
 880                 return KERN_INVALID_VALUE;
 881
 882         kr = port_name_to_semaphore(name, &semaphore);
 883         if (kr == KERN_SUCCESS) {
 884                 int                             option = SEMAPHORE_OPTION_NONE;
 885                 uint64_t                deadline = 0;
 886
 887                 if (sec == 0 && nsec == 0)
 888                         option = SEMAPHORE_TIMEOUT_NOBLOCK;
 889                 else
 890                         deadline = semaphore_deadline(sec, nsec);
 891
 892                 kr = semaphore_wait_internal(semaphore,
 893                                 SEMAPHORE_NULL,
 894                                 deadline, option,
 895                                 caller_cont);
 896                 semaphore_dereference(semaphore);
 897         }
 898         return kr;
 899 }
 900
 901 /*
 902  *      Routine:        semaphore_wait_signal
 903  *
 904  *      Atomically register a wait on a semaphore and THEN signal
 905  *      another.  This is the in-kernel entry point that does not
 906  *      block at a continuation and does not free a signal_semaphore
 907  *      reference.
 908  */
 909 kern_return_t
 910 semaphore_wait_signal(
 911         semaphore_t             wait_semaphore,
 912         semaphore_t             signal_semaphore)
 913 {
 914         if (wait_semaphore == SEMAPHORE_NULL)
 915                 return KERN_INVALID_ARGUMENT;
 916
 917         return(semaphore_wait_internal(wait_semaphore,
 918                                        signal_semaphore,
 919                                            0ULL, SEMAPHORE_OPTION_NONE,
 920                                        (void(*)(kern_return_t))0));
 921 }
 922
 923 /*
 924  *      Trap:   semaphore_wait_signal_trap
 925  *
 926  *      Atomically register a wait on a semaphore and THEN signal
 927  *      another.  This is the trap version from user space.
 928  */
 929 kern_return_t
 930 semaphore_wait_signal_trap(
 931         struct semaphore_wait_signal_trap_args *args)
 932 {
 933         return(semaphore_wait_signal_trap_internal(args->wait_name, args->signal_name, thread_syscall_return));
 934 }
 935
 936 kern_return_t
 937 semaphore_wait_signal_trap_internal(
 938         mach_port_name_t wait_name,
 939         mach_port_name_t signal_name,
 940         void (*caller_cont)(kern_return_t))
 941 {
 942         semaphore_t wait_semaphore;
 943         semaphore_t signal_semaphore;
 944         kern_return_t kr;
 945
 946         kr = port_name_to_semaphore(signal_name, &signal_semaphore);
 947         if (kr == KERN_SUCCESS) {
 948                 kr = port_name_to_semaphore(wait_name, &wait_semaphore);
 949                 if (kr == KERN_SUCCESS) {
 950                         kr = semaphore_wait_internal(wait_semaphore,
 951                                         signal_semaphore,
 952                                         0ULL, SEMAPHORE_OPTION_NONE,
 953                                         caller_cont);
 954                         semaphore_dereference(wait_semaphore);
 955                 }
 956                 semaphore_dereference(signal_semaphore);
 957         }
 958         return kr;
 959 }
 960
 961
 962 /*
 963  *      Routine:        semaphore_timedwait_signal
 964  *
 965  *      Atomically register a wait on a semaphore and THEN signal
 966  *      another.  This is the in-kernel entry point that does not
 967  *      block at a continuation.
 968  *
 969  *      A timeout of {0,0} is considered non-blocking.
 970  */
 971 kern_return_t
 972 semaphore_timedwait_signal(
 973         semaphore_t             wait_semaphore,
 974         semaphore_t             signal_semaphore,
 975         mach_timespec_t         wait_time)
 976 {
 977         int                             option = SEMAPHORE_OPTION_NONE;
 978         uint64_t                deadline = 0;
 979
 980         if (wait_semaphore == SEMAPHORE_NULL)
 981                 return KERN_INVALID_ARGUMENT;
 982
 983         if(BAD_MACH_TIMESPEC(&wait_time))
 984                 return KERN_INVALID_VALUE;
 985
 986         if (wait_time.tv_sec == 0 && wait_time.tv_nsec == 0)
 987                 option = SEMAPHORE_TIMEOUT_NOBLOCK;
 988         else
 989                 deadline = semaphore_deadline(wait_time.tv_sec, wait_time.tv_nsec);
 990
 991         return(semaphore_wait_internal(wait_semaphore,
 992                                        signal_semaphore,
 993                                            deadline, option,
 994                                        (void(*)(kern_return_t))0));
 995 }
 996
 997 /*
 998  *      Trap:   semaphore_timedwait_signal_trap
 999  *
1000  *      Atomically register a timed wait on a semaphore and THEN signal
1001  *      another.  This is the trap version from user space.
1002  */
1003 kern_return_t
1004 semaphore_timedwait_signal_trap(
1005         struct semaphore_timedwait_signal_trap_args *args)
1006 {
1007         return(semaphore_timedwait_signal_trap_internal(args->wait_name, args->signal_name, args->sec, args->nsec, thread_syscall_return));
1008 }
1009
1010 kern_return_t
1011 semaphore_timedwait_signal_trap_internal(
1012         mach_port_name_t wait_name,
1013         mach_port_name_t signal_name,
1014         unsigned int sec,
1015         clock_res_t nsec,
1016         void (*caller_cont)(kern_return_t))
1017 {
1018         semaphore_t wait_semaphore;
1019         semaphore_t signal_semaphore;
1020         mach_timespec_t wait_time;
1021         kern_return_t kr;
1022
1023         wait_time.tv_sec = sec;
1024         wait_time.tv_nsec = nsec;
1025         if(BAD_MACH_TIMESPEC(&wait_time))
1026                 return KERN_INVALID_VALUE;
1027
1028         kr = port_name_to_semaphore(signal_name, &signal_semaphore);
1029         if (kr == KERN_SUCCESS) {
1030                 kr = port_name_to_semaphore(wait_name, &wait_semaphore);
1031                 if (kr == KERN_SUCCESS) {
1032                         int                             option = SEMAPHORE_OPTION_NONE;
1033                         uint64_t                deadline = 0;
1034
1035                         if (sec == 0 && nsec == 0)
1036                                 option = SEMAPHORE_TIMEOUT_NOBLOCK;
1037                         else
1038                                 deadline = semaphore_deadline(sec, nsec);
1039
1040                         kr = semaphore_wait_internal(wait_semaphore,
1041                                         signal_semaphore,
1042                                         deadline, option,
1043                                         caller_cont);
1044                         semaphore_dereference(wait_semaphore);
1045                 }
1046                 semaphore_dereference(signal_semaphore);
1047         }
1048         return kr;
1049 }
1050
1051
1052 /*
1053  *      Routine:        semaphore_reference
1054  *
1055  *      Take out a reference on a semaphore.  This keeps the data structure
1056  *      in existence (but the semaphore may be deactivated).
1057  */
1058 void
1059 semaphore_reference(
1060         semaphore_t             semaphore)
1061 {
1062         (void)hw_atomic_add(&semaphore->ref_count, 1);
1063 }
1064
1065 /*
1066  *      Routine:        semaphore_dereference
1067  *
1068  *      Release a reference on a semaphore.  If this is the last reference,
1069  *      the semaphore data structure is deallocated.
1070  */
1071 void
1072 semaphore_dereference(
1073         semaphore_t             semaphore)
1074 {
1075         if (semaphore == NULL)
1076                 return;
1077
1078         if (hw_atomic_sub(&semaphore->ref_count, 1) != 0)
1079                 return;
1080
1081         /*
1082          * Last ref, clean up the port [if any]
1083          * associated with the semaphore, destroy
1084          * it (if still active) and then free
1085          * the semaphore.
1086          */
1087         ipc_port_t port = semaphore->port;
1088
1089         if (IP_VALID(port)) {
1090                 assert(!port->ip_srights);
1091                 ipc_port_dealloc_kernel(port);
1092         }
1093         if (semaphore->active) {
1094                 assert(semaphore->owner != TASK_NULL);
1095                 semaphore_destroy_internal(semaphore->owner, semaphore);
1096         }
1097         zfree(semaphore_zone, semaphore);
1098 }
1099
1100