osfmk/kern/sync_sema.c

   1 /*
   2  * Copyright (c) 2000-2009 Apple Inc. All rights reserved.
   3  *
   4  * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
   5  *
   6  * This file contains Original Code and/or Modifications of Original Code
   7  * as defined in and that are subject to the Apple Public Source License
   8  * Version 2.0 (the 'License'). You may not use this file except in
   9  * compliance with the License. The rights granted to you under the License
  10  * may not be used to create, or enable the creation or redistribution of,
  11  * unlawful or unlicensed copies of an Apple operating system, or to
  12  * circumvent, violate, or enable the circumvention or violation of, any
  13  * terms of an Apple operating system software license agreement.
  14  *
  15  * Please obtain a copy of the License at
  16  * http://www.opensource.apple.com/apsl/ and read it before using this file.
  17  *
  18  * The Original Code and all software distributed under the License are
  19  * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  20  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  21  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
  22  * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
  23  * Please see the License for the specific language governing rights and
  24  * limitations under the License.
  25  *
  26  * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  27  */
  28 /*
  29  * @OSF_COPYRIGHT@
  30  *
  31  */
  32 /*
  33  *      File:   kern/sync_sema.c
  34  *      Author: Joseph CaraDonna
  35  *
  36  *      Contains RT distributed semaphore synchronization services.
  37  */
  38
  39 #include <mach/mach_types.h>
  40 #include <mach/mach_traps.h>
  41 #include <mach/kern_return.h>
  42 #include <mach/semaphore.h>
  43 #include <mach/sync_policy.h>
  44 #include <mach/task.h>
  45
  46 #include <kern/misc_protos.h>
  47 #include <kern/sync_sema.h>
  48 #include <kern/spl.h>
  49 #include <kern/ipc_kobject.h>
  50 #include <kern/ipc_sync.h>
  51 #include <kern/ipc_tt.h>
  52 #include <kern/thread.h>
  53 #include <kern/clock.h>
  54 #include <ipc/ipc_port.h>
  55 #include <ipc/ipc_space.h>
  56 #include <kern/host.h>
  57 #include <kern/waitq.h>
  58 #include <kern/zalloc.h>
  59 #include <kern/mach_param.h>
  60
  61 #include <libkern/OSAtomic.h>
  62
  63 static unsigned int semaphore_event;
  64 #define SEMAPHORE_EVENT CAST_EVENT64_T(&semaphore_event)
  65
  66 zone_t semaphore_zone;
  67 unsigned int semaphore_max;
  68
  69 os_refgrp_decl(static, sema_refgrp, "semaphore", NULL);
  70
  71 /* Forward declarations */
  72
  73
  74 kern_return_t
  75 semaphore_wait_trap_internal(
  76         mach_port_name_t name,
  77         void (*caller_cont)(kern_return_t));
  78
  79 kern_return_t
  80 semaphore_wait_signal_trap_internal(
  81         mach_port_name_t wait_name,
  82         mach_port_name_t signal_name,
  83         void (*caller_cont)(kern_return_t));
  84
  85 kern_return_t
  86 semaphore_timedwait_trap_internal(
  87         mach_port_name_t name,
  88         unsigned int sec,
  89         clock_res_t nsec,
  90         void (*caller_cont)(kern_return_t));
  91
  92 kern_return_t
  93 semaphore_timedwait_signal_trap_internal(
  94         mach_port_name_t wait_name,
  95         mach_port_name_t signal_name,
  96         unsigned int sec,
  97         clock_res_t nsec,
  98         void (*caller_cont)(kern_return_t));
  99
 100 kern_return_t
 101 semaphore_signal_internal_trap(mach_port_name_t sema_name);
 102
 103 kern_return_t
 104 semaphore_signal_internal(
 105         semaphore_t             semaphore,
 106         thread_t                        thread,
 107         int                             options);
 108
 109 kern_return_t
 110 semaphore_convert_wait_result(
 111         int                             wait_result);
 112
 113 void
 114 semaphore_wait_continue(void);
 115
 116 static kern_return_t
 117 semaphore_wait_internal(
 118         semaphore_t             wait_semaphore,
 119         semaphore_t             signal_semaphore,
 120         uint64_t                deadline,
 121         int                             option,
 122         void (*caller_cont)(kern_return_t));
 123
 124 static __inline__ uint64_t
 125 semaphore_deadline(
 126         unsigned int            sec,
 127         clock_res_t                     nsec)
 128 {
 129         uint64_t        abstime;
 130
 131         nanoseconds_to_absolutetime((uint64_t)sec *     NSEC_PER_SEC + nsec, &abstime);
 132         clock_absolutetime_interval_to_deadline(abstime, &abstime);
 133
 134         return abstime;
 135 }
 136
 137 /*
 138  *      ROUTINE:        semaphore_init          [private]
 139  *
 140  *      Initialize the semaphore mechanisms.
 141  *      Right now, we only need to initialize the semaphore zone.
 142  */
 143 void
 144 semaphore_init(void)
 145 {
 146         semaphore_zone = zinit(sizeof(struct semaphore),
 147             semaphore_max * sizeof(struct semaphore),
 148             sizeof(struct semaphore),
 149             "semaphores");
 150         zone_change(semaphore_zone, Z_NOENCRYPT, TRUE);
 151 }
 152
 153 /*
 154  *      Routine:        semaphore_create
 155  *
 156  *      Creates a semaphore.
 157  *      The port representing the semaphore is returned as a parameter.
 158  */
 159 kern_return_t
 160 semaphore_create(
 161         task_t                  task,
 162         semaphore_t             *new_semaphore,
 163         int                             policy,
 164         int                             value)
 165 {
 166         semaphore_t              s = SEMAPHORE_NULL;
 167         kern_return_t           kret;
 168
 169
 170         *new_semaphore = SEMAPHORE_NULL;
 171         if (task == TASK_NULL || value < 0 || policy > SYNC_POLICY_MAX) {
 172                 return KERN_INVALID_ARGUMENT;
 173         }
 174
 175         s = (semaphore_t) zalloc(semaphore_zone);
 176
 177         if (s == SEMAPHORE_NULL) {
 178                 return KERN_RESOURCE_SHORTAGE;
 179         }
 180
 181         kret = waitq_init(&s->waitq, policy | SYNC_POLICY_DISABLE_IRQ); /* also inits lock */
 182         if (kret != KERN_SUCCESS) {
 183                 zfree(semaphore_zone, s);
 184                 return kret;
 185         }
 186
 187         /*
 188          * Initialize the semaphore values.
 189          */
 190         s->port = IP_NULL;
 191         os_ref_init(&s->ref_count, &sema_refgrp);
 192         s->count = value;
 193         s->active = TRUE;
 194         s->owner = task;
 195
 196         /*
 197          *  Associate the new semaphore with the task by adding
 198          *  the new semaphore to the task's semaphore list.
 199          */
 200         task_lock(task);
 201         enqueue_head(&task->semaphore_list, (queue_entry_t) s);
 202         task->semaphores_owned++;
 203         task_unlock(task);
 204
 205         *new_semaphore = s;
 206
 207         return KERN_SUCCESS;
 208 }
 209
 210 /*
 211  *      Routine:        semaphore_destroy_internal
 212  *
 213  *      Disassociate a semaphore from its owning task, mark it inactive,
 214  *      and set any waiting threads running with THREAD_RESTART.
 215  *
 216  *      Conditions:
 217  *                      task is locked
 218  *                      semaphore is locked
 219  *                      semaphore is owned by the specified task
 220  *      Returns:
 221  *                      with semaphore unlocked
 222  */
 223 static void
 224 semaphore_destroy_internal(
 225         task_t                  task,
 226         semaphore_t             semaphore)
 227 {
 228         int                     old_count;
 229
 230         /* unlink semaphore from owning task */
 231         assert(semaphore->owner == task);
 232         remqueue((queue_entry_t) semaphore);
 233         semaphore->owner = TASK_NULL;
 234         task->semaphores_owned--;
 235
 236         /*
 237          *  Deactivate semaphore
 238          */
 239         assert(semaphore->active);
 240         semaphore->active = FALSE;
 241
 242         /*
 243          *  Wakeup blocked threads
 244          */
 245         old_count = semaphore->count;
 246         semaphore->count = 0;
 247
 248         if (old_count < 0) {
 249                 waitq_wakeup64_all_locked(&semaphore->waitq,
 250                     SEMAPHORE_EVENT,
 251                     THREAD_RESTART, NULL,
 252                     WAITQ_ALL_PRIORITIES,
 253                     WAITQ_UNLOCK);
 254                 /* waitq/semaphore is unlocked */
 255         } else {
 256                 semaphore_unlock(semaphore);
 257         }
 258 }
 259
 260 /*
 261  *      Routine:        semaphore_destroy
 262  *
 263  *      Destroys a semaphore and consume the caller's reference on the
 264  *      semaphore.
 265  */
 266 kern_return_t
 267 semaphore_destroy(
 268         task_t                  task,
 269         semaphore_t             semaphore)
 270 {
 271         spl_t spl_level;
 272
 273         if (semaphore == SEMAPHORE_NULL) {
 274                 return KERN_INVALID_ARGUMENT;
 275         }
 276
 277         if (task == TASK_NULL) {
 278                 semaphore_dereference(semaphore);
 279                 return KERN_INVALID_ARGUMENT;
 280         }
 281
 282         task_lock(task);
 283         spl_level = splsched();
 284         semaphore_lock(semaphore);
 285
 286         if (semaphore->owner != task) {
 287                 semaphore_unlock(semaphore);
 288                 semaphore_dereference(semaphore);
 289                 splx(spl_level);
 290                 task_unlock(task);
 291                 return KERN_INVALID_ARGUMENT;
 292         }
 293
 294         semaphore_destroy_internal(task, semaphore);
 295         /* semaphore unlocked */
 296
 297         splx(spl_level);
 298         task_unlock(task);
 299
 300         semaphore_dereference(semaphore);
 301         return KERN_SUCCESS;
 302 }
 303
 304 /*
 305  *      Routine:        semaphore_destroy_all
 306  *
 307  *      Destroy all the semaphores associated with a given task.
 308  */
 309 #define SEMASPERSPL 20  /* max number of semaphores to destroy per spl hold */
 310
 311 void
 312 semaphore_destroy_all(
 313         task_t                  task)
 314 {
 315         uint32_t count;
 316         spl_t spl_level;
 317
 318         count = 0;
 319         task_lock(task);
 320         while (!queue_empty(&task->semaphore_list)) {
 321                 semaphore_t semaphore;
 322
 323                 semaphore = (semaphore_t) queue_first(&task->semaphore_list);
 324
 325                 if (count == 0) {
 326                         spl_level = splsched();
 327                 }
 328                 semaphore_lock(semaphore);
 329
 330                 semaphore_destroy_internal(task, semaphore);
 331                 /* semaphore unlocked */
 332
 333                 /* throttle number of semaphores per interrupt disablement */
 334                 if (++count == SEMASPERSPL) {
 335                         count = 0;
 336                         splx(spl_level);
 337                 }
 338         }
 339         if (count != 0) {
 340                 splx(spl_level);
 341         }
 342
 343         task_unlock(task);
 344 }
 345
 346 /*
 347  *      Routine:        semaphore_signal_internal
 348  *
 349  *              Signals the semaphore as direct.
 350  *      Assumptions:
 351  *              Semaphore is locked.
 352  */
 353 kern_return_t
 354 semaphore_signal_internal(
 355         semaphore_t             semaphore,
 356         thread_t                thread,
 357         int                             options)
 358 {
 359         kern_return_t kr;
 360         spl_t  spl_level;
 361
 362         spl_level = splsched();
 363         semaphore_lock(semaphore);
 364
 365         if (!semaphore->active) {
 366                 semaphore_unlock(semaphore);
 367                 splx(spl_level);
 368                 return KERN_TERMINATED;
 369         }
 370
 371         if (thread != THREAD_NULL) {
 372                 if (semaphore->count < 0) {
 373                         kr = waitq_wakeup64_thread_locked(
 374                                 &semaphore->waitq,
 375                                 SEMAPHORE_EVENT,
 376                                 thread,
 377                                 THREAD_AWAKENED,
 378                                 WAITQ_UNLOCK);
 379                         /* waitq/semaphore is unlocked */
 380                 } else {
 381                         kr = KERN_NOT_WAITING;
 382                         semaphore_unlock(semaphore);
 383                 }
 384                 splx(spl_level);
 385                 return kr;
 386         }
 387
 388         if (options & SEMAPHORE_SIGNAL_ALL) {
 389                 int old_count = semaphore->count;
 390
 391                 kr = KERN_NOT_WAITING;
 392                 if (old_count < 0) {
 393                         semaphore->count = 0;  /* always reset */
 394                         kr = waitq_wakeup64_all_locked(
 395                                 &semaphore->waitq,
 396                                 SEMAPHORE_EVENT,
 397                                 THREAD_AWAKENED, NULL,
 398                                 WAITQ_ALL_PRIORITIES,
 399                                 WAITQ_UNLOCK);
 400                         /* waitq / semaphore is unlocked */
 401                 } else {
 402                         if (options & SEMAPHORE_SIGNAL_PREPOST) {
 403                                 semaphore->count++;
 404                         }
 405                         kr = KERN_SUCCESS;
 406                         semaphore_unlock(semaphore);
 407                 }
 408                 splx(spl_level);
 409                 return kr;
 410         }
 411
 412         if (semaphore->count < 0) {
 413                 kr = waitq_wakeup64_one_locked(
 414                         &semaphore->waitq,
 415                         SEMAPHORE_EVENT,
 416                         THREAD_AWAKENED, NULL,
 417                         WAITQ_ALL_PRIORITIES,
 418                         WAITQ_KEEP_LOCKED);
 419                 if (kr == KERN_SUCCESS) {
 420                         semaphore_unlock(semaphore);
 421                         splx(spl_level);
 422                         return KERN_SUCCESS;
 423                 } else {
 424                         semaphore->count = 0;  /* all waiters gone */
 425                 }
 426         }
 427
 428         if (options & SEMAPHORE_SIGNAL_PREPOST) {
 429                 semaphore->count++;
 430         }
 431
 432         semaphore_unlock(semaphore);
 433         splx(spl_level);
 434         return KERN_NOT_WAITING;
 435 }
 436
 437 /*
 438  *      Routine:        semaphore_signal_thread
 439  *
 440  *      If the specified thread is blocked on the semaphore, it is
 441  *      woken up.  If a NULL thread was supplied, then any one
 442  *      thread is woken up.  Otherwise the caller gets KERN_NOT_WAITING
 443  *      and the semaphore is unchanged.
 444  */
 445 kern_return_t
 446 semaphore_signal_thread(
 447         semaphore_t     semaphore,
 448         thread_t        thread)
 449 {
 450         kern_return_t           ret;
 451
 452         if (semaphore == SEMAPHORE_NULL) {
 453                 return KERN_INVALID_ARGUMENT;
 454         }
 455
 456         ret = semaphore_signal_internal(semaphore,
 457             thread,
 458             SEMAPHORE_OPTION_NONE);
 459         return ret;
 460 }
 461
 462 /*
 463  *      Routine:        semaphore_signal_thread_trap
 464  *
 465  *      Trap interface to the semaphore_signal_thread function.
 466  */
 467 kern_return_t
 468 semaphore_signal_thread_trap(
 469         struct semaphore_signal_thread_trap_args *args)
 470 {
 471         mach_port_name_t sema_name = args->signal_name;
 472         mach_port_name_t thread_name = args->thread_name;
 473         semaphore_t     semaphore;
 474         thread_t        thread;
 475         kern_return_t   kr;
 476
 477         /*
 478          * MACH_PORT_NULL is not an error. It means that we want to
 479          * select any one thread that is already waiting, but not to
 480          * pre-post the semaphore.
 481          */
 482         if (thread_name != MACH_PORT_NULL) {
 483                 thread = port_name_to_thread(thread_name, PORT_TO_THREAD_NONE);
 484                 if (thread == THREAD_NULL) {
 485                         return KERN_INVALID_ARGUMENT;
 486                 }
 487         } else {
 488                 thread = THREAD_NULL;
 489         }
 490
 491         kr = port_name_to_semaphore(sema_name, &semaphore);
 492         if (kr == KERN_SUCCESS) {
 493                 kr = semaphore_signal_internal(semaphore,
 494                     thread,
 495                     SEMAPHORE_OPTION_NONE);
 496                 semaphore_dereference(semaphore);
 497         }
 498         if (thread != THREAD_NULL) {
 499                 thread_deallocate(thread);
 500         }
 501         return kr;
 502 }
 503
 504
 505
 506 /*
 507  *      Routine:        semaphore_signal
 508  *
 509  *              Traditional (in-kernel client and MIG interface) semaphore
 510  *              signal routine.  Most users will access the trap version.
 511  *
 512  *              This interface in not defined to return info about whether
 513  *              this call found a thread waiting or not.  The internal
 514  *              routines (and future external routines) do.  We have to
 515  *              convert those into plain KERN_SUCCESS returns.
 516  */
 517 kern_return_t
 518 semaphore_signal(
 519         semaphore_t             semaphore)
 520 {
 521         kern_return_t           kr;
 522
 523         if (semaphore == SEMAPHORE_NULL) {
 524                 return KERN_INVALID_ARGUMENT;
 525         }
 526
 527         kr = semaphore_signal_internal(semaphore,
 528             THREAD_NULL,
 529             SEMAPHORE_SIGNAL_PREPOST);
 530         if (kr == KERN_NOT_WAITING) {
 531                 return KERN_SUCCESS;
 532         }
 533         return kr;
 534 }
 535
 536 /*
 537  *      Routine:        semaphore_signal_trap
 538  *
 539  *      Trap interface to the semaphore_signal function.
 540  */
 541 kern_return_t
 542 semaphore_signal_trap(
 543         struct semaphore_signal_trap_args *args)
 544 {
 545         mach_port_name_t sema_name = args->signal_name;
 546
 547         return semaphore_signal_internal_trap(sema_name);
 548 }
 549
 550 kern_return_t
 551 semaphore_signal_internal_trap(mach_port_name_t sema_name)
 552 {
 553         semaphore_t     semaphore;
 554         kern_return_t kr;
 555
 556         kr = port_name_to_semaphore(sema_name, &semaphore);
 557         if (kr == KERN_SUCCESS) {
 558                 kr = semaphore_signal_internal(semaphore,
 559                     THREAD_NULL,
 560                     SEMAPHORE_SIGNAL_PREPOST);
 561                 semaphore_dereference(semaphore);
 562                 if (kr == KERN_NOT_WAITING) {
 563                         kr = KERN_SUCCESS;
 564                 }
 565         }
 566         return kr;
 567 }
 568
 569 /*
 570  *      Routine:        semaphore_signal_all
 571  *
 572  *      Awakens ALL threads currently blocked on the semaphore.
 573  *      The semaphore count returns to zero.
 574  */
 575 kern_return_t
 576 semaphore_signal_all(
 577         semaphore_t             semaphore)
 578 {
 579         kern_return_t kr;
 580
 581         if (semaphore == SEMAPHORE_NULL) {
 582                 return KERN_INVALID_ARGUMENT;
 583         }
 584
 585         kr = semaphore_signal_internal(semaphore,
 586             THREAD_NULL,
 587             SEMAPHORE_SIGNAL_ALL);
 588         if (kr == KERN_NOT_WAITING) {
 589                 return KERN_SUCCESS;
 590         }
 591         return kr;
 592 }
 593
 594 /*
 595  *      Routine:        semaphore_signal_all_trap
 596  *
 597  *      Trap interface to the semaphore_signal_all function.
 598  */
 599 kern_return_t
 600 semaphore_signal_all_trap(
 601         struct semaphore_signal_all_trap_args *args)
 602 {
 603         mach_port_name_t sema_name = args->signal_name;
 604         semaphore_t     semaphore;
 605         kern_return_t kr;
 606
 607         kr = port_name_to_semaphore(sema_name, &semaphore);
 608         if (kr == KERN_SUCCESS) {
 609                 kr = semaphore_signal_internal(semaphore,
 610                     THREAD_NULL,
 611                     SEMAPHORE_SIGNAL_ALL);
 612                 semaphore_dereference(semaphore);
 613                 if (kr == KERN_NOT_WAITING) {
 614                         kr = KERN_SUCCESS;
 615                 }
 616         }
 617         return kr;
 618 }
 619
 620 /*
 621  *      Routine:        semaphore_convert_wait_result
 622  *
 623  *      Generate the return code after a semaphore wait/block.  It
 624  *      takes the wait result as an input and coverts that to an
 625  *      appropriate result.
 626  */
 627 kern_return_t
 628 semaphore_convert_wait_result(int wait_result)
 629 {
 630         switch (wait_result) {
 631         case THREAD_AWAKENED:
 632                 return KERN_SUCCESS;
 633
 634         case THREAD_TIMED_OUT:
 635                 return KERN_OPERATION_TIMED_OUT;
 636
 637         case THREAD_INTERRUPTED:
 638                 return KERN_ABORTED;
 639
 640         case THREAD_RESTART:
 641                 return KERN_TERMINATED;
 642
 643         default:
 644                 panic("semaphore_block\n");
 645                 return KERN_FAILURE;
 646         }
 647 }
 648
 649 /*
 650  *      Routine:        semaphore_wait_continue
 651  *
 652  *      Common continuation routine after waiting on a semphore.
 653  *      It returns directly to user space.
 654  */
 655 void
 656 semaphore_wait_continue(void)
 657 {
 658         thread_t self = current_thread();
 659         int wait_result = self->wait_result;
 660         void (*caller_cont)(kern_return_t) = self->sth_continuation;
 661
 662         assert(self->sth_waitsemaphore != SEMAPHORE_NULL);
 663         semaphore_dereference(self->sth_waitsemaphore);
 664         if (self->sth_signalsemaphore != SEMAPHORE_NULL) {
 665                 semaphore_dereference(self->sth_signalsemaphore);
 666         }
 667
 668         assert(caller_cont != (void (*)(kern_return_t))0);
 669         (*caller_cont)(semaphore_convert_wait_result(wait_result));
 670 }
 671
 672 /*
 673  *      Routine:        semaphore_wait_internal
 674  *
 675  *              Decrements the semaphore count by one.  If the count is
 676  *              negative after the decrement, the calling thread blocks
 677  *              (possibly at a continuation and/or with a timeout).
 678  *
 679  *      Assumptions:
 680  *              The reference
 681  *              A reference is held on the signal semaphore.
 682  */
 683 static kern_return_t
 684 semaphore_wait_internal(
 685         semaphore_t             wait_semaphore,
 686         semaphore_t             signal_semaphore,
 687         uint64_t                deadline,
 688         int                             option,
 689         void                    (*caller_cont)(kern_return_t))
 690 {
 691         int                                     wait_result;
 692         spl_t                           spl_level;
 693         kern_return_t           kr = KERN_ALREADY_WAITING;
 694
 695         spl_level = splsched();
 696         semaphore_lock(wait_semaphore);
 697
 698         if (!wait_semaphore->active) {
 699                 kr = KERN_TERMINATED;
 700         } else if (wait_semaphore->count > 0) {
 701                 wait_semaphore->count--;
 702                 kr = KERN_SUCCESS;
 703         } else if (option & SEMAPHORE_TIMEOUT_NOBLOCK) {
 704                 kr = KERN_OPERATION_TIMED_OUT;
 705         } else {
 706                 thread_t        self = current_thread();
 707
 708                 wait_semaphore->count = -1;  /* we don't keep an actual count */
 709
 710                 thread_set_pending_block_hint(self, kThreadWaitSemaphore);
 711                 (void)waitq_assert_wait64_locked(
 712                         &wait_semaphore->waitq,
 713                         SEMAPHORE_EVENT,
 714                         THREAD_ABORTSAFE,
 715                         TIMEOUT_URGENCY_USER_NORMAL,
 716                         deadline, TIMEOUT_NO_LEEWAY,
 717                         self);
 718         }
 719         semaphore_unlock(wait_semaphore);
 720         splx(spl_level);
 721
 722         /*
 723          * wait_semaphore is unlocked so we are free to go ahead and
 724          * signal the signal_semaphore (if one was provided).
 725          */
 726         if (signal_semaphore != SEMAPHORE_NULL) {
 727                 kern_return_t signal_kr;
 728
 729                 /*
 730                  * lock the signal semaphore reference we got and signal it.
 731                  * This will NOT block (we cannot block after having asserted
 732                  * our intention to wait above).
 733                  */
 734                 signal_kr = semaphore_signal_internal(signal_semaphore,
 735                     THREAD_NULL,
 736                     SEMAPHORE_SIGNAL_PREPOST);
 737
 738                 if (signal_kr == KERN_NOT_WAITING) {
 739                         signal_kr = KERN_SUCCESS;
 740                 } else if (signal_kr == KERN_TERMINATED) {
 741                         /*
 742                          * Uh!Oh!  The semaphore we were to signal died.
 743                          * We have to get ourselves out of the wait in
 744                          * case we get stuck here forever (it is assumed
 745                          * that the semaphore we were posting is gating
 746                          * the decision by someone else to post the
 747                          * semaphore we are waiting on).  People will
 748                          * discover the other dead semaphore soon enough.
 749                          * If we got out of the wait cleanly (someone
 750                          * already posted a wakeup to us) then return that
 751                          * (most important) result.  Otherwise,
 752                          * return the KERN_TERMINATED status.
 753                          */
 754                         thread_t self = current_thread();
 755
 756                         clear_wait(self, THREAD_INTERRUPTED);
 757                         kr = semaphore_convert_wait_result(self->wait_result);
 758                         if (kr == KERN_ABORTED) {
 759                                 kr = KERN_TERMINATED;
 760                         }
 761                 }
 762         }
 763
 764         /*
 765          * If we had an error, or we didn't really need to wait we can
 766          * return now that we have signalled the signal semaphore.
 767          */
 768         if (kr != KERN_ALREADY_WAITING) {
 769                 return kr;
 770         }
 771
 772         /*
 773          * Now, we can block.  If the caller supplied a continuation
 774          * pointer of his own for after the block, block with the
 775          * appropriate semaphore continuation.  Thiswill gather the
 776          * semaphore results, release references on the semaphore(s),
 777          * and then call the caller's continuation.
 778          */
 779         if (caller_cont) {
 780                 thread_t self = current_thread();
 781
 782                 self->sth_continuation = caller_cont;
 783                 self->sth_waitsemaphore = wait_semaphore;
 784                 self->sth_signalsemaphore = signal_semaphore;
 785                 wait_result = thread_block((thread_continue_t)semaphore_wait_continue);
 786         } else {
 787                 wait_result = thread_block(THREAD_CONTINUE_NULL);
 788         }
 789
 790         return semaphore_convert_wait_result(wait_result);
 791 }
 792
 793
 794 /*
 795  *      Routine:        semaphore_wait
 796  *
 797  *      Traditional (non-continuation) interface presented to
 798  *      in-kernel clients to wait on a semaphore.
 799  */
 800 kern_return_t
 801 semaphore_wait(
 802         semaphore_t             semaphore)
 803 {
 804         if (semaphore == SEMAPHORE_NULL) {
 805                 return KERN_INVALID_ARGUMENT;
 806         }
 807
 808         return semaphore_wait_internal(semaphore,
 809                    SEMAPHORE_NULL,
 810                    0ULL, SEMAPHORE_OPTION_NONE,
 811                    (void (*)(kern_return_t))0);
 812 }
 813
 814 kern_return_t
 815 semaphore_wait_noblock(
 816         semaphore_t             semaphore)
 817 {
 818         if (semaphore == SEMAPHORE_NULL) {
 819                 return KERN_INVALID_ARGUMENT;
 820         }
 821
 822         return semaphore_wait_internal(semaphore,
 823                    SEMAPHORE_NULL,
 824                    0ULL, SEMAPHORE_TIMEOUT_NOBLOCK,
 825                    (void (*)(kern_return_t))0);
 826 }
 827
 828 kern_return_t
 829 semaphore_wait_deadline(
 830         semaphore_t             semaphore,
 831         uint64_t                deadline)
 832 {
 833         if (semaphore == SEMAPHORE_NULL) {
 834                 return KERN_INVALID_ARGUMENT;
 835         }
 836
 837         return semaphore_wait_internal(semaphore,
 838                    SEMAPHORE_NULL,
 839                    deadline, SEMAPHORE_OPTION_NONE,
 840                    (void (*)(kern_return_t))0);
 841 }
 842
 843 /*
 844  *      Trap:   semaphore_wait_trap
 845  *
 846  *      Trap version of semaphore wait.  Called on behalf of user-level
 847  *      clients.
 848  */
 849
 850 kern_return_t
 851 semaphore_wait_trap(
 852         struct semaphore_wait_trap_args *args)
 853 {
 854         return semaphore_wait_trap_internal(args->wait_name, thread_syscall_return);
 855 }
 856
 857
 858
 859 kern_return_t
 860 semaphore_wait_trap_internal(
 861         mach_port_name_t name,
 862         void (*caller_cont)(kern_return_t))
 863 {
 864         semaphore_t     semaphore;
 865         kern_return_t kr;
 866
 867         kr = port_name_to_semaphore(name, &semaphore);
 868         if (kr == KERN_SUCCESS) {
 869                 kr = semaphore_wait_internal(semaphore,
 870                     SEMAPHORE_NULL,
 871                     0ULL, SEMAPHORE_OPTION_NONE,
 872                     caller_cont);
 873                 semaphore_dereference(semaphore);
 874         }
 875         return kr;
 876 }
 877
 878 /*
 879  *      Routine:        semaphore_timedwait
 880  *
 881  *      Traditional (non-continuation) interface presented to
 882  *      in-kernel clients to wait on a semaphore with a timeout.
 883  *
 884  *      A timeout of {0,0} is considered non-blocking.
 885  */
 886 kern_return_t
 887 semaphore_timedwait(
 888         semaphore_t             semaphore,
 889         mach_timespec_t         wait_time)
 890 {
 891         int                             option = SEMAPHORE_OPTION_NONE;
 892         uint64_t                deadline = 0;
 893
 894         if (semaphore == SEMAPHORE_NULL) {
 895                 return KERN_INVALID_ARGUMENT;
 896         }
 897
 898         if (BAD_MACH_TIMESPEC(&wait_time)) {
 899                 return KERN_INVALID_VALUE;
 900         }
 901
 902         if (wait_time.tv_sec == 0 && wait_time.tv_nsec == 0) {
 903                 option = SEMAPHORE_TIMEOUT_NOBLOCK;
 904         } else {
 905                 deadline = semaphore_deadline(wait_time.tv_sec, wait_time.tv_nsec);
 906         }
 907
 908         return semaphore_wait_internal(semaphore,
 909                    SEMAPHORE_NULL,
 910                    deadline, option,
 911                    (void (*)(kern_return_t))0);
 912 }
 913
 914 /*
 915  *      Trap:   semaphore_timedwait_trap
 916  *
 917  *      Trap version of a semaphore_timedwait.  The timeout parameter
 918  *      is passed in two distinct parts and re-assembled on this side
 919  *      of the trap interface (to accomodate calling conventions that
 920  *      pass structures as pointers instead of inline in registers without
 921  *      having to add a copyin).
 922  *
 923  *      A timeout of {0,0} is considered non-blocking.
 924  */
 925 kern_return_t
 926 semaphore_timedwait_trap(
 927         struct semaphore_timedwait_trap_args *args)
 928 {
 929         return semaphore_timedwait_trap_internal(args->wait_name, args->sec, args->nsec, thread_syscall_return);
 930 }
 931
 932
 933 kern_return_t
 934 semaphore_timedwait_trap_internal(
 935         mach_port_name_t name,
 936         unsigned int            sec,
 937         clock_res_t             nsec,
 938         void (*caller_cont)(kern_return_t))
 939 {
 940         semaphore_t semaphore;
 941         mach_timespec_t wait_time;
 942         kern_return_t kr;
 943
 944         wait_time.tv_sec = sec;
 945         wait_time.tv_nsec = nsec;
 946         if (BAD_MACH_TIMESPEC(&wait_time)) {
 947                 return KERN_INVALID_VALUE;
 948         }
 949
 950         kr = port_name_to_semaphore(name, &semaphore);
 951         if (kr == KERN_SUCCESS) {
 952                 int                             option = SEMAPHORE_OPTION_NONE;
 953                 uint64_t                deadline = 0;
 954
 955                 if (sec == 0 && nsec == 0) {
 956                         option = SEMAPHORE_TIMEOUT_NOBLOCK;
 957                 } else {
 958                         deadline = semaphore_deadline(sec, nsec);
 959                 }
 960
 961                 kr = semaphore_wait_internal(semaphore,
 962                     SEMAPHORE_NULL,
 963                     deadline, option,
 964                     caller_cont);
 965                 semaphore_dereference(semaphore);
 966         }
 967         return kr;
 968 }
 969
 970 /*
 971  *      Routine:        semaphore_wait_signal
 972  *
 973  *      Atomically register a wait on a semaphore and THEN signal
 974  *      another.  This is the in-kernel entry point that does not
 975  *      block at a continuation and does not free a signal_semaphore
 976  *      reference.
 977  */
 978 kern_return_t
 979 semaphore_wait_signal(
 980         semaphore_t             wait_semaphore,
 981         semaphore_t             signal_semaphore)
 982 {
 983         if (wait_semaphore == SEMAPHORE_NULL) {
 984                 return KERN_INVALID_ARGUMENT;
 985         }
 986
 987         return semaphore_wait_internal(wait_semaphore,
 988                    signal_semaphore,
 989                    0ULL, SEMAPHORE_OPTION_NONE,
 990                    (void (*)(kern_return_t))0);
 991 }
 992
 993 /*
 994  *      Trap:   semaphore_wait_signal_trap
 995  *
 996  *      Atomically register a wait on a semaphore and THEN signal
 997  *      another.  This is the trap version from user space.
 998  */
 999 kern_return_t
1000 semaphore_wait_signal_trap(
1001         struct semaphore_wait_signal_trap_args *args)
1002 {
1003         return semaphore_wait_signal_trap_internal(args->wait_name, args->signal_name, thread_syscall_return);
1004 }
1005
1006 kern_return_t
1007 semaphore_wait_signal_trap_internal(
1008         mach_port_name_t wait_name,
1009         mach_port_name_t signal_name,
1010         void (*caller_cont)(kern_return_t))
1011 {
1012         semaphore_t wait_semaphore;
1013         semaphore_t signal_semaphore;
1014         kern_return_t kr;
1015
1016         kr = port_name_to_semaphore(signal_name, &signal_semaphore);
1017         if (kr == KERN_SUCCESS) {
1018                 kr = port_name_to_semaphore(wait_name, &wait_semaphore);
1019                 if (kr == KERN_SUCCESS) {
1020                         kr = semaphore_wait_internal(wait_semaphore,
1021                             signal_semaphore,
1022                             0ULL, SEMAPHORE_OPTION_NONE,
1023                             caller_cont);
1024                         semaphore_dereference(wait_semaphore);
1025                 }
1026                 semaphore_dereference(signal_semaphore);
1027         }
1028         return kr;
1029 }
1030
1031
1032 /*
1033  *      Routine:        semaphore_timedwait_signal
1034  *
1035  *      Atomically register a wait on a semaphore and THEN signal
1036  *      another.  This is the in-kernel entry point that does not
1037  *      block at a continuation.
1038  *
1039  *      A timeout of {0,0} is considered non-blocking.
1040  */
1041 kern_return_t
1042 semaphore_timedwait_signal(
1043         semaphore_t             wait_semaphore,
1044         semaphore_t             signal_semaphore,
1045         mach_timespec_t         wait_time)
1046 {
1047         int                             option = SEMAPHORE_OPTION_NONE;
1048         uint64_t                deadline = 0;
1049
1050         if (wait_semaphore == SEMAPHORE_NULL) {
1051                 return KERN_INVALID_ARGUMENT;
1052         }
1053
1054         if (BAD_MACH_TIMESPEC(&wait_time)) {
1055                 return KERN_INVALID_VALUE;
1056         }
1057
1058         if (wait_time.tv_sec == 0 && wait_time.tv_nsec == 0) {
1059                 option = SEMAPHORE_TIMEOUT_NOBLOCK;
1060         } else {
1061                 deadline = semaphore_deadline(wait_time.tv_sec, wait_time.tv_nsec);
1062         }
1063
1064         return semaphore_wait_internal(wait_semaphore,
1065                    signal_semaphore,
1066                    deadline, option,
1067                    (void (*)(kern_return_t))0);
1068 }
1069
1070 /*
1071  *      Trap:   semaphore_timedwait_signal_trap
1072  *
1073  *      Atomically register a timed wait on a semaphore and THEN signal
1074  *      another.  This is the trap version from user space.
1075  */
1076 kern_return_t
1077 semaphore_timedwait_signal_trap(
1078         struct semaphore_timedwait_signal_trap_args *args)
1079 {
1080         return semaphore_timedwait_signal_trap_internal(args->wait_name, args->signal_name, args->sec, args->nsec, thread_syscall_return);
1081 }
1082
1083 kern_return_t
1084 semaphore_timedwait_signal_trap_internal(
1085         mach_port_name_t wait_name,
1086         mach_port_name_t signal_name,
1087         unsigned int sec,
1088         clock_res_t nsec,
1089         void (*caller_cont)(kern_return_t))
1090 {
1091         semaphore_t wait_semaphore;
1092         semaphore_t signal_semaphore;
1093         mach_timespec_t wait_time;
1094         kern_return_t kr;
1095
1096         wait_time.tv_sec = sec;
1097         wait_time.tv_nsec = nsec;
1098         if (BAD_MACH_TIMESPEC(&wait_time)) {
1099                 return KERN_INVALID_VALUE;
1100         }
1101
1102         kr = port_name_to_semaphore(signal_name, &signal_semaphore);
1103         if (kr == KERN_SUCCESS) {
1104                 kr = port_name_to_semaphore(wait_name, &wait_semaphore);
1105                 if (kr == KERN_SUCCESS) {
1106                         int                             option = SEMAPHORE_OPTION_NONE;
1107                         uint64_t                deadline = 0;
1108
1109                         if (sec == 0 && nsec == 0) {
1110                                 option = SEMAPHORE_TIMEOUT_NOBLOCK;
1111                         } else {
1112                                 deadline = semaphore_deadline(sec, nsec);
1113                         }
1114
1115                         kr = semaphore_wait_internal(wait_semaphore,
1116                             signal_semaphore,
1117                             deadline, option,
1118                             caller_cont);
1119                         semaphore_dereference(wait_semaphore);
1120                 }
1121                 semaphore_dereference(signal_semaphore);
1122         }
1123         return kr;
1124 }
1125
1126
1127 /*
1128  *      Routine:        semaphore_reference
1129  *
1130  *      Take out a reference on a semaphore.  This keeps the data structure
1131  *      in existence (but the semaphore may be deactivated).
1132  */
1133 void
1134 semaphore_reference(
1135         semaphore_t             semaphore)
1136 {
1137         os_ref_retain(&semaphore->ref_count);
1138 }
1139
1140 /*
1141  *      Routine:        semaphore_dereference
1142  *
1143  *      Release a reference on a semaphore.  If this is the last reference,
1144  *      the semaphore data structure is deallocated.
1145  */
1146 void
1147 semaphore_dereference(
1148         semaphore_t             semaphore)
1149 {
1150         uint32_t collisions;
1151         spl_t spl_level;
1152
1153         if (semaphore == NULL) {
1154                 return;
1155         }
1156
1157         if (os_ref_release(&semaphore->ref_count) > 0) {
1158                 return;
1159         }
1160
1161         /*
1162          * Last ref, clean up the port [if any]
1163          * associated with the semaphore, destroy
1164          * it (if still active) and then free
1165          * the semaphore.
1166          */
1167         ipc_port_t port = semaphore->port;
1168
1169         if (IP_VALID(port)) {
1170                 assert(!port->ip_srights);
1171                 ipc_port_dealloc_kernel(port);
1172         }
1173
1174         /*
1175          * Lock the semaphore to lock in the owner task reference.
1176          * Then continue to try to lock the task (inverse order).
1177          */
1178         spl_level = splsched();
1179         semaphore_lock(semaphore);
1180         for (collisions = 0; semaphore->active; collisions++) {
1181                 task_t task = semaphore->owner;
1182
1183                 assert(task != TASK_NULL);
1184
1185                 if (task_lock_try(task)) {
1186                         semaphore_destroy_internal(task, semaphore);
1187                         /* semaphore unlocked */
1188                         splx(spl_level);
1189                         task_unlock(task);
1190                         goto out;
1191                 }
1192
1193                 /* failed to get out-of-order locks */
1194                 semaphore_unlock(semaphore);
1195                 splx(spl_level);
1196                 mutex_pause(collisions);
1197                 spl_level = splsched();
1198                 semaphore_lock(semaphore);
1199         }
1200         semaphore_unlock(semaphore);
1201         splx(spl_level);
1202
1203 out:
1204         zfree(semaphore_zone, semaphore);
1205 }
1206
1207 #define WAITQ_TO_SEMA(wq) ((semaphore_t) ((uintptr_t)(wq) - offsetof(struct semaphore, waitq)))
1208 void
1209 kdp_sema_find_owner(struct waitq * waitq, __assert_only event64_t event, thread_waitinfo_t * waitinfo)
1210 {
1211         semaphore_t sem = WAITQ_TO_SEMA(waitq);
1212         assert(event == SEMAPHORE_EVENT);
1213         assert(kdp_is_in_zone(sem, "semaphores"));
1214
1215         waitinfo->context = VM_KERNEL_UNSLIDE_OR_PERM(sem->port);
1216         if (sem->owner) {
1217                 waitinfo->owner = pid_from_task(sem->owner);
1218         }
1219 }