osfmk/kern/thread_call.c

   1 /*
   2  * Copyright (c) 1993-1995, 1999-2005 Apple Computer, Inc.
   3  * All rights reserved.
   4  *
   5  * @APPLE_LICENSE_HEADER_START@
   6  *
   7  * This file contains Original Code and/or Modifications of Original Code
   8  * as defined in and that are subject to the Apple Public Source License
   9  * Version 2.0 (the 'License'). You may not use this file except in
  10  * compliance with the License. Please obtain a copy of the License at
  11  * http://www.opensource.apple.com/apsl/ and read it before using this
  12  * file.
  13  *
  14  * The Original Code and all software distributed under the License are
  15  * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  16  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  17  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
  18  * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
  19  * Please see the License for the specific language governing rights and
  20  * limitations under the License.
  21  *
  22  * @APPLE_LICENSE_HEADER_END@
  23  */
  24
  25 #include <mach/mach_types.h>
  26 #include <mach/thread_act.h>
  27
  28 #include <kern/kern_types.h>
  29 #include <kern/kalloc.h>
  30 #include <kern/sched_prim.h>
  31 #include <kern/clock.h>
  32 #include <kern/task.h>
  33 #include <kern/thread.h>
  34 #include <kern/wait_queue.h>
  35
  36 #include <vm/vm_pageout.h>
  37
  38 #include <kern/thread_call.h>
  39 #include <kern/call_entry.h>
  40
  41 #include <kern/timer_call.h>
  42
  43 #include <sys/kdebug.h>
  44
  45 #define internal_call_num       768
  46
  47 #define thread_call_thread_min  4
  48
  49 static
  50 thread_call_data_t
  51         internal_call_storage[internal_call_num];
  52
  53 decl_simple_lock_data(static,thread_call_lock)
  54
  55 static
  56 timer_call_data_t
  57         thread_call_delaytimer;
  58
  59 static
  60 queue_head_t
  61         thread_call_xxx_queue,
  62         thread_call_pending_queue, thread_call_delayed_queue;
  63
  64 static
  65 struct wait_queue
  66         call_thread_waitqueue;
  67
  68 static
  69 boolean_t
  70         activate_thread_awake;
  71
  72 static struct {
  73         int             pending_num,
  74                         pending_hiwat;
  75         int             active_num,
  76                         active_hiwat,
  77                         active_lowat;
  78         int             delayed_num,
  79                         delayed_hiwat;
  80         int             idle_thread_num;
  81         int             thread_num,
  82                         thread_hiwat,
  83                         thread_lowat;
  84 } thread_call_vars;
  85
  86 static __inline__ thread_call_t
  87         _internal_call_allocate(void);
  88
  89 static __inline__ void
  90 _internal_call_release(
  91         thread_call_t           call
  92 );
  93
  94 static __inline__ void
  95 _pending_call_enqueue(
  96         thread_call_t           call
  97 ),
  98 _pending_call_dequeue(
  99         thread_call_t           call
 100 ),
 101 _delayed_call_enqueue(
 102         thread_call_t           call
 103 ),
 104 _delayed_call_dequeue(
 105         thread_call_t           call
 106 );
 107
 108 static __inline__ void
 109 _set_delayed_call_timer(
 110         thread_call_t           call
 111 );
 112
 113 static boolean_t
 114 _remove_from_pending_queue(
 115         thread_call_func_t      func,
 116         thread_call_param_t     param0,
 117         boolean_t                       remove_all
 118 ),
 119 _remove_from_delayed_queue(
 120         thread_call_func_t      func,
 121         thread_call_param_t     param0,
 122         boolean_t                       remove_all
 123 );
 124
 125 static __inline__ void
 126         _call_thread_wake(void);
 127
 128 static void
 129         _call_thread(void),
 130         _activate_thread(void);
 131
 132 static void
 133 _delayed_call_timer(
 134         timer_call_param_t              p0,
 135         timer_call_param_t              p1
 136 );
 137
 138 #define qe(x)           ((queue_entry_t)(x))
 139 #define TC(x)           ((thread_call_t)(x))
 140
 141 /*
 142  * Routine:     thread_call_initialize [public]
 143  *
 144  * Description: Initialize this module, called
 145  *              early during system initialization.
 146  *
 147  * Preconditions:       None.
 148  *
 149  * Postconditions:      None.
 150  */
 151
 152 void
 153 thread_call_initialize(void)
 154 {
 155         kern_return_t   result;
 156         thread_t                thread;
 157     thread_call_t       call;
 158         spl_t                   s;
 159
 160     simple_lock_init(&thread_call_lock, 0);
 161
 162         s = splsched();
 163         simple_lock(&thread_call_lock);
 164
 165     queue_init(&thread_call_pending_queue);
 166     queue_init(&thread_call_delayed_queue);
 167
 168     queue_init(&thread_call_xxx_queue);
 169     for (
 170                 call = internal_call_storage;
 171                         call < &internal_call_storage[internal_call_num];
 172                         call++) {
 173
 174                 enqueue_tail(&thread_call_xxx_queue, qe(call));
 175     }
 176
 177         timer_call_setup(&thread_call_delaytimer, _delayed_call_timer, NULL);
 178
 179         wait_queue_init(&call_thread_waitqueue, SYNC_POLICY_FIFO);
 180         thread_call_vars.thread_lowat = thread_call_thread_min;
 181
 182         activate_thread_awake = TRUE;
 183
 184         simple_unlock(&thread_call_lock);
 185         splx(s);
 186
 187         result = kernel_thread_start_priority((thread_continue_t)_activate_thread, NULL, MAXPRI_KERNEL - 2, &thread);
 188         if (result != KERN_SUCCESS)
 189                 panic("thread_call_initialize");
 190
 191         thread_deallocate(thread);
 192 }
 193
 194 void
 195 thread_call_setup(
 196         thread_call_t                   call,
 197         thread_call_func_t              func,
 198         thread_call_param_t             param0
 199 )
 200 {
 201         call_entry_setup(call, func, param0);
 202 }
 203
 204 /*
 205  * Routine:     _internal_call_allocate [private, inline]
 206  *
 207  * Purpose:     Allocate an internal callout entry.
 208  *
 209  * Preconditions:       thread_call_lock held.
 210  *
 211  * Postconditions:      None.
 212  */
 213
 214 static __inline__ thread_call_t
 215 _internal_call_allocate(void)
 216 {
 217     thread_call_t               call;
 218
 219     if (queue_empty(&thread_call_xxx_queue))
 220         panic("_internal_call_allocate");
 221
 222     call = TC(dequeue_head(&thread_call_xxx_queue));
 223
 224     return (call);
 225 }
 226
 227 /*
 228  * Routine:     _internal_call_release [private, inline]
 229  *
 230  * Purpose:     Release an internal callout entry which
 231  *              is no longer pending (or delayed).
 232  *
 233  * Preconditions:       thread_call_lock held.
 234  *
 235  * Postconditions:      None.
 236  */
 237
 238 static __inline__
 239 void
 240 _internal_call_release(
 241     thread_call_t               call
 242 )
 243 {
 244     if (    call >= internal_call_storage                                               &&
 245                     call < &internal_call_storage[internal_call_num]            )
 246                 enqueue_head(&thread_call_xxx_queue, qe(call));
 247 }
 248
 249 /*
 250  * Routine:     _pending_call_enqueue [private, inline]
 251  *
 252  * Purpose:     Place an entry at the end of the
 253  *              pending queue, to be executed soon.
 254  *
 255  * Preconditions:       thread_call_lock held.
 256  *
 257  * Postconditions:      None.
 258  */
 259
 260 static __inline__
 261 void
 262 _pending_call_enqueue(
 263     thread_call_t               call
 264 )
 265 {
 266     enqueue_tail(&thread_call_pending_queue, qe(call));
 267         if (++thread_call_vars.pending_num > thread_call_vars.pending_hiwat)
 268                 thread_call_vars.pending_hiwat = thread_call_vars.pending_num;
 269
 270     call->state = PENDING;
 271 }
 272
 273 /*
 274  * Routine:     _pending_call_dequeue [private, inline]
 275  *
 276  * Purpose:     Remove an entry from the pending queue,
 277  *              effectively unscheduling it.
 278  *
 279  * Preconditions:       thread_call_lock held.
 280  *
 281  * Postconditions:      None.
 282  */
 283
 284 static __inline__
 285 void
 286 _pending_call_dequeue(
 287     thread_call_t               call
 288 )
 289 {
 290     (void)remque(qe(call));
 291         thread_call_vars.pending_num--;
 292
 293     call->state = IDLE;
 294 }
 295
 296 /*
 297  * Routine:     _delayed_call_enqueue [private, inline]
 298  *
 299  * Purpose:     Place an entry on the delayed queue,
 300  *              after existing entries with an earlier
 301  *              (or identical) deadline.
 302  *
 303  * Preconditions:       thread_call_lock held.
 304  *
 305  * Postconditions:      None.
 306  */
 307
 308 static __inline__
 309 void
 310 _delayed_call_enqueue(
 311     thread_call_t               call
 312 )
 313 {
 314     thread_call_t               current;
 315
 316     current = TC(queue_first(&thread_call_delayed_queue));
 317
 318     while (TRUE) {
 319         if (    queue_end(&thread_call_delayed_queue, qe(current))              ||
 320                                         call->deadline < current->deadline                      ) {
 321                         current = TC(queue_prev(qe(current)));
 322                         break;
 323                 }
 324
 325                 current = TC(queue_next(qe(current)));
 326     }
 327
 328     insque(qe(call), qe(current));
 329         if (++thread_call_vars.delayed_num > thread_call_vars.delayed_hiwat)
 330                 thread_call_vars.delayed_hiwat = thread_call_vars.delayed_num;
 331
 332     call->state = DELAYED;
 333 }
 334
 335 /*
 336  * Routine:     _delayed_call_dequeue [private, inline]
 337  *
 338  * Purpose:     Remove an entry from the delayed queue,
 339  *              effectively unscheduling it.
 340  *
 341  * Preconditions:       thread_call_lock held.
 342  *
 343  * Postconditions:      None.
 344  */
 345
 346 static __inline__
 347 void
 348 _delayed_call_dequeue(
 349     thread_call_t               call
 350 )
 351 {
 352     (void)remque(qe(call));
 353         thread_call_vars.delayed_num--;
 354
 355     call->state = IDLE;
 356 }
 357
 358 /*
 359  * Routine:     _set_delayed_call_timer [private]
 360  *
 361  * Purpose:     Reset the timer so that it
 362  *              next expires when the entry is due.
 363  *
 364  * Preconditions:       thread_call_lock held.
 365  *
 366  * Postconditions:      None.
 367  */
 368
 369 static __inline__ void
 370 _set_delayed_call_timer(
 371     thread_call_t               call
 372 )
 373 {
 374     timer_call_enter(&thread_call_delaytimer, call->deadline);
 375 }
 376
 377 /*
 378  * Routine:     _remove_from_pending_queue [private]
 379  *
 380  * Purpose:     Remove the first (or all) matching
 381  *              entries from the pending queue,
 382  *              effectively unscheduling them.
 383  *              Returns whether any matching entries
 384  *              were found.
 385  *
 386  * Preconditions:       thread_call_lock held.
 387  *
 388  * Postconditions:      None.
 389  */
 390
 391 static
 392 boolean_t
 393 _remove_from_pending_queue(
 394     thread_call_func_t          func,
 395     thread_call_param_t         param0,
 396     boolean_t                           remove_all
 397 )
 398 {
 399         boolean_t                       call_removed = FALSE;
 400         thread_call_t           call;
 401
 402     call = TC(queue_first(&thread_call_pending_queue));
 403
 404     while (!queue_end(&thread_call_pending_queue, qe(call))) {
 405         if (    call->func == func                      &&
 406                                 call->param0 == param0                  ) {
 407                         thread_call_t   next = TC(queue_next(qe(call)));
 408
 409                         _pending_call_dequeue(call);
 410
 411                         _internal_call_release(call);
 412
 413                         call_removed = TRUE;
 414                         if (!remove_all)
 415                                 break;
 416
 417                         call = next;
 418                 }
 419                 else
 420                         call = TC(queue_next(qe(call)));
 421     }
 422
 423     return (call_removed);
 424 }
 425
 426 /*
 427  * Routine:     _remove_from_delayed_queue [private]
 428  *
 429  * Purpose:     Remove the first (or all) matching
 430  *              entries from the delayed queue,
 431  *              effectively unscheduling them.
 432  *              Returns whether any matching entries
 433  *              were found.
 434  *
 435  * Preconditions:       thread_call_lock held.
 436  *
 437  * Postconditions:      None.
 438  */
 439
 440 static
 441 boolean_t
 442 _remove_from_delayed_queue(
 443     thread_call_func_t          func,
 444     thread_call_param_t         param0,
 445     boolean_t                           remove_all
 446 )
 447 {
 448     boolean_t                   call_removed = FALSE;
 449     thread_call_t               call;
 450
 451     call = TC(queue_first(&thread_call_delayed_queue));
 452
 453     while (!queue_end(&thread_call_delayed_queue, qe(call))) {
 454         if (    call->func == func                      &&
 455                                 call->param0 == param0                  ) {
 456                         thread_call_t   next = TC(queue_next(qe(call)));
 457
 458                         _delayed_call_dequeue(call);
 459
 460                         _internal_call_release(call);
 461
 462                         call_removed = TRUE;
 463                         if (!remove_all)
 464                                 break;
 465
 466                         call = next;
 467                 }
 468                 else
 469                         call = TC(queue_next(qe(call)));
 470     }
 471
 472     return (call_removed);
 473 }
 474
 475 /*
 476  * Routine:     thread_call_func [public]
 477  *
 478  * Purpose:     Schedule a function callout.
 479  *              Guarantees { function, argument }
 480  *              uniqueness if unique_call is TRUE.
 481  *
 482  * Preconditions:       Callable from an interrupt context
 483  *                                      below splsched.
 484  *
 485  * Postconditions:      None.
 486  */
 487
 488 void
 489 thread_call_func(
 490     thread_call_func_t          func,
 491     thread_call_param_t         param,
 492     boolean_t                           unique_call
 493 )
 494 {
 495     thread_call_t               call;
 496     spl_t                               s;
 497
 498     s = splsched();
 499     simple_lock(&thread_call_lock);
 500
 501     call = TC(queue_first(&thread_call_pending_queue));
 502
 503         while (unique_call && !queue_end(&thread_call_pending_queue, qe(call))) {
 504         if (    call->func == func                      &&
 505                                 call->param0 == param                   ) {
 506                         break;
 507                 }
 508
 509                 call = TC(queue_next(qe(call)));
 510     }
 511
 512     if (!unique_call || queue_end(&thread_call_pending_queue, qe(call))) {
 513                 call = _internal_call_allocate();
 514                 call->func                      = func;
 515                 call->param0            = param;
 516                 call->param1            = 0;
 517
 518                 _pending_call_enqueue(call);
 519
 520                 if (thread_call_vars.active_num <= 0)
 521                         _call_thread_wake();
 522     }
 523
 524         simple_unlock(&thread_call_lock);
 525     splx(s);
 526 }
 527
 528 /*
 529  * Routine:     thread_call_func_delayed [public]
 530  *
 531  * Purpose:     Schedule a function callout to
 532  *              occur at the stated time.
 533  *
 534  * Preconditions:       Callable from an interrupt context
 535  *                                      below splsched.
 536  *
 537  * Postconditions:      None.
 538  */
 539
 540 void
 541 thread_call_func_delayed(
 542     thread_call_func_t          func,
 543     thread_call_param_t         param,
 544     uint64_t                            deadline
 545 )
 546 {
 547     thread_call_t               call;
 548     spl_t                               s;
 549
 550     s = splsched();
 551     simple_lock(&thread_call_lock);
 552
 553     call = _internal_call_allocate();
 554     call->func                  = func;
 555     call->param0                = param;
 556     call->param1                = 0;
 557     call->deadline              = deadline;
 558
 559     _delayed_call_enqueue(call);
 560
 561     if (queue_first(&thread_call_delayed_queue) == qe(call))
 562         _set_delayed_call_timer(call);
 563
 564     simple_unlock(&thread_call_lock);
 565     splx(s);
 566 }
 567
 568 /*
 569  * Routine:     thread_call_func_cancel [public]
 570  *
 571  * Purpose:     Unschedule a function callout.
 572  *              Removes one (or all)
 573  *              { function, argument }
 574  *              instance(s) from either (or both)
 575  *              the pending and the delayed queue,
 576  *              in that order.  Returns a boolean
 577  *              indicating whether any calls were
 578  *              cancelled.
 579  *
 580  * Preconditions:       Callable from an interrupt context
 581  *                                      below splsched.
 582  *
 583  * Postconditions:      None.
 584  */
 585
 586 boolean_t
 587 thread_call_func_cancel(
 588     thread_call_func_t          func,
 589     thread_call_param_t         param,
 590     boolean_t                           cancel_all
 591 )
 592 {
 593         boolean_t                       result;
 594     spl_t                               s;
 595
 596     s = splsched();
 597     simple_lock(&thread_call_lock);
 598
 599     if (cancel_all)
 600                 result = _remove_from_pending_queue(func, param, cancel_all) |
 601                                                 _remove_from_delayed_queue(func, param, cancel_all);
 602         else
 603                 result = _remove_from_pending_queue(func, param, cancel_all) ||
 604                                                 _remove_from_delayed_queue(func, param, cancel_all);
 605
 606     simple_unlock(&thread_call_lock);
 607     splx(s);
 608
 609         return (result);
 610 }
 611
 612 /*
 613  * Routine:     thread_call_allocate [public]
 614  *
 615  * Purpose:     Allocate an external callout
 616  *              entry.
 617  *
 618  * Preconditions:       None.
 619  *
 620  * Postconditions:      None.
 621  */
 622
 623 thread_call_t
 624 thread_call_allocate(
 625     thread_call_func_t          func,
 626     thread_call_param_t         param0
 627 )
 628 {
 629     thread_call_t               call = (void *)kalloc(sizeof (thread_call_data_t));
 630
 631     call->func                  = func;
 632     call->param0                = param0;
 633     call->state                 = IDLE;
 634
 635     return (call);
 636 }
 637
 638 /*
 639  * Routine:     thread_call_free [public]
 640  *
 641  * Purpose:     Free an external callout
 642  *              entry.
 643  *
 644  * Preconditions:       None.
 645  *
 646  * Postconditions:      None.
 647  */
 648
 649 boolean_t
 650 thread_call_free(
 651     thread_call_t               call
 652 )
 653 {
 654     spl_t               s;
 655
 656     s = splsched();
 657     simple_lock(&thread_call_lock);
 658
 659     if (call->state != IDLE) {
 660         simple_unlock(&thread_call_lock);
 661                 splx(s);
 662
 663                 return (FALSE);
 664     }
 665
 666     simple_unlock(&thread_call_lock);
 667     splx(s);
 668
 669     kfree(call, sizeof (thread_call_data_t));
 670
 671         return (TRUE);
 672 }
 673
 674 /*
 675  * Routine:     thread_call_enter [public]
 676  *
 677  * Purpose:     Schedule an external callout
 678  *              entry to occur "soon".  Returns a
 679  *              boolean indicating whether the call
 680  *              had been already scheduled.
 681  *
 682  * Preconditions:       Callable from an interrupt context
 683  *                                      below splsched.
 684  *
 685  * Postconditions:      None.
 686  */
 687
 688 boolean_t
 689 thread_call_enter(
 690     thread_call_t               call
 691 )
 692 {
 693         boolean_t               result = TRUE;
 694     spl_t                       s;
 695
 696     s = splsched();
 697     simple_lock(&thread_call_lock);
 698
 699     if (call->state != PENDING) {
 700                 if (call->state == DELAYED)
 701                         _delayed_call_dequeue(call);
 702                 else if (call->state == IDLE)
 703                         result = FALSE;
 704
 705         _pending_call_enqueue(call);
 706
 707                 if (thread_call_vars.active_num <= 0)
 708                         _call_thread_wake();
 709         }
 710
 711         call->param1 = 0;
 712
 713         simple_unlock(&thread_call_lock);
 714     splx(s);
 715
 716         return (result);
 717 }
 718
 719 boolean_t
 720 thread_call_enter1(
 721     thread_call_t                       call,
 722     thread_call_param_t         param1
 723 )
 724 {
 725         boolean_t                       result = TRUE;
 726     spl_t                               s;
 727
 728     s = splsched();
 729     simple_lock(&thread_call_lock);
 730
 731     if (call->state != PENDING) {
 732                 if (call->state == DELAYED)
 733                         _delayed_call_dequeue(call);
 734                 else if (call->state == IDLE)
 735                         result = FALSE;
 736
 737         _pending_call_enqueue(call);
 738
 739                 if (thread_call_vars.active_num <= 0)
 740                         _call_thread_wake();
 741     }
 742
 743         call->param1 = param1;
 744
 745         simple_unlock(&thread_call_lock);
 746     splx(s);
 747
 748         return (result);
 749 }
 750
 751 /*
 752  * Routine:     thread_call_enter_delayed [public]
 753  *
 754  * Purpose:     Schedule an external callout
 755  *              entry to occur at the stated time.
 756  *              Returns a boolean indicating whether
 757  *              the call had been already scheduled.
 758  *
 759  * Preconditions:       Callable from an interrupt context
 760  *                                      below splsched.
 761  *
 762  * Postconditions:      None.
 763  */
 764
 765 boolean_t
 766 thread_call_enter_delayed(
 767     thread_call_t               call,
 768     uint64_t                    deadline
 769 )
 770 {
 771         boolean_t               result = TRUE;
 772     spl_t                       s;
 773
 774     s = splsched();
 775     simple_lock(&thread_call_lock);
 776
 777         if (call->state == PENDING)
 778                 _pending_call_dequeue(call);
 779         else if (call->state == DELAYED)
 780                 _delayed_call_dequeue(call);
 781         else if (call->state == IDLE)
 782                 result = FALSE;
 783
 784         call->param1    = 0;
 785         call->deadline  = deadline;
 786
 787         _delayed_call_enqueue(call);
 788
 789         if (queue_first(&thread_call_delayed_queue) == qe(call))
 790                 _set_delayed_call_timer(call);
 791
 792     simple_unlock(&thread_call_lock);
 793     splx(s);
 794
 795         return (result);
 796 }
 797
 798 boolean_t
 799 thread_call_enter1_delayed(
 800     thread_call_t                       call,
 801     thread_call_param_t         param1,
 802     uint64_t                            deadline
 803 )
 804 {
 805         boolean_t                       result = TRUE;
 806     spl_t                               s;
 807
 808     s = splsched();
 809     simple_lock(&thread_call_lock);
 810
 811         if (call->state == PENDING)
 812                 _pending_call_dequeue(call);
 813         else if (call->state == DELAYED)
 814                 _delayed_call_dequeue(call);
 815         else if (call->state == IDLE)
 816                 result = FALSE;
 817
 818         call->param1    = param1;
 819         call->deadline  = deadline;
 820
 821         _delayed_call_enqueue(call);
 822
 823         if (queue_first(&thread_call_delayed_queue) == qe(call))
 824                 _set_delayed_call_timer(call);
 825
 826     simple_unlock(&thread_call_lock);
 827     splx(s);
 828
 829         return (result);
 830 }
 831
 832 /*
 833  * Routine:     thread_call_cancel [public]
 834  *
 835  * Purpose:     Unschedule a callout entry.
 836  *              Returns a boolean indicating
 837  *              whether the call had actually
 838  *              been scheduled.
 839  *
 840  * Preconditions:       Callable from an interrupt context
 841  *                                      below splsched.
 842  *
 843  * Postconditions:      None.
 844  */
 845
 846 boolean_t
 847 thread_call_cancel(
 848     thread_call_t               call
 849 )
 850 {
 851         boolean_t               result = TRUE;
 852     spl_t                       s;
 853
 854     s = splsched();
 855     simple_lock(&thread_call_lock);
 856
 857     if (call->state == PENDING)
 858         _pending_call_dequeue(call);
 859     else if (call->state == DELAYED)
 860         _delayed_call_dequeue(call);
 861     else
 862         result = FALSE;
 863
 864     simple_unlock(&thread_call_lock);
 865     splx(s);
 866
 867         return (result);
 868 }
 869
 870 /*
 871  * Routine:     thread_call_is_delayed [public]
 872  *
 873  * Purpose:     Returns a boolean indicating
 874  *              whether a call is currently scheduled
 875  *              to occur at a later time.  Optionally
 876  *              returns the expiration time.
 877  *
 878  * Preconditions:       Callable from an interrupt context
 879  *                                      below splsched.
 880  *
 881  * Postconditions:      None.
 882  */
 883
 884 boolean_t
 885 thread_call_is_delayed(
 886         thread_call_t           call,
 887         uint64_t                        *deadline)
 888 {
 889         boolean_t               result = FALSE;
 890         spl_t                   s;
 891
 892         s = splsched();
 893         simple_lock(&thread_call_lock);
 894
 895         if (call->state == DELAYED) {
 896                 if (deadline != NULL)
 897                         *deadline = call->deadline;
 898                 result = TRUE;
 899         }
 900
 901         simple_unlock(&thread_call_lock);
 902         splx(s);
 903
 904         return (result);
 905 }
 906
 907 /*
 908  * Routine:     _call_thread_wake [private, inline]
 909  *
 910  * Purpose:     Wake a callout thread to service
 911  *              pending callout entries.  May wake
 912  *              the activate thread in order to
 913  *              create additional callout threads.
 914  *
 915  * Preconditions:       thread_call_lock held.
 916  *
 917  * Postconditions:      None.
 918  */
 919
 920 static __inline__
 921 void
 922 _call_thread_wake(void)
 923 {
 924         if (wait_queue_wakeup_one(&call_thread_waitqueue, NULL, THREAD_AWAKENED) == KERN_SUCCESS) {
 925                 thread_call_vars.idle_thread_num--;
 926
 927                 if (++thread_call_vars.active_num > thread_call_vars.active_hiwat)
 928                         thread_call_vars.active_hiwat = thread_call_vars.active_num;
 929         }
 930         else
 931         if (!activate_thread_awake) {
 932                 thread_wakeup_one(&activate_thread_awake);
 933                 activate_thread_awake = TRUE;
 934         }
 935 }
 936
 937 /*
 938  * Routine:     call_thread_block [private]
 939  *
 940  * Purpose:     Hook via thread dispatch on
 941  *              the occasion of a callout blocking.
 942  *
 943  * Preconditions:       splsched.
 944  *
 945  * Postconditions:      None.
 946  */
 947
 948 void
 949 call_thread_block(void)
 950 {
 951         simple_lock(&thread_call_lock);
 952
 953         if (--thread_call_vars.active_num < thread_call_vars.active_lowat)
 954                 thread_call_vars.active_lowat = thread_call_vars.active_num;
 955
 956         if (    thread_call_vars.active_num <= 0        &&
 957                         thread_call_vars.pending_num > 0                )
 958                 _call_thread_wake();
 959
 960         simple_unlock(&thread_call_lock);
 961 }
 962
 963 /*
 964  * Routine:     call_thread_unblock [private]
 965  *
 966  * Purpose:     Hook via thread wakeup on
 967  *              the occasion of a callout unblocking.
 968  *
 969  * Preconditions:       splsched.
 970  *
 971  * Postconditions:      None.
 972  */
 973
 974 void
 975 call_thread_unblock(void)
 976 {
 977         simple_lock(&thread_call_lock);
 978
 979         if (++thread_call_vars.active_num > thread_call_vars.active_hiwat)
 980                 thread_call_vars.active_hiwat = thread_call_vars.active_num;
 981
 982         simple_unlock(&thread_call_lock);
 983 }
 984
 985 /*
 986  * Routine:     _call_thread [private]
 987  *
 988  * Purpose:     Executed by a callout thread.
 989  *
 990  * Preconditions:       None.
 991  *
 992  * Postconditions:      None.
 993  */
 994
 995 static
 996 void
 997 _call_thread_continue(void)
 998 {
 999         thread_t                self = current_thread();
1000
1001     (void) splsched();
1002     simple_lock(&thread_call_lock);
1003
1004         self->options |= TH_OPT_CALLOUT;
1005
1006     while (thread_call_vars.pending_num > 0) {
1007                 thread_call_t                   call;
1008                 thread_call_func_t              func;
1009                 thread_call_param_t             param0, param1;
1010
1011                 call = TC(dequeue_head(&thread_call_pending_queue));
1012                 thread_call_vars.pending_num--;
1013
1014                 func = call->func;
1015                 param0 = call->param0;
1016                 param1 = call->param1;
1017
1018                 call->state = IDLE;
1019
1020                 _internal_call_release(call);
1021
1022                 simple_unlock(&thread_call_lock);
1023                 (void) spllo();
1024
1025                 KERNEL_DEBUG_CONSTANT(
1026                         MACHDBG_CODE(DBG_MACH_SCHED,MACH_CALLOUT) | DBG_FUNC_NONE,
1027                                 (int)func, (int)param0, (int)param1, 0, 0);
1028
1029                 (*func)(param0, param1);
1030
1031                 (void)thread_funnel_set(self->funnel_lock, FALSE);
1032
1033                 (void) splsched();
1034                 simple_lock(&thread_call_lock);
1035     }
1036
1037         self->options &= ~TH_OPT_CALLOUT;
1038
1039         if (--thread_call_vars.active_num < thread_call_vars.active_lowat)
1040                 thread_call_vars.active_lowat = thread_call_vars.active_num;
1041
1042     if (thread_call_vars.idle_thread_num < thread_call_vars.thread_lowat) {
1043                 thread_call_vars.idle_thread_num++;
1044
1045                 wait_queue_assert_wait(&call_thread_waitqueue, NULL, THREAD_UNINT, 0);
1046
1047                 simple_unlock(&thread_call_lock);
1048                 (void) spllo();
1049
1050                 thread_block((thread_continue_t)_call_thread_continue);
1051                 /* NOTREACHED */
1052     }
1053
1054     thread_call_vars.thread_num--;
1055
1056     simple_unlock(&thread_call_lock);
1057     (void) spllo();
1058
1059     thread_terminate(self);
1060         /* NOTREACHED */
1061 }
1062
1063 static
1064 void
1065 _call_thread(void)
1066 {
1067     _call_thread_continue();
1068     /* NOTREACHED */
1069 }
1070
1071 /*
1072  * Routine:     _activate_thread [private]
1073  *
1074  * Purpose:     Executed by the activate thread.
1075  *
1076  * Preconditions:       None.
1077  *
1078  * Postconditions:      Never terminates.
1079  */
1080
1081 static
1082 void
1083 _activate_thread_continue(void)
1084 {
1085         kern_return_t   result;
1086         thread_t                thread;
1087
1088     (void) splsched();
1089     simple_lock(&thread_call_lock);
1090
1091         while (         thread_call_vars.active_num <= 0        &&
1092                                 thread_call_vars.pending_num > 0                ) {
1093
1094                 if (++thread_call_vars.active_num > thread_call_vars.active_hiwat)
1095                         thread_call_vars.active_hiwat = thread_call_vars.active_num;
1096
1097                 if (++thread_call_vars.thread_num > thread_call_vars.thread_hiwat)
1098                         thread_call_vars.thread_hiwat = thread_call_vars.thread_num;
1099
1100                 simple_unlock(&thread_call_lock);
1101                 (void) spllo();
1102
1103                 result = kernel_thread_start_priority((thread_continue_t)_call_thread, NULL, MAXPRI_KERNEL - 1, &thread);
1104                 if (result != KERN_SUCCESS)
1105                         panic("activate_thread");
1106
1107                 thread_deallocate(thread);
1108
1109                 (void) splsched();
1110                 simple_lock(&thread_call_lock);
1111     }
1112
1113     assert_wait(&activate_thread_awake, THREAD_INTERRUPTIBLE);
1114         activate_thread_awake = FALSE;
1115
1116     simple_unlock(&thread_call_lock);
1117         (void) spllo();
1118
1119         thread_block((thread_continue_t)_activate_thread_continue);
1120         /* NOTREACHED */
1121 }
1122
1123 static
1124 void
1125 _activate_thread(void)
1126 {
1127         thread_t        self = current_thread();
1128
1129         self->options |= TH_OPT_VMPRIV;
1130         vm_page_free_reserve(2);        /* XXX */
1131
1132     _activate_thread_continue();
1133     /* NOTREACHED */
1134 }
1135
1136 static
1137 void
1138 _delayed_call_timer(
1139         __unused timer_call_param_t             p0,
1140         __unused timer_call_param_t             p1
1141 )
1142 {
1143         uint64_t                        timestamp;
1144     thread_call_t               call;
1145         boolean_t                       new_pending = FALSE;
1146     spl_t                               s;
1147
1148     s = splsched();
1149     simple_lock(&thread_call_lock);
1150
1151         clock_get_uptime(&timestamp);
1152
1153     call = TC(queue_first(&thread_call_delayed_queue));
1154
1155     while (!queue_end(&thread_call_delayed_queue, qe(call))) {
1156         if (call->deadline <= timestamp) {
1157                         _delayed_call_dequeue(call);
1158
1159                         _pending_call_enqueue(call);
1160                         new_pending = TRUE;
1161                 }
1162                 else
1163                         break;
1164
1165                 call = TC(queue_first(&thread_call_delayed_queue));
1166     }
1167
1168         if (!queue_end(&thread_call_delayed_queue, qe(call)))
1169                 _set_delayed_call_timer(call);
1170
1171     if (new_pending && thread_call_vars.active_num <= 0)
1172                 _call_thread_wake();
1173
1174     simple_unlock(&thread_call_lock);
1175     splx(s);
1176 }