osfmk/kern/thread_call.c

   1 /*
   2  * Copyright (c) 1993-1995, 1999-2005 Apple Computer, Inc.
   3  * All rights reserved.
   4  *
   5  * @APPLE_LICENSE_OSREFERENCE_HEADER_START@
   6  *
   7  * This file contains Original Code and/or Modifications of Original Code
   8  * as defined in and that are subject to the Apple Public Source License
   9  * Version 2.0 (the 'License'). You may not use this file except in
  10  * compliance with the License.  The rights granted to you under the
  11  * License may not be used to create, or enable the creation or
  12  * redistribution of, unlawful or unlicensed copies of an Apple operating
  13  * system, or to circumvent, violate, or enable the circumvention or
  14  * violation of, any terms of an Apple operating system software license
  15  * agreement.
  16  *
  17  * Please obtain a copy of the License at
  18  * http://www.opensource.apple.com/apsl/ and read it before using this
  19  * file.
  20  *
  21  * The Original Code and all software distributed under the License are
  22  * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  23  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  24  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
  25  * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
  26  * Please see the License for the specific language governing rights and
  27  * limitations under the License.
  28  *
  29  * @APPLE_LICENSE_OSREFERENCE_HEADER_END@
  30  */
  31
  32 #include <mach/mach_types.h>
  33 #include <mach/thread_act.h>
  34
  35 #include <kern/kern_types.h>
  36 #include <kern/kalloc.h>
  37 #include <kern/sched_prim.h>
  38 #include <kern/clock.h>
  39 #include <kern/task.h>
  40 #include <kern/thread.h>
  41 #include <kern/wait_queue.h>
  42
  43 #include <vm/vm_pageout.h>
  44
  45 #include <kern/thread_call.h>
  46 #include <kern/call_entry.h>
  47
  48 #include <kern/timer_call.h>
  49
  50 #include <sys/kdebug.h>
  51
  52 #define internal_call_num       768
  53
  54 #define thread_call_thread_min  4
  55
  56 static
  57 thread_call_data_t
  58         internal_call_storage[internal_call_num];
  59
  60 decl_simple_lock_data(static,thread_call_lock)
  61
  62 static
  63 timer_call_data_t
  64         thread_call_delaytimer;
  65
  66 static
  67 queue_head_t
  68         thread_call_xxx_queue,
  69         thread_call_pending_queue, thread_call_delayed_queue;
  70
  71 static
  72 struct wait_queue
  73         call_thread_waitqueue;
  74
  75 static
  76 boolean_t
  77         activate_thread_awake;
  78
  79 static struct {
  80         int             pending_num,
  81                         pending_hiwat;
  82         int             active_num,
  83                         active_hiwat,
  84                         active_lowat;
  85         int             delayed_num,
  86                         delayed_hiwat;
  87         int             idle_thread_num;
  88         int             thread_num,
  89                         thread_hiwat,
  90                         thread_lowat;
  91 } thread_call_vars;
  92
  93 static __inline__ thread_call_t
  94         _internal_call_allocate(void);
  95
  96 static __inline__ void
  97 _internal_call_release(
  98         thread_call_t           call
  99 );
 100
 101 static __inline__ void
 102 _pending_call_enqueue(
 103         thread_call_t           call
 104 ),
 105 _pending_call_dequeue(
 106         thread_call_t           call
 107 ),
 108 _delayed_call_enqueue(
 109         thread_call_t           call
 110 ),
 111 _delayed_call_dequeue(
 112         thread_call_t           call
 113 );
 114
 115 static __inline__ void
 116 _set_delayed_call_timer(
 117         thread_call_t           call
 118 );
 119
 120 static boolean_t
 121 _remove_from_pending_queue(
 122         thread_call_func_t      func,
 123         thread_call_param_t     param0,
 124         boolean_t                       remove_all
 125 ),
 126 _remove_from_delayed_queue(
 127         thread_call_func_t      func,
 128         thread_call_param_t     param0,
 129         boolean_t                       remove_all
 130 );
 131
 132 static __inline__ void
 133         _call_thread_wake(void);
 134
 135 static void
 136         _call_thread(void),
 137         _activate_thread(void);
 138
 139 static void
 140 _delayed_call_timer(
 141         timer_call_param_t              p0,
 142         timer_call_param_t              p1
 143 );
 144
 145 #define qe(x)           ((queue_entry_t)(x))
 146 #define TC(x)           ((thread_call_t)(x))
 147
 148 /*
 149  * Routine:     thread_call_initialize [public]
 150  *
 151  * Description: Initialize this module, called
 152  *              early during system initialization.
 153  *
 154  * Preconditions:       None.
 155  *
 156  * Postconditions:      None.
 157  */
 158
 159 void
 160 thread_call_initialize(void)
 161 {
 162         kern_return_t   result;
 163         thread_t                thread;
 164     thread_call_t       call;
 165         spl_t                   s;
 166
 167     simple_lock_init(&thread_call_lock, 0);
 168
 169         s = splsched();
 170         simple_lock(&thread_call_lock);
 171
 172     queue_init(&thread_call_pending_queue);
 173     queue_init(&thread_call_delayed_queue);
 174
 175     queue_init(&thread_call_xxx_queue);
 176     for (
 177                 call = internal_call_storage;
 178                         call < &internal_call_storage[internal_call_num];
 179                         call++) {
 180
 181                 enqueue_tail(&thread_call_xxx_queue, qe(call));
 182     }
 183
 184         timer_call_setup(&thread_call_delaytimer, _delayed_call_timer, NULL);
 185
 186         wait_queue_init(&call_thread_waitqueue, SYNC_POLICY_FIFO);
 187         thread_call_vars.thread_lowat = thread_call_thread_min;
 188
 189         activate_thread_awake = TRUE;
 190
 191         simple_unlock(&thread_call_lock);
 192         splx(s);
 193
 194         result = kernel_thread_start_priority((thread_continue_t)_activate_thread, NULL, MAXPRI_KERNEL - 2, &thread);
 195         if (result != KERN_SUCCESS)
 196                 panic("thread_call_initialize");
 197
 198         thread_deallocate(thread);
 199 }
 200
 201 void
 202 thread_call_setup(
 203         thread_call_t                   call,
 204         thread_call_func_t              func,
 205         thread_call_param_t             param0
 206 )
 207 {
 208         call_entry_setup(call, func, param0);
 209 }
 210
 211 /*
 212  * Routine:     _internal_call_allocate [private, inline]
 213  *
 214  * Purpose:     Allocate an internal callout entry.
 215  *
 216  * Preconditions:       thread_call_lock held.
 217  *
 218  * Postconditions:      None.
 219  */
 220
 221 static __inline__ thread_call_t
 222 _internal_call_allocate(void)
 223 {
 224     thread_call_t               call;
 225
 226     if (queue_empty(&thread_call_xxx_queue))
 227         panic("_internal_call_allocate");
 228
 229     call = TC(dequeue_head(&thread_call_xxx_queue));
 230
 231     return (call);
 232 }
 233
 234 /*
 235  * Routine:     _internal_call_release [private, inline]
 236  *
 237  * Purpose:     Release an internal callout entry which
 238  *              is no longer pending (or delayed).
 239  *
 240  * Preconditions:       thread_call_lock held.
 241  *
 242  * Postconditions:      None.
 243  */
 244
 245 static __inline__
 246 void
 247 _internal_call_release(
 248     thread_call_t               call
 249 )
 250 {
 251     if (    call >= internal_call_storage                                               &&
 252                     call < &internal_call_storage[internal_call_num]            )
 253                 enqueue_head(&thread_call_xxx_queue, qe(call));
 254 }
 255
 256 /*
 257  * Routine:     _pending_call_enqueue [private, inline]
 258  *
 259  * Purpose:     Place an entry at the end of the
 260  *              pending queue, to be executed soon.
 261  *
 262  * Preconditions:       thread_call_lock held.
 263  *
 264  * Postconditions:      None.
 265  */
 266
 267 static __inline__
 268 void
 269 _pending_call_enqueue(
 270     thread_call_t               call
 271 )
 272 {
 273     enqueue_tail(&thread_call_pending_queue, qe(call));
 274         if (++thread_call_vars.pending_num > thread_call_vars.pending_hiwat)
 275                 thread_call_vars.pending_hiwat = thread_call_vars.pending_num;
 276
 277     call->state = PENDING;
 278 }
 279
 280 /*
 281  * Routine:     _pending_call_dequeue [private, inline]
 282  *
 283  * Purpose:     Remove an entry from the pending queue,
 284  *              effectively unscheduling it.
 285  *
 286  * Preconditions:       thread_call_lock held.
 287  *
 288  * Postconditions:      None.
 289  */
 290
 291 static __inline__
 292 void
 293 _pending_call_dequeue(
 294     thread_call_t               call
 295 )
 296 {
 297     (void)remque(qe(call));
 298         thread_call_vars.pending_num--;
 299
 300     call->state = IDLE;
 301 }
 302
 303 /*
 304  * Routine:     _delayed_call_enqueue [private, inline]
 305  *
 306  * Purpose:     Place an entry on the delayed queue,
 307  *              after existing entries with an earlier
 308  *              (or identical) deadline.
 309  *
 310  * Preconditions:       thread_call_lock held.
 311  *
 312  * Postconditions:      None.
 313  */
 314
 315 static __inline__
 316 void
 317 _delayed_call_enqueue(
 318     thread_call_t               call
 319 )
 320 {
 321     thread_call_t               current;
 322
 323     current = TC(queue_first(&thread_call_delayed_queue));
 324
 325     while (TRUE) {
 326         if (    queue_end(&thread_call_delayed_queue, qe(current))              ||
 327                                         call->deadline < current->deadline                      ) {
 328                         current = TC(queue_prev(qe(current)));
 329                         break;
 330                 }
 331
 332                 current = TC(queue_next(qe(current)));
 333     }
 334
 335     insque(qe(call), qe(current));
 336         if (++thread_call_vars.delayed_num > thread_call_vars.delayed_hiwat)
 337                 thread_call_vars.delayed_hiwat = thread_call_vars.delayed_num;
 338
 339     call->state = DELAYED;
 340 }
 341
 342 /*
 343  * Routine:     _delayed_call_dequeue [private, inline]
 344  *
 345  * Purpose:     Remove an entry from the delayed queue,
 346  *              effectively unscheduling it.
 347  *
 348  * Preconditions:       thread_call_lock held.
 349  *
 350  * Postconditions:      None.
 351  */
 352
 353 static __inline__
 354 void
 355 _delayed_call_dequeue(
 356     thread_call_t               call
 357 )
 358 {
 359     (void)remque(qe(call));
 360         thread_call_vars.delayed_num--;
 361
 362     call->state = IDLE;
 363 }
 364
 365 /*
 366  * Routine:     _set_delayed_call_timer [private]
 367  *
 368  * Purpose:     Reset the timer so that it
 369  *              next expires when the entry is due.
 370  *
 371  * Preconditions:       thread_call_lock held.
 372  *
 373  * Postconditions:      None.
 374  */
 375
 376 static __inline__ void
 377 _set_delayed_call_timer(
 378     thread_call_t               call
 379 )
 380 {
 381     timer_call_enter(&thread_call_delaytimer, call->deadline);
 382 }
 383
 384 /*
 385  * Routine:     _remove_from_pending_queue [private]
 386  *
 387  * Purpose:     Remove the first (or all) matching
 388  *              entries from the pending queue,
 389  *              effectively unscheduling them.
 390  *              Returns whether any matching entries
 391  *              were found.
 392  *
 393  * Preconditions:       thread_call_lock held.
 394  *
 395  * Postconditions:      None.
 396  */
 397
 398 static
 399 boolean_t
 400 _remove_from_pending_queue(
 401     thread_call_func_t          func,
 402     thread_call_param_t         param0,
 403     boolean_t                           remove_all
 404 )
 405 {
 406         boolean_t                       call_removed = FALSE;
 407         thread_call_t           call;
 408
 409     call = TC(queue_first(&thread_call_pending_queue));
 410
 411     while (!queue_end(&thread_call_pending_queue, qe(call))) {
 412         if (    call->func == func                      &&
 413                                 call->param0 == param0                  ) {
 414                         thread_call_t   next = TC(queue_next(qe(call)));
 415
 416                         _pending_call_dequeue(call);
 417
 418                         _internal_call_release(call);
 419
 420                         call_removed = TRUE;
 421                         if (!remove_all)
 422                                 break;
 423
 424                         call = next;
 425                 }
 426                 else
 427                         call = TC(queue_next(qe(call)));
 428     }
 429
 430     return (call_removed);
 431 }
 432
 433 /*
 434  * Routine:     _remove_from_delayed_queue [private]
 435  *
 436  * Purpose:     Remove the first (or all) matching
 437  *              entries from the delayed queue,
 438  *              effectively unscheduling them.
 439  *              Returns whether any matching entries
 440  *              were found.
 441  *
 442  * Preconditions:       thread_call_lock held.
 443  *
 444  * Postconditions:      None.
 445  */
 446
 447 static
 448 boolean_t
 449 _remove_from_delayed_queue(
 450     thread_call_func_t          func,
 451     thread_call_param_t         param0,
 452     boolean_t                           remove_all
 453 )
 454 {
 455     boolean_t                   call_removed = FALSE;
 456     thread_call_t               call;
 457
 458     call = TC(queue_first(&thread_call_delayed_queue));
 459
 460     while (!queue_end(&thread_call_delayed_queue, qe(call))) {
 461         if (    call->func == func                      &&
 462                                 call->param0 == param0                  ) {
 463                         thread_call_t   next = TC(queue_next(qe(call)));
 464
 465                         _delayed_call_dequeue(call);
 466
 467                         _internal_call_release(call);
 468
 469                         call_removed = TRUE;
 470                         if (!remove_all)
 471                                 break;
 472
 473                         call = next;
 474                 }
 475                 else
 476                         call = TC(queue_next(qe(call)));
 477     }
 478
 479     return (call_removed);
 480 }
 481
 482 /*
 483  * Routine:     thread_call_func [public]
 484  *
 485  * Purpose:     Schedule a function callout.
 486  *              Guarantees { function, argument }
 487  *              uniqueness if unique_call is TRUE.
 488  *
 489  * Preconditions:       Callable from an interrupt context
 490  *                                      below splsched.
 491  *
 492  * Postconditions:      None.
 493  */
 494
 495 void
 496 thread_call_func(
 497     thread_call_func_t          func,
 498     thread_call_param_t         param,
 499     boolean_t                           unique_call
 500 )
 501 {
 502     thread_call_t               call;
 503     spl_t                               s;
 504
 505     s = splsched();
 506     simple_lock(&thread_call_lock);
 507
 508     call = TC(queue_first(&thread_call_pending_queue));
 509
 510         while (unique_call && !queue_end(&thread_call_pending_queue, qe(call))) {
 511         if (    call->func == func                      &&
 512                                 call->param0 == param                   ) {
 513                         break;
 514                 }
 515
 516                 call = TC(queue_next(qe(call)));
 517     }
 518
 519     if (!unique_call || queue_end(&thread_call_pending_queue, qe(call))) {
 520                 call = _internal_call_allocate();
 521                 call->func                      = func;
 522                 call->param0            = param;
 523                 call->param1            = 0;
 524
 525                 _pending_call_enqueue(call);
 526
 527                 if (thread_call_vars.active_num <= 0)
 528                         _call_thread_wake();
 529     }
 530
 531         simple_unlock(&thread_call_lock);
 532     splx(s);
 533 }
 534
 535 /*
 536  * Routine:     thread_call_func_delayed [public]
 537  *
 538  * Purpose:     Schedule a function callout to
 539  *              occur at the stated time.
 540  *
 541  * Preconditions:       Callable from an interrupt context
 542  *                                      below splsched.
 543  *
 544  * Postconditions:      None.
 545  */
 546
 547 void
 548 thread_call_func_delayed(
 549     thread_call_func_t          func,
 550     thread_call_param_t         param,
 551     uint64_t                            deadline
 552 )
 553 {
 554     thread_call_t               call;
 555     spl_t                               s;
 556
 557     s = splsched();
 558     simple_lock(&thread_call_lock);
 559
 560     call = _internal_call_allocate();
 561     call->func                  = func;
 562     call->param0                = param;
 563     call->param1                = 0;
 564     call->deadline              = deadline;
 565
 566     _delayed_call_enqueue(call);
 567
 568     if (queue_first(&thread_call_delayed_queue) == qe(call))
 569         _set_delayed_call_timer(call);
 570
 571     simple_unlock(&thread_call_lock);
 572     splx(s);
 573 }
 574
 575 /*
 576  * Routine:     thread_call_func_cancel [public]
 577  *
 578  * Purpose:     Unschedule a function callout.
 579  *              Removes one (or all)
 580  *              { function, argument }
 581  *              instance(s) from either (or both)
 582  *              the pending and the delayed queue,
 583  *              in that order.  Returns a boolean
 584  *              indicating whether any calls were
 585  *              cancelled.
 586  *
 587  * Preconditions:       Callable from an interrupt context
 588  *                                      below splsched.
 589  *
 590  * Postconditions:      None.
 591  */
 592
 593 boolean_t
 594 thread_call_func_cancel(
 595     thread_call_func_t          func,
 596     thread_call_param_t         param,
 597     boolean_t                           cancel_all
 598 )
 599 {
 600         boolean_t                       result;
 601     spl_t                               s;
 602
 603     s = splsched();
 604     simple_lock(&thread_call_lock);
 605
 606     if (cancel_all)
 607                 result = _remove_from_pending_queue(func, param, cancel_all) |
 608                                                 _remove_from_delayed_queue(func, param, cancel_all);
 609         else
 610                 result = _remove_from_pending_queue(func, param, cancel_all) ||
 611                                                 _remove_from_delayed_queue(func, param, cancel_all);
 612
 613     simple_unlock(&thread_call_lock);
 614     splx(s);
 615
 616         return (result);
 617 }
 618
 619 /*
 620  * Routine:     thread_call_allocate [public]
 621  *
 622  * Purpose:     Allocate an external callout
 623  *              entry.
 624  *
 625  * Preconditions:       None.
 626  *
 627  * Postconditions:      None.
 628  */
 629
 630 thread_call_t
 631 thread_call_allocate(
 632     thread_call_func_t          func,
 633     thread_call_param_t         param0
 634 )
 635 {
 636     thread_call_t               call = (void *)kalloc(sizeof (thread_call_data_t));
 637
 638     call->func                  = func;
 639     call->param0                = param0;
 640     call->state                 = IDLE;
 641
 642     return (call);
 643 }
 644
 645 /*
 646  * Routine:     thread_call_free [public]
 647  *
 648  * Purpose:     Free an external callout
 649  *              entry.
 650  *
 651  * Preconditions:       None.
 652  *
 653  * Postconditions:      None.
 654  */
 655
 656 boolean_t
 657 thread_call_free(
 658     thread_call_t               call
 659 )
 660 {
 661     spl_t               s;
 662
 663     s = splsched();
 664     simple_lock(&thread_call_lock);
 665
 666     if (call->state != IDLE) {
 667         simple_unlock(&thread_call_lock);
 668                 splx(s);
 669
 670                 return (FALSE);
 671     }
 672
 673     simple_unlock(&thread_call_lock);
 674     splx(s);
 675
 676     kfree(call, sizeof (thread_call_data_t));
 677
 678         return (TRUE);
 679 }
 680
 681 /*
 682  * Routine:     thread_call_enter [public]
 683  *
 684  * Purpose:     Schedule an external callout
 685  *              entry to occur "soon".  Returns a
 686  *              boolean indicating whether the call
 687  *              had been already scheduled.
 688  *
 689  * Preconditions:       Callable from an interrupt context
 690  *                                      below splsched.
 691  *
 692  * Postconditions:      None.
 693  */
 694
 695 boolean_t
 696 thread_call_enter(
 697     thread_call_t               call
 698 )
 699 {
 700         boolean_t               result = TRUE;
 701     spl_t                       s;
 702
 703     s = splsched();
 704     simple_lock(&thread_call_lock);
 705
 706     if (call->state != PENDING) {
 707                 if (call->state == DELAYED)
 708                         _delayed_call_dequeue(call);
 709                 else if (call->state == IDLE)
 710                         result = FALSE;
 711
 712         _pending_call_enqueue(call);
 713
 714                 if (thread_call_vars.active_num <= 0)
 715                         _call_thread_wake();
 716         }
 717
 718         call->param1 = 0;
 719
 720         simple_unlock(&thread_call_lock);
 721     splx(s);
 722
 723         return (result);
 724 }
 725
 726 boolean_t
 727 thread_call_enter1(
 728     thread_call_t                       call,
 729     thread_call_param_t         param1
 730 )
 731 {
 732         boolean_t                       result = TRUE;
 733     spl_t                               s;
 734
 735     s = splsched();
 736     simple_lock(&thread_call_lock);
 737
 738     if (call->state != PENDING) {
 739                 if (call->state == DELAYED)
 740                         _delayed_call_dequeue(call);
 741                 else if (call->state == IDLE)
 742                         result = FALSE;
 743
 744         _pending_call_enqueue(call);
 745
 746                 if (thread_call_vars.active_num <= 0)
 747                         _call_thread_wake();
 748     }
 749
 750         call->param1 = param1;
 751
 752         simple_unlock(&thread_call_lock);
 753     splx(s);
 754
 755         return (result);
 756 }
 757
 758 /*
 759  * Routine:     thread_call_enter_delayed [public]
 760  *
 761  * Purpose:     Schedule an external callout
 762  *              entry to occur at the stated time.
 763  *              Returns a boolean indicating whether
 764  *              the call had been already scheduled.
 765  *
 766  * Preconditions:       Callable from an interrupt context
 767  *                                      below splsched.
 768  *
 769  * Postconditions:      None.
 770  */
 771
 772 boolean_t
 773 thread_call_enter_delayed(
 774     thread_call_t               call,
 775     uint64_t                    deadline
 776 )
 777 {
 778         boolean_t               result = TRUE;
 779     spl_t                       s;
 780
 781     s = splsched();
 782     simple_lock(&thread_call_lock);
 783
 784         if (call->state == PENDING)
 785                 _pending_call_dequeue(call);
 786         else if (call->state == DELAYED)
 787                 _delayed_call_dequeue(call);
 788         else if (call->state == IDLE)
 789                 result = FALSE;
 790
 791         call->param1    = 0;
 792         call->deadline  = deadline;
 793
 794         _delayed_call_enqueue(call);
 795
 796         if (queue_first(&thread_call_delayed_queue) == qe(call))
 797                 _set_delayed_call_timer(call);
 798
 799     simple_unlock(&thread_call_lock);
 800     splx(s);
 801
 802         return (result);
 803 }
 804
 805 boolean_t
 806 thread_call_enter1_delayed(
 807     thread_call_t                       call,
 808     thread_call_param_t         param1,
 809     uint64_t                            deadline
 810 )
 811 {
 812         boolean_t                       result = TRUE;
 813     spl_t                               s;
 814
 815     s = splsched();
 816     simple_lock(&thread_call_lock);
 817
 818         if (call->state == PENDING)
 819                 _pending_call_dequeue(call);
 820         else if (call->state == DELAYED)
 821                 _delayed_call_dequeue(call);
 822         else if (call->state == IDLE)
 823                 result = FALSE;
 824
 825         call->param1    = param1;
 826         call->deadline  = deadline;
 827
 828         _delayed_call_enqueue(call);
 829
 830         if (queue_first(&thread_call_delayed_queue) == qe(call))
 831                 _set_delayed_call_timer(call);
 832
 833     simple_unlock(&thread_call_lock);
 834     splx(s);
 835
 836         return (result);
 837 }
 838
 839 /*
 840  * Routine:     thread_call_cancel [public]
 841  *
 842  * Purpose:     Unschedule a callout entry.
 843  *              Returns a boolean indicating
 844  *              whether the call had actually
 845  *              been scheduled.
 846  *
 847  * Preconditions:       Callable from an interrupt context
 848  *                                      below splsched.
 849  *
 850  * Postconditions:      None.
 851  */
 852
 853 boolean_t
 854 thread_call_cancel(
 855     thread_call_t               call
 856 )
 857 {
 858         boolean_t               result = TRUE;
 859     spl_t                       s;
 860
 861     s = splsched();
 862     simple_lock(&thread_call_lock);
 863
 864     if (call->state == PENDING)
 865         _pending_call_dequeue(call);
 866     else if (call->state == DELAYED)
 867         _delayed_call_dequeue(call);
 868     else
 869         result = FALSE;
 870
 871     simple_unlock(&thread_call_lock);
 872     splx(s);
 873
 874         return (result);
 875 }
 876
 877 /*
 878  * Routine:     thread_call_is_delayed [public]
 879  *
 880  * Purpose:     Returns a boolean indicating
 881  *              whether a call is currently scheduled
 882  *              to occur at a later time.  Optionally
 883  *              returns the expiration time.
 884  *
 885  * Preconditions:       Callable from an interrupt context
 886  *                                      below splsched.
 887  *
 888  * Postconditions:      None.
 889  */
 890
 891 boolean_t
 892 thread_call_is_delayed(
 893         thread_call_t           call,
 894         uint64_t                        *deadline)
 895 {
 896         boolean_t               result = FALSE;
 897         spl_t                   s;
 898
 899         s = splsched();
 900         simple_lock(&thread_call_lock);
 901
 902         if (call->state == DELAYED) {
 903                 if (deadline != NULL)
 904                         *deadline = call->deadline;
 905                 result = TRUE;
 906         }
 907
 908         simple_unlock(&thread_call_lock);
 909         splx(s);
 910
 911         return (result);
 912 }
 913
 914 /*
 915  * Routine:     _call_thread_wake [private, inline]
 916  *
 917  * Purpose:     Wake a callout thread to service
 918  *              pending callout entries.  May wake
 919  *              the activate thread in order to
 920  *              create additional callout threads.
 921  *
 922  * Preconditions:       thread_call_lock held.
 923  *
 924  * Postconditions:      None.
 925  */
 926
 927 static __inline__
 928 void
 929 _call_thread_wake(void)
 930 {
 931         if (wait_queue_wakeup_one(&call_thread_waitqueue, NULL, THREAD_AWAKENED) == KERN_SUCCESS) {
 932                 thread_call_vars.idle_thread_num--;
 933
 934                 if (++thread_call_vars.active_num > thread_call_vars.active_hiwat)
 935                         thread_call_vars.active_hiwat = thread_call_vars.active_num;
 936         }
 937         else
 938         if (!activate_thread_awake) {
 939                 thread_wakeup_one(&activate_thread_awake);
 940                 activate_thread_awake = TRUE;
 941         }
 942 }
 943
 944 /*
 945  * Routine:     call_thread_block [private]
 946  *
 947  * Purpose:     Hook via thread dispatch on
 948  *              the occasion of a callout blocking.
 949  *
 950  * Preconditions:       splsched.
 951  *
 952  * Postconditions:      None.
 953  */
 954
 955 void
 956 call_thread_block(void)
 957 {
 958         simple_lock(&thread_call_lock);
 959
 960         if (--thread_call_vars.active_num < thread_call_vars.active_lowat)
 961                 thread_call_vars.active_lowat = thread_call_vars.active_num;
 962
 963         if (    thread_call_vars.active_num <= 0        &&
 964                         thread_call_vars.pending_num > 0                )
 965                 _call_thread_wake();
 966
 967         simple_unlock(&thread_call_lock);
 968 }
 969
 970 /*
 971  * Routine:     call_thread_unblock [private]
 972  *
 973  * Purpose:     Hook via thread wakeup on
 974  *              the occasion of a callout unblocking.
 975  *
 976  * Preconditions:       splsched.
 977  *
 978  * Postconditions:      None.
 979  */
 980
 981 void
 982 call_thread_unblock(void)
 983 {
 984         simple_lock(&thread_call_lock);
 985
 986         if (++thread_call_vars.active_num > thread_call_vars.active_hiwat)
 987                 thread_call_vars.active_hiwat = thread_call_vars.active_num;
 988
 989         simple_unlock(&thread_call_lock);
 990 }
 991
 992 /*
 993  * Routine:     _call_thread [private]
 994  *
 995  * Purpose:     Executed by a callout thread.
 996  *
 997  * Preconditions:       None.
 998  *
 999  * Postconditions:      None.
1000  */
1001
1002 static
1003 void
1004 _call_thread_continue(void)
1005 {
1006         thread_t                self = current_thread();
1007
1008     (void) splsched();
1009     simple_lock(&thread_call_lock);
1010
1011         self->options |= TH_OPT_CALLOUT;
1012
1013     while (thread_call_vars.pending_num > 0) {
1014                 thread_call_t                   call;
1015                 thread_call_func_t              func;
1016                 thread_call_param_t             param0, param1;
1017
1018                 call = TC(dequeue_head(&thread_call_pending_queue));
1019                 thread_call_vars.pending_num--;
1020
1021                 func = call->func;
1022                 param0 = call->param0;
1023                 param1 = call->param1;
1024
1025                 call->state = IDLE;
1026
1027                 _internal_call_release(call);
1028
1029                 simple_unlock(&thread_call_lock);
1030                 (void) spllo();
1031
1032                 KERNEL_DEBUG_CONSTANT(
1033                         MACHDBG_CODE(DBG_MACH_SCHED,MACH_CALLOUT) | DBG_FUNC_NONE,
1034                                 (int)func, (int)param0, (int)param1, 0, 0);
1035
1036                 (*func)(param0, param1);
1037
1038                 (void)thread_funnel_set(self->funnel_lock, FALSE);
1039
1040                 (void) splsched();
1041                 simple_lock(&thread_call_lock);
1042     }
1043
1044         self->options &= ~TH_OPT_CALLOUT;
1045
1046         if (--thread_call_vars.active_num < thread_call_vars.active_lowat)
1047                 thread_call_vars.active_lowat = thread_call_vars.active_num;
1048
1049     if (thread_call_vars.idle_thread_num < thread_call_vars.thread_lowat) {
1050                 thread_call_vars.idle_thread_num++;
1051
1052                 wait_queue_assert_wait(&call_thread_waitqueue, NULL, THREAD_UNINT, 0);
1053
1054                 simple_unlock(&thread_call_lock);
1055                 (void) spllo();
1056
1057                 thread_block((thread_continue_t)_call_thread_continue);
1058                 /* NOTREACHED */
1059     }
1060
1061     thread_call_vars.thread_num--;
1062
1063     simple_unlock(&thread_call_lock);
1064     (void) spllo();
1065
1066     thread_terminate(self);
1067         /* NOTREACHED */
1068 }
1069
1070 static
1071 void
1072 _call_thread(void)
1073 {
1074     _call_thread_continue();
1075     /* NOTREACHED */
1076 }
1077
1078 /*
1079  * Routine:     _activate_thread [private]
1080  *
1081  * Purpose:     Executed by the activate thread.
1082  *
1083  * Preconditions:       None.
1084  *
1085  * Postconditions:      Never terminates.
1086  */
1087
1088 static
1089 void
1090 _activate_thread_continue(void)
1091 {
1092         kern_return_t   result;
1093         thread_t                thread;
1094
1095     (void) splsched();
1096     simple_lock(&thread_call_lock);
1097
1098         while (         thread_call_vars.active_num <= 0        &&
1099                                 thread_call_vars.pending_num > 0                ) {
1100
1101                 if (++thread_call_vars.active_num > thread_call_vars.active_hiwat)
1102                         thread_call_vars.active_hiwat = thread_call_vars.active_num;
1103
1104                 if (++thread_call_vars.thread_num > thread_call_vars.thread_hiwat)
1105                         thread_call_vars.thread_hiwat = thread_call_vars.thread_num;
1106
1107                 simple_unlock(&thread_call_lock);
1108                 (void) spllo();
1109
1110                 result = kernel_thread_start_priority((thread_continue_t)_call_thread, NULL, MAXPRI_KERNEL - 1, &thread);
1111                 if (result != KERN_SUCCESS)
1112                         panic("activate_thread");
1113
1114                 thread_deallocate(thread);
1115
1116                 (void) splsched();
1117                 simple_lock(&thread_call_lock);
1118     }
1119
1120     assert_wait(&activate_thread_awake, THREAD_INTERRUPTIBLE);
1121         activate_thread_awake = FALSE;
1122
1123     simple_unlock(&thread_call_lock);
1124         (void) spllo();
1125
1126         thread_block((thread_continue_t)_activate_thread_continue);
1127         /* NOTREACHED */
1128 }
1129
1130 static
1131 void
1132 _activate_thread(void)
1133 {
1134         thread_t        self = current_thread();
1135
1136         self->options |= TH_OPT_VMPRIV;
1137         vm_page_free_reserve(2);        /* XXX */
1138
1139     _activate_thread_continue();
1140     /* NOTREACHED */
1141 }
1142
1143 static
1144 void
1145 _delayed_call_timer(
1146         __unused timer_call_param_t             p0,
1147         __unused timer_call_param_t             p1
1148 )
1149 {
1150         uint64_t                        timestamp;
1151     thread_call_t               call;
1152         boolean_t                       new_pending = FALSE;
1153     spl_t                               s;
1154
1155     s = splsched();
1156     simple_lock(&thread_call_lock);
1157
1158         clock_get_uptime(&timestamp);
1159
1160     call = TC(queue_first(&thread_call_delayed_queue));
1161
1162     while (!queue_end(&thread_call_delayed_queue, qe(call))) {
1163         if (call->deadline <= timestamp) {
1164                         _delayed_call_dequeue(call);
1165
1166                         _pending_call_enqueue(call);
1167                         new_pending = TRUE;
1168                 }
1169                 else
1170                         break;
1171
1172                 call = TC(queue_first(&thread_call_delayed_queue));
1173     }
1174
1175         if (!queue_end(&thread_call_delayed_queue, qe(call)))
1176                 _set_delayed_call_timer(call);
1177
1178     if (new_pending && thread_call_vars.active_num <= 0)
1179                 _call_thread_wake();
1180
1181     simple_unlock(&thread_call_lock);
1182     splx(s);
1183 }