osfmk/kern/turnstile.h

   1 /*
   2  * Copyright (c) 2017 Apple Inc. All rights reserved.
   3  *
   4  * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
   5  *
   6  * This file contains Original Code and/or Modifications of Original Code
   7  * as defined in and that are subject to the Apple Public Source License
   8  * Version 2.0 (the 'License'). You may not use this file except in
   9  * compliance with the License. The rights granted to you under the License
  10  * may not be used to create, or enable the creation or redistribution of,
  11  * unlawful or unlicensed copies of an Apple operating system, or to
  12  * circumvent, violate, or enable the circumvention or violation of, any
  13  * terms of an Apple operating system software license agreement.
  14  *
  15  * Please obtain a copy of the License at
  16  * http://www.opensource.apple.com/apsl/ and read it before using this file.
  17  *
  18  * The Original Code and all software distributed under the License are
  19  * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  20  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  21  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
  22  * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
  23  * Please see the License for the specific language governing rights and
  24  * limitations under the License.
  25  *
  26  * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  27  */
  28
  29 #ifndef _TURNSTILE_H_
  30 #define _TURNSTILE_H_
  31
  32 #include <mach/mach_types.h>
  33 #include <mach/kern_return.h>
  34 #include <sys/cdefs.h>
  35
  36 #if PRIVATE
  37 #define TURNSTILE_MAX_HOP_DEFAULT (10)
  38 struct turnstile_stats {
  39         uint64_t ts_priority_propagation;
  40         uint64_t ts_no_inheritor;
  41         uint64_t ts_thread_runnable;
  42         uint64_t ts_no_priority_change_required;
  43         uint64_t ts_above_ui_pri_change;
  44         uint64_t ts_no_turnstile;
  45 };
  46 #endif
  47
  48 #ifdef KERNEL_PRIVATE
  49 #include <kern/queue.h>
  50 #include <sys/queue.h>
  51 #include <kern/waitq.h>
  52 #include <kern/priority_queue.h>
  53 #include <os/refcnt.h>
  54 #include <kern/assert.h>
  55 #include <kern/kern_types.h>
  56 #include <kern/mpsc_queue.h>
  57 #include <kern/locks.h>
  58
  59 /*
  60  * turnstile_type_t : Indicates the type of primitive the turnstile is associated with
  61  *                    Please populate turnstile_promote_policy array if a new type is added here.
  62  */
  63 typedef enum __attribute__((packed)) turnstile_type {
  64         TURNSTILE_NONE = 0,
  65         TURNSTILE_KERNEL_MUTEX = 1,
  66         TURNSTILE_ULOCK = 2,
  67         TURNSTILE_PTHREAD_MUTEX = 3,
  68         TURNSTILE_SYNC_IPC = 4,
  69         TURNSTILE_WORKLOOPS = 5,
  70         TURNSTILE_WORKQS = 6,
  71         TURNSTILE_KNOTE = 7,
  72         TURNSTILE_SLEEP_INHERITOR = 8,
  73         TURNSTILE_TOTAL_TYPES = 9,
  74 } turnstile_type_t;
  75
  76 /*
  77  * For each type of turnstile, following are the type of
  78  * inheritors passed:
  79  *
  80  * TURNSTILE_KERNEL_MUTEX
  81  *    Interlock: kernel mutex interlock.
  82  *    Inheritor: threads.
  83  *    Lock order: turnstile lock, thread lock.
  84  *
  85  * TURNSTILE_ULOCK
  86  *    Interlock: ulocks interlock.
  87  *    Inheritor: threads.
  88  *    Lock order: turnstile lock, thread lock.
  89  *
  90  * TURNSTILE_PTHREAD_MUTEX
  91  *    Interlock: pthread mtx interlock.
  92  *    Inheritor: threads.
  93  *    Lock order: turnstile lock, thread lock.
  94  *
  95  * TURNSTILE_SYNC_IPC
  96  *    Interlock: port's mqueue lock
  97  *    Inheritor: turnstile (of port in which we are enqueued or WL turnstile.
  98  *    Lock order: Our turnstile, then turnstile of the port we are enqueued in.
  99  *                Port circularity will make sure there is never a cycle formation
 100  *                and lock order is maintained.
 101  *
 102  * TURNSTILE_WORKLOOPS
 103  *    Interlock:
 104  *    - kq req lock
 105  *    - wq lock when "filt_wlworkq_interlock_needed() is true"
 106  *    Inheritor: thread, turnstile (of workq)
 107  *    Lock order: turnstile lock, thread lock
 108  *                WL turnstile lock, Workq turnstile lock
 109  *
 110  * TURNSTILE_WORKQS
 111  *    Interlock: workqueue lock
 112  *    Inheritor: thread
 113  *    Lock order: turnstile lock, thread lock.
 114  *
 115  * TURNSTILE_KNOTE
 116  *    Interlock: the knote lock
 117  *    Inheritor: WL turnstile
 118  *
 119  * TURNSTILE_SLEEP_INHERITOR
 120  *    Interlock: turnstile_htable bucket spinlock.
 121  *    Inheritor: threads.
 122  *    Lock order: turnstile lock, thread lock.
 123  *
 124  */
 125
 126 typedef enum __attribute__((flag_enum)) turnstile_promote_policy {
 127         TURNSTILE_PROMOTE_NONE = 0,
 128         TURNSTILE_KERNEL_PROMOTE = 0x1,
 129         TURNSTILE_USER_PROMOTE = 0x2,
 130         TURNSTILE_USER_IPC_PROMOTE = 0x4,
 131 } turnstile_promote_policy_t;
 132
 133 typedef enum __attribute__((flag_enum)) turnstile_hash_lock_policy {
 134         TURNSTILE_HASH_LOCK_POLICY_NONE = 0,
 135         TURNSTILE_IRQ_UNSAFE_HASH = 0x1,
 136         TURNSTILE_LOCKED_HASH = 0x2,
 137 } turnstile_hash_lock_policy_t;
 138
 139 /*
 140  * Turnstile state flags
 141  *
 142  * The turnstile state flags represent the current ownership of a turnstile.
 143  * The supported flags are:
 144  * - TURNSTILE_STATE_THREAD     : Turnstile is attached to a thread
 145  * - TURNSTILE_STATE_FREELIST   : Turnstile is hanging off the freelist of another turnstile
 146  * - TURNSTILE_STATE_HASHTABLE  : Turnstile is in the global hash table as the turnstile for a primitive
 147  * - TURNSTILE_STATE_PROPRIETOR : Turnstile is attached to a proprietor
 148  *
 149  * The flag updates are done while holding the primitive interlock.
 150  * */
 151
 152 #define TURNSTILE_STATE_THREAD          0x1
 153 #define TURNSTILE_STATE_FREELIST        0x2
 154 #define TURNSTILE_STATE_HASHTABLE       0x4
 155 #define TURNSTILE_STATE_PROPRIETOR      0x8
 156
 157 /* Helper macros to set/unset turnstile state flags */
 158 #if DEVELOPMENT || DEBUG
 159
 160 #define turnstile_state_init(ts, state)         \
 161 MACRO_BEGIN                                     \
 162                 ts->ts_state = state;           \
 163 MACRO_END
 164
 165 #define turnstile_state_add(ts, state)          \
 166 MACRO_BEGIN                                     \
 167                 assert((ts->ts_state & (state)) == 0);  \
 168                 ts->ts_state |= state;                  \
 169 MACRO_END
 170
 171 #define turnstile_state_remove(ts, state)       \
 172 MACRO_BEGIN                                     \
 173                 assert(ts->ts_state & (state));         \
 174                 ts->ts_state &= ~(state);               \
 175 MACRO_END
 176
 177 #else  /* DEVELOPMENT || DEBUG */
 178
 179 #define turnstile_state_init(ts, state)         \
 180 MACRO_BEGIN                                     \
 181                 (void)ts;                       \
 182 MACRO_END
 183
 184 #define turnstile_state_add(ts, state)          \
 185 MACRO_BEGIN                                     \
 186                 (void)ts;                       \
 187 MACRO_END
 188
 189 #define turnstile_state_remove(ts, state)       \
 190 MACRO_BEGIN                                     \
 191                 (void)ts;                       \
 192 MACRO_END
 193
 194 #endif /* DEVELOPMENT || DEBUG */
 195
 196 struct knote;
 197 struct turnstile;
 198
 199 /*
 200  * Turnstile update flags
 201  *
 202  * TURNSTILE_IMMEDIATE_UPDATE
 203  *    When passed to turnstile_update_inheritor
 204  *    update the inheritor of the turnstile in
 205  *    the same call.
 206  *
 207  * TURNSTILE_DELAYED_UPDATE
 208  *    When passed to turnstile_update_inheritor
 209  *    it stashed the inheritor on the thread and
 210  *    turnstile's inheritor is updated in
 211  *    assert wait.
 212  *
 213  * TURNSTILE_INHERITOR_THREAD
 214  *    The turnstile inheritor is of type thread.
 215  *
 216  * TURNSTILE_INHERITOR_TURNSTILE
 217  *    The turnstile inheritor is of type turnstile.
 218  *
 219  * TURNSTILE_INHERITOR_WORKQ
 220  *    The turnstile inheritor is of type workqueue
 221  *
 222  * TURNSTILE_INHERITOR_NEEDS_PRI_UPDATE
 223  *    The inheritor needs a chain priority update.
 224  *
 225  * TURNSTILE_NEEDS_PRI_UPDATE
 226  *    Current turnstile needs a chain priority update.
 227  *
 228  * Locking order for passing thread and turnstile as inheritor
 229  *
 230  * Thread as an inheritor:
 231  *    When thread is passed as an inheritor of a turnstile
 232  *    turnstile lock is taken and then thread lock.
 233  *
 234  * Turnstile as in inheritor:
 235  *    When turnstile (T1) is passed as an inheritor of
 236  *    a turnstile (T2), turnstile lock of T2 is taken
 237  *    and then turnstile lock of T1 is taken.
 238  *
 239  * Caution: While passing turnstile as an inheritor, its
 240  *    job of the adopter to make sure that there is no
 241  *    lock inversion.
 242  */
 243 typedef enum __attribute__((flag_enum)) __attribute__((packed)) turnstile_update_flags {
 244         TURNSTILE_UPDATE_FLAGS_NONE = 0,
 245         TURNSTILE_IMMEDIATE_UPDATE = 0x1,
 246         TURNSTILE_DELAYED_UPDATE = 0x2,
 247         TURNSTILE_INHERITOR_THREAD = 0x4,
 248         TURNSTILE_INHERITOR_TURNSTILE = 0x8,
 249         TURNSTILE_INHERITOR_NEEDS_PRI_UPDATE = 0x10,
 250         TURNSTILE_NEEDS_PRI_UPDATE = 0x20,
 251         TURNSTILE_INHERITOR_WORKQ = 0x40,
 252         TURNSTILE_UPDATE_BOOST = 0x80,
 253 } turnstile_update_flags_t;
 254
 255 #define TURNSTILE_NULL ((struct turnstile *)0)
 256
 257 typedef void * turnstile_inheritor_t;
 258
 259 #define TURNSTILE_INHERITOR_NULL NULL
 260
 261 #ifdef XNU_KERNEL_PRIVATE
 262
 263 /* Turnstile stats update flags
 264  *
 265  * TSU_TURNSTILE_BLOCK_COUNT
 266  *    thread blocking on turnstile waitq, increment global
 267  *    thread block on turnstile count.
 268  *
 269  * TSU_REGULAR_WAITQ_BLOCK_COUNT
 270  *    thread blocking on regular waitq, increment global
 271  *    thread block on regular waitq count.
 272  *
 273  * TSU_PRI_PROPAGATION
 274  *    turnstile propagation update stopped at nth hop, update
 275  *    priority change count for nth element in stats array.
 276  *
 277  * TSU_NO_INHERITOR
 278  *    turnstile propagation update stopped due to turnstile
 279  *    not having an inheritor after nth hop, update the no
 280  *    inheritor count for nth element in the stats array.
 281  *
 282  * TSU_NO_TURNSTILE
 283  *    turnstile propagation update stopped due to thread
 284  *    not blocked on a turnstile waitq after nth hop, update
 285  *    the no turnstile count for the nth element in the stats
 286  *    array.
 287  *
 288  * TSU_NO_PRI_CHANGE_NEEDED
 289  *    turnstile propagation update stopped due to thread or
 290  *    turnstile having the correct priority or not blocked.
 291  *    update the no priority change count for the nth element
 292  *    in the stats array.
 293  *
 294  * TSU_THREAD_RUNNABLE
 295  *    turnstile propagation update stopped due to thread
 296  *    being runnable, update the thread runnable count for
 297  *    the nth element in the stats array.
 298  *
 299  * TSU_ABOVE_UI_PRI_CHANGE
 300  *    turnstile propagation caused an above UI priority change.
 301  */
 302 typedef enum __attribute__((flag_enum)) turnstile_stats_update_flags {
 303         TSU_FLAGS_NONE = 0,
 304         TSU_TURNSTILE_BLOCK_COUNT = 0x1,
 305         TSU_REGULAR_WAITQ_BLOCK_COUNT = 0x2,
 306         TSU_PRI_PROPAGATION = 0x4,
 307         TSU_NO_INHERITOR = 0x8,
 308         TSU_NO_TURNSTILE = 0x10,
 309         TSU_NO_PRI_CHANGE_NEEDED = 0x20,
 310         TSU_THREAD_RUNNABLE = 0x40,
 311         TSU_ABOVE_UI_PRI_CHANGE = 0x80,
 312         TSU_THREAD_ARG = 0x100,
 313         TSU_TURNSTILE_ARG = 0x200,
 314         TSU_BOOST_ARG = 0x400,
 315 } turnstile_stats_update_flags_t;
 316
 317 SLIST_HEAD(turnstile_list, turnstile);
 318
 319 struct turnstile {
 320         struct waitq                  ts_waitq;              /* waitq embedded in turnstile */
 321         turnstile_inheritor_t         ts_inheritor;          /* thread/turnstile inheriting the priority (IL, WL) */
 322         union {
 323                 struct turnstile_list ts_free_turnstiles;    /* turnstile free list (IL) */
 324                 SLIST_ENTRY(turnstile) ts_free_elm;          /* turnstile free list element (IL) */
 325         };
 326         struct priority_queue_sched_max ts_inheritor_queue;    /* Queue of turnstile with us as an inheritor (WL) */
 327         union {
 328                 struct priority_queue_entry_sched ts_inheritor_links;    /* Inheritor queue links */
 329                 struct mpsc_queue_chain   ts_deallocate_link;    /* thread deallocate link */
 330         };
 331         SLIST_ENTRY(turnstile)        ts_htable_link;        /* linkage for turnstile in global hash table */
 332         uintptr_t                     ts_proprietor;         /* hash key lookup turnstile (IL) */
 333         os_refcnt_t                   ts_refcount;           /* reference count for turnstiles */
 334         _Atomic uint32_t              ts_type_gencount;      /* gen count used for priority chaining (IL), type of turnstile (IL) */
 335         uint32_t                      ts_port_ref;           /* number of explicit refs from ports on send turnstile */
 336         turnstile_update_flags_t      ts_inheritor_flags;    /* flags for turnstile inheritor (IL, WL) */
 337         uint8_t                       ts_priority;           /* priority of turnstile (WL) */
 338
 339 #if DEVELOPMENT || DEBUG
 340         uint8_t                       ts_state;              /* current state of turnstile (IL) */
 341         queue_chain_t                 ts_global_elm;         /* global turnstile chain */
 342         thread_t                      ts_thread;             /* thread the turnstile is attached to */
 343         thread_t                      ts_prev_thread;        /* thread the turnstile was attached before donation */
 344 #endif
 345 };
 346
 347 #define waitq_to_turnstile(waitq) __container_of(waitq, struct turnstile, ts_waitq)
 348
 349 /* IL - interlock, WL - turnstile lock i.e. waitq lock */
 350
 351 #define TURNSTILE_PROPRIETOR_NULL 0ul
 352
 353 /*
 354  * Name: turnstiles_init
 355  *
 356  * Description: Initialize turnstile sub system.
 357  *
 358  * Args: None.
 359  *
 360  * Returns: None.
 361  */
 362 void
 363 turnstiles_init(void);
 364
 365 /*
 366  * Name: turnstile_alloc
 367  *
 368  * Description: Allocate a turnstile.
 369  *
 370  * Args: None.
 371  *
 372  * Returns:
 373  *   turnstile on Success.
 374  */
 375 struct turnstile *
 376 turnstile_alloc(void);
 377
 378 /*
 379  * Name: turnstile_destroy
 380  *
 381  * Description: Deallocates the turnstile.
 382  *
 383  * Args:
 384  *   Arg1: turnstile
 385  *
 386  * Returns: None.
 387  */
 388 void
 389 turnstile_destroy(struct turnstile *turnstile);
 390
 391 /*
 392  * Name: turnstile_reference
 393  *
 394  * Description: Take a reference on the turnstile.
 395  *
 396  * Arg1: turnstile
 397  *
 398  * Returns: None.
 399  */
 400 void
 401 turnstile_reference(struct turnstile *turnstile);
 402
 403 /*
 404  * Name: turnstile_deallocate
 405  *
 406  * Description: Drop a reference on the turnstile.
 407  *              Destroy the turnstile if the last ref.
 408  *
 409  * Arg1: turnstile
 410  *
 411  * Returns: None.
 412  */
 413 void
 414 turnstile_deallocate(struct turnstile *turnstile);
 415
 416 /*
 417  * Name: turnstile_waitq_add_thread_priority_queue
 418  *
 419  * Description: add thread to the turnstile waitq
 420  *
 421  * Arg1: waitq
 422  * Arg2: thread
 423  *
 424  * Conditions: waitq locked
 425  */
 426 void
 427 turnstile_waitq_add_thread_priority_queue(
 428         struct waitq* wq,
 429         thread_t thread);
 430
 431 /*
 432  * Name: turnstile_deallocate_safe
 433  *
 434  * Description: Drop a reference on the turnstile safely without triggering zfree.
 435  *
 436  * Arg1: turnstile
 437  *
 438  * Returns: None.
 439  */
 440 void
 441 turnstile_deallocate_safe(struct turnstile *turnstile);
 442
 443 /*
 444  * Name: turnstile_recompute_priority_locked
 445  *
 446  * Description: Update turnstile priority based
 447  *              on highest waiter thread and highest blocking
 448  *              turnstile.
 449  *
 450  * Args: turnstile
 451  *
 452  * Returns: TRUE: if the turnstile priority changed and needs propagation.
 453  *          FALSE: if the turnstile priority did not change or it does not need propagation.
 454  *
 455  * Condition: turnstile locked
 456  */
 457 boolean_t
 458 turnstile_recompute_priority_locked(
 459         struct turnstile *turnstile);
 460
 461 /*
 462  * Name: turnstile_recompute_priority
 463  *
 464  * Description: Update turnstile priority based
 465  *              on highest waiter thread and highest blocking
 466  *              turnstile.
 467  *
 468  * Args: turnstile
 469  *
 470  * Returns: TRUE: if the turnstile priority changed and needs propagation.
 471  *          FALSE: if the turnstile priority did not change or it does not need propagation.
 472  */
 473 boolean_t
 474 turnstile_recompute_priority(
 475         struct turnstile *turnstile);
 476
 477 /*
 478  * Name: turnstile_workq_proprietor_of_max_turnstile
 479  *
 480  * Description: Returns the highest priority and proprietor of a turnstile
 481  *              pushing on a workqueue turnstile.
 482  *
 483  *              This will not return waiters that are at priority
 484  *              MAXPRI_THROTTLE or lower.
 485  *
 486  * Args: turnstile
 487  *
 488  * Returns:
 489  *    Priority of the max entry, or 0
 490  *    Pointer to the max entry proprietor
 491  */
 492 int
 493 turnstile_workq_proprietor_of_max_turnstile(
 494         struct turnstile *turnstile,
 495         uintptr_t *proprietor);
 496
 497 /*
 498  * Name: turnstile_workloop_pusher_info
 499  *
 500  * Description: Returns the priority of the turnstile push for a workloop,
 501  *              and the thread or knote responsible for this push.
 502  *
 503  * Args: workloop turnstile
 504  *
 505  * Returns:
 506  *    Priority of the push or 0
 507  *    Thread (with a +1 reference) with that push or THREAD_NULL.
 508  *    Port (with a +1 reference) with that push, or IP_NULL.
 509  *    Sync IPC knote with the highest push (or NULL)
 510  */
 511 int
 512 turnstile_workloop_pusher_info(
 513         struct turnstile *turnstile,
 514         thread_t *thread,
 515         ipc_port_t *port,
 516         struct knote **knote_out);
 517
 518 /*
 519  * Name: turnstile_cleanup
 520  *
 521  * Description: Update priority of a turnstile inheritor
 522  *              if needed.
 523  *
 524  * Args: inheritor and flags passed on thread struct.
 525  *
 526  * Returns: None.
 527  */
 528 void
 529 turnstile_cleanup(void);
 530
 531 /*
 532  * Name: turnstile_update_thread_priority_chain
 533  *
 534  * Description: Priority of a thread blocked on a turnstile
 535  *              has changed, update the turnstile priority.
 536  *
 537  * Arg1: thread: thread whose priority has changed.
 538  *
 539  * Returns: None.
 540  */
 541 void
 542 turnstile_update_thread_priority_chain(thread_t thread);
 543
 544 /*
 545  * Name: turnstile_update_inheritor_locked
 546  *
 547  * Description: Update the inheritor of the turnstile and boost the
 548  *              inheritor, called with turnstile locked.
 549  *
 550  * Args:
 551  *   Arg1: turnstile
 552  *   Implicit arg: new inheritor value is stashed in current thread's struct
 553  *
 554  * Returns:
 555  *   old inheritor reference is returned on current thread's struct.
 556  */
 557 void
 558 turnstile_update_inheritor_locked(struct turnstile *turnstile);
 559
 560 /*
 561  * Name: thread_get_inheritor_turnstile_base_priority
 562  *
 563  * Description: Get the max base priority of all the inheritor turnstiles
 564  *
 565  * Arg1: thread
 566  *
 567  * Returns: Max base priority of all the inheritor turnstiles.
 568  *
 569  * Condition: thread locked
 570  */
 571 int
 572 thread_get_inheritor_turnstile_base_priority(thread_t thread);
 573
 574 /*
 575  * Name: thread_get_inheritor_turnstile_sched_priority
 576  *
 577  * Description: Get the max sched priority of all the inheritor turnstiles
 578  *
 579  * Arg1: thread
 580  *
 581  * Returns: Max sched priority of all the inheritor turnstiles.
 582  *
 583  * Condition: thread locked
 584  */
 585 int
 586 thread_get_inheritor_turnstile_sched_priority(thread_t thread);
 587
 588 /*
 589  * Name: thread_get_waiting_turnstile
 590  *
 591  * Description: Get the turnstile if the thread is waiting on a turnstile.
 592  *
 593  * Arg1: thread
 594  *
 595  * Returns: turnstile: if the thread is blocked on a turnstile.
 596  *          TURNSTILE_NULL: otherwise.
 597  *
 598  * Condition: thread locked.
 599  */
 600 struct turnstile *
 601 thread_get_waiting_turnstile(thread_t thread);
 602
 603 /*
 604  * Name: turnstile_lookup_by_proprietor
 605  *
 606  * Description: Get turnstile for a proprietor from global
 607  *              turnstile hash.
 608  *
 609  * Arg1: port
 610  * Arg2: turnstile_type_t type
 611  *
 612  * Returns: turnstile: if the proprietor has a turnstile.
 613  *          TURNSTILE_NULL: otherwise.
 614  *
 615  * Condition: proprietor interlock held.
 616  */
 617 struct turnstile *
 618 turnstile_lookup_by_proprietor(uintptr_t proprietor, turnstile_type_t type);
 619
 620 /*
 621  * Name: turnstile_has_waiters
 622  *
 623  * Description: returns if there are waiters on the turnstile
 624  *
 625  * Arg1: turnstile: turnstile
 626  *
 627  * Returns: TRUE if there are waiters, FALSE otherwise.
 628  */
 629
 630 boolean_t
 631 turnstile_has_waiters(struct turnstile *turnstile);
 632
 633 /*
 634  * Name: turnstile_stats_update
 635  *
 636  * Description: Function to update turnstile stats for dev kernel.
 637  *
 638  * Arg1: hops : number of thread hops in priority propagation
 639  * Arg2: flags : turnstile stats update flags
 640  * Arg3: inheritor: inheritor
 641  *
 642  * Returns: Nothing
 643  */
 644 void
 645 turnstile_stats_update(
 646         int hop __assert_only,
 647         turnstile_stats_update_flags_t flags __assert_only,
 648         turnstile_inheritor_t inheritor __assert_only);
 649
 650 #if DEVELOPMENT || DEBUG
 651
 652 #define SYSCTL_TURNSTILE_TEST_USER_DEFAULT              1
 653 #define SYSCTL_TURNSTILE_TEST_USER_HASHTABLE            2
 654 #define SYSCTL_TURNSTILE_TEST_KERNEL_DEFAULT            3
 655 #define SYSCTL_TURNSTILE_TEST_KERNEL_HASHTABLE          4
 656
 657 /* Functions used by debug test primitive exported by sysctls */
 658 int
 659 tstile_test_prim_lock(int val);
 660
 661 int
 662 tstile_test_prim_unlock(int val);
 663
 664 int
 665 turnstile_get_boost_stats_sysctl(void *req);
 666 int
 667 turnstile_get_unboost_stats_sysctl(void *req);
 668 #endif /* DEVELOPMENT || DEBUG */
 669 #endif /* XNU_KERNEL_PRIVATE */
 670
 671 /* Interface */
 672
 673 /*
 674  * Name: turnstile_hash_bucket_lock
 675  *
 676  * Description: locks the spinlock associated with proprietor's bucket.
 677  *              if proprietor is specified the index for the hash will be
 678  *              recomputed and returned in index_proprietor,
 679  *              otherwise the value save in index_proprietor is used as index.
 680  *
 681  * Args:
 682  *   Arg1: proprietor (key) for hashing
 683  *   Arg2: index for proprietor in the hash
 684  *   Arg3: turnstile type
 685  *
 686  * Returns: old value of irq if irq were disabled before acquiring the lock.
 687  */
 688 unsigned
 689 turnstile_hash_bucket_lock(uintptr_t proprietor, uint32_t *index_proprietor, turnstile_type_t type);
 690
 691 /*
 692  * Name: turnstile_hash_bucket_unlock
 693  *
 694  * Description: unlocks the spinlock associated with proprietor's bucket.
 695  *              if proprietor is specified the index for the hash will be
 696  *              recomputed and returned in index_proprietor,
 697  *              otherwise the value save in index_proprietor is used as index.
 698  *
 699  * Args:
 700  *   Arg1: proprietor (key) for hashing
 701  *   Arg2: index for proprietor in the hash
 702  *   Arg3: turnstile type
 703  *   Arg4: irq value returned by turnstile_hash_bucket_lock
 704  *
 705  */
 706 void
 707 turnstile_hash_bucket_unlock(uintptr_t proprietor, uint32_t *index_proprietor, turnstile_type_t type, unsigned s);
 708
 709 /*
 710  * Name: turnstile_prepare
 711  *
 712  * Description: Transfer current thread's turnstile to primitive or it's free turnstile list.
 713  *              Function is called holding the interlock (spinlock) of the primitive.
 714  *              The turnstile returned by this function is safe to use untill the thread calls turnstile_complete.
 715  *              When no turnstile is provided explicitly, the calling thread will not have a turnstile attached to
 716  *              it untill it calls turnstile_complete.
 717  *
 718  * Args:
 719  *   Arg1: proprietor
 720  *   Arg2: pointer in primitive struct to store turnstile
 721  *   Arg3: turnstile to use instead of taking it from thread.
 722  *   Arg4: type of primitive
 723  *
 724  * Returns:
 725  *   turnstile.
 726  */
 727 struct turnstile *
 728 turnstile_prepare(
 729         uintptr_t proprietor,
 730         struct turnstile **tstore,
 731         struct turnstile *turnstile,
 732         turnstile_type_t type);
 733
 734 /*
 735  * Name: turnstile_complete
 736  *
 737  * Description: Transfer the primitive's turnstile or from it's freelist to current thread.
 738  *              Function is called holding the interlock (spinlock) of the primitive.
 739  *              Current thread will have a turnstile attached to it after this call.
 740  *
 741  * Args:
 742  *   Arg1: proprietor
 743  *   Arg2: pointer in primitive struct to update turnstile
 744  *   Arg3: pointer to store the returned turnstile instead of attaching it to thread
 745  *   Arg4: type of primitive
 746  *
 747  * Returns:
 748  *   None.
 749  */
 750 void
 751 turnstile_complete(
 752         uintptr_t proprietor,
 753         struct turnstile **tstore,
 754         struct turnstile **turnstile,
 755         turnstile_type_t type);
 756
 757 /*
 758  * Name: turnstile_update_inheritor
 759  *
 760  * Description: Update the inheritor of the turnstile and boost the
 761  *              inheritor. It will take a thread reference on the inheritor.
 762  *              Called with the interlock of the primitive held.
 763  *
 764  * Args:
 765  *   Arg1: turnstile
 766  *   Arg2: inheritor
 767  *   Arg3: flags - TURNSTILE_DELAYED_UPDATE - update will happen later in assert_wait
 768  *
 769  * Returns:
 770  *   old inheritor reference is stashed on current thread's struct.
 771  */
 772 void
 773 turnstile_update_inheritor(
 774         struct turnstile *turnstile,
 775         turnstile_inheritor_t new_inheritor,
 776         turnstile_update_flags_t flags);
 777
 778 typedef enum turnstile_update_complete_flags {
 779         TURNSTILE_INTERLOCK_NOT_HELD = 0x1,
 780         TURNSTILE_INTERLOCK_HELD = 0x2,
 781 } turnstile_update_complete_flags_t;
 782
 783 /*
 784  * Name: turnstile_update_inheritor_complete
 785  *
 786  * Description: Update turnstile inheritor's priority and propagate the
 787  *              priority if the inheritor is blocked on a turnstile.
 788  *              Consumes thread ref of old inheritor returned by
 789  *              turnstile_update_inheritor. Recursive priority update
 790  *              will only happen when called with interlock dropped.
 791  *
 792  * Args:
 793  *   Arg1: turnstile
 794  *   Arg2: interlock held
 795  *
 796  * Returns: None.
 797  */
 798 void
 799 turnstile_update_inheritor_complete(
 800         struct turnstile *turnstile,
 801         turnstile_update_complete_flags_t flags);
 802
 803
 804 /*
 805  * Name: turnstile_kernel_update_inheritor_on_wake_locked
 806  *
 807  * Description: Set thread as the inheritor of the turnstile and
 808  *              boost the inheritor.
 809  * Args:
 810  *   Arg1: turnstile
 811  *   Arg2: new_inheritor
 812  *   Arg3: flags
 813  *
 814  * Called with turnstile locked
 815  */
 816 void
 817 turnstile_kernel_update_inheritor_on_wake_locked(
 818         struct turnstile *turnstile,
 819         turnstile_inheritor_t new_inheritor,
 820         turnstile_update_flags_t flags);
 821
 822 /*
 823  * Internal KPI for sleep_with_inheritor, wakeup_with_inheritor, change_sleep_inheritor
 824  * meant to allow specifing the turnstile type to use to have different policy
 825  * on how to push on the inheritor.
 826  *
 827  * Differently from the "standard" KPI in locks.h these are meant to be used only
 828  * if you know what you are doing with turnstile.
 829  */
 830
 831 extern wait_result_t
 832 lck_mtx_sleep_with_inheritor_and_turnstile_type(lck_mtx_t *lock, lck_sleep_action_t lck_sleep_action, event_t event, thread_t inheritor, wait_interrupt_t interruptible, uint64_t deadline, turnstile_type_t type);
 833
 834 extern wait_result_t
 835 lck_rw_sleep_with_inheritor_and_turnstile_type(lck_rw_t *lock, lck_sleep_action_t lck_sleep_action, event_t event, thread_t inheritor, wait_interrupt_t interruptible, uint64_t deadline, turnstile_type_t type);
 836
 837 extern kern_return_t
 838 wakeup_with_inheritor_and_turnstile_type(event_t event, turnstile_type_t type, wait_result_t result, bool wake_one, lck_wake_action_t action, thread_t *thread_wokenup);
 839
 840 extern kern_return_t
 841 change_sleep_inheritor_and_turnstile_type(event_t event, thread_t inheritor, turnstile_type_t type);
 842
 843 #endif /* KERNEL_PRIVATE */
 844 #if XNU_KERNEL_PRIVATE
 845
 846 struct workqueue;
 847
 848 /* pthread_workqueue.c */
 849 extern void workq_reference(struct workqueue *wq);
 850 extern void workq_deallocate_safe(struct workqueue *wq);
 851 extern bool workq_is_current_thread_updating_turnstile(struct workqueue *wq);
 852 extern void workq_schedule_creator_turnstile_redrive(struct workqueue *wq,
 853     bool locked);
 854
 855 #endif /* XNU_KERNEL_PRIVATE */
 856
 857 #endif /* _TURNSTILE_H_ */