osfmk/kern/thread.h

   1 /*
   2  * Copyright (c) 2000-2019 Apple Inc. All rights reserved.
   3  *
   4  * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
   5  *
   6  * This file contains Original Code and/or Modifications of Original Code
   7  * as defined in and that are subject to the Apple Public Source License
   8  * Version 2.0 (the 'License'). You may not use this file except in
   9  * compliance with the License. The rights granted to you under the License
  10  * may not be used to create, or enable the creation or redistribution of,
  11  * unlawful or unlicensed copies of an Apple operating system, or to
  12  * circumvent, violate, or enable the circumvention or violation of, any
  13  * terms of an Apple operating system software license agreement.
  14  *
  15  * Please obtain a copy of the License at
  16  * http://www.opensource.apple.com/apsl/ and read it before using this file.
  17  *
  18  * The Original Code and all software distributed under the License are
  19  * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  20  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  21  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
  22  * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
  23  * Please see the License for the specific language governing rights and
  24  * limitations under the License.
  25  *
  26  * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  27  */
  28 /*
  29  * @OSF_FREE_COPYRIGHT@
  30  */
  31 /*
  32  * Mach Operating System
  33  * Copyright (c) 1991,1990,1989,1988,1987 Carnegie Mellon University
  34  * All Rights Reserved.
  35  *
  36  * Permission to use, copy, modify and distribute this software and its
  37  * documentation is hereby granted, provided that both the copyright
  38  * notice and this permission notice appear in all copies of the
  39  * software, derivative works or modified versions, and any portions
  40  * thereof, and that both notices appear in supporting documentation.
  41  *
  42  * CARNEGIE MELLON ALLOWS FREE USE OF THIS SOFTWARE IN ITS "AS IS"
  43  * CONDITION.  CARNEGIE MELLON DISCLAIMS ANY LIABILITY OF ANY KIND FOR
  44  * ANY DAMAGES WHATSOEVER RESULTING FROM THE USE OF THIS SOFTWARE.
  45  *
  46  * Carnegie Mellon requests users of this software to return to
  47  *
  48  *  Software Distribution Coordinator  or  Software.Distribution@CS.CMU.EDU
  49  *  School of Computer Science
  50  *  Carnegie Mellon University
  51  *  Pittsburgh PA 15213-3890
  52  *
  53  * any improvements or extensions that they make and grant Carnegie Mellon
  54  * the rights to redistribute these changes.
  55  */
  56 /*
  57  */
  58 /*
  59  *      File:   thread.h
  60  *      Author: Avadis Tevanian, Jr.
  61  *
  62  *      This file contains the structure definitions for threads.
  63  *
  64  */
  65 /*
  66  * Copyright (c) 1993 The University of Utah and
  67  * the Computer Systems Laboratory (CSL).  All rights reserved.
  68  *
  69  * Permission to use, copy, modify and distribute this software and its
  70  * documentation is hereby granted, provided that both the copyright
  71  * notice and this permission notice appear in all copies of the
  72  * software, derivative works or modified versions, and any portions
  73  * thereof, and that both notices appear in supporting documentation.
  74  *
  75  * THE UNIVERSITY OF UTAH AND CSL ALLOW FREE USE OF THIS SOFTWARE IN ITS "AS
  76  * IS" CONDITION.  THE UNIVERSITY OF UTAH AND CSL DISCLAIM ANY LIABILITY OF
  77  * ANY KIND FOR ANY DAMAGES WHATSOEVER RESULTING FROM THE USE OF THIS SOFTWARE.
  78  *
  79  * CSL requests users of this software to return to csl-dist@cs.utah.edu any
  80  * improvements that they make and grant CSL redistribution rights.
  81  *
  82  */
  83
  84 #ifndef _KERN_THREAD_H_
  85 #define _KERN_THREAD_H_
  86
  87 #include <mach/kern_return.h>
  88 #include <mach/mach_types.h>
  89 #include <mach/message.h>
  90 #include <mach/boolean.h>
  91 #include <mach/vm_param.h>
  92 #include <mach/thread_info.h>
  93 #include <mach/thread_status.h>
  94 #include <mach/exception_types.h>
  95
  96 #include <kern/kern_types.h>
  97 #include <vm/vm_kern.h>
  98
  99 #include <sys/cdefs.h>
 100
 101 #ifdef XNU_KERNEL_PRIVATE
 102 /* Thread tags; for easy identification. */
 103 __options_closed_decl(thread_tag_t, uint16_t, {
 104         THREAD_TAG_MAINTHREAD   = 0x01,
 105         THREAD_TAG_CALLOUT      = 0x02,
 106         THREAD_TAG_IOWORKLOOP   = 0x04,
 107         THREAD_TAG_PTHREAD      = 0x10,
 108         THREAD_TAG_WORKQUEUE    = 0x20,
 109 });
 110 #endif /* XNU_KERNEL_PRIVATE */
 111
 112 #ifdef  MACH_KERNEL_PRIVATE
 113
 114 #include <mach_assert.h>
 115 #include <mach_ldebug.h>
 116
 117 #include <ipc/ipc_types.h>
 118
 119 #include <mach/port.h>
 120 #include <kern/cpu_number.h>
 121 #include <kern/smp.h>
 122 #include <kern/queue.h>
 123
 124 #include <kern/timer.h>
 125 #include <kern/simple_lock.h>
 126 #include <kern/locks.h>
 127 #include <kern/sched.h>
 128 #include <kern/sched_prim.h>
 129 #include <mach/sfi_class.h>
 130 #include <kern/thread_call.h>
 131 #include <kern/thread_group.h>
 132 #include <kern/timer_call.h>
 133 #include <kern/task.h>
 134 #include <kern/exception.h>
 135 #include <kern/affinity.h>
 136 #include <kern/debug.h>
 137 #include <kern/block_hint.h>
 138 #include <kern/turnstile.h>
 139 #include <kern/mpsc_queue.h>
 140
 141 #include <kern/waitq.h>
 142 #include <san/kasan.h>
 143 #include <os/refcnt.h>
 144
 145 #include <ipc/ipc_kmsg.h>
 146
 147 #include <machine/atomic.h>
 148 #include <machine/cpu_data.h>
 149 #include <machine/thread.h>
 150
 151 #ifdef XNU_KERNEL_PRIVATE
 152 /* priority queue static asserts fail for __ARM64_ARCH_8_32__ kext builds */
 153 #include <kern/priority_queue.h>
 154 #endif /* XNU_KERNEL_PRIVATE */
 155
 156 #if MONOTONIC
 157 #include <stdatomic.h>
 158 #include <machine/monotonic.h>
 159 #endif /* MONOTONIC */
 160
 161 #if CONFIG_TASKWATCH
 162 /* Taskwatch related. TODO: find this a better home */
 163 typedef struct task_watcher task_watch_t;
 164 #endif /* CONFIG_TASKWATCH */
 165
 166 __options_decl(thread_work_interval_flags_t, uint32_t, {
 167         TH_WORK_INTERVAL_FLAGS_NONE           = 0x0,
 168 #if CONFIG_SCHED_AUTO_JOIN
 169         /* Flags to indicate status about work interval thread is currently part of */
 170         TH_WORK_INTERVAL_FLAGS_AUTO_JOIN_LEAK = 0x1,
 171 #endif /* CONFIG_SCHED_AUTO_JOIN */
 172 });
 173
 174 struct thread {
 175 #if MACH_ASSERT
 176 #define THREAD_MAGIC 0x1234ABCDDCBA4321ULL
 177         /* Ensure nothing uses &thread as a queue entry */
 178         uint64_t                thread_magic;
 179 #endif /* MACH_ASSERT */
 180
 181         /*
 182          *      NOTE:   The runq field in the thread structure has an unusual
 183          *      locking protocol.  If its value is PROCESSOR_NULL, then it is
 184          *      locked by the thread_lock, but if its value is something else
 185          *      then it is locked by the associated run queue lock. It is
 186          *      set to PROCESSOR_NULL without holding the thread lock, but the
 187          *      transition from PROCESSOR_NULL to non-null must be done
 188          *      under the thread lock and the run queue lock.
 189          *
 190          *      New waitq APIs allow the 'links' and 'runq' fields to be
 191          *      anywhere in the thread structure.
 192          */
 193         union {
 194                 queue_chain_t                   runq_links;             /* run queue links */
 195                 queue_chain_t                   wait_links;             /* wait queue links */
 196                 struct mpsc_queue_chain         mpsc_links;             /* thread daemon mpsc links */
 197                 struct priority_queue_entry_sched wait_prioq_links;       /* priority ordered waitq links */
 198         };
 199
 200         event64_t               wait_event;     /* wait queue event */
 201         processor_t             runq;           /* run queue assignment */
 202         struct waitq           *waitq;          /* wait queue this thread is enqueued on */
 203         struct turnstile       *turnstile;      /* thread's turnstile, protected by primitives interlock */
 204         void                   *inheritor;      /* inheritor of the primitive the thread will block on */
 205         struct priority_queue_sched_max sched_inheritor_queue; /* Inheritor queue for kernel promotion */
 206         struct priority_queue_sched_max base_inheritor_queue; /* Inheritor queue for user promotion */
 207
 208 #if CONFIG_SCHED_EDGE
 209         boolean_t               th_bound_cluster_enqueued;
 210 #endif /* CONFIG_SCHED_EDGE */
 211
 212 #if CONFIG_SCHED_CLUTCH
 213         /*
 214          * In the clutch scheduler, the threads are maintained in runqs at the clutch_bucket
 215          * level (clutch_bucket defines a unique thread group and scheduling bucket pair). The
 216          * thread is linked via a couple of linkages in the clutch bucket:
 217          *
 218          * - A stable priority queue linkage which is the main runqueue (based on sched_pri) for the clutch bucket
 219          * - A regular priority queue linkage which is based on thread's base/promoted pri (used for clutch bucket priority calculation)
 220          * - A queue linkage used for timesharing operations of threads at the scheduler tick
 221          */
 222         struct priority_queue_entry_stable      th_clutch_runq_link;
 223         struct priority_queue_entry_sched       th_clutch_pri_link;
 224         queue_chain_t                           th_clutch_timeshare_link;
 225 #endif /* CONFIG_SCHED_CLUTCH */
 226
 227         /* Data updated during assert_wait/thread_wakeup */
 228         decl_simple_lock_data(, sched_lock);     /* scheduling lock (thread_lock()) */
 229         decl_simple_lock_data(, wake_lock);      /* for thread stop / wait (wake_lock()) */
 230         uint16_t                options;        /* options set by thread itself */
 231 #define TH_OPT_INTMASK          0x0003          /* interrupt / abort level */
 232 #define TH_OPT_VMPRIV           0x0004          /* may allocate reserved memory */
 233 #define TH_OPT_SYSTEM_CRITICAL  0x0010          /* Thread must always be allowed to run - even under heavy load */
 234 #define TH_OPT_PROC_CPULIMIT    0x0020          /* Thread has a task-wide CPU limit applied to it */
 235 #define TH_OPT_PRVT_CPULIMIT    0x0040          /* Thread has a thread-private CPU limit applied to it */
 236 #define TH_OPT_IDLE_THREAD      0x0080          /* Thread is a per-processor idle thread */
 237 #define TH_OPT_GLOBAL_FORCED_IDLE       0x0100  /* Thread performs forced idle for thermal control */
 238 #define TH_OPT_SCHED_VM_GROUP   0x0200          /* Thread belongs to special scheduler VM group */
 239 #define TH_OPT_HONOR_QLIMIT     0x0400          /* Thread will honor qlimit while sending mach_msg, regardless of MACH_SEND_ALWAYS */
 240 #define TH_OPT_SEND_IMPORTANCE  0x0800          /* Thread will allow importance donation from kernel rpc */
 241 #define TH_OPT_ZONE_PRIV        0x1000          /* Thread may use the zone replenish reserve */
 242 #define TH_OPT_IPC_TG_BLOCKED   0x2000          /* Thread blocked in sync IPC and has made the thread group blocked callout */
 243
 244         bool                    wake_active;    /* wake event on stop */
 245         bool                    at_safe_point;  /* thread_abort_safely allowed */
 246         uint8_t                 sched_saved_run_weight;
 247         ast_t                   reason;         /* why we blocked */
 248         uint32_t                quantum_remaining;
 249         wait_result_t           wait_result;    /* outcome of wait -
 250                                                 * may be examined by this thread
 251                                                 * WITHOUT locking */
 252         thread_continue_t       continuation;   /* continue here next dispatch */
 253         void                   *parameter;      /* continuation parameter */
 254
 255         /* Data updated/used in thread_invoke */
 256         vm_offset_t             kernel_stack;   /* current kernel stack */
 257         vm_offset_t             reserved_stack; /* reserved kernel stack */
 258
 259 #if KASAN
 260         struct kasan_thread_data kasan_data;
 261 #endif
 262 #if CONFIG_KSANCOV
 263         void                   *ksancov_data;
 264 #endif
 265
 266         /* Thread state: */
 267         int                     state;
 268 /*
 269  *      Thread states [bits or'ed]
 270  */
 271 #define TH_WAIT                 0x01            /* queued for waiting */
 272 #define TH_SUSP                 0x02            /* stopped or requested to stop */
 273 #define TH_RUN                  0x04            /* running or on runq */
 274 #define TH_UNINT                0x08            /* waiting uninteruptibly */
 275 #define TH_TERMINATE            0x10            /* halted at termination */
 276 #define TH_TERMINATE2           0x20            /* added to termination queue */
 277 #define TH_WAIT_REPORT          0x40            /* the wait is using the sched_call,
 278                                                 * only set if TH_WAIT is also set */
 279 #define TH_IDLE                 0x80            /* idling processor */
 280
 281         /* Scheduling information */
 282         sched_mode_t            sched_mode;     /* scheduling mode */
 283         sched_mode_t            saved_mode;     /* saved mode during forced mode demotion */
 284
 285         /* This thread's contribution to global sched counters */
 286         sched_bucket_t          th_sched_bucket;
 287
 288         sfi_class_id_t          sfi_class;      /* SFI class (XXX Updated on CSW/QE/AST) */
 289         sfi_class_id_t          sfi_wait_class; /* Currently in SFI wait for this class, protected by sfi_lock */
 290
 291         uint32_t                sched_flags;            /* current flag bits */
 292 #define TH_SFLAG_NO_SMT                 0x0001          /* On an SMT CPU, this thread must be scheduled alone */
 293 #define TH_SFLAG_FAILSAFE               0x0002          /* fail-safe has tripped */
 294 #define TH_SFLAG_THROTTLED              0x0004          /* throttled thread forced to timeshare mode (may be applied in addition to failsafe) */
 295 #define TH_SFLAG_DEMOTED_MASK      (TH_SFLAG_THROTTLED | TH_SFLAG_FAILSAFE)     /* saved_mode contains previous sched_mode */
 296
 297 #define TH_SFLAG_PROMOTED               0x0008          /* sched pri has been promoted by kernel mutex priority promotion */
 298 #define TH_SFLAG_ABORT                  0x0010          /* abort interruptible waits */
 299 #define TH_SFLAG_ABORTSAFELY            0x0020          /* ... but only those at safe point */
 300 #define TH_SFLAG_ABORTED_MASK           (TH_SFLAG_ABORT | TH_SFLAG_ABORTSAFELY)
 301 #define TH_SFLAG_DEPRESS                0x0040          /* normal depress yield */
 302 #define TH_SFLAG_POLLDEPRESS            0x0080          /* polled depress yield */
 303 #define TH_SFLAG_DEPRESSED_MASK         (TH_SFLAG_DEPRESS | TH_SFLAG_POLLDEPRESS)
 304 /* unused TH_SFLAG_PRI_UPDATE           0x0100 */
 305 #define TH_SFLAG_EAGERPREEMPT           0x0200          /* Any preemption of this thread should be treated as if AST_URGENT applied */
 306 #define TH_SFLAG_RW_PROMOTED            0x0400          /* promote reason: blocking with RW lock held */
 307 #define TH_SFLAG_BASE_PRI_FROZEN        0x0800          /* (effective) base_pri is frozen */
 308 #define TH_SFLAG_WAITQ_PROMOTED         0x1000          /* promote reason: waitq wakeup (generally for IPC receive) */
 309
 310 #if __AMP__
 311 #define TH_SFLAG_ECORE_ONLY             0x2000          /* Bind thread to E core processor set */
 312 #define TH_SFLAG_PCORE_ONLY             0x4000          /* Bind thread to P core processor set */
 313 #endif
 314
 315 #define TH_SFLAG_EXEC_PROMOTED          0x8000          /* promote reason: thread is in an exec */
 316
 317 #define TH_SFLAG_THREAD_GROUP_AUTO_JOIN 0x10000         /* thread has been auto-joined to thread group */
 318 #if __AMP__
 319 #define TH_SFLAG_BOUND_SOFT             0x20000         /* thread is soft bound to a cluster; can run anywhere if bound cluster unavailable */
 320 #endif /* __AMP__ */
 321 /* 'promote reasons' that request a priority floor only, not a custom priority */
 322 #define TH_SFLAG_PROMOTE_REASON_MASK    (TH_SFLAG_RW_PROMOTED | TH_SFLAG_WAITQ_PROMOTED | TH_SFLAG_EXEC_PROMOTED)
 323
 324 #define TH_SFLAG_RW_PROMOTED_BIT        (10)    /* 0x400 */
 325
 326         int16_t                 sched_pri;              /* scheduled (current) priority */
 327         int16_t                 base_pri;               /* effective base priority (equal to req_base_pri unless TH_SFLAG_BASE_PRI_FROZEN) */
 328         int16_t                 req_base_pri;           /* requested base priority */
 329         int16_t                 max_priority;           /* copy of max base priority */
 330         int16_t                 task_priority;          /* copy of task base priority */
 331         int16_t                 promotion_priority;     /* priority thread is currently promoted to */
 332
 333         int                     iotier_override;        /* atomic operations to set, cleared on ret to user */
 334         os_refcnt_t             ref_count;              /* number of references to me */
 335
 336         lck_mtx_t*              waiting_for_mutex;      /* points to mutex we're waiting for until we acquire it */
 337
 338         uint32_t                rwlock_count;           /* Number of lck_rw_t locks held by thread */
 339         uint32_t                t_temp_alloc_count;     /* In flight temporary allocations */
 340 #if DEBUG || DEVELOPMENT
 341         queue_head_t            t_temp_alloc_list;
 342 #endif /* DEBUG || DEVELOPMENT */
 343
 344         integer_t               importance;             /* task-relative importance */
 345
 346         /* Priority depression expiration */
 347         integer_t               depress_timer_active;
 348         timer_call_data_t       depress_timer;
 349
 350         /* real-time parameters */
 351         struct {                                        /* see mach/thread_policy.h */
 352                 uint32_t            period;
 353                 uint32_t            computation;
 354                 uint32_t            constraint;
 355                 boolean_t           preemptible;
 356                 uint64_t            deadline;
 357         }                       realtime;
 358
 359         uint64_t                last_run_time;          /* time when thread was switched away from */
 360         uint64_t                last_made_runnable_time;        /* time when thread was unblocked or preempted */
 361         uint64_t                last_basepri_change_time;       /* time when thread was last changed in basepri while runnable */
 362         uint64_t                same_pri_latency;
 363 #define THREAD_NOT_RUNNABLE (~0ULL)
 364
 365 #if CONFIG_THREAD_GROUPS
 366         struct thread_group     *thread_group;
 367 #endif
 368
 369 #if defined(CONFIG_SCHED_MULTIQ)
 370         sched_group_t           sched_group;
 371 #endif /* defined(CONFIG_SCHED_MULTIQ) */
 372
 373         /* Data used during setrun/dispatch */
 374         timer_data_t            system_timer;           /* system mode timer */
 375         processor_t             bound_processor;        /* bound to a processor? */
 376         processor_t             last_processor;         /* processor last dispatched on */
 377         processor_t             chosen_processor;       /* Where we want to run this thread */
 378
 379         /* Fail-safe computation since last unblock or qualifying yield */
 380         uint64_t                computation_metered;
 381         uint64_t                computation_epoch;
 382         uint64_t                safe_release;           /* when to release fail-safe */
 383
 384         /* Call out from scheduler */
 385         void                  (*sched_call)(int type, thread_t thread);
 386
 387 #if defined(CONFIG_SCHED_PROTO)
 388         uint32_t                runqueue_generation;    /* last time runqueue was drained */
 389 #endif
 390
 391         /* Statistics and timesharing calculations */
 392 #if defined(CONFIG_SCHED_TIMESHARE_CORE)
 393         natural_t               sched_stamp;            /* last scheduler tick */
 394         natural_t               sched_usage;            /* timesharing cpu usage [sched] */
 395         natural_t               pri_shift;              /* usage -> priority from pset */
 396         natural_t               cpu_usage;              /* instrumented cpu usage [%cpu] */
 397         natural_t               cpu_delta;              /* accumulated cpu_usage delta */
 398 #endif /* CONFIG_SCHED_TIMESHARE_CORE */
 399
 400         uint32_t                c_switch;               /* total context switches */
 401         uint32_t                p_switch;               /* total processor switches */
 402         uint32_t                ps_switch;              /* total pset switches */
 403
 404         integer_t mutex_count;  /* total count of locks held */
 405         /* Timing data structures */
 406         int                     precise_user_kernel_time; /* precise user/kernel enabled for this thread */
 407         timer_data_t            user_timer;             /* user mode timer */
 408         uint64_t                user_timer_save;        /* saved user timer value */
 409         uint64_t                system_timer_save;      /* saved system timer value */
 410         uint64_t                vtimer_user_save;       /* saved values for vtimers */
 411         uint64_t                vtimer_prof_save;
 412         uint64_t                vtimer_rlim_save;
 413         uint64_t                vtimer_qos_save;
 414
 415         timer_data_t            ptime;                  /* time executing in P mode */
 416         timer_data_t            runnable_timer;         /* time the thread is runnable (including running) */
 417
 418 #if CONFIG_SCHED_SFI
 419         /* Timing for wait state */
 420         uint64_t                wait_sfi_begin_time;    /* start time for thread waiting in SFI */
 421 #endif
 422
 423         /*
 424          * Processor/cache affinity
 425          * - affinity_threads links task threads with the same affinity set
 426          */
 427         queue_chain_t           affinity_threads;
 428         affinity_set_t          affinity_set;
 429
 430 #if CONFIG_TASKWATCH
 431         task_watch_t           *taskwatch;              /* task watch */
 432 #endif /* CONFIG_TASKWATCH */
 433
 434         /* Various bits of state to stash across a continuation, exclusive to the current thread block point */
 435         union {
 436                 struct {
 437                         mach_msg_return_t       state;          /* receive state */
 438                         mach_port_seqno_t       seqno;          /* seqno of recvd message */
 439                         ipc_object_t            object;         /* object received on */
 440                         vm_address_t            msg_addr;       /* receive buffer pointer */
 441                         mach_msg_size_t         rsize;          /* max size for recvd msg */
 442                         mach_msg_size_t         msize;          /* actual size for recvd msg */
 443                         mach_msg_option_t       option;         /* options for receive */
 444                         mach_port_name_t        receiver_name;  /* the receive port name */
 445                         struct knote            *knote;         /* knote fired for rcv */
 446                         union {
 447                                 struct ipc_kmsg   *kmsg;        /* received message */
 448                                 struct ipc_mqueue *peekq;       /* mqueue to peek at */
 449                                 struct {
 450                                         uint32_t       ppri;    /* received message pthread_priority_t */
 451                                         mach_msg_qos_t oqos;    /* override qos for message */
 452                                 } received_qos;
 453                         };
 454                         mach_msg_continue_t     continuation;
 455                 } receive;
 456                 struct {
 457                         struct semaphore        *waitsemaphore;         /* semaphore ref */
 458                         struct semaphore        *signalsemaphore;       /* semaphore ref */
 459                         int                     options;                /* semaphore options */
 460                         kern_return_t           result;                 /* primary result */
 461                         mach_msg_continue_t continuation;
 462                 } sema;
 463                 struct {
 464 #define THREAD_SAVE_IOKIT_TLS_COUNT     8
 465                         void                    *tls[THREAD_SAVE_IOKIT_TLS_COUNT];
 466                 } iokit;
 467         } saved;
 468
 469         /* Only user threads can cause guard exceptions, only kernel threads can be thread call threads */
 470         union {
 471                 /* Thread call thread's state structure, stored on its stack */
 472                 struct thread_call_thread_state *thc_state;
 473
 474                 /* Structure to save information about guard exception */
 475                 struct {
 476                         mach_exception_code_t           code;
 477                         mach_exception_subcode_t        subcode;
 478                 } guard_exc_info;
 479         };
 480
 481         /* Kernel holds on this thread  */
 482         int16_t                 suspend_count;
 483         /* User level suspensions */
 484         int16_t                 user_stop_count;
 485
 486         /* IPC data structures */
 487 #if IMPORTANCE_INHERITANCE
 488         natural_t ith_assertions;                       /* assertions pending drop */
 489 #endif
 490         struct ipc_kmsg_queue ith_messages;             /* messages to reap */
 491         mach_port_t ith_rpc_reply;                      /* reply port for kernel RPCs */
 492
 493         /* Pending thread ast(s) */
 494         ast_t                   ast;
 495
 496         /* Ast/Halt data structures */
 497         vm_offset_t             recover;                /* page fault recover(copyin/out) */
 498
 499         queue_chain_t           threads;                /* global list of all threads */
 500
 501         /* Activation */
 502         queue_chain_t           task_threads;
 503
 504         /* Task membership */
 505         struct task             *task;
 506         vm_map_t                map;
 507         thread_t                handoff_thread;
 508 #if DEVELOPMENT || DEBUG
 509         bool      pmap_footprint_suspended;
 510 #endif /* DEVELOPMENT || DEBUG */
 511
 512         /* Timed wait expiration */
 513         timer_call_data_t       wait_timer;
 514         uint16_t                wait_timer_active;
 515         bool                    wait_timer_is_set;
 516
 517         /* Miscellaneous bits guarded by mutex */
 518         uint32_t
 519             active:1,                                   /* Thread is active and has not been terminated */
 520             started:1,                                  /* Thread has been started after creation */
 521             static_param:1,                             /* Disallow policy parameter changes */
 522             inspection:1,                               /* TRUE when task is being inspected by crash reporter */
 523             policy_reset:1,                             /* Disallow policy parameter changes on terminating threads */
 524             suspend_parked:1,                           /* thread parked in thread_suspended */
 525             corpse_dup:1,                               /* TRUE when thread is an inactive duplicate in a corpse */
 526         :0;
 527
 528         decl_lck_mtx_data(, mutex);
 529
 530         /*
 531          * Different flavors of thread port.
 532          * These flavors THREAD_FLAVOR_* are defined in mach_types.h
 533          */
 534         struct ipc_port         *ith_self[THREAD_SELF_PORT_COUNT];        /* does not hold right */
 535         struct ipc_port         *ith_settable_self;        /* a send right */
 536         struct ipc_port         *ith_special_reply_port;   /* ref to special reply port */
 537         struct exception_action *exc_actions;
 538
 539 #ifdef  MACH_BSD
 540         void                    *uthread;
 541 #endif
 542
 543 #if CONFIG_DTRACE
 544         uint16_t                t_dtrace_flags;         /* DTrace thread states */
 545 #define TH_DTRACE_EXECSUCCESS   0x01
 546         uint16_t                t_dtrace_inprobe;       /* Executing under dtrace_probe */
 547         uint32_t                t_dtrace_predcache;     /* DTrace per thread predicate value hint */
 548         int64_t                 t_dtrace_tracing;       /* Thread time under dtrace_probe() */
 549         int64_t                 t_dtrace_vtime;
 550 #endif
 551
 552         clock_sec_t             t_page_creation_time;
 553         uint32_t                t_page_creation_count;
 554         uint32_t                t_page_creation_throttled;
 555 #if (DEVELOPMENT || DEBUG)
 556         uint64_t                t_page_creation_throttled_hard;
 557         uint64_t                t_page_creation_throttled_soft;
 558 #endif /* DEVELOPMENT || DEBUG */
 559         int                     t_pagein_error;         /* for vm_fault(), holds error from vnop_pagein() */
 560
 561 #ifdef KPERF
 562 /* The high 8 bits are the number of frames to sample of a user callstack. */
 563 #define T_KPERF_CALLSTACK_DEPTH_OFFSET     (24)
 564 #define T_KPERF_SET_CALLSTACK_DEPTH(DEPTH) (((uint32_t)(DEPTH)) << T_KPERF_CALLSTACK_DEPTH_OFFSET)
 565 #define T_KPERF_GET_CALLSTACK_DEPTH(FLAGS) ((FLAGS) >> T_KPERF_CALLSTACK_DEPTH_OFFSET)
 566 #define T_KPERF_ACTIONID_OFFSET            (18)
 567 #define T_KPERF_SET_ACTIONID(AID)          (((uint32_t)(AID)) << T_KPERF_ACTIONID_OFFSET)
 568 #define T_KPERF_GET_ACTIONID(FLAGS)        ((FLAGS) >> T_KPERF_ACTIONID_OFFSET)
 569 #endif
 570
 571 #define T_KPERF_AST_CALLSTACK 0x1 /* dump a callstack on thread's next AST */
 572 #define T_KPERF_AST_DISPATCH  0x2 /* dump a name on thread's next AST */
 573 #define T_KPC_ALLOC           0x4 /* thread needs a kpc_buf allocated */
 574
 575 #define T_KPERF_AST_ALL \
 576     (T_KPERF_AST_CALLSTACK | T_KPERF_AST_DISPATCH | T_KPC_ALLOC)
 577 /* only go up to T_KPERF_ACTIONID_OFFSET - 1 */
 578
 579 #ifdef KPERF
 580         uint32_t                kperf_ast;
 581         uint32_t                kperf_pet_gen;  /* last generation of PET that sampled this thread*/
 582         uint32_t                kperf_c_switch; /* last dispatch detection */
 583         uint32_t                kperf_pet_cnt;  /* how many times a thread has been sampled by PET */
 584 #endif
 585
 586 #ifdef KPC
 587         /* accumulated performance counters for this thread */
 588         uint64_t               *kpc_buf;
 589 #endif
 590
 591 #if HYPERVISOR
 592         /* hypervisor virtual CPU object associated with this thread */
 593         void                   *hv_thread_target;
 594 #endif /* HYPERVISOR */
 595
 596         /*** Machine-dependent state ***/
 597         struct machine_thread   machine;
 598
 599         /* Statistics accumulated per-thread and aggregated per-task */
 600         uint32_t                syscalls_unix;
 601         uint32_t                syscalls_mach;
 602         ledger_t                t_ledger;
 603         ledger_t                t_threadledger; /* per thread ledger */
 604         ledger_t                t_bankledger;                /* ledger to charge someone */
 605         uint64_t                t_deduct_bank_ledger_time;   /* cpu time to be deducted from bank ledger */
 606         uint64_t                t_deduct_bank_ledger_energy; /* energy to be deducted from bank ledger */
 607
 608         uint64_t                thread_id;             /*system wide unique thread-id*/
 609
 610 #if MONOTONIC
 611         struct mt_thread        t_monotonic;
 612 #endif /* MONOTONIC */
 613
 614         /* policy is protected by the thread mutex */
 615         struct thread_requested_policy  requested_policy;
 616         struct thread_effective_policy  effective_policy;
 617
 618         /* usynch override is protected by the task lock, eventually will be thread mutex */
 619         struct thread_qos_override {
 620                 struct thread_qos_override      *override_next;
 621                 uint32_t        override_contended_resource_count;
 622                 int16_t         override_qos;
 623                 int16_t         override_resource_type;
 624                 user_addr_t     override_resource;
 625         } *overrides;
 626
 627         uint32_t                kevent_overrides;
 628         uint8_t                 user_promotion_basepri;
 629         uint8_t                 kern_promotion_schedpri;
 630         _Atomic uint16_t        kevent_ast_bits;
 631
 632         io_stat_info_t          thread_io_stats; /* per-thread I/O statistics */
 633
 634         uint32_t                thread_callout_interrupt_wakeups;
 635         uint32_t                thread_callout_platform_idle_wakeups;
 636         uint32_t                thread_timer_wakeups_bin_1;
 637         uint32_t                thread_timer_wakeups_bin_2;
 638         thread_tag_t            thread_tag;
 639
 640         /*
 641          * callout_* fields are only set for thread call threads whereas guard_exc_fatal is set
 642          * by user threads on themselves while taking a guard exception. So it's okay for them to
 643          * share this bitfield.
 644          */
 645         uint16_t
 646             callout_woken_from_icontext:1,
 647             callout_woken_from_platform_idle:1,
 648             callout_woke_thread:1,
 649             guard_exc_fatal:1,
 650             thread_bitfield_unused:12;
 651
 652         mach_port_name_t        ith_voucher_name;
 653         ipc_voucher_t           ith_voucher;
 654 #if CONFIG_IOSCHED
 655         void                   *decmp_upl;
 656 #endif /* CONFIG_IOSCHED */
 657
 658         /* work interval (if any) associated with the thread. Uses thread mutex */
 659         struct work_interval            *th_work_interval;
 660         thread_work_interval_flags_t    th_work_interval_flags;
 661
 662 #if     SCHED_TRACE_THREAD_WAKEUPS
 663         uintptr_t               thread_wakeup_bt[64];
 664 #endif
 665         turnstile_update_flags_t inheritor_flags; /* inheritor flags for inheritor field */
 666         block_hint_t            pending_block_hint;
 667         block_hint_t            block_hint;      /* What type of primitive last caused us to block. */
 668         integer_t               decompressions;  /* Per-thread decompressions counter to be added to per-task decompressions counter */
 669         int                     thread_region_page_shift; /* Page shift that this thread would like to use when */
 670                                                           /* introspecting a task. This is currently being used */
 671                                                           /* by footprint which uses a thread for each task being inspected. */
 672 };
 673
 674 #define ith_state           saved.receive.state
 675 #define ith_object          saved.receive.object
 676 #define ith_msg_addr        saved.receive.msg_addr
 677 #define ith_rsize           saved.receive.rsize
 678 #define ith_msize           saved.receive.msize
 679 #define ith_option          saved.receive.option
 680 #define ith_receiver_name   saved.receive.receiver_name
 681 #define ith_continuation    saved.receive.continuation
 682 #define ith_kmsg            saved.receive.kmsg
 683 #define ith_peekq           saved.receive.peekq
 684 #define ith_knote           saved.receive.knote
 685 #define ith_ppriority       saved.receive.received_qos.ppri
 686 #define ith_qos_override    saved.receive.received_qos.oqos
 687 #define ith_seqno           saved.receive.seqno
 688
 689 #define sth_waitsemaphore   saved.sema.waitsemaphore
 690 #define sth_signalsemaphore saved.sema.signalsemaphore
 691 #define sth_options         saved.sema.options
 692 #define sth_result          saved.sema.result
 693 #define sth_continuation    saved.sema.continuation
 694
 695 #define ITH_KNOTE_NULL      ((void *)NULL)
 696 #define ITH_KNOTE_PSEUDO    ((void *)0xdeadbeef)
 697 /*
 698  * The ith_knote is used during message delivery, and can safely be interpreted
 699  * only when used for one of these codepaths, which the test for the msgt_name
 700  * being RECEIVE or SEND_ONCE is about.
 701  */
 702 #define ITH_KNOTE_VALID(kn, msgt_name) \
 703                 (((kn) != ITH_KNOTE_NULL && (kn) != ITH_KNOTE_PSEUDO) && \
 704                  ((msgt_name) == MACH_MSG_TYPE_PORT_RECEIVE || \
 705                  (msgt_name) == MACH_MSG_TYPE_PORT_SEND_ONCE))
 706
 707 #if MACH_ASSERT
 708 #define assert_thread_magic(thread) assertf((thread)->thread_magic == THREAD_MAGIC, \
 709                                             "bad thread magic 0x%llx for thread %p, expected 0x%llx", \
 710                                             (thread)->thread_magic, (thread), THREAD_MAGIC)
 711 #else
 712 #define assert_thread_magic(thread) do { (void)(thread); } while (0)
 713 #endif
 714
 715 extern thread_t                 thread_bootstrap(void);
 716
 717 extern void                     thread_machine_init_template(void);
 718
 719 extern void                     thread_init(void);
 720
 721 extern void                     thread_daemon_init(void);
 722
 723 #define thread_reference_internal(thread)       \
 724                         os_ref_retain(&(thread)->ref_count);
 725
 726 #define thread_reference(thread)                                        \
 727 MACRO_BEGIN                                                                                     \
 728         if ((thread) != THREAD_NULL)                                    \
 729                 thread_reference_internal(thread);              \
 730 MACRO_END
 731
 732 extern void                     thread_deallocate(
 733         thread_t                thread);
 734
 735 extern void                     thread_inspect_deallocate(
 736         thread_inspect_t        thread);
 737
 738 extern void                     thread_read_deallocate(
 739         thread_read_t        thread);
 740
 741 extern void                     thread_terminate_self(void);
 742
 743 extern kern_return_t    thread_terminate_internal(
 744         thread_t                thread);
 745
 746 extern void                     thread_start(
 747         thread_t                        thread) __attribute__ ((noinline));
 748
 749 extern void                     thread_start_in_assert_wait(
 750         thread_t                        thread,
 751         event_t             event,
 752         wait_interrupt_t    interruptible) __attribute__ ((noinline));
 753
 754 extern void                     thread_terminate_enqueue(
 755         thread_t                thread);
 756
 757 extern void                     thread_exception_enqueue(
 758         task_t          task,
 759         thread_t        thread,
 760         exception_type_t etype);
 761
 762 extern void                     thread_copy_resource_info(
 763         thread_t dst_thread,
 764         thread_t src_thread);
 765
 766 extern void                     thread_terminate_crashed_threads(void);
 767
 768 extern void                     thread_stack_enqueue(
 769         thread_t                thread);
 770
 771 extern void                     thread_hold(
 772         thread_t        thread);
 773
 774 extern void                     thread_release(
 775         thread_t        thread);
 776
 777 extern void                     thread_corpse_continue(void) __dead2;
 778
 779 extern boolean_t                thread_is_active(thread_t thread);
 780
 781 extern lck_grp_t                thread_lck_grp;
 782
 783 /* Locking for scheduler state, always acquired with interrupts disabled (splsched()) */
 784 #define thread_lock_init(th)    simple_lock_init(&(th)->sched_lock, 0)
 785 #define thread_lock(th)                 simple_lock(&(th)->sched_lock, &thread_lck_grp)
 786 #define thread_unlock(th)               simple_unlock(&(th)->sched_lock)
 787
 788 #define wake_lock_init(th)              simple_lock_init(&(th)->wake_lock, 0)
 789 #define wake_lock(th)                   simple_lock(&(th)->wake_lock, &thread_lck_grp)
 790 #define wake_unlock(th)                 simple_unlock(&(th)->wake_lock)
 791
 792 #define thread_should_halt_fast(thread)         (!(thread)->active)
 793
 794 extern void                             stack_alloc(
 795         thread_t                thread);
 796
 797 extern void                     stack_handoff(
 798         thread_t                from,
 799         thread_t                to);
 800
 801 extern void                             stack_free(
 802         thread_t                thread);
 803
 804 extern void                             stack_free_reserved(
 805         thread_t                thread);
 806
 807 extern boolean_t                stack_alloc_try(
 808         thread_t            thread);
 809
 810 extern void                             stack_collect(void);
 811
 812 extern void                             stack_init(void);
 813
 814
 815 extern kern_return_t    thread_info_internal(
 816         thread_t                                thread,
 817         thread_flavor_t                 flavor,
 818         thread_info_t                   thread_info_out,
 819         mach_msg_type_number_t  *thread_info_count);
 820
 821
 822
 823 extern kern_return_t    kernel_thread_create(
 824         thread_continue_t       continuation,
 825         void                            *parameter,
 826         integer_t                       priority,
 827         thread_t                        *new_thread);
 828
 829 extern kern_return_t    kernel_thread_start_priority(
 830         thread_continue_t       continuation,
 831         void                            *parameter,
 832         integer_t                       priority,
 833         thread_t                        *new_thread);
 834
 835 extern void                             machine_stack_attach(
 836         thread_t                thread,
 837         vm_offset_t             stack);
 838
 839 extern vm_offset_t              machine_stack_detach(
 840         thread_t                thread);
 841
 842 extern void                             machine_stack_handoff(
 843         thread_t                old,
 844         thread_t                new);
 845
 846 extern thread_t                 machine_switch_context(
 847         thread_t                        old_thread,
 848         thread_continue_t       continuation,
 849         thread_t                        new_thread);
 850
 851 extern void                             machine_load_context(
 852         thread_t                thread) __attribute__((noreturn));
 853
 854 extern kern_return_t    machine_thread_state_initialize(
 855         thread_t                                thread);
 856
 857 extern kern_return_t    machine_thread_set_state(
 858         thread_t                                thread,
 859         thread_flavor_t                 flavor,
 860         thread_state_t                  state,
 861         mach_msg_type_number_t  count);
 862
 863 extern mach_vm_address_t machine_thread_pc(
 864         thread_t                thread);
 865
 866 extern void machine_thread_reset_pc(
 867         thread_t                thread,
 868         mach_vm_address_t       pc);
 869
 870 extern boolean_t        machine_thread_on_core(
 871         thread_t                thread);
 872
 873 extern kern_return_t    machine_thread_get_state(
 874         thread_t                                thread,
 875         thread_flavor_t                 flavor,
 876         thread_state_t                  state,
 877         mach_msg_type_number_t  *count);
 878
 879 extern kern_return_t    machine_thread_state_convert_from_user(
 880         thread_t                                thread,
 881         thread_flavor_t                 flavor,
 882         thread_state_t                  tstate,
 883         mach_msg_type_number_t  count);
 884
 885 extern kern_return_t    machine_thread_state_convert_to_user(
 886         thread_t                                thread,
 887         thread_flavor_t                 flavor,
 888         thread_state_t                  tstate,
 889         mach_msg_type_number_t  *count);
 890
 891 extern kern_return_t    machine_thread_dup(
 892         thread_t                self,
 893         thread_t                target,
 894         boolean_t               is_corpse);
 895
 896 extern void             machine_thread_init(void);
 897
 898 extern void             machine_thread_template_init(thread_t thr_template);
 899
 900
 901 extern kern_return_t    machine_thread_create(
 902         thread_t                thread,
 903         task_t                  task);
 904 extern void             machine_thread_switch_addrmode(
 905         thread_t                 thread);
 906
 907 extern void                 machine_thread_destroy(
 908         thread_t                thread);
 909
 910 extern void                             machine_set_current_thread(
 911         thread_t                        thread);
 912
 913 extern kern_return_t    machine_thread_get_kern_state(
 914         thread_t                                thread,
 915         thread_flavor_t                 flavor,
 916         thread_state_t                  tstate,
 917         mach_msg_type_number_t  *count);
 918
 919 extern kern_return_t    machine_thread_inherit_taskwide(
 920         thread_t                thread,
 921         task_t                  parent_task);
 922
 923 extern kern_return_t    machine_thread_set_tsd_base(
 924         thread_t                                thread,
 925         mach_vm_offset_t                tsd_base);
 926
 927 #define thread_mtx_lock(thread)                 lck_mtx_lock(&(thread)->mutex)
 928 #define thread_mtx_try(thread)                  lck_mtx_try_lock(&(thread)->mutex)
 929 #define thread_mtx_unlock(thread)               lck_mtx_unlock(&(thread)->mutex)
 930 #define thread_mtx_held(thread)                 lck_mtx_assert(&(thread)->mutex, LCK_MTX_ASSERT_OWNED)
 931
 932 extern void thread_apc_ast(thread_t thread);
 933
 934 extern void thread_update_qos_cpu_time(thread_t thread);
 935
 936 void act_machine_sv_free(thread_t, int);
 937
 938 vm_offset_t                     min_valid_stack_address(void);
 939 vm_offset_t                     max_valid_stack_address(void);
 940
 941 extern bool thread_no_smt(thread_t thread);
 942 extern bool processor_active_thread_no_smt(processor_t processor);
 943
 944 extern void thread_set_options(uint32_t thopt);
 945
 946 #if CONFIG_THREAD_GROUPS
 947 struct thread_group *thread_get_current_voucher_thread_group(thread_t thread);
 948 #endif /* CONFIG_THREAD_GROUPS */
 949
 950 #else   /* MACH_KERNEL_PRIVATE */
 951
 952 __BEGIN_DECLS
 953
 954 extern void thread_mtx_lock(thread_t thread);
 955
 956 extern void thread_mtx_unlock(thread_t thread);
 957
 958 extern thread_t         current_thread(void) __attribute__((const));
 959
 960 extern void                     thread_reference(
 961         thread_t        thread);
 962
 963 extern void                     thread_deallocate(
 964         thread_t        thread);
 965
 966 #if BSD_KERNEL_PRIVATE
 967 /* Duplicated from osfmk/kern/ipc_tt.h */
 968 __options_decl(port_to_thread_options_t, uint32_t, {
 969         PORT_TO_THREAD_NONE               = 0x0000,
 970         PORT_TO_THREAD_IN_CURRENT_TASK    = 0x0001,
 971         PORT_TO_THREAD_NOT_CURRENT_THREAD = 0x0002,
 972 });
 973
 974 extern thread_t port_name_to_thread(
 975         mach_port_name_t            port_name,
 976         port_to_thread_options_t    options);
 977 #endif /* BSD_KERNEL_PRIVATE */
 978
 979 __END_DECLS
 980
 981 #endif  /* MACH_KERNEL_PRIVATE */
 982
 983 #ifdef  KERNEL_PRIVATE
 984
 985 __BEGIN_DECLS
 986
 987 extern void                     thread_deallocate_safe(
 988         thread_t                thread);
 989
 990 extern uint64_t                 thread_dispatchqaddr(
 991         thread_t thread);
 992
 993 extern uint64_t                 thread_rettokern_addr(
 994         thread_t thread);
 995
 996 extern integer_t        thread_kern_get_pri(thread_t thr) __attribute__((const));
 997
 998 extern void             thread_kern_set_pri(thread_t thr, integer_t pri);
 999
1000 extern integer_t        thread_kern_get_kernel_maxpri(void) __attribute__((const));
1001
1002 __END_DECLS
1003
1004 #endif  /* KERNEL_PRIVATE */
1005
1006 #ifdef KERNEL
1007 __BEGIN_DECLS
1008
1009 extern uint64_t                 thread_tid(thread_t thread);
1010
1011 __END_DECLS
1012
1013 #endif  /* KERNEL */
1014
1015 __BEGIN_DECLS
1016
1017 #ifdef  XNU_KERNEL_PRIVATE
1018
1019 uint16_t        thread_set_tag(thread_t thread, uint16_t tag);
1020 uint16_t        thread_get_tag(thread_t thread);
1021
1022 #ifdef MACH_KERNEL_PRIVATE
1023 static inline thread_tag_t
1024 thread_set_tag_internal(thread_t thread, thread_tag_t tag)
1025 {
1026         return os_atomic_or_orig(&thread->thread_tag, tag, relaxed);
1027 }
1028
1029 static inline thread_tag_t
1030 thread_get_tag_internal(thread_t thread)
1031 {
1032         return thread->thread_tag;
1033 }
1034 #endif /* MACH_KERNEL_PRIVATE */
1035
1036 uint64_t        thread_last_run_time(thread_t thread);
1037
1038 extern kern_return_t    thread_state_initialize(
1039         thread_t                                thread);
1040
1041 extern kern_return_t    thread_setstatus(
1042         thread_t                                thread,
1043         int                                             flavor,
1044         thread_state_t                  tstate,
1045         mach_msg_type_number_t  count);
1046
1047 extern kern_return_t    thread_setstatus_from_user(
1048         thread_t                                thread,
1049         int                                             flavor,
1050         thread_state_t                  tstate,
1051         mach_msg_type_number_t  count);
1052
1053 extern kern_return_t    thread_getstatus(
1054         thread_t                                thread,
1055         int                                             flavor,
1056         thread_state_t                  tstate,
1057         mach_msg_type_number_t  *count);
1058
1059 extern kern_return_t    thread_getstatus_to_user(
1060         thread_t                                thread,
1061         int                                             flavor,
1062         thread_state_t                  tstate,
1063         mach_msg_type_number_t  *count);
1064
1065 extern kern_return_t    thread_create_with_continuation(
1066         task_t task,
1067         thread_t *new_thread,
1068         thread_continue_t continuation);
1069
1070 extern kern_return_t thread_create_waiting(task_t               task,
1071     thread_continue_t    continuation,
1072     event_t              event,
1073     thread_t             *new_thread);
1074
1075 extern kern_return_t    thread_create_workq_waiting(
1076         task_t                  task,
1077         thread_continue_t       thread_return,
1078         thread_t                *new_thread);
1079
1080 extern  void    thread_yield_internal(
1081         mach_msg_timeout_t      interval);
1082
1083 extern void             thread_yield_to_preemption(void);
1084
1085 /*
1086  * Thread-private CPU limits: apply a private CPU limit to this thread only. Available actions are:
1087  *
1088  * 1) Block. Prevent CPU consumption of the thread from exceeding the limit.
1089  * 2) Exception. Generate a resource consumption exception when the limit is exceeded.
1090  * 3) Disable. Remove any existing CPU limit.
1091  */
1092 #define THREAD_CPULIMIT_BLOCK           0x1
1093 #define THREAD_CPULIMIT_EXCEPTION       0x2
1094 #define THREAD_CPULIMIT_DISABLE         0x3
1095
1096 struct _thread_ledger_indices {
1097         int cpu_time;
1098 };
1099
1100 extern struct _thread_ledger_indices thread_ledgers;
1101
1102 extern int thread_get_cpulimit(int *action, uint8_t *percentage, uint64_t *interval_ns);
1103 extern int thread_set_cpulimit(int action, uint8_t percentage, uint64_t interval_ns);
1104
1105 extern void                     thread_read_times(
1106         thread_t                thread,
1107         time_value_t    *user_time,
1108         time_value_t    *system_time,
1109         time_value_t    *runnable_time);
1110
1111 extern uint64_t         thread_get_runtime_self(void);
1112
1113 extern void                     thread_setuserstack(
1114         thread_t                thread,
1115         mach_vm_offset_t        user_stack);
1116
1117 extern user_addr_t         thread_adjuserstack(
1118         thread_t                thread,
1119         int                             adjust);
1120
1121
1122 extern void                     thread_setentrypoint(
1123         thread_t                thread,
1124         mach_vm_offset_t        entry);
1125
1126 extern kern_return_t    thread_set_tsd_base(
1127         thread_t        thread,
1128         mach_vm_offset_t tsd_base);
1129
1130 extern kern_return_t    thread_setsinglestep(
1131         thread_t                thread,
1132         int                     on);
1133
1134 extern kern_return_t    thread_userstack(
1135         thread_t,
1136         int,
1137         thread_state_t,
1138         unsigned int,
1139         mach_vm_offset_t *,
1140         int *,
1141         boolean_t);
1142
1143 extern kern_return_t    thread_entrypoint(
1144         thread_t,
1145         int,
1146         thread_state_t,
1147         unsigned int,
1148         mach_vm_offset_t *);
1149
1150 extern kern_return_t    thread_userstackdefault(
1151         mach_vm_offset_t *,
1152         boolean_t);
1153
1154 extern kern_return_t    thread_wire_internal(
1155         host_priv_t             host_priv,
1156         thread_t                thread,
1157         boolean_t               wired,
1158         boolean_t               *prev_state);
1159
1160
1161 extern kern_return_t    thread_dup(thread_t);
1162
1163 extern kern_return_t thread_dup2(thread_t, thread_t);
1164
1165 #if !defined(_SCHED_CALL_T_DEFINED)
1166 #define _SCHED_CALL_T_DEFINED
1167 typedef void    (*sched_call_t)(
1168         int                             type,
1169         thread_t                thread);
1170 #endif
1171
1172 #define SCHED_CALL_BLOCK                0x1
1173 #define SCHED_CALL_UNBLOCK              0x2
1174
1175 extern void             thread_sched_call(
1176         thread_t                thread,
1177         sched_call_t    call);
1178
1179 extern boolean_t        thread_is_static_param(
1180         thread_t                thread);
1181
1182 extern task_t   get_threadtask(thread_t);
1183
1184 /*
1185  * Thread is running within a 64-bit address space.
1186  */
1187 #define thread_is_64bit_addr(thd)       \
1188         task_has_64Bit_addr(get_threadtask(thd))
1189
1190 /*
1191  * Thread is using 64-bit machine state.
1192  */
1193 #define thread_is_64bit_data(thd)       \
1194         task_has_64Bit_data(get_threadtask(thd))
1195
1196 #if defined(__x86_64__)
1197 extern int              thread_task_has_ldt(thread_t);
1198 #endif
1199 extern void             *get_bsdthread_info(thread_t);
1200 extern void             set_bsdthread_info(thread_t, void *);
1201 extern void             set_thread_pagein_error(thread_t, int);
1202 extern void             *uthread_alloc(task_t, thread_t, int);
1203 extern event_t  workq_thread_init_and_wq_lock(task_t, thread_t); // bsd/pthread/
1204 extern void             uthread_cleanup_name(void *uthread);
1205 extern void             uthread_cleanup(task_t, void *, void *);
1206 extern void             uthread_zone_free(void *);
1207 extern void             uthread_cred_free(void *);
1208 extern void             uthread_reset_proc_refcount(void *);
1209 #if PROC_REF_DEBUG
1210 extern int              uthread_get_proc_refcount(void *);
1211 extern int              proc_ref_tracking_disabled;
1212 #endif
1213
1214 extern boolean_t        thread_should_halt(
1215         thread_t                thread);
1216
1217 extern boolean_t        thread_should_abort(
1218         thread_t);
1219
1220 extern int is_64signalregset(void);
1221
1222 extern void act_set_kperf(thread_t);
1223 extern void act_set_astledger(thread_t thread);
1224 extern void act_set_astledger_async(thread_t thread);
1225 extern void act_set_io_telemetry_ast(thread_t);
1226
1227 extern uint32_t dtrace_get_thread_predcache(thread_t);
1228 extern int64_t dtrace_get_thread_vtime(thread_t);
1229 extern int64_t dtrace_get_thread_tracing(thread_t);
1230 extern uint16_t dtrace_get_thread_inprobe(thread_t);
1231 extern int dtrace_get_thread_last_cpu_id(thread_t);
1232 extern vm_offset_t dtrace_get_kernel_stack(thread_t);
1233 extern void dtrace_set_thread_predcache(thread_t, uint32_t);
1234 extern void dtrace_set_thread_vtime(thread_t, int64_t);
1235 extern void dtrace_set_thread_tracing(thread_t, int64_t);
1236 extern void dtrace_set_thread_inprobe(thread_t, uint16_t);
1237 extern vm_offset_t dtrace_set_thread_recover(thread_t, vm_offset_t);
1238 extern vm_offset_t dtrace_sign_and_set_thread_recover(thread_t, vm_offset_t);
1239 extern void dtrace_thread_bootstrap(void);
1240 extern void dtrace_thread_didexec(thread_t);
1241
1242 extern int64_t dtrace_calc_thread_recent_vtime(thread_t);
1243
1244
1245 extern kern_return_t    thread_set_wq_state32(
1246         thread_t          thread,
1247         thread_state_t    tstate);
1248
1249 extern kern_return_t    thread_set_wq_state64(
1250         thread_t          thread,
1251         thread_state_t    tstate);
1252
1253 extern vm_offset_t      kernel_stack_mask;
1254 extern vm_offset_t      kernel_stack_size;
1255 extern vm_offset_t      kernel_stack_depth_max;
1256
1257 extern void guard_ast(thread_t);
1258 extern void fd_guard_ast(thread_t,
1259     mach_exception_code_t, mach_exception_subcode_t);
1260 #if CONFIG_VNGUARD
1261 extern void vn_guard_ast(thread_t,
1262     mach_exception_code_t, mach_exception_subcode_t);
1263 #endif
1264 extern void mach_port_guard_ast(thread_t,
1265     mach_exception_code_t, mach_exception_subcode_t);
1266 extern void virt_memory_guard_ast(thread_t,
1267     mach_exception_code_t, mach_exception_subcode_t);
1268 extern void thread_guard_violation(thread_t,
1269     mach_exception_code_t, mach_exception_subcode_t, boolean_t);
1270 extern void thread_update_io_stats(thread_t, int size, int io_flags);
1271
1272 extern kern_return_t    thread_set_voucher_name(mach_port_name_t name);
1273 extern kern_return_t thread_get_current_voucher_origin_pid(int32_t *pid);
1274
1275 extern void set_thread_rwlock_boost(void);
1276 extern void clear_thread_rwlock_boost(void);
1277
1278 extern void thread_enable_send_importance(thread_t thread, boolean_t enable);
1279
1280 /*
1281  * Translate signal context data pointer to userspace representation
1282  */
1283
1284 extern kern_return_t    machine_thread_siguctx_pointer_convert_to_user(
1285         thread_t thread,
1286         user_addr_t *uctxp);
1287
1288 extern void machine_tecs(thread_t thr);
1289
1290 typedef enum cpuvn {
1291         CPUVN_CI = 1
1292 } cpuvn_e;
1293
1294 extern int machine_csv(cpuvn_e cve);
1295
1296 /*
1297  * Translate array of function pointer syscall arguments from userspace representation
1298  */
1299
1300 extern kern_return_t    machine_thread_function_pointers_convert_from_user(
1301         thread_t thread,
1302         user_addr_t *fptrs,
1303         uint32_t count);
1304
1305 /*
1306  * Get a backtrace for a threads kernel or user stack (user_p), using fp to start the
1307  * backtrace if provided.
1308  *
1309  * Returns bytes added to buffer, and kThreadTruncatedBT in thread_trace_flags if a
1310  * user page is not present after kdp_lightweight_fault() is * called.
1311  */
1312
1313 extern int machine_trace_thread(
1314         thread_t thread,
1315         char *tracepos,
1316         char *tracebound,
1317         int nframes,
1318         boolean_t user_p,
1319         uint32_t *thread_trace_flags);
1320
1321 extern int machine_trace_thread64(thread_t thread,
1322     char *tracepos,
1323     char *tracebound,
1324     int nframes,
1325     boolean_t user_p,
1326     uint32_t *thread_trace_flags,
1327     uint64_t *sp,
1328     vm_offset_t fp);
1329
1330 /*
1331  * Get the duration of the given thread's last wait.
1332  */
1333 uint64_t thread_get_last_wait_duration(thread_t thread);
1334
1335 extern bool thread_get_no_smt(void);
1336
1337 #endif  /* XNU_KERNEL_PRIVATE */
1338
1339 #ifdef KERNEL_PRIVATE
1340 extern void thread_set_no_smt(bool set);
1341 #endif /* KERNEL_PRIVATE */
1342
1343 /*! @function thread_has_thread_name
1344  *   @abstract Checks if a thread has a name.
1345  *   @discussion This function takes one input, a thread, and returns a boolean value indicating if that thread already has a name associated with it.
1346  *   @param th The thread to inspect.
1347  *   @result TRUE if the thread has a name, FALSE otherwise.
1348  */
1349 extern boolean_t thread_has_thread_name(thread_t th);
1350
1351 /*! @function thread_set_thread_name
1352  *   @abstract Set a thread's name.
1353  *   @discussion This function takes two input parameters: a thread to name, and the name to apply to the thread.  The name will be copied over to the thread in order to better identify the thread.  If the name is longer than MAXTHREADNAMESIZE - 1, it will be truncated.
1354  *   @param th The thread to be named.
1355  *   @param name The name to apply to the thread.
1356  */
1357 extern void thread_set_thread_name(thread_t th, const char* name);
1358
1359 #ifdef XNU_KERNEL_PRIVATE
1360 extern void
1361 thread_get_thread_name(thread_t th, char* name);
1362 #endif /* XNU_KERNEL_PRIVATE */
1363
1364 /*! @function kernel_thread_start
1365  *   @abstract Create a kernel thread.
1366  *   @discussion This function takes three input parameters, namely reference to the function that the thread should execute, caller specified data and a reference which is used to return the newly created kernel thread. The function returns KERN_SUCCESS on success or an appropriate kernel code type indicating the error. It may be noted that the caller is responsible for explicitly releasing the reference to the created thread when no longer needed. This should be done by calling thread_deallocate(new_thread).
1367  *   @param continuation A C-function pointer where the thread will begin execution.
1368  *   @param parameter Caller specified data to be passed to the new thread.
1369  *   @param new_thread Reference to the new thread is returned in this parameter.
1370  *   @result Returns KERN_SUCCESS on success or an appropriate kernel code type.
1371  */
1372
1373 extern kern_return_t    kernel_thread_start(
1374         thread_continue_t       continuation,
1375         void                            *parameter,
1376         thread_t                        *new_thread);
1377
1378 #ifdef KERNEL_PRIVATE
1379 void thread_set_eager_preempt(thread_t thread);
1380 void thread_clear_eager_preempt(thread_t thread);
1381 void thread_set_honor_qlimit(thread_t thread);
1382 void thread_clear_honor_qlimit(thread_t thread);
1383 extern ipc_port_t convert_thread_to_port(thread_t);
1384 extern ipc_port_t convert_thread_inspect_to_port(thread_inspect_t);
1385 extern ipc_port_t convert_thread_read_to_port(thread_read_t);
1386 extern boolean_t is_vm_privileged(void);
1387 extern boolean_t set_vm_privilege(boolean_t);
1388 extern kern_allocation_name_t thread_set_allocation_name(kern_allocation_name_t new_name);
1389 extern void *thread_iokit_tls_get(uint32_t index);
1390 extern void thread_iokit_tls_set(uint32_t index, void * data);
1391 extern void thread_port_with_flavor_notify(mach_msg_header_t *msg);
1392 extern int thread_self_region_page_shift(void);
1393 extern void thread_self_region_page_shift_set(int pgshift);
1394 #endif /* KERNEL_PRIVATE */
1395
1396 __END_DECLS
1397
1398 #endif  /* _KERN_THREAD_H_ */