osfmk/kern/thread.h

   1 /*
   2  * Copyright (c) 2000-2019 Apple Inc. All rights reserved.
   3  *
   4  * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
   5  *
   6  * This file contains Original Code and/or Modifications of Original Code
   7  * as defined in and that are subject to the Apple Public Source License
   8  * Version 2.0 (the 'License'). You may not use this file except in
   9  * compliance with the License. The rights granted to you under the License
  10  * may not be used to create, or enable the creation or redistribution of,
  11  * unlawful or unlicensed copies of an Apple operating system, or to
  12  * circumvent, violate, or enable the circumvention or violation of, any
  13  * terms of an Apple operating system software license agreement.
  14  *
  15  * Please obtain a copy of the License at
  16  * http://www.opensource.apple.com/apsl/ and read it before using this file.
  17  *
  18  * The Original Code and all software distributed under the License are
  19  * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  20  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  21  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
  22  * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
  23  * Please see the License for the specific language governing rights and
  24  * limitations under the License.
  25  *
  26  * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  27  */
  28 /*
  29  * @OSF_FREE_COPYRIGHT@
  30  */
  31 /*
  32  * Mach Operating System
  33  * Copyright (c) 1991,1990,1989,1988,1987 Carnegie Mellon University
  34  * All Rights Reserved.
  35  *
  36  * Permission to use, copy, modify and distribute this software and its
  37  * documentation is hereby granted, provided that both the copyright
  38  * notice and this permission notice appear in all copies of the
  39  * software, derivative works or modified versions, and any portions
  40  * thereof, and that both notices appear in supporting documentation.
  41  *
  42  * CARNEGIE MELLON ALLOWS FREE USE OF THIS SOFTWARE IN ITS "AS IS"
  43  * CONDITION.  CARNEGIE MELLON DISCLAIMS ANY LIABILITY OF ANY KIND FOR
  44  * ANY DAMAGES WHATSOEVER RESULTING FROM THE USE OF THIS SOFTWARE.
  45  *
  46  * Carnegie Mellon requests users of this software to return to
  47  *
  48  *  Software Distribution Coordinator  or  Software.Distribution@CS.CMU.EDU
  49  *  School of Computer Science
  50  *  Carnegie Mellon University
  51  *  Pittsburgh PA 15213-3890
  52  *
  53  * any improvements or extensions that they make and grant Carnegie Mellon
  54  * the rights to redistribute these changes.
  55  */
  56 /*
  57  */
  58 /*
  59  *      File:   thread.h
  60  *      Author: Avadis Tevanian, Jr.
  61  *
  62  *      This file contains the structure definitions for threads.
  63  *
  64  */
  65 /*
  66  * Copyright (c) 1993 The University of Utah and
  67  * the Computer Systems Laboratory (CSL).  All rights reserved.
  68  *
  69  * Permission to use, copy, modify and distribute this software and its
  70  * documentation is hereby granted, provided that both the copyright
  71  * notice and this permission notice appear in all copies of the
  72  * software, derivative works or modified versions, and any portions
  73  * thereof, and that both notices appear in supporting documentation.
  74  *
  75  * THE UNIVERSITY OF UTAH AND CSL ALLOW FREE USE OF THIS SOFTWARE IN ITS "AS
  76  * IS" CONDITION.  THE UNIVERSITY OF UTAH AND CSL DISCLAIM ANY LIABILITY OF
  77  * ANY KIND FOR ANY DAMAGES WHATSOEVER RESULTING FROM THE USE OF THIS SOFTWARE.
  78  *
  79  * CSL requests users of this software to return to csl-dist@cs.utah.edu any
  80  * improvements that they make and grant CSL redistribution rights.
  81  *
  82  */
  83
  84 #ifndef _KERN_THREAD_H_
  85 #define _KERN_THREAD_H_
  86
  87 #include <mach/kern_return.h>
  88 #include <mach/mach_types.h>
  89 #include <mach/message.h>
  90 #include <mach/boolean.h>
  91 #include <mach/vm_param.h>
  92 #include <mach/thread_info.h>
  93 #include <mach/thread_status.h>
  94 #include <mach/exception_types.h>
  95
  96 #include <kern/kern_types.h>
  97 #include <vm/vm_kern.h>
  98
  99 #include <sys/cdefs.h>
 100
 101 #ifdef  MACH_KERNEL_PRIVATE
 102
 103 #include <mach_assert.h>
 104 #include <mach_ldebug.h>
 105
 106 #include <ipc/ipc_types.h>
 107
 108 #include <mach/port.h>
 109 #include <kern/cpu_number.h>
 110 #include <kern/smp.h>
 111 #include <kern/queue.h>
 112
 113 #include <kern/timer.h>
 114 #include <kern/simple_lock.h>
 115 #include <kern/locks.h>
 116 #include <kern/sched.h>
 117 #include <kern/sched_prim.h>
 118 #include <mach/sfi_class.h>
 119 #include <kern/thread_call.h>
 120 #include <kern/thread_group.h>
 121 #include <kern/timer_call.h>
 122 #include <kern/task.h>
 123 #include <kern/exception.h>
 124 #include <kern/affinity.h>
 125 #include <kern/debug.h>
 126 #include <kern/block_hint.h>
 127 #include <kern/turnstile.h>
 128 #include <kern/mpsc_queue.h>
 129
 130 #include <kern/waitq.h>
 131 #include <san/kasan.h>
 132 #include <os/refcnt.h>
 133
 134 #include <ipc/ipc_kmsg.h>
 135
 136 #include <machine/atomic.h>
 137 #include <machine/cpu_data.h>
 138 #include <machine/thread.h>
 139
 140 #ifdef XNU_KERNEL_PRIVATE
 141 /* priority queue static asserts fail for __ARM64_ARCH_8_32__ kext builds */
 142 #include <kern/priority_queue.h>
 143 #endif /* XNU_KERNEL_PRIVATE */
 144
 145 #if MONOTONIC
 146 #include <stdatomic.h>
 147 #include <machine/monotonic.h>
 148 #endif /* MONOTONIC */
 149
 150 #if CONFIG_EMBEDDED
 151 /* Taskwatch related. TODO: find this a better home */
 152 typedef struct task_watcher task_watch_t;
 153 #endif /* CONFIG_EMBEDDED */
 154
 155 struct thread {
 156 #if MACH_ASSERT
 157 #define THREAD_MAGIC 0x1234ABCDDCBA4321ULL
 158         /* Ensure nothing uses &thread as a queue entry */
 159         uint64_t                thread_magic;
 160 #endif /* MACH_ASSERT */
 161
 162         /*
 163          *      NOTE:   The runq field in the thread structure has an unusual
 164          *      locking protocol.  If its value is PROCESSOR_NULL, then it is
 165          *      locked by the thread_lock, but if its value is something else
 166          *      then it is locked by the associated run queue lock. It is
 167          *      set to PROCESSOR_NULL without holding the thread lock, but the
 168          *      transition from PROCESSOR_NULL to non-null must be done
 169          *      under the thread lock and the run queue lock.
 170          *
 171          *      New waitq APIs allow the 'links' and 'runq' fields to be
 172          *      anywhere in the thread structure.
 173          */
 174         union {
 175                 queue_chain_t                   runq_links;             /* run queue links */
 176                 queue_chain_t                   wait_links;             /* wait queue links */
 177                 struct mpsc_queue_chain         mpsc_links;             /* thread daemon mpsc links */
 178                 struct priority_queue_entry     wait_prioq_links;       /* priority ordered waitq links */
 179         };
 180
 181         processor_t             runq;           /* run queue assignment */
 182
 183         event64_t               wait_event;     /* wait queue event */
 184         struct waitq           *waitq;          /* wait queue this thread is enqueued on */
 185         struct turnstile       *turnstile;      /* thread's turnstile, protected by primitives interlock */
 186         void                   *inheritor;      /* inheritor of the primitive the thread will block on */
 187         struct priority_queue  sched_inheritor_queue; /* Inheritor queue for kernel promotion */
 188         struct priority_queue  base_inheritor_queue; /* Inheritor queue for user promotion */
 189
 190 #if CONFIG_SCHED_CLUTCH
 191         /*
 192          * In the clutch scheduler, the threads are maintained in runqs at the clutch_bucket
 193          * level (clutch_bucket defines a unique thread group and scheduling bucket pair). In
 194          * order to determine the priority of the clutch bucket as a whole, it is necessary to
 195          * find the highest thread in it. The thread could be present in the clutch bucket due
 196          * to its base_pri or its promoted pri. This link is used to maintain that queue.
 197          */
 198         struct priority_queue_entry sched_clutchpri_link;
 199
 200 #endif /* CONFIG_SCHED_CLUTCH */
 201
 202         /* Data updated during assert_wait/thread_wakeup */
 203 #if __SMP__
 204         decl_simple_lock_data(, sched_lock);     /* scheduling lock (thread_lock()) */
 205         decl_simple_lock_data(, wake_lock);      /* for thread stop / wait (wake_lock()) */
 206 #endif
 207         integer_t               options;                        /* options set by thread itself */
 208 #define TH_OPT_INTMASK          0x0003          /* interrupt / abort level */
 209 #define TH_OPT_VMPRIV           0x0004          /* may allocate reserved memory */
 210 #define TH_OPT_SYSTEM_CRITICAL  0x0010          /* Thread must always be allowed to run - even under heavy load */
 211 #define TH_OPT_PROC_CPULIMIT    0x0020          /* Thread has a task-wide CPU limit applied to it */
 212 #define TH_OPT_PRVT_CPULIMIT    0x0040          /* Thread has a thread-private CPU limit applied to it */
 213 #define TH_OPT_IDLE_THREAD      0x0080          /* Thread is a per-processor idle thread */
 214 #define TH_OPT_GLOBAL_FORCED_IDLE       0x0100  /* Thread performs forced idle for thermal control */
 215 #define TH_OPT_SCHED_VM_GROUP   0x0200          /* Thread belongs to special scheduler VM group */
 216 #define TH_OPT_HONOR_QLIMIT     0x0400          /* Thread will honor qlimit while sending mach_msg, regardless of MACH_SEND_ALWAYS */
 217 #define TH_OPT_SEND_IMPORTANCE  0x0800          /* Thread will allow importance donation from kernel rpc */
 218 #define TH_OPT_ZONE_GC          0x1000          /* zone_gc() called on this thread */
 219
 220         boolean_t                       wake_active;    /* wake event on stop */
 221         int                                     at_safe_point;  /* thread_abort_safely allowed */
 222         ast_t                           reason;                 /* why we blocked */
 223         uint32_t                        quantum_remaining;
 224         wait_result_t                   wait_result;    /* outcome of wait -
 225                                                         * may be examined by this thread
 226                                                         * WITHOUT locking */
 227         thread_continue_t       continuation;   /* continue here next dispatch */
 228         void                            *parameter;             /* continuation parameter */
 229
 230         /* Data updated/used in thread_invoke */
 231         vm_offset_t             kernel_stack;           /* current kernel stack */
 232         vm_offset_t                     reserved_stack;         /* reserved kernel stack */
 233
 234 #if KASAN
 235         struct kasan_thread_data kasan_data;
 236 #endif
 237
 238 #if CONFIG_KSANCOV
 239         void *ksancov_data;
 240 #endif
 241
 242         /* Thread state: */
 243         int                                     state;
 244 /*
 245  *      Thread states [bits or'ed]
 246  */
 247 #define TH_WAIT                 0x01                    /* queued for waiting */
 248 #define TH_SUSP                 0x02                    /* stopped or requested to stop */
 249 #define TH_RUN                  0x04                    /* running or on runq */
 250 #define TH_UNINT                0x08                    /* waiting uninteruptibly */
 251 #define TH_TERMINATE    0x10                    /* halted at termination */
 252 #define TH_TERMINATE2   0x20                    /* added to termination queue */
 253 #define TH_WAIT_REPORT  0x40                    /* the wait is using the sched_call,
 254                                                  *                                  only set if TH_WAIT is also set */
 255 #define TH_IDLE                 0x80                    /* idling processor */
 256
 257         /* Scheduling information */
 258         sched_mode_t                    sched_mode;             /* scheduling mode */
 259         sched_mode_t                    saved_mode;             /* saved mode during forced mode demotion */
 260
 261         /* This thread's contribution to global sched counters */
 262         sched_bucket_t                  th_sched_bucket;
 263
 264         sfi_class_id_t                  sfi_class;              /* SFI class (XXX Updated on CSW/QE/AST) */
 265         sfi_class_id_t                  sfi_wait_class; /* Currently in SFI wait for this class, protected by sfi_lock */
 266
 267
 268         uint32_t                        sched_flags;            /* current flag bits */
 269 #define TH_SFLAG_NO_SMT                 0x0001          /* On an SMT CPU, this thread must be scheduled alone */
 270 #define TH_SFLAG_FAILSAFE               0x0002          /* fail-safe has tripped */
 271 #define TH_SFLAG_THROTTLED              0x0004          /* throttled thread forced to timeshare mode (may be applied in addition to failsafe) */
 272 #define TH_SFLAG_DEMOTED_MASK      (TH_SFLAG_THROTTLED | TH_SFLAG_FAILSAFE)     /* saved_mode contains previous sched_mode */
 273
 274 #define TH_SFLAG_PROMOTED               0x0008          /* sched pri has been promoted by kernel mutex priority promotion */
 275 #define TH_SFLAG_ABORT                  0x0010          /* abort interruptible waits */
 276 #define TH_SFLAG_ABORTSAFELY            0x0020          /* ... but only those at safe point */
 277 #define TH_SFLAG_ABORTED_MASK           (TH_SFLAG_ABORT | TH_SFLAG_ABORTSAFELY)
 278 #define TH_SFLAG_DEPRESS                0x0040          /* normal depress yield */
 279 #define TH_SFLAG_POLLDEPRESS            0x0080          /* polled depress yield */
 280 #define TH_SFLAG_DEPRESSED_MASK         (TH_SFLAG_DEPRESS | TH_SFLAG_POLLDEPRESS)
 281 /* unused TH_SFLAG_PRI_UPDATE           0x0100 */
 282 #define TH_SFLAG_EAGERPREEMPT           0x0200          /* Any preemption of this thread should be treated as if AST_URGENT applied */
 283 #define TH_SFLAG_RW_PROMOTED            0x0400          /* promote reason: blocking with RW lock held */
 284 #define TH_SFLAG_BASE_PRI_FROZEN        0x0800          /* (effective) base_pri is frozen */
 285 #define TH_SFLAG_WAITQ_PROMOTED         0x1000          /* promote reason: waitq wakeup (generally for IPC receive) */
 286
 287
 288 #define TH_SFLAG_EXEC_PROMOTED          0x8000          /* promote reason: thread is in an exec */
 289
 290 /* 'promote reasons' that request a priority floor only, not a custom priority */
 291 #define TH_SFLAG_PROMOTE_REASON_MASK    (TH_SFLAG_RW_PROMOTED | TH_SFLAG_WAITQ_PROMOTED | TH_SFLAG_EXEC_PROMOTED)
 292
 293 #define TH_SFLAG_RW_PROMOTED_BIT        (10)    /* 0x400 */
 294
 295         int16_t                         sched_pri;              /* scheduled (current) priority */
 296         int16_t                         base_pri;               /* effective base priority (equal to req_base_pri unless TH_SFLAG_BASE_PRI_FROZEN) */
 297         int16_t                         req_base_pri;           /* requested base priority */
 298         int16_t                         max_priority;           /* copy of max base priority */
 299         int16_t                         task_priority;          /* copy of task base priority */
 300         int16_t                         promotion_priority;     /* priority thread is currently promoted to */
 301
 302 #if defined(CONFIG_SCHED_GRRR)
 303 #if 0
 304         uint16_t                        grrr_deficit;           /* fixed point (1/1000th quantum) fractional deficit */
 305 #endif
 306 #endif
 307
 308         int                             iotier_override; /* atomic operations to set, cleared on ret to user */
 309         os_refcnt_t                     ref_count;              /* number of references to me */
 310
 311         lck_mtx_t*                      waiting_for_mutex;      /* points to mutex we're waiting for until we acquire it */
 312
 313         uint32_t                        rwlock_count;   /* Number of lck_rw_t locks held by thread */
 314
 315         integer_t                       importance;                     /* task-relative importance */
 316
 317         /* Priority depression expiration */
 318         integer_t                       depress_timer_active;
 319         timer_call_data_t       depress_timer;
 320         /* real-time parameters */
 321         struct {                                                                /* see mach/thread_policy.h */
 322                 uint32_t                        period;
 323                 uint32_t                        computation;
 324                 uint32_t                        constraint;
 325                 boolean_t                       preemptible;
 326                 uint64_t                        deadline;
 327         }                                       realtime;
 328
 329         uint64_t                        last_run_time;          /* time when thread was switched away from */
 330         uint64_t                        last_made_runnable_time;        /* time when thread was unblocked or preempted */
 331         uint64_t                        last_basepri_change_time;       /* time when thread was last changed in basepri while runnable */
 332         uint64_t                        same_pri_latency;
 333 #define THREAD_NOT_RUNNABLE (~0ULL)
 334
 335
 336 #if defined(CONFIG_SCHED_MULTIQ)
 337         sched_group_t                   sched_group;
 338 #endif /* defined(CONFIG_SCHED_MULTIQ) */
 339
 340         /* Data used during setrun/dispatch */
 341         timer_data_t            system_timer;           /* system mode timer */
 342         processor_t                     bound_processor;        /* bound to a processor? */
 343         processor_t                     last_processor;         /* processor last dispatched on */
 344         processor_t                     chosen_processor;       /* Where we want to run this thread */
 345
 346         /* Fail-safe computation since last unblock or qualifying yield */
 347         uint64_t                        computation_metered;
 348         uint64_t                        computation_epoch;
 349         uint64_t                        safe_release;   /* when to release fail-safe */
 350
 351         /* Call out from scheduler */
 352         void                            (*sched_call)(
 353                 int                     type,
 354                 thread_t        thread);
 355 #if defined(CONFIG_SCHED_PROTO)
 356         uint32_t                        runqueue_generation;    /* last time runqueue was drained */
 357 #endif
 358
 359         /* Statistics and timesharing calculations */
 360 #if defined(CONFIG_SCHED_TIMESHARE_CORE)
 361         natural_t                       sched_stamp;    /* last scheduler tick */
 362         natural_t                       sched_usage;    /* timesharing cpu usage [sched] */
 363         natural_t                       pri_shift;              /* usage -> priority from pset */
 364         natural_t                       cpu_usage;              /* instrumented cpu usage [%cpu] */
 365         natural_t                       cpu_delta;              /* accumulated cpu_usage delta */
 366 #endif /* CONFIG_SCHED_TIMESHARE_CORE */
 367
 368         uint32_t                        c_switch;               /* total context switches */
 369         uint32_t                        p_switch;               /* total processor switches */
 370         uint32_t                        ps_switch;              /* total pset switches */
 371
 372         integer_t mutex_count;  /* total count of locks held */
 373         /* Timing data structures */
 374         int                                     precise_user_kernel_time; /* precise user/kernel enabled for this thread */
 375         timer_data_t            user_timer;                     /* user mode timer */
 376         uint64_t                        user_timer_save;        /* saved user timer value */
 377         uint64_t                        system_timer_save;      /* saved system timer value */
 378         uint64_t                        vtimer_user_save;       /* saved values for vtimers */
 379         uint64_t                        vtimer_prof_save;
 380         uint64_t                        vtimer_rlim_save;
 381         uint64_t                        vtimer_qos_save;
 382
 383         timer_data_t            ptime;                  /* time executing in P mode */
 384         timer_data_t            runnable_timer;         /* time the thread is runnable (including running) */
 385
 386 #if CONFIG_SCHED_SFI
 387         /* Timing for wait state */
 388         uint64_t                wait_sfi_begin_time;    /* start time for thread waiting in SFI */
 389 #endif
 390
 391         /* Timed wait expiration */
 392         timer_call_data_t       wait_timer;
 393         integer_t                       wait_timer_active;
 394         boolean_t                       wait_timer_is_set;
 395
 396
 397         /*
 398          * Processor/cache affinity
 399          * - affinity_threads links task threads with the same affinity set
 400          */
 401         affinity_set_t                  affinity_set;
 402         queue_chain_t                   affinity_threads;
 403
 404         /* Various bits of state to stash across a continuation, exclusive to the current thread block point */
 405         union {
 406                 struct {
 407                         mach_msg_return_t       state;          /* receive state */
 408                         mach_port_seqno_t       seqno;          /* seqno of recvd message */
 409                         ipc_object_t            object;         /* object received on */
 410                         mach_vm_address_t       msg_addr;       /* receive buffer pointer */
 411                         mach_msg_size_t         rsize;          /* max size for recvd msg */
 412                         mach_msg_size_t         msize;          /* actual size for recvd msg */
 413                         mach_msg_option_t       option;         /* options for receive */
 414                         mach_port_name_t        receiver_name;  /* the receive port name */
 415                         struct knote            *knote;         /* knote fired for rcv */
 416                         union {
 417                                 struct ipc_kmsg   *kmsg;        /* received message */
 418                                 struct ipc_mqueue *peekq;       /* mqueue to peek at */
 419                                 struct {
 420                                         mach_msg_priority_t qos;        /* received message qos */
 421                                         mach_msg_priority_t oqos;       /* override qos for message */
 422                                 } received_qos;
 423                         };
 424                         mach_msg_continue_t     continuation;
 425                 } receive;
 426                 struct {
 427                         struct semaphore        *waitsemaphore;         /* semaphore ref */
 428                         struct semaphore        *signalsemaphore;       /* semaphore ref */
 429                         int                                     options;                        /* semaphore options */
 430                         kern_return_t           result;                         /* primary result */
 431                         mach_msg_continue_t continuation;
 432                 } sema;
 433                 struct {
 434 #define THREAD_SAVE_IOKIT_TLS_COUNT     8
 435                         void                    *tls[THREAD_SAVE_IOKIT_TLS_COUNT];
 436                 } iokit;
 437         } saved;
 438
 439         /* Only user threads can cause guard exceptions, only kernel threads can be thread call threads */
 440         union {
 441                 /* Group and call this thread is working on behalf of */
 442                 struct {
 443                         struct thread_call_group * thc_group;
 444                         struct thread_call *       thc_call;                    /* debug only, may be deallocated */
 445                 } thc_state;
 446
 447                 /* Structure to save information about guard exception */
 448                 struct {
 449                         mach_exception_code_t           code;
 450                         mach_exception_subcode_t        subcode;
 451                 } guard_exc_info;
 452         };
 453
 454         /* Kernel holds on this thread  */
 455         int16_t                                         suspend_count;
 456         /* User level suspensions */
 457         int16_t                                         user_stop_count;
 458
 459         /* IPC data structures */
 460 #if IMPORTANCE_INHERITANCE
 461         natural_t ith_assertions;                       /* assertions pending drop */
 462 #endif
 463         struct ipc_kmsg_queue ith_messages;             /* messages to reap */
 464         mach_port_t ith_rpc_reply;                      /* reply port for kernel RPCs */
 465
 466         /* Ast/Halt data structures */
 467         vm_offset_t                                     recover;                /* page fault recover(copyin/out) */
 468
 469         queue_chain_t                           threads;                /* global list of all threads */
 470
 471         /* Activation */
 472         queue_chain_t                   task_threads;
 473
 474         /* Task membership */
 475         struct task                             *task;
 476         vm_map_t                                map;
 477 #if DEVELOPMENT || DEBUG
 478         boolean_t pmap_footprint_suspended;
 479 #endif /* DEVELOPMENT || DEBUG */
 480
 481         decl_lck_mtx_data(, mutex);
 482
 483
 484         /* Pending thread ast(s) */
 485         ast_t                                   ast;
 486
 487         /* Miscellaneous bits guarded by mutex */
 488         uint32_t
 489             active:1,                                   /* Thread is active and has not been terminated */
 490             started:1,                                  /* Thread has been started after creation */
 491             static_param:1,                             /* Disallow policy parameter changes */
 492             inspection:1,                               /* TRUE when task is being inspected by crash reporter */
 493             policy_reset:1,                             /* Disallow policy parameter changes on terminating threads */
 494             suspend_parked:1,                           /* thread parked in thread_suspended */
 495             corpse_dup:1,                               /* TRUE when thread is an inactive duplicate in a corpse */
 496         :0;
 497
 498         /* Ports associated with this thread */
 499         struct ipc_port                 *ith_self;                      /* not a right, doesn't hold ref */
 500         struct ipc_port                 *ith_sself;                     /* a send right */
 501         struct ipc_port                 *ith_special_reply_port;         /* ref to special reply port */
 502         struct exception_action *exc_actions;
 503
 504 #ifdef  MACH_BSD
 505         void                                    *uthread;
 506 #endif
 507
 508 #if CONFIG_DTRACE
 509         uint16_t t_dtrace_flags;                /* DTrace thread states */
 510 #define TH_DTRACE_EXECSUCCESS   0x01
 511         uint16_t t_dtrace_inprobe;          /* Executing under dtrace_probe */
 512         uint32_t t_dtrace_predcache;        /* DTrace per thread predicate value hint */
 513         int64_t t_dtrace_tracing;               /* Thread time under dtrace_probe() */
 514         int64_t t_dtrace_vtime;
 515 #endif
 516
 517         clock_sec_t t_page_creation_time;
 518         uint32_t    t_page_creation_count;
 519         uint32_t    t_page_creation_throttled;
 520 #if (DEVELOPMENT || DEBUG)
 521         uint64_t    t_page_creation_throttled_hard;
 522         uint64_t    t_page_creation_throttled_soft;
 523 #endif /* DEVELOPMENT || DEBUG */
 524         int         t_pagein_error;            /* for vm_fault(), holds error from vnop_pagein() */
 525
 526 #ifdef KPERF
 527 /* The high 8 bits are the number of frames to sample of a user callstack. */
 528 #define T_KPERF_CALLSTACK_DEPTH_OFFSET     (24)
 529 #define T_KPERF_SET_CALLSTACK_DEPTH(DEPTH) (((uint32_t)(DEPTH)) << T_KPERF_CALLSTACK_DEPTH_OFFSET)
 530 #define T_KPERF_GET_CALLSTACK_DEPTH(FLAGS) ((FLAGS) >> T_KPERF_CALLSTACK_DEPTH_OFFSET)
 531 #endif
 532
 533 #define T_KPERF_AST_CALLSTACK (1U << 0) /* dump a callstack on thread's next AST */
 534 #define T_KPERF_AST_DISPATCH  (1U << 1) /* dump a name on thread's next AST */
 535 #define T_KPC_ALLOC           (1U << 2) /* thread needs a kpc_buf allocated */
 536 /* only go up to T_KPERF_CALLSTACK_DEPTH_OFFSET - 1 */
 537
 538 #ifdef KPERF
 539         uint32_t kperf_flags;
 540         uint32_t kperf_pet_gen;  /* last generation of PET that sampled this thread*/
 541         uint32_t kperf_c_switch; /* last dispatch detection */
 542         uint32_t kperf_pet_cnt;  /* how many times a thread has been sampled by PET */
 543 #endif
 544
 545 #ifdef KPC
 546         /* accumulated performance counters for this thread */
 547         uint64_t *kpc_buf;
 548 #endif
 549
 550 #if HYPERVISOR
 551         /* hypervisor virtual CPU object associated with this thread */
 552         void *hv_thread_target;
 553 #endif /* HYPERVISOR */
 554
 555         uint64_t thread_id;             /*system wide unique thread-id*/
 556
 557         /* Statistics accumulated per-thread and aggregated per-task */
 558         uint32_t                syscalls_unix;
 559         uint32_t                syscalls_mach;
 560         ledger_t                t_ledger;
 561         ledger_t                t_threadledger; /* per thread ledger */
 562         ledger_t                t_bankledger;                /* ledger to charge someone */
 563         uint64_t                t_deduct_bank_ledger_time;   /* cpu time to be deducted from bank ledger */
 564         uint64_t                t_deduct_bank_ledger_energy; /* energy to be deducted from bank ledger */
 565
 566 #if MONOTONIC
 567         struct mt_thread t_monotonic;
 568 #endif /* MONOTONIC */
 569
 570         /*** Machine-dependent state ***/
 571         struct machine_thread   machine;
 572
 573         /* policy is protected by the thread mutex */
 574         struct thread_requested_policy  requested_policy;
 575         struct thread_effective_policy  effective_policy;
 576
 577         /* usynch override is protected by the task lock, eventually will be thread mutex */
 578         struct thread_qos_override {
 579                 struct thread_qos_override      *override_next;
 580                 uint32_t        override_contended_resource_count;
 581                 int16_t         override_qos;
 582                 int16_t         override_resource_type;
 583                 user_addr_t     override_resource;
 584         } *overrides;
 585
 586         uint32_t        kevent_overrides;
 587         uint16_t        user_promotion_basepri;
 588         uint16_t         kern_promotion_schedpri;
 589         _Atomic uint16_t kevent_ast_bits;
 590
 591         io_stat_info_t                  thread_io_stats; /* per-thread I/O statistics */
 592
 593 #if CONFIG_EMBEDDED
 594         task_watch_t *  taskwatch;              /* task watch */
 595 #endif /* CONFIG_EMBEDDED */
 596
 597         uint32_t                        thread_callout_interrupt_wakeups;
 598         uint32_t                        thread_callout_platform_idle_wakeups;
 599         uint32_t                        thread_timer_wakeups_bin_1;
 600         uint32_t                        thread_timer_wakeups_bin_2;
 601         uint16_t                        thread_tag;
 602         /*
 603          * callout_* fields are only set for thread call threads whereas guard_exc_fatal is set
 604          * by user threads on themselves while taking a guard exception. So it's okay for them to
 605          * share this bitfield.
 606          */
 607         uint16_t                        callout_woken_from_icontext:1,
 608             callout_woken_from_platform_idle:1,
 609             callout_woke_thread:1,
 610             guard_exc_fatal:1,
 611             thread_bitfield_unused:12;
 612
 613         mach_port_name_t                ith_voucher_name;
 614         ipc_voucher_t                   ith_voucher;
 615 #if CONFIG_IOSCHED
 616         void                            *decmp_upl;
 617 #endif /* CONFIG_IOSCHED */
 618
 619         /* work interval (if any) associated with the thread. Uses thread mutex */
 620         struct work_interval            *th_work_interval;
 621
 622 #if     SCHED_TRACE_THREAD_WAKEUPS
 623         uintptr_t               thread_wakeup_bt[64];
 624 #endif
 625         turnstile_update_flags_t inheritor_flags; /* inheritor flags for inheritor field */
 626         block_hint_t    pending_block_hint;
 627         block_hint_t    block_hint;      /* What type of primitive last caused us to block. */
 628         integer_t       decompressions;  /* Per-thread decompressions counter to be added to per-task decompressions counter */
 629 };
 630
 631 #define ith_state           saved.receive.state
 632 #define ith_object          saved.receive.object
 633 #define ith_msg_addr        saved.receive.msg_addr
 634 #define ith_rsize           saved.receive.rsize
 635 #define ith_msize           saved.receive.msize
 636 #define ith_option          saved.receive.option
 637 #define ith_receiver_name   saved.receive.receiver_name
 638 #define ith_continuation    saved.receive.continuation
 639 #define ith_kmsg            saved.receive.kmsg
 640 #define ith_peekq           saved.receive.peekq
 641 #define ith_knote           saved.receive.knote
 642 #define ith_qos             saved.receive.received_qos.qos
 643 #define ith_qos_override    saved.receive.received_qos.oqos
 644 #define ith_seqno           saved.receive.seqno
 645
 646 #define sth_waitsemaphore   saved.sema.waitsemaphore
 647 #define sth_signalsemaphore saved.sema.signalsemaphore
 648 #define sth_options         saved.sema.options
 649 #define sth_result          saved.sema.result
 650 #define sth_continuation    saved.sema.continuation
 651
 652 #define ITH_KNOTE_NULL      ((void *)NULL)
 653 #define ITH_KNOTE_PSEUDO    ((void *)0xdeadbeef)
 654 /*
 655  * The ith_knote is used during message delivery, and can safely be interpreted
 656  * only when used for one of these codepaths, which the test for the msgt_name
 657  * being RECEIVE or SEND_ONCE is about.
 658  */
 659 #define ITH_KNOTE_VALID(kn, msgt_name) \
 660                 (((kn) != ITH_KNOTE_NULL && (kn) != ITH_KNOTE_PSEUDO) && \
 661                  ((msgt_name) == MACH_MSG_TYPE_PORT_RECEIVE || \
 662                  (msgt_name) == MACH_MSG_TYPE_PORT_SEND_ONCE))
 663
 664 #if MACH_ASSERT
 665 #define assert_thread_magic(thread) assertf((thread)->thread_magic == THREAD_MAGIC, \
 666                                             "bad thread magic 0x%llx for thread %p, expected 0x%llx", \
 667                                             (thread)->thread_magic, (thread), THREAD_MAGIC)
 668 #else
 669 #define assert_thread_magic(thread) do { (void)(thread); } while (0)
 670 #endif
 671
 672 extern void                     thread_bootstrap(void);
 673
 674 extern void                     thread_init(void);
 675
 676 extern void                     thread_daemon_init(void);
 677
 678 #define thread_reference_internal(thread)       \
 679                         os_ref_retain(&(thread)->ref_count);
 680
 681 #define thread_reference(thread)                                        \
 682 MACRO_BEGIN                                                                                     \
 683         if ((thread) != THREAD_NULL)                                    \
 684                 thread_reference_internal(thread);              \
 685 MACRO_END
 686
 687 extern void                     thread_deallocate(
 688         thread_t                thread);
 689
 690 extern void                     thread_inspect_deallocate(
 691         thread_inspect_t        thread);
 692
 693 extern void                     thread_terminate_self(void);
 694
 695 extern kern_return_t    thread_terminate_internal(
 696         thread_t                thread);
 697
 698 extern void                     thread_start(
 699         thread_t                        thread) __attribute__ ((noinline));
 700
 701 extern void                     thread_start_in_assert_wait(
 702         thread_t                        thread,
 703         event_t             event,
 704         wait_interrupt_t    interruptible) __attribute__ ((noinline));
 705
 706 extern void                     thread_terminate_enqueue(
 707         thread_t                thread);
 708
 709 extern void                     thread_exception_enqueue(
 710         task_t          task,
 711         thread_t        thread,
 712         exception_type_t etype);
 713
 714 extern void                     thread_copy_resource_info(
 715         thread_t dst_thread,
 716         thread_t src_thread);
 717
 718 extern void                     thread_terminate_crashed_threads(void);
 719
 720 extern void                     thread_stack_enqueue(
 721         thread_t                thread);
 722
 723 extern void                     thread_hold(
 724         thread_t        thread);
 725
 726 extern void                     thread_release(
 727         thread_t        thread);
 728
 729 extern void                     thread_corpse_continue(void) __dead2;
 730
 731 extern boolean_t                thread_is_active(thread_t thread);
 732
 733 extern lck_grp_t                thread_lck_grp;
 734
 735 /* Locking for scheduler state, always acquired with interrupts disabled (splsched()) */
 736 #if __SMP__
 737 #define thread_lock_init(th)    simple_lock_init(&(th)->sched_lock, 0)
 738 #define thread_lock(th)                 simple_lock(&(th)->sched_lock, &thread_lck_grp)
 739 #define thread_unlock(th)               simple_unlock(&(th)->sched_lock)
 740
 741 #define wake_lock_init(th)              simple_lock_init(&(th)->wake_lock, 0)
 742 #define wake_lock(th)                   simple_lock(&(th)->wake_lock, &thread_lck_grp)
 743 #define wake_unlock(th)                 simple_unlock(&(th)->wake_lock)
 744 #else
 745 #define thread_lock_init(th)    do { (void)th; } while(0)
 746 #define thread_lock(th)                 do { (void)th; } while(0)
 747 #define thread_unlock(th)               do { (void)th; } while(0)
 748
 749 #define wake_lock_init(th)              do { (void)th; } while(0)
 750 #define wake_lock(th)                   do { (void)th; } while(0)
 751 #define wake_unlock(th)                 do { (void)th; } while(0)
 752 #endif
 753
 754 #define thread_should_halt_fast(thread)         (!(thread)->active)
 755
 756 extern void                             stack_alloc(
 757         thread_t                thread);
 758
 759 extern void                     stack_handoff(
 760         thread_t                from,
 761         thread_t                to);
 762
 763 extern void                             stack_free(
 764         thread_t                thread);
 765
 766 extern void                             stack_free_reserved(
 767         thread_t                thread);
 768
 769 extern boolean_t                stack_alloc_try(
 770         thread_t            thread);
 771
 772 extern void                             stack_collect(void);
 773
 774 extern void                             stack_init(void);
 775
 776
 777 extern kern_return_t    thread_info_internal(
 778         thread_t                                thread,
 779         thread_flavor_t                 flavor,
 780         thread_info_t                   thread_info_out,
 781         mach_msg_type_number_t  *thread_info_count);
 782
 783
 784
 785 extern kern_return_t    kernel_thread_create(
 786         thread_continue_t       continuation,
 787         void                            *parameter,
 788         integer_t                       priority,
 789         thread_t                        *new_thread);
 790
 791 extern kern_return_t    kernel_thread_start_priority(
 792         thread_continue_t       continuation,
 793         void                            *parameter,
 794         integer_t                       priority,
 795         thread_t                        *new_thread);
 796
 797 extern void                             machine_stack_attach(
 798         thread_t                thread,
 799         vm_offset_t             stack);
 800
 801 extern vm_offset_t              machine_stack_detach(
 802         thread_t                thread);
 803
 804 extern void                             machine_stack_handoff(
 805         thread_t                old,
 806         thread_t                new);
 807
 808 extern thread_t                 machine_switch_context(
 809         thread_t                        old_thread,
 810         thread_continue_t       continuation,
 811         thread_t                        new_thread);
 812
 813 extern void                             machine_load_context(
 814         thread_t                thread) __attribute__((noreturn));
 815
 816 extern kern_return_t    machine_thread_state_initialize(
 817         thread_t                                thread);
 818
 819 extern kern_return_t    machine_thread_set_state(
 820         thread_t                                thread,
 821         thread_flavor_t                 flavor,
 822         thread_state_t                  state,
 823         mach_msg_type_number_t  count);
 824
 825 extern mach_vm_address_t machine_thread_pc(
 826         thread_t                thread);
 827
 828 extern void machine_thread_reset_pc(
 829         thread_t                thread,
 830         mach_vm_address_t       pc);
 831
 832 extern boolean_t        machine_thread_on_core(
 833         thread_t                thread);
 834
 835 extern kern_return_t    machine_thread_get_state(
 836         thread_t                                thread,
 837         thread_flavor_t                 flavor,
 838         thread_state_t                  state,
 839         mach_msg_type_number_t  *count);
 840
 841 extern kern_return_t    machine_thread_state_convert_from_user(
 842         thread_t                                thread,
 843         thread_flavor_t                 flavor,
 844         thread_state_t                  tstate,
 845         mach_msg_type_number_t  count);
 846
 847 extern kern_return_t    machine_thread_state_convert_to_user(
 848         thread_t                                thread,
 849         thread_flavor_t                 flavor,
 850         thread_state_t                  tstate,
 851         mach_msg_type_number_t  *count);
 852
 853 extern kern_return_t    machine_thread_dup(
 854         thread_t                self,
 855         thread_t                target,
 856         boolean_t               is_corpse);
 857
 858 extern void                             machine_thread_init(void);
 859
 860 extern kern_return_t    machine_thread_create(
 861         thread_t                thread,
 862         task_t                  task);
 863 extern void             machine_thread_switch_addrmode(
 864         thread_t                 thread);
 865
 866 extern void                 machine_thread_destroy(
 867         thread_t                thread);
 868
 869 extern void                             machine_set_current_thread(
 870         thread_t                        thread);
 871
 872 extern kern_return_t    machine_thread_get_kern_state(
 873         thread_t                                thread,
 874         thread_flavor_t                 flavor,
 875         thread_state_t                  tstate,
 876         mach_msg_type_number_t  *count);
 877
 878 extern kern_return_t    machine_thread_inherit_taskwide(
 879         thread_t                thread,
 880         task_t                  parent_task);
 881
 882 extern kern_return_t    machine_thread_set_tsd_base(
 883         thread_t                                thread,
 884         mach_vm_offset_t                tsd_base);
 885
 886 #define thread_mtx_lock(thread)                 lck_mtx_lock(&(thread)->mutex)
 887 #define thread_mtx_try(thread)                  lck_mtx_try_lock(&(thread)->mutex)
 888 #define thread_mtx_unlock(thread)               lck_mtx_unlock(&(thread)->mutex)
 889
 890 extern void thread_apc_ast(thread_t thread);
 891
 892 extern void thread_update_qos_cpu_time(thread_t thread);
 893
 894 void act_machine_sv_free(thread_t, int);
 895
 896 vm_offset_t                     min_valid_stack_address(void);
 897 vm_offset_t                     max_valid_stack_address(void);
 898
 899 static inline uint16_t
 900 thread_set_tag_internal(thread_t        thread, uint16_t tag)
 901 {
 902         return os_atomic_or_orig(&thread->thread_tag, tag, relaxed);
 903 }
 904
 905 static inline uint16_t
 906 thread_get_tag_internal(thread_t        thread)
 907 {
 908         return thread->thread_tag;
 909 }
 910
 911 extern bool thread_no_smt(thread_t thread);
 912 extern bool processor_active_thread_no_smt(processor_t processor);
 913
 914 extern void thread_set_options(uint32_t thopt);
 915
 916
 917 #else   /* MACH_KERNEL_PRIVATE */
 918
 919 __BEGIN_DECLS
 920
 921 extern void thread_mtx_lock(thread_t thread);
 922
 923 extern void thread_mtx_unlock(thread_t thread);
 924
 925 extern thread_t         current_thread(void) __attribute__((const));
 926
 927 extern void                     thread_reference(
 928         thread_t        thread);
 929
 930 extern void                     thread_deallocate(
 931         thread_t        thread);
 932
 933 #if BSD_KERNEL_PRIVATE
 934 /* Duplicated from osfmk/kern/ipc_tt.h */
 935 __options_decl(port_to_thread_options_t, uint32_t, {
 936         PORT_TO_THREAD_NONE               = 0x0000,
 937         PORT_TO_THREAD_IN_CURRENT_TASK    = 0x0001,
 938         PORT_TO_THREAD_NOT_CURRENT_THREAD = 0x0002,
 939 });
 940
 941 extern thread_t port_name_to_thread(
 942         mach_port_name_t            port_name,
 943         port_to_thread_options_t    options);
 944 #endif /* BSD_KERNEL_PRIVATE */
 945
 946 __END_DECLS
 947
 948 #endif  /* MACH_KERNEL_PRIVATE */
 949
 950 #ifdef  KERNEL_PRIVATE
 951
 952 __BEGIN_DECLS
 953
 954 extern void                     thread_deallocate_safe(
 955         thread_t                thread);
 956
 957 extern uint64_t                 thread_dispatchqaddr(
 958         thread_t thread);
 959
 960 extern uint64_t                 thread_rettokern_addr(
 961         thread_t thread);
 962
 963 extern integer_t        thread_kern_get_pri(thread_t thr) __attribute__((const));
 964
 965 extern void             thread_kern_set_pri(thread_t thr, integer_t pri);
 966
 967 extern integer_t        thread_kern_get_kernel_maxpri(void) __attribute__((const));
 968
 969 __END_DECLS
 970
 971 #endif  /* KERNEL_PRIVATE */
 972
 973 #ifdef KERNEL
 974 __BEGIN_DECLS
 975
 976 extern uint64_t                 thread_tid(thread_t thread);
 977
 978 __END_DECLS
 979
 980 #endif  /* KERNEL */
 981
 982 __BEGIN_DECLS
 983
 984 #ifdef  XNU_KERNEL_PRIVATE
 985
 986 /*
 987  * Thread tags; for easy identification.
 988  */
 989 #define THREAD_TAG_MAINTHREAD 0x1
 990 #define THREAD_TAG_CALLOUT 0x2
 991 #define THREAD_TAG_IOWORKLOOP 0x4
 992
 993 #define THREAD_TAG_PTHREAD 0x10
 994 #define THREAD_TAG_WORKQUEUE 0x20
 995
 996 uint16_t        thread_set_tag(thread_t, uint16_t);
 997 uint16_t        thread_get_tag(thread_t);
 998 uint64_t        thread_last_run_time(thread_t);
 999
1000 extern kern_return_t    thread_state_initialize(
1001         thread_t                                thread);
1002
1003 extern kern_return_t    thread_setstatus(
1004         thread_t                                thread,
1005         int                                             flavor,
1006         thread_state_t                  tstate,
1007         mach_msg_type_number_t  count);
1008
1009 extern kern_return_t    thread_setstatus_from_user(
1010         thread_t                                thread,
1011         int                                             flavor,
1012         thread_state_t                  tstate,
1013         mach_msg_type_number_t  count);
1014
1015 extern kern_return_t    thread_getstatus(
1016         thread_t                                thread,
1017         int                                             flavor,
1018         thread_state_t                  tstate,
1019         mach_msg_type_number_t  *count);
1020
1021 extern kern_return_t    thread_getstatus_to_user(
1022         thread_t                                thread,
1023         int                                             flavor,
1024         thread_state_t                  tstate,
1025         mach_msg_type_number_t  *count);
1026
1027 extern kern_return_t    thread_create_with_continuation(
1028         task_t task,
1029         thread_t *new_thread,
1030         thread_continue_t continuation);
1031
1032 extern kern_return_t thread_create_waiting(task_t               task,
1033     thread_continue_t    continuation,
1034     event_t              event,
1035     thread_t             *new_thread);
1036
1037 extern kern_return_t    thread_create_workq_waiting(
1038         task_t                  task,
1039         thread_continue_t       thread_return,
1040         thread_t                *new_thread);
1041
1042 extern  void    thread_yield_internal(
1043         mach_msg_timeout_t      interval);
1044
1045 extern void             thread_yield_to_preemption(void);
1046
1047 /*
1048  * Thread-private CPU limits: apply a private CPU limit to this thread only. Available actions are:
1049  *
1050  * 1) Block. Prevent CPU consumption of the thread from exceeding the limit.
1051  * 2) Exception. Generate a resource consumption exception when the limit is exceeded.
1052  * 3) Disable. Remove any existing CPU limit.
1053  */
1054 #define THREAD_CPULIMIT_BLOCK           0x1
1055 #define THREAD_CPULIMIT_EXCEPTION       0x2
1056 #define THREAD_CPULIMIT_DISABLE         0x3
1057
1058 struct _thread_ledger_indices {
1059         int cpu_time;
1060 };
1061
1062 extern struct _thread_ledger_indices thread_ledgers;
1063
1064 extern int thread_get_cpulimit(int *action, uint8_t *percentage, uint64_t *interval_ns);
1065 extern int thread_set_cpulimit(int action, uint8_t percentage, uint64_t interval_ns);
1066
1067 extern void                     thread_read_times(
1068         thread_t                thread,
1069         time_value_t    *user_time,
1070         time_value_t    *system_time,
1071         time_value_t    *runnable_time);
1072
1073 extern uint64_t         thread_get_runtime_self(void);
1074
1075 extern void                     thread_setuserstack(
1076         thread_t                thread,
1077         mach_vm_offset_t        user_stack);
1078
1079 extern uint64_t         thread_adjuserstack(
1080         thread_t                thread,
1081         int                             adjust);
1082
1083 extern void                     thread_setentrypoint(
1084         thread_t                thread,
1085         mach_vm_offset_t        entry);
1086
1087 extern kern_return_t    thread_set_tsd_base(
1088         thread_t        thread,
1089         mach_vm_offset_t tsd_base);
1090
1091 extern kern_return_t    thread_setsinglestep(
1092         thread_t                thread,
1093         int                     on);
1094
1095 extern kern_return_t    thread_userstack(
1096         thread_t,
1097         int,
1098         thread_state_t,
1099         unsigned int,
1100         mach_vm_offset_t *,
1101         int *,
1102         boolean_t);
1103
1104 extern kern_return_t    thread_entrypoint(
1105         thread_t,
1106         int,
1107         thread_state_t,
1108         unsigned int,
1109         mach_vm_offset_t *);
1110
1111 extern kern_return_t    thread_userstackdefault(
1112         mach_vm_offset_t *,
1113         boolean_t);
1114
1115 extern kern_return_t    thread_wire_internal(
1116         host_priv_t             host_priv,
1117         thread_t                thread,
1118         boolean_t               wired,
1119         boolean_t               *prev_state);
1120
1121
1122 extern kern_return_t    thread_dup(thread_t);
1123
1124 extern kern_return_t thread_dup2(thread_t, thread_t);
1125
1126 #if !defined(_SCHED_CALL_T_DEFINED)
1127 #define _SCHED_CALL_T_DEFINED
1128 typedef void    (*sched_call_t)(
1129         int                             type,
1130         thread_t                thread);
1131 #endif
1132
1133 #define SCHED_CALL_BLOCK                0x1
1134 #define SCHED_CALL_UNBLOCK              0x2
1135
1136 extern void             thread_sched_call(
1137         thread_t                thread,
1138         sched_call_t    call);
1139
1140 extern boolean_t        thread_is_static_param(
1141         thread_t                thread);
1142
1143 extern task_t   get_threadtask(thread_t);
1144
1145 /*
1146  * Thread is running within a 64-bit address space.
1147  */
1148 #define thread_is_64bit_addr(thd)       \
1149         task_has_64Bit_addr(get_threadtask(thd))
1150
1151 /*
1152  * Thread is using 64-bit machine state.
1153  */
1154 #define thread_is_64bit_data(thd)       \
1155         task_has_64Bit_data(get_threadtask(thd))
1156
1157 #if defined(__x86_64__)
1158 extern int              thread_task_has_ldt(thread_t);
1159 #endif
1160 extern void             *get_bsdthread_info(thread_t);
1161 extern void             set_bsdthread_info(thread_t, void *);
1162 extern void             set_thread_pagein_error(thread_t, int);
1163 extern void             *uthread_alloc(task_t, thread_t, int);
1164 extern event_t  workq_thread_init_and_wq_lock(task_t, thread_t); // bsd/pthread/
1165 extern void             uthread_cleanup_name(void *uthread);
1166 extern void             uthread_cleanup(task_t, void *, void *);
1167 extern void             uthread_zone_free(void *);
1168 extern void             uthread_cred_free(void *);
1169
1170 extern void             uthread_reset_proc_refcount(void *);
1171 #if PROC_REF_DEBUG
1172 extern int              uthread_get_proc_refcount(void *);
1173 extern int              proc_ref_tracking_disabled;
1174 #endif
1175
1176 extern boolean_t        thread_should_halt(
1177         thread_t                thread);
1178
1179 extern boolean_t        thread_should_abort(
1180         thread_t);
1181
1182 extern int is_64signalregset(void);
1183
1184 extern void act_set_kperf(thread_t);
1185 extern void act_set_astledger(thread_t thread);
1186 extern void act_set_astledger_async(thread_t thread);
1187 extern void act_set_io_telemetry_ast(thread_t);
1188
1189 extern uint32_t dtrace_get_thread_predcache(thread_t);
1190 extern int64_t dtrace_get_thread_vtime(thread_t);
1191 extern int64_t dtrace_get_thread_tracing(thread_t);
1192 extern uint16_t dtrace_get_thread_inprobe(thread_t);
1193 extern int dtrace_get_thread_last_cpu_id(thread_t);
1194 extern vm_offset_t dtrace_get_kernel_stack(thread_t);
1195 extern void dtrace_set_thread_predcache(thread_t, uint32_t);
1196 extern void dtrace_set_thread_vtime(thread_t, int64_t);
1197 extern void dtrace_set_thread_tracing(thread_t, int64_t);
1198 extern void dtrace_set_thread_inprobe(thread_t, uint16_t);
1199 extern vm_offset_t dtrace_set_thread_recover(thread_t, vm_offset_t);
1200 extern vm_offset_t dtrace_sign_and_set_thread_recover(thread_t, vm_offset_t);
1201 extern void dtrace_thread_bootstrap(void);
1202 extern void dtrace_thread_didexec(thread_t);
1203
1204 extern int64_t dtrace_calc_thread_recent_vtime(thread_t);
1205
1206
1207 extern kern_return_t    thread_set_wq_state32(
1208         thread_t          thread,
1209         thread_state_t    tstate);
1210
1211 extern kern_return_t    thread_set_wq_state64(
1212         thread_t          thread,
1213         thread_state_t    tstate);
1214
1215 extern vm_offset_t      kernel_stack_mask;
1216 extern vm_offset_t      kernel_stack_size;
1217 extern vm_offset_t      kernel_stack_depth_max;
1218
1219 extern void guard_ast(thread_t);
1220 extern void fd_guard_ast(thread_t,
1221     mach_exception_code_t, mach_exception_subcode_t);
1222 #if CONFIG_VNGUARD
1223 extern void vn_guard_ast(thread_t,
1224     mach_exception_code_t, mach_exception_subcode_t);
1225 #endif
1226 extern void mach_port_guard_ast(thread_t,
1227     mach_exception_code_t, mach_exception_subcode_t);
1228 extern void virt_memory_guard_ast(thread_t,
1229     mach_exception_code_t, mach_exception_subcode_t);
1230 extern void thread_guard_violation(thread_t,
1231     mach_exception_code_t, mach_exception_subcode_t, boolean_t);
1232 extern void thread_update_io_stats(thread_t, int size, int io_flags);
1233
1234 extern kern_return_t    thread_set_voucher_name(mach_port_name_t name);
1235 extern kern_return_t thread_get_current_voucher_origin_pid(int32_t *pid);
1236
1237 extern void set_thread_rwlock_boost(void);
1238 extern void clear_thread_rwlock_boost(void);
1239
1240 extern void thread_enable_send_importance(thread_t thread, boolean_t enable);
1241
1242 /*
1243  * Translate signal context data pointer to userspace representation
1244  */
1245
1246 extern kern_return_t    machine_thread_siguctx_pointer_convert_to_user(
1247         thread_t thread,
1248         user_addr_t *uctxp);
1249
1250 extern void machine_tecs(thread_t thr);
1251
1252 typedef enum cpuvn {
1253         CPUVN_CI = 1
1254 } cpuvn_e;
1255
1256 extern int machine_csv(cpuvn_e cve);
1257
1258 /*
1259  * Translate array of function pointer syscall arguments from userspace representation
1260  */
1261
1262 extern kern_return_t    machine_thread_function_pointers_convert_from_user(
1263         thread_t thread,
1264         user_addr_t *fptrs,
1265         uint32_t count);
1266
1267 /* Get a backtrace for a threads kernel or user stack (user_p), with pc and optionally
1268  * frame pointer (getfp). Returns bytes added to buffer, and kThreadTruncatedBT in
1269  * thread_trace_flags if a user page is not present after kdp_lightweight_fault() is
1270  * called.
1271  */
1272
1273 extern int                              machine_trace_thread(
1274         thread_t thread,
1275         char *tracepos,
1276         char *tracebound,
1277         int nframes,
1278         boolean_t user_p,
1279         boolean_t getfp,
1280         uint32_t *thread_trace_flags);
1281
1282 extern int                              machine_trace_thread64(thread_t thread,
1283     char *tracepos,
1284     char *tracebound,
1285     int nframes,
1286     boolean_t user_p,
1287     boolean_t getfp,
1288     uint32_t *thread_trace_flags,
1289     uint64_t *sp);
1290
1291 /*
1292  * Get the duration of the given thread's last wait.
1293  */
1294 uint64_t thread_get_last_wait_duration(thread_t thread);
1295
1296 extern void thread_set_no_smt(bool set);
1297 extern bool thread_get_no_smt(void);
1298
1299 #endif  /* XNU_KERNEL_PRIVATE */
1300
1301 /*! @function thread_has_thread_name
1302  *   @abstract Checks if a thread has a name.
1303  *   @discussion This function takes one input, a thread, and returns a boolean value indicating if that thread already has a name associated with it.
1304  *   @param th The thread to inspect.
1305  *   @result TRUE if the thread has a name, FALSE otherwise.
1306  */
1307 extern boolean_t thread_has_thread_name(thread_t th);
1308
1309 /*! @function thread_set_thread_name
1310  *   @abstract Set a thread's name.
1311  *   @discussion This function takes two input parameters: a thread to name, and the name to apply to the thread.  The name will be copied over to the thread in order to better identify the thread.  If the name is longer than MAXTHREADNAMESIZE - 1, it will be truncated.
1312  *   @param th The thread to be named.
1313  *   @param name The name to apply to the thread.
1314  */
1315 extern void thread_set_thread_name(thread_t th, const char* name);
1316
1317 /*! @function kernel_thread_start
1318  *   @abstract Create a kernel thread.
1319  *   @discussion This function takes three input parameters, namely reference to the function that the thread should execute, caller specified data and a reference which is used to return the newly created kernel thread. The function returns KERN_SUCCESS on success or an appropriate kernel code type indicating the error. It may be noted that the caller is responsible for explicitly releasing the reference to the created thread when no longer needed. This should be done by calling thread_deallocate(new_thread).
1320  *   @param continuation A C-function pointer where the thread will begin execution.
1321  *   @param parameter Caller specified data to be passed to the new thread.
1322  *   @param new_thread Reference to the new thread is returned in this parameter.
1323  *   @result Returns KERN_SUCCESS on success or an appropriate kernel code type.
1324  */
1325
1326 extern kern_return_t    kernel_thread_start(
1327         thread_continue_t       continuation,
1328         void                            *parameter,
1329         thread_t                        *new_thread);
1330
1331 #ifdef KERNEL_PRIVATE
1332 void thread_set_eager_preempt(thread_t thread);
1333 void thread_clear_eager_preempt(thread_t thread);
1334 void thread_set_honor_qlimit(thread_t thread);
1335 void thread_clear_honor_qlimit(thread_t thread);
1336 extern ipc_port_t convert_thread_to_port(thread_t);
1337 extern ipc_port_t convert_thread_inspect_to_port(thread_inspect_t);
1338 extern boolean_t is_vm_privileged(void);
1339 extern boolean_t set_vm_privilege(boolean_t);
1340 extern kern_allocation_name_t thread_set_allocation_name(kern_allocation_name_t new_name);
1341 extern void *thread_iokit_tls_get(uint32_t index);
1342 extern void thread_iokit_tls_set(uint32_t index, void * data);
1343 #endif /* KERNEL_PRIVATE */
1344
1345 __END_DECLS
1346
1347 #endif  /* _KERN_THREAD_H_ */