]> git.saurik.com Git - apple/xnu.git/blobdiff - osfmk/kern/thread.h
projects / apple / xnu.git / blobdiff
? search:
summary | shortlog | log | commit | commitdiff | tree
raw | inline | side by side
xnu-4570.71.2.tar.gz
[apple/xnu.git] / osfmk / kern / thread.h
diff --git a/osfmk/kern/thread.h b/osfmk/kern/thread.h
index a81bc6c8ddf44c2e6b13edb17aa739798edae589..ca2217584b3ad5fa91efa71b055004e77bc2238a 100644 (file)
--- a/osfmk/kern/thread.h
+++ b/osfmk/kern/thread.h
@@ -1,5 +1,5 @@
 /*
 /*
- * Copyright (c) 2000-2014 Apple Inc. All rights reserved.
+ * Copyright (c) 2000-2016 Apple Inc. All rights reserved.
  *
  * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
  *
  * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
@@ -94,6 +94,7 @@
 #include <mach/exception_types.h>
 
 #include <kern/kern_types.h>
 #include <mach/exception_types.h>
 
 #include <kern/kern_types.h>
+#include <vm/vm_kern.h>
 
 #include <sys/cdefs.h>
 
 
 #include <sys/cdefs.h>
 
@@ -106,6 +107,7 @@
 
 #include <mach/port.h>
 #include <kern/cpu_number.h>
 
 #include <mach/port.h>
 #include <kern/cpu_number.h>
+#include <kern/smp.h>
 #include <kern/queue.h>
 #include <kern/timer.h>
 #include <kern/simple_lock.h>
 #include <kern/queue.h>
 #include <kern/timer.h>
 #include <kern/simple_lock.h>
@@ -114,17 +116,40 @@
 #include <kern/sched_prim.h>
 #include <mach/sfi_class.h>
 #include <kern/thread_call.h>
 #include <kern/sched_prim.h>
 #include <mach/sfi_class.h>
 #include <kern/thread_call.h>
+#include <kern/thread_group.h>
 #include <kern/timer_call.h>
 #include <kern/task.h>
 #include <kern/exception.h>
 #include <kern/affinity.h>
 #include <kern/timer_call.h>
 #include <kern/task.h>
 #include <kern/exception.h>
 #include <kern/affinity.h>
+#include <kern/debug.h>
+#include <kern/block_hint.h>
+
+#include <kern/waitq.h>
+#include <san/kasan.h>
 
 #include <ipc/ipc_kmsg.h>
 
 #include <machine/cpu_data.h>
 #include <machine/thread.h>
 
 
 #include <ipc/ipc_kmsg.h>
 
 #include <machine/cpu_data.h>
 #include <machine/thread.h>
 
+#if MONOTONIC
+#include <stdatomic.h>
+#include <machine/monotonic.h>
+#endif /* MONOTONIC */
+
+#if CONFIG_EMBEDDED
+/* Taskwatch related. TODO: find this a better home */
+typedef struct task_watcher task_watch_t;
+#endif /* CONFIG_EMBEDDED */
+
 struct thread {
 struct thread {
+
+#if MACH_ASSERT
+#define THREAD_MAGIC 0x1234ABCDDCBA4321ULL
+       /* Ensure nothing uses &thread as a queue entry */
+       uint64_t                thread_magic;
+#endif /* MACH_ASSERT */
+
        /*
         *      NOTE:   The runq field in the thread structure has an unusual
         *      locking protocol.  If its value is PROCESSOR_NULL, then it is
        /*
         *      NOTE:   The runq field in the thread structure has an unusual
         *      locking protocol.  If its value is PROCESSOR_NULL, then it is
@@ -134,19 +159,24 @@ struct thread {
         *      transition from PROCESSOR_NULL to non-null must be done
         *      under the thread lock and the run queue lock.
         *
         *      transition from PROCESSOR_NULL to non-null must be done
         *      under the thread lock and the run queue lock.
         *
-        *      When the thread is on a wait queue, these first three fields
-        *      are treated as an unofficial union with a wait_queue_element.
-        *      If you change these, you must change that definition as well
-        *      (kern/wait_queue.h).
+        *      New waitq APIs allow the 'links' and 'runq' fields to be
+        *      anywhere in the thread structure.
         */
         */
-       /* Items examined often, modified infrequently */
-       queue_chain_t   links;                          /* run/wait queue links */
-       processor_t             runq;                           /* run queue assignment */
-       wait_queue_t    wait_queue;                     /* wait queue we are currently on */
-       event64_t               wait_event;                     /* wait queue event */
+       union {
+               queue_chain_t           runq_links;     /* run queue links */
+               queue_chain_t           wait_links;     /* wait queue links */
+       };
+
+       processor_t             runq;           /* run queue assignment */
+
+       event64_t               wait_event;     /* wait queue event */
+       struct waitq           *waitq;          /* wait queue this thread is enqueued on */
+
        /* Data updated during assert_wait/thread_wakeup */
        /* Data updated during assert_wait/thread_wakeup */
+#if __SMP__
        decl_simple_lock_data(,sched_lock)      /* scheduling lock (thread_lock()) */
        decl_simple_lock_data(,wake_lock)       /* for thread stop / wait (wake_lock()) */
        decl_simple_lock_data(,sched_lock)      /* scheduling lock (thread_lock()) */
        decl_simple_lock_data(,wake_lock)       /* for thread stop / wait (wake_lock()) */
+#endif
        integer_t               options;                        /* options set by thread itself */
 #define TH_OPT_INTMASK         0x0003          /* interrupt / abort level */
 #define TH_OPT_VMPRIV          0x0004          /* may allocate reserved memory */
        integer_t               options;                        /* options set by thread itself */
 #define TH_OPT_INTMASK         0x0003          /* interrupt / abort level */
 #define TH_OPT_VMPRIV          0x0004          /* may allocate reserved memory */
@@ -154,21 +184,31 @@ struct thread {
 #define TH_OPT_SYSTEM_CRITICAL 0x0010          /* Thread must always be allowed to run - even under heavy load */
 #define TH_OPT_PROC_CPULIMIT   0x0020          /* Thread has a task-wide CPU limit applied to it */
 #define TH_OPT_PRVT_CPULIMIT   0x0040          /* Thread has a thread-private CPU limit applied to it */
 #define TH_OPT_SYSTEM_CRITICAL 0x0010          /* Thread must always be allowed to run - even under heavy load */
 #define TH_OPT_PROC_CPULIMIT   0x0020          /* Thread has a task-wide CPU limit applied to it */
 #define TH_OPT_PRVT_CPULIMIT   0x0040          /* Thread has a thread-private CPU limit applied to it */
-#define TH_OPT_IDLE_THREAD             0x0080          /* Thread is a per-processor idle thread */
+#define TH_OPT_IDLE_THREAD     0x0080          /* Thread is a per-processor idle thread */
+#define TH_OPT_GLOBAL_FORCED_IDLE      0x0100  /* Thread performs forced idle for thermal control */
+#define TH_OPT_SCHED_VM_GROUP  0x0200          /* Thread belongs to special scheduler VM group */
+#define TH_OPT_HONOR_QLIMIT    0x0400          /* Thread will honor qlimit while sending mach_msg, regardless of MACH_SEND_ALWAYS */
+#define TH_OPT_SEND_IMPORTANCE 0x0800          /* Thread will allow importance donation from kernel rpc */
+#define TH_OPT_ZONE_GC         0x1000          /* zone_gc() called on this thread */
 
        boolean_t                       wake_active;    /* wake event on stop */
        int                                     at_safe_point;  /* thread_abort_safely allowed */
        ast_t                           reason;                 /* why we blocked */
 
        boolean_t                       wake_active;    /* wake event on stop */
        int                                     at_safe_point;  /* thread_abort_safely allowed */
        ast_t                           reason;                 /* why we blocked */
+       uint32_t                        quantum_remaining;
+       wait_result_t                   wait_result;    /* outcome of wait -
+                                                        * may be examined by this thread
+                                                        * WITHOUT locking */
        thread_continue_t       continuation;   /* continue here next dispatch */
        void                            *parameter;             /* continuation parameter */
        thread_continue_t       continuation;   /* continue here next dispatch */
        void                            *parameter;             /* continuation parameter */
-       wait_result_t           wait_result;    /* outcome of wait -
-                                                                                * may be examined by this thread
-                                                                                * WITHOUT locking */
 
        /* Data updated/used in thread_invoke */
        vm_offset_t             kernel_stack;           /* current kernel stack */
        vm_offset_t                     reserved_stack;         /* reserved kernel stack */
 
 
        /* Data updated/used in thread_invoke */
        vm_offset_t             kernel_stack;           /* current kernel stack */
        vm_offset_t                     reserved_stack;         /* reserved kernel stack */
 
+#if KASAN
+       struct kasan_thread_data kasan_data;
+#endif
+
        /* Thread state: */
        int                                     state;
 /*
        /* Thread state: */
        int                                     state;
 /*
@@ -178,8 +218,8 @@ struct thread {
 #define TH_SUSP                        0x02                    /* stopped or requested to stop */
 #define TH_RUN                 0x04                    /* running or on runq */
 #define TH_UNINT               0x08                    /* waiting uninteruptibly */
 #define TH_SUSP                        0x02                    /* stopped or requested to stop */
 #define TH_RUN                 0x04                    /* running or on runq */
 #define TH_UNINT               0x08                    /* waiting uninteruptibly */
-#define        TH_TERMINATE    0x10                    /* halted at termination */
-#define        TH_TERMINATE2   0x20                    /* added to termination queue */
+#define TH_TERMINATE           0x10                    /* halted at termination */
+#define TH_TERMINATE2          0x20                    /* added to termination queue */
 
 #define TH_IDLE                        0x80                    /* idling processor */
 
 
 #define TH_IDLE                        0x80                    /* idling processor */
 
@@ -187,14 +227,18 @@ struct thread {
        sched_mode_t                    sched_mode;             /* scheduling mode */
        sched_mode_t                    saved_mode;             /* saved mode during forced mode demotion */
 
        sched_mode_t                    sched_mode;             /* scheduling mode */
        sched_mode_t                    saved_mode;             /* saved mode during forced mode demotion */
 
+       /* This thread's contribution to global sched counters */
+       sched_bucket_t                  th_sched_bucket;
+
        sfi_class_id_t                  sfi_class;              /* SFI class (XXX Updated on CSW/QE/AST) */
        sfi_class_id_t                  sfi_wait_class; /* Currently in SFI wait for this class, protected by sfi_lock */
        sfi_class_id_t                  sfi_class;              /* SFI class (XXX Updated on CSW/QE/AST) */
        sfi_class_id_t                  sfi_wait_class; /* Currently in SFI wait for this class, protected by sfi_lock */
-       
+
+
        uint32_t                        sched_flags;            /* current flag bits */
        uint32_t                        sched_flags;            /* current flag bits */
-#define TH_SFLAG_FAIRSHARE_TRIPPED     0x0001          /* fairshare scheduling activated */
+/* TH_SFLAG_FAIRSHARE_TRIPPED (unused) 0x0001 */
 #define TH_SFLAG_FAILSAFE              0x0002          /* fail-safe has tripped */
 #define TH_SFLAG_FAILSAFE              0x0002          /* fail-safe has tripped */
-#define TH_SFLAG_THROTTLED             0x0004          /* thread treated as background for scheduler decay purposes */
-#define TH_SFLAG_DEMOTED_MASK      (TH_SFLAG_THROTTLE_DEMOTED | TH_SFLAG_FAILSAFE | TH_SFLAG_FAIRSHARE_TRIPPED)        /* saved_mode contains previous sched_mode */
+#define TH_SFLAG_THROTTLED             0x0004          /* throttled thread forced to timeshare mode (may be applied in addition to failsafe) */
+#define TH_SFLAG_DEMOTED_MASK      (TH_SFLAG_THROTTLED | TH_SFLAG_FAILSAFE)    /* saved_mode contains previous sched_mode */
 
 #define        TH_SFLAG_PROMOTED               0x0008          /* sched pri has been promoted */
 #define TH_SFLAG_ABORT                 0x0010          /* abort interruptible waits */
 
 #define        TH_SFLAG_PROMOTED               0x0008          /* sched pri has been promoted */
 #define TH_SFLAG_ABORT                 0x0010          /* abort interruptible waits */
@@ -203,18 +247,23 @@ struct thread {
 #define        TH_SFLAG_DEPRESS                0x0040          /* normal depress yield */
 #define TH_SFLAG_POLLDEPRESS           0x0080          /* polled depress yield */
 #define TH_SFLAG_DEPRESSED_MASK                (TH_SFLAG_DEPRESS | TH_SFLAG_POLLDEPRESS)
 #define        TH_SFLAG_DEPRESS                0x0040          /* normal depress yield */
 #define TH_SFLAG_POLLDEPRESS           0x0080          /* polled depress yield */
 #define TH_SFLAG_DEPRESSED_MASK                (TH_SFLAG_DEPRESS | TH_SFLAG_POLLDEPRESS)
-#define TH_SFLAG_PRI_UPDATE            0x0100          /* Updating priority */
+/* unused TH_SFLAG_PRI_UPDATE           0x0100 */
 #define TH_SFLAG_EAGERPREEMPT          0x0200          /* Any preemption of this thread should be treated as if AST_URGENT applied */
 #define TH_SFLAG_RW_PROMOTED           0x0400          /* sched pri has been promoted due to blocking with RW lock held */
 #define TH_SFLAG_EAGERPREEMPT          0x0200          /* Any preemption of this thread should be treated as if AST_URGENT applied */
 #define TH_SFLAG_RW_PROMOTED           0x0400          /* sched pri has been promoted due to blocking with RW lock held */
-#define TH_SFLAG_PROMOTED_MASK         (TH_SFLAG_PROMOTED | TH_SFLAG_RW_PROMOTED)
-#define TH_SFLAG_THROTTLE_DEMOTED      0x0800          /* throttled thread forced to timeshare mode (may be applied in addition to failsafe) */
+/* unused TH_SFLAG_THROTTLE_DEMOTED     0x0800 */
+#define TH_SFLAG_WAITQ_PROMOTED                0x1000          /* sched pri promoted from waitq wakeup (generally for IPC receive) */
+
+
+#define TH_SFLAG_EXEC_PROMOTED          0x8000         /* sched pri has been promoted since thread is in an exec */
+#define TH_SFLAG_PROMOTED_MASK         (TH_SFLAG_PROMOTED | TH_SFLAG_RW_PROMOTED | TH_SFLAG_WAITQ_PROMOTED | TH_SFLAG_EXEC_PROMOTED)
 
 #define TH_SFLAG_RW_PROMOTED_BIT       (10)    /* 0x400 */
 
 
 #define TH_SFLAG_RW_PROMOTED_BIT       (10)    /* 0x400 */
 
-       int16_t                         sched_pri;                      /* scheduled (current) priority */
-       int16_t                         priority;                       /* base priority */
-       int16_t                         max_priority;           /* copy of max base priority */
-       int16_t                         task_priority;          /* copy of task base priority */
+       int16_t                         sched_pri;              /* scheduled (current) priority */
+       int16_t                         base_pri;               /* base priority */
+       int16_t                         max_priority;           /* copy of max base priority */
+       int16_t                         task_priority;          /* copy of task base priority */
+
 #if defined(CONFIG_SCHED_GRRR)
 #if 0
        uint16_t                        grrr_deficit;           /* fixed point (1/1000th quantum) fractional deficit */
 #if defined(CONFIG_SCHED_GRRR)
 #if 0
        uint16_t                        grrr_deficit;           /* fixed point (1/1000th quantum) fractional deficit */
@@ -223,16 +272,13 @@ struct thread {
        
        int16_t                         promotions;                     /* level of promotion */
        int16_t                         pending_promoter_index;
        
        int16_t                         promotions;                     /* level of promotion */
        int16_t                         pending_promoter_index;
-       uint32_t                        ref_count;              /* number of references to me */
+       _Atomic uint32_t                ref_count;              /* number of references to me */
        void                            *pending_promoter[2];
 
        uint32_t                        rwlock_count;   /* Number of lck_rw_t locks held by thread */
 
        void                            *pending_promoter[2];
 
        uint32_t                        rwlock_count;   /* Number of lck_rw_t locks held by thread */
 
-#if MACH_ASSERT
-       uint32_t                        SHARE_COUNT, BG_COUNT; /* This thread's contribution to global sched counters (temporary debugging) */
-#endif /* MACH_ASSERT */
-
        integer_t                       importance;                     /* task-relative importance */
        integer_t                       importance;                     /* task-relative importance */
+       uint32_t                        was_promoted_on_wakeup;
 
        /* Priority depression expiration */
        integer_t                       depress_timer_active;
 
        /* Priority depression expiration */
        integer_t                       depress_timer_active;
@@ -246,9 +292,12 @@ struct thread {
                uint64_t                        deadline;
        }                                       realtime;
 
                uint64_t                        deadline;
        }                                       realtime;
 
-       uint32_t                        was_promoted_on_wakeup;
        uint64_t                        last_run_time;          /* time when thread was switched away from */
        uint64_t                        last_run_time;          /* time when thread was switched away from */
-       uint32_t                        quantum_remaining;                      /* duration of current quantum remaining */
+       uint64_t                        last_made_runnable_time;        /* time when thread was unblocked or preempted */
+       uint64_t                        last_basepri_change_time;       /* time when thread was last changed in basepri while runnable */
+       uint64_t                        same_pri_latency;
+#define THREAD_NOT_RUNNABLE (~0ULL)
+
 
 #if defined(CONFIG_SCHED_MULTIQ)
        sched_group_t                   sched_group;
 
 #if defined(CONFIG_SCHED_MULTIQ)
        sched_group_t                   sched_group;
@@ -286,6 +335,7 @@ struct thread {
        uint32_t                        p_switch;               /* total processor switches */
        uint32_t                        ps_switch;              /* total pset switches */
 
        uint32_t                        p_switch;               /* total processor switches */
        uint32_t                        ps_switch;              /* total pset switches */
 
+       integer_t mutex_count;  /* total count of locks held */
        /* Timing data structures */
        int                                     precise_user_kernel_time; /* precise user/kernel enabled for this thread */
        timer_data_t            user_timer;                     /* user mode timer */
        /* Timing data structures */
        int                                     precise_user_kernel_time; /* precise user/kernel enabled for this thread */
        timer_data_t            user_timer;                     /* user mode timer */
@@ -294,9 +344,14 @@ struct thread {
        uint64_t                        vtimer_user_save;       /* saved values for vtimers */
        uint64_t                        vtimer_prof_save;
        uint64_t                        vtimer_rlim_save;
        uint64_t                        vtimer_user_save;       /* saved values for vtimers */
        uint64_t                        vtimer_prof_save;
        uint64_t                        vtimer_rlim_save;
+       uint64_t                        vtimer_qos_save;
+
+       timer_data_t            ptime;                  /* time executing in P mode */
 
 
+#if CONFIG_SCHED_SFI
        /* Timing for wait state */
        uint64_t                wait_sfi_begin_time;    /* start time for thread waiting in SFI */
        /* Timing for wait state */
        uint64_t                wait_sfi_begin_time;    /* start time for thread waiting in SFI */
+#endif
 
        /* Timed wait expiration */
        timer_call_data_t       wait_timer;
 
        /* Timed wait expiration */
        timer_call_data_t       wait_timer;
@@ -311,18 +366,26 @@ struct thread {
        affinity_set_t                  affinity_set;
        queue_chain_t                   affinity_threads;
 
        affinity_set_t                  affinity_set;
        queue_chain_t                   affinity_threads;
 
-       /* Various bits of stashed state */
+       /* Various bits of state to stash across a continuation, exclusive to the current thread block point */
        union {
                struct {
                        mach_msg_return_t       state;          /* receive state */
                        mach_port_seqno_t       seqno;          /* seqno of recvd message */
                        ipc_object_t            object;         /* object received on */
                        mach_vm_address_t       msg_addr;       /* receive buffer pointer */
        union {
                struct {
                        mach_msg_return_t       state;          /* receive state */
                        mach_port_seqno_t       seqno;          /* seqno of recvd message */
                        ipc_object_t            object;         /* object received on */
                        mach_vm_address_t       msg_addr;       /* receive buffer pointer */
-                       mach_msg_size_t         msize;          /* max size for recvd msg */
+                       mach_msg_size_t         rsize;          /* max size for recvd msg */
+                       mach_msg_size_t         msize;          /* actual size for recvd msg */
                        mach_msg_option_t       option;         /* options for receive */
                        mach_msg_option_t       option;         /* options for receive */
-                       mach_msg_size_t         slist_size;     /* scatter list size */
                        mach_port_name_t        receiver_name;  /* the receive port name */
                        mach_port_name_t        receiver_name;  /* the receive port name */
-                       struct ipc_kmsg         *kmsg;          /* received message */
+                       struct knote            *knote;         /* knote fired for rcv */
+                       union {
+                               struct ipc_kmsg   *kmsg;        /* received message */
+                               struct ipc_mqueue *peekq;       /* mqueue to peek at */
+                               struct {
+                                       mach_msg_priority_t qos;        /* received message qos */
+                                       mach_msg_priority_t oqos;       /* override qos for message */
+                               } received_qos;
+                       };
                        mach_msg_continue_t     continuation;
                } receive;
                struct {
                        mach_msg_continue_t     continuation;
                } receive;
                struct {
@@ -336,16 +399,27 @@ struct thread {
                        int                                     option;         /* switch option */
                        boolean_t                               reenable_workq_callback;        /* on entry, callbacks were suspended */
                } swtch;
                        int                                     option;         /* switch option */
                        boolean_t                               reenable_workq_callback;        /* on entry, callbacks were suspended */
                } swtch;
-               int                                             misc;           /* catch-all for other state */
        } saved;
 
        } saved;
 
-       /* Structure to save information about guard exception */
-       struct {
-               unsigned                                type;           /* EXC_GUARD reason/type */
-               mach_exception_data_type_t              code;           /* Exception code */
-               mach_exception_data_type_t              subcode;        /* Exception sub-code */
-       } guard_exc_info;
+       /* Only user threads can cause guard exceptions, only kernel threads can be thread call threads */
+       union {
+               /* Group and call this thread is working on behalf of */
+               struct {
+                       struct thread_call_group * thc_group;
+                       struct thread_call *       thc_call;                    /* debug only, may be deallocated */
+               } thc_state;
 
 
+               /* Structure to save information about guard exception */
+               struct {
+                       mach_exception_code_t           code;
+                       mach_exception_subcode_t        subcode;
+               } guard_exc_info;
+       };
+
+       /* Kernel holds on this thread  */
+       int16_t                                         suspend_count;
+       /* User level suspensions */
+       int16_t                                         user_stop_count;
 
        /* IPC data structures */
 #if IMPORTANCE_INHERITANCE
 
        /* IPC data structures */
 #if IMPORTANCE_INHERITANCE
@@ -362,9 +436,6 @@ struct thread {
        /* Activation */
                queue_chain_t                   task_threads;
 
        /* Activation */
                queue_chain_t                   task_threads;
 
-               /*** Machine-dependent state ***/
-               struct machine_thread   machine;
-
                /* Task membership */
                struct task                             *task;
                vm_map_t                                map;
                /* Task membership */
                struct task                             *task;
                vm_map_t                                map;
@@ -377,23 +448,19 @@ struct thread {
 
                /* Miscellaneous bits guarded by mutex */
                uint32_t
 
                /* Miscellaneous bits guarded by mutex */
                uint32_t
-                       active:1,                               /* Thread is active and has not been terminated */
-                       started:1,                              /* Thread has been started after creation */
-                       static_param:1,                 /* Disallow policy parameter changes */
-                       policy_reset:1,                 /* Disallow policy parameter changes on terminating threads */
+                       active:1,                       /* Thread is active and has not been terminated */
+                       started:1,                      /* Thread has been started after creation */
+                       static_param:1,                 /* Disallow policy parameter changes */
+                       inspection:1,                   /* TRUE when task is being inspected by crash reporter */
+                       policy_reset:1,                 /* Disallow policy parameter changes on terminating threads */
+                       suspend_parked:1,               /* thread parked in thread_suspended */
+                       corpse_dup:1,                   /* TRUE when thread is an inactive duplicate in a corpse */
                        :0;
 
                        :0;
 
-               /* Return Handers */
-               struct ReturnHandler {
-                       struct ReturnHandler    *next;
-                       void            (*handler)(
-                                                       struct ReturnHandler            *rh,
-                                                       struct thread                           *thread);
-               } *handlers, special_handler;
-
                /* Ports associated with this thread */
                struct ipc_port                 *ith_self;              /* not a right, doesn't hold ref */
                struct ipc_port                 *ith_sself;             /* a send right */
                /* Ports associated with this thread */
                struct ipc_port                 *ith_self;              /* not a right, doesn't hold ref */
                struct ipc_port                 *ith_sself;             /* a send right */
+               struct ipc_port                 *ith_special_reply_port; /* ref to special reply port */
                struct exception_action *exc_actions;
 
 #ifdef MACH_BSD
                struct exception_action *exc_actions;
 
 #ifdef MACH_BSD
@@ -410,35 +477,36 @@ struct thread {
 
                clock_sec_t t_page_creation_time;
                uint32_t    t_page_creation_count;
 
                clock_sec_t t_page_creation_time;
                uint32_t    t_page_creation_count;
+               uint32_t    t_page_creation_throttled;
+#if (DEVELOPMENT || DEBUG)
+               uint64_t    t_page_creation_throttled_hard;
+               uint64_t    t_page_creation_throttled_soft;
+#endif /* DEVELOPMENT || DEBUG */
 
 
-#define T_CHUD_MARKED           0x01          /* this thread is marked by CHUD */
-#define T_IN_CHUD               0x02          /* this thread is already in a CHUD handler */
-#define THREAD_PMC_FLAG         0x04          /* Bit in "t_chud" signifying PMC interest */    
-#define T_AST_CALLSTACK         0x08          /* Thread scheduled to dump a
-                                              * callstack on its next
-                                              * AST */
-#define T_AST_NAME              0x10          /* Thread scheduled to dump
-                                              * its name on its next
-                                              * AST */
-#define T_NAME_DONE             0x20          /* Thread has previously
-                                              * recorded its name */
-#define T_KPC_ALLOC             0x40          /* Thread needs a kpc_buf */
+#ifdef KPERF
+/* The high 7 bits are the number of frames to sample of a user callstack. */
+#define T_KPERF_CALLSTACK_DEPTH_OFFSET     (25)
+#define T_KPERF_SET_CALLSTACK_DEPTH(DEPTH) (((uint32_t)(DEPTH)) << T_KPERF_CALLSTACK_DEPTH_OFFSET)
+#define T_KPERF_GET_CALLSTACK_DEPTH(FLAGS) ((FLAGS) >> T_KPERF_CALLSTACK_DEPTH_OFFSET)
+#endif
 
 
-               uint32_t t_chud;        /* CHUD flags, used for Shark */
-               uint32_t chud_c_switch; /* last dispatch detection */
+#define T_KPERF_AST_CALLSTACK (1U << 0) /* dump a callstack on thread's next AST */
+#define T_KPERF_AST_DISPATCH  (1U << 1) /* dump a name on thread's next AST */
+#define T_KPC_ALLOC           (1U << 2) /* thread needs a kpc_buf allocated */
+/* only go up to T_KPERF_CALLSTACK_DEPTH_OFFSET - 1 */
 
 
-               integer_t mutex_count;  /* total count of locks held */
+#ifdef KPERF
+       uint32_t kperf_flags;
+       uint32_t kperf_pet_gen;  /* last generation of PET that sampled this thread*/
+       uint32_t kperf_c_switch; /* last dispatch detection */
+       uint32_t kperf_pet_cnt;  /* how many times a thread has been sampled by PET */
+#endif
 
 #ifdef KPC
        /* accumulated performance counters for this thread */
        uint64_t *kpc_buf;
 #endif
 
 
 #ifdef KPC
        /* accumulated performance counters for this thread */
        uint64_t *kpc_buf;
 #endif
 
-#ifdef KPERF
-       /* count of how many times a thread has been sampled since it was last scheduled */
-       uint64_t kperf_pet_cnt;
-#endif
-
 #if HYPERVISOR
        /* hypervisor virtual CPU object associated with this thread */
        void *hv_thread_target;
 #if HYPERVISOR
        /* hypervisor virtual CPU object associated with this thread */
        void *hv_thread_target;
@@ -451,16 +519,20 @@ struct thread {
        uint32_t                syscalls_mach;
        ledger_t                t_ledger;
        ledger_t                t_threadledger; /* per thread ledger */
        uint32_t                syscalls_mach;
        ledger_t                t_ledger;
        ledger_t                t_threadledger; /* per thread ledger */
-       uint64_t                cpu_time_last_qos;
-#ifdef CONFIG_BANK
-       ledger_t                t_bankledger;              /* ledger to charge someone */
-       uint64_t                t_deduct_bank_ledger_time; /* cpu time to be deducted from bank ledger */
-#endif
+       ledger_t                t_bankledger;                /* ledger to charge someone */
+       uint64_t                t_deduct_bank_ledger_time;   /* cpu time to be deducted from bank ledger */
+       uint64_t                t_deduct_bank_ledger_energy; /* energy to be deducted from bank ledger */
 
 
-       /* policy is protected by the task lock */
-       struct task_requested_policy     requested_policy;
-       struct task_effective_policy     effective_policy;
-       struct task_pended_policy        pended_policy;
+#if MONOTONIC
+       struct mt_thread t_monotonic;
+#endif /* MONOTONIC */
+
+       /*** Machine-dependent state ***/
+       struct machine_thread   machine;
+
+       /* policy is protected by the thread mutex */
+       struct thread_requested_policy  requested_policy;
+       struct thread_effective_policy  effective_policy;
 
        /* usynch override is protected by the task lock, eventually will be thread mutex */
        struct thread_qos_override {
 
        /* usynch override is protected by the task lock, eventually will be thread mutex */
        struct thread_qos_override {
@@ -471,11 +543,22 @@ struct thread {
                user_addr_t     override_resource;
        } *overrides;
 
                user_addr_t     override_resource;
        } *overrides;
 
+       _Atomic uint32_t kqwl_owning_count;
+       uint32_t        ipc_overrides;
+       uint32_t        sync_ipc_overrides;
+       uint32_t        user_promotions;
+       uint16_t        user_promotion_basepri;
+       _Atomic uint16_t kevent_ast_bits;
+
+       block_hint_t    pending_block_hint;
+       block_hint_t    block_hint;      /* What type of primitive last caused us to block. */
+
        int     iotier_override; /* atomic operations to set, cleared on ret to user */
        io_stat_info_t                  thread_io_stats; /* per-thread I/O statistics */
 
        int     iotier_override; /* atomic operations to set, cleared on ret to user */
        io_stat_info_t                  thread_io_stats; /* per-thread I/O statistics */
 
-
-       integer_t               saved_importance;               /* saved task-relative importance */
+#if CONFIG_EMBEDDED
+       task_watch_t *  taskwatch;              /* task watch */
+#endif /* CONFIG_EMBEDDED */
 
        uint32_t                        thread_callout_interrupt_wakeups;
        uint32_t                        thread_callout_platform_idle_wakeups;
 
        uint32_t                        thread_callout_interrupt_wakeups;
        uint32_t                        thread_callout_platform_idle_wakeups;
@@ -486,34 +569,53 @@ struct thread {
                                        callout_woken_from_platform_idle:1,
                                        callout_woke_thread:1,
                                        thread_bitfield_unused:13;
                                        callout_woken_from_platform_idle:1,
                                        callout_woke_thread:1,
                                        thread_bitfield_unused:13;
-       /* Kernel holds on this thread  */
-       int16_t                                         suspend_count;
-       /* User level suspensions */
-       int16_t                                         user_stop_count;
 
        mach_port_name_t                ith_voucher_name;
        ipc_voucher_t                   ith_voucher;
 #if CONFIG_IOSCHED
        void                            *decmp_upl;
 #endif /* CONFIG_IOSCHED */
 
        mach_port_name_t                ith_voucher_name;
        ipc_voucher_t                   ith_voucher;
 #if CONFIG_IOSCHED
        void                            *decmp_upl;
 #endif /* CONFIG_IOSCHED */
+
+       /* work interval (if any) associated with the thread. Uses thread mutex */
+       struct work_interval            *th_work_interval;
+
+#if    SCHED_TRACE_THREAD_WAKEUPS
+       uintptr_t               thread_wakeup_bt[64];
+#endif
 };
 
 };
 
-#define ith_state              saved.receive.state
-#define ith_object             saved.receive.object
-#define ith_msg_addr                   saved.receive.msg_addr
-#define ith_msize              saved.receive.msize
-#define        ith_option              saved.receive.option
-#define ith_scatter_list_size  saved.receive.slist_size
-#define ith_receiver_name      saved.receive.receiver_name
-#define ith_continuation       saved.receive.continuation
-#define ith_kmsg               saved.receive.kmsg
-#define ith_seqno              saved.receive.seqno
-
-#define sth_waitsemaphore      saved.sema.waitsemaphore
-#define sth_signalsemaphore    saved.sema.signalsemaphore
-#define sth_options            saved.sema.options
-#define sth_result             saved.sema.result
-#define sth_continuation       saved.sema.continuation
+#define ith_state           saved.receive.state
+#define ith_object          saved.receive.object
+#define ith_msg_addr        saved.receive.msg_addr
+#define ith_rsize           saved.receive.rsize
+#define ith_msize           saved.receive.msize
+#define        ith_option          saved.receive.option
+#define ith_receiver_name   saved.receive.receiver_name
+#define ith_continuation    saved.receive.continuation
+#define ith_kmsg            saved.receive.kmsg
+#define ith_peekq           saved.receive.peekq
+#define ith_knote           saved.receive.knote
+#define ith_qos             saved.receive.received_qos.qos
+#define ith_qos_override    saved.receive.received_qos.oqos
+#define ith_seqno           saved.receive.seqno
+
+#define sth_waitsemaphore   saved.sema.waitsemaphore
+#define sth_signalsemaphore saved.sema.signalsemaphore
+#define sth_options         saved.sema.options
+#define sth_result          saved.sema.result
+#define sth_continuation    saved.sema.continuation
+
+#define ITH_KNOTE_NULL      ((void *)NULL)
+#define ITH_KNOTE_PSEUDO    ((void *)0xdeadbeef)
+#define ITH_KNOTE_VALID(kn) ((kn) != ITH_KNOTE_NULL && (kn) != ITH_KNOTE_PSEUDO)
+
+#if MACH_ASSERT
+#define assert_thread_magic(thread) assertf((thread)->thread_magic == THREAD_MAGIC, \
+                                            "bad thread magic 0x%llx for thread %p, expected 0x%llx", \
+                                            (thread)->thread_magic, (thread), THREAD_MAGIC)
+#else
+#define assert_thread_magic(thread) do { (void)(thread); } while (0)
+#endif
 
 extern void                    thread_bootstrap(void);
 
 
 extern void                    thread_bootstrap(void);
 
@@ -522,10 +624,7 @@ extern void                        thread_init(void);
 extern void                    thread_daemon_init(void);
 
 #define        thread_reference_internal(thread)       \
 extern void                    thread_daemon_init(void);
 
 #define        thread_reference_internal(thread)       \
-                       (void)hw_atomic_add(&(thread)->ref_count, 1)
-
-#define thread_deallocate_internal(thread)     \
-                       hw_atomic_sub(&(thread)->ref_count, 1)
+                       (void)atomic_fetch_add_explicit(&(thread)->ref_count, 1, memory_order_relaxed)
 
 #define thread_reference(thread)                                       \
 MACRO_BEGIN                                                                                    \
 
 #define thread_reference(thread)                                       \
 MACRO_BEGIN                                                                                    \
@@ -536,17 +635,39 @@ MACRO_END
 extern void                    thread_deallocate(
                                                thread_t                thread);
 
 extern void                    thread_deallocate(
                                                thread_t                thread);
 
+extern void                    thread_deallocate_safe(
+                                               thread_t                thread);
+
+extern void                    thread_inspect_deallocate(
+                                               thread_inspect_t        thread);
+
 extern void                    thread_terminate_self(void);
 
 extern kern_return_t   thread_terminate_internal(
                                                        thread_t                thread);
 
 extern void                    thread_terminate_self(void);
 
 extern kern_return_t   thread_terminate_internal(
                                                        thread_t                thread);
 
-extern void                    thread_start_internal(
+extern void                    thread_start(
                                                        thread_t                        thread) __attribute__ ((noinline));
 
                                                        thread_t                        thread) __attribute__ ((noinline));
 
+extern void                    thread_start_in_assert_wait(
+                                                       thread_t                        thread,
+                                                       event_t             event,
+                                                       wait_interrupt_t    interruptible) __attribute__ ((noinline));
+
 extern void                    thread_terminate_enqueue(
                                                thread_t                thread);
 
 extern void                    thread_terminate_enqueue(
                                                thread_t                thread);
 
+extern void                    thread_exception_enqueue(
+                                               task_t          task,
+                                               thread_t        thread,
+                                               exception_type_t etype);
+
+extern void                    thread_copy_resource_info(
+                                               thread_t dst_thread,
+                                               thread_t src_thread);
+
+extern void                    thread_terminate_crashed_threads(void);
+
 extern void                    thread_stack_enqueue(
                                                thread_t                thread);
 
 extern void                    thread_stack_enqueue(
                                                thread_t                thread);
 
@@ -556,7 +677,12 @@ extern void                        thread_hold(
 extern void                    thread_release(
                                                thread_t        thread);
 
 extern void                    thread_release(
                                                thread_t        thread);
 
+extern void                    thread_corpse_continue(void);
+
+extern boolean_t               thread_is_active(thread_t thread);
 
 
+/* Locking for scheduler state, always acquired with interrupts disabled (splsched()) */
+#if __SMP__
 #define        thread_lock_init(th)    simple_lock_init(&(th)->sched_lock, 0)
 #define thread_lock(th)                        simple_lock(&(th)->sched_lock)
 #define thread_unlock(th)              simple_unlock(&(th)->sched_lock)
 #define        thread_lock_init(th)    simple_lock_init(&(th)->sched_lock, 0)
 #define thread_lock(th)                        simple_lock(&(th)->sched_lock)
 #define thread_unlock(th)              simple_unlock(&(th)->sched_lock)
@@ -564,6 +690,15 @@ extern void                        thread_release(
 #define wake_lock_init(th)             simple_lock_init(&(th)->wake_lock, 0)
 #define wake_lock(th)                  simple_lock(&(th)->wake_lock)
 #define wake_unlock(th)                        simple_unlock(&(th)->wake_lock)
 #define wake_lock_init(th)             simple_lock_init(&(th)->wake_lock, 0)
 #define wake_lock(th)                  simple_lock(&(th)->wake_lock)
 #define wake_unlock(th)                        simple_unlock(&(th)->wake_lock)
+#else
+#define thread_lock_init(th)   do { (void)th; } while(0)
+#define thread_lock(th)                        do { (void)th; } while(0)
+#define thread_unlock(th)              do { (void)th; } while(0)
+
+#define wake_lock_init(th)             do { (void)th; } while(0)
+#define wake_lock(th)                  do { (void)th; } while(0)
+#define wake_unlock(th)                        do { (void)th; } while(0)
+#endif
 
 #define thread_should_halt_fast(thread)                (!(thread)->active)
 
 
 #define thread_should_halt_fast(thread)                (!(thread)->active)
 
@@ -594,18 +729,7 @@ extern kern_return_t       thread_info_internal(
                                                        thread_info_t                   thread_info_out,
                                                        mach_msg_type_number_t  *thread_info_count);
 
                                                        thread_info_t                   thread_info_out,
                                                        mach_msg_type_number_t  *thread_info_count);
 
-extern void                            thread_task_priority(
-                                                       thread_t                thread,
-                                                       integer_t               priority,
-                                                       integer_t               max_priority);
-
-extern kern_return_t                    thread_set_mode_and_absolute_pri(
-                                                        thread_t       thread,
-                                                        integer_t      policy,
-                                                        integer_t      priority);
 
 
-extern void                            thread_policy_reset(
-                                                       thread_t                thread);
 
 extern kern_return_t   kernel_thread_create(
                                                        thread_continue_t       continuation,
 
 extern kern_return_t   kernel_thread_create(
                                                        thread_continue_t       continuation,
@@ -636,7 +760,8 @@ extern thread_t                     machine_switch_context(
                                                        thread_t                        new_thread);
 
 extern void                            machine_load_context(
                                                        thread_t                        new_thread);
 
 extern void                            machine_load_context(
-                                                       thread_t                thread);
+                                                       thread_t                thread) __attribute__((noreturn));
+
 
 extern kern_return_t   machine_thread_state_initialize(
                                                        thread_t                                thread);
 
 extern kern_return_t   machine_thread_state_initialize(
                                                        thread_t                                thread);
@@ -685,25 +810,13 @@ extern kern_return_t      machine_thread_set_tsd_base(
                                                        thread_t                                thread,
                                                        mach_vm_offset_t                tsd_base);
 
                                                        thread_t                                thread,
                                                        mach_vm_offset_t                tsd_base);
 
-typedef struct ReturnHandler           ReturnHandler;
-
 #define        thread_mtx_lock(thread)                 lck_mtx_lock(&(thread)->mutex)
 #define        thread_mtx_try(thread)                  lck_mtx_try_lock(&(thread)->mutex)
 #define        thread_mtx_unlock(thread)               lck_mtx_unlock(&(thread)->mutex)
 
 #define        thread_mtx_lock(thread)                 lck_mtx_lock(&(thread)->mutex)
 #define        thread_mtx_try(thread)                  lck_mtx_try_lock(&(thread)->mutex)
 #define        thread_mtx_unlock(thread)               lck_mtx_unlock(&(thread)->mutex)
 
-extern void                    act_execute_returnhandlers(void);
-
-extern void                    install_special_handler(
-                                               thread_t                thread);
-
-extern void                    special_handler(
-                                               ReturnHandler   *rh,
-                                               thread_t                thread);
+extern void thread_apc_ast(thread_t thread);
 
 
-extern void
-thread_update_qos_cpu_time(
-                       thread_t thread,
-                       boolean_t lock_needed);
+extern void thread_update_qos_cpu_time(thread_t thread);
 
 void act_machine_sv_free(thread_t, int);
 
 
 void act_machine_sv_free(thread_t, int);
 
@@ -718,12 +831,9 @@ static inline uint16_t     thread_get_tag_internal(thread_t        thread) {
        return thread->thread_tag;
 }
 
        return thread->thread_tag;
 }
 
-typedef struct {
-       int             qos_pri[THREAD_QOS_LAST];
-       int             qos_iotier[THREAD_QOS_LAST];
-       uint32_t        qos_through_qos[THREAD_QOS_LAST];
-       uint32_t        qos_latency_qos[THREAD_QOS_LAST];
-} qos_policy_params_t;
+
+extern void thread_set_options(uint32_t thopt);
+
 
 #else  /* MACH_KERNEL_PRIVATE */
 
 
 #else  /* MACH_KERNEL_PRIVATE */
 
@@ -745,7 +855,20 @@ __END_DECLS
 
 __BEGIN_DECLS
 
 
 __BEGIN_DECLS
 
-extern uint64_t                        thread_dispatchqaddr(
+extern void                    thread_starts_owning_workloop(
+                                               thread_t                thread);
+
+extern void                    thread_ends_owning_workloop(
+                                               thread_t                thread);
+
+extern uint32_t                thread_owned_workloops_count(
+                                               thread_t                thread);
+
+
+extern uint64_t                        thread_dispatchqaddr(
+                                               thread_t thread);
+
+extern uint64_t                        thread_rettokern_addr(
                                                thread_t thread);
 
 __END_DECLS
                                                thread_t thread);
 
 __END_DECLS
@@ -755,7 +878,7 @@ __END_DECLS
 #ifdef KERNEL
 __BEGIN_DECLS
 
 #ifdef KERNEL
 __BEGIN_DECLS
 
-extern uint64_t                        thread_tid(thread_t thread);
+extern uint64_t                        thread_tid(thread_t thread);
 
 __END_DECLS
 
 
 __END_DECLS
 
@@ -772,10 +895,12 @@ __BEGIN_DECLS
 #define        THREAD_TAG_CALLOUT 0x2
 #define        THREAD_TAG_IOWORKLOOP 0x4
 
 #define        THREAD_TAG_CALLOUT 0x2
 #define        THREAD_TAG_IOWORKLOOP 0x4
 
+#define        THREAD_TAG_PTHREAD 0x10
+#define        THREAD_TAG_WORKQUEUE 0x20
+
 uint16_t       thread_set_tag(thread_t, uint16_t);
 uint16_t       thread_get_tag(thread_t);
 
 uint16_t       thread_set_tag(thread_t, uint16_t);
 uint16_t       thread_get_tag(thread_t);
 
-
 extern kern_return_t    thread_state_initialize(
                                                        thread_t                                thread);
 
 extern kern_return_t    thread_state_initialize(
                                                        thread_t                                thread);
 
@@ -791,14 +916,32 @@ extern kern_return_t      thread_getstatus(
                                                        thread_state_t                  tstate,
                                                        mach_msg_type_number_t  *count);
 
                                                        thread_state_t                  tstate,
                                                        mach_msg_type_number_t  *count);
 
+extern kern_return_t   thread_create_with_continuation(
+                                                       task_t task,
+                                                       thread_t *new_thread,
+                                                       thread_continue_t continuation);
+
+extern kern_return_t thread_create_waiting(task_t               task,
+                                           thread_continue_t    continuation,
+                                           event_t              event,
+                                           thread_t             *new_thread);
+
 extern kern_return_t   thread_create_workq(
                                                        task_t                  task,
                                                        thread_continue_t       thread_return,
                                                        thread_t                *new_thread);
 
 extern kern_return_t   thread_create_workq(
                                                        task_t                  task,
                                                        thread_continue_t       thread_return,
                                                        thread_t                *new_thread);
 
+extern kern_return_t   thread_create_workq_waiting(
+                                                       task_t                  task,
+                                                       thread_continue_t       thread_return,
+                                                       event_t         event,
+                                                       thread_t                *new_thread);
+
 extern void    thread_yield_internal(
        mach_msg_timeout_t      interval);
 
 extern void    thread_yield_internal(
        mach_msg_timeout_t      interval);
 
+extern void    thread_yield_to_preemption(void);
+
 /*
  * Thread-private CPU limits: apply a private CPU limit to this thread only. Available actions are:
  * 
 /*
  * Thread-private CPU limits: apply a private CPU limit to this thread only. Available actions are:
  * 
@@ -852,7 +995,8 @@ extern kern_return_t        thread_userstack(
                                                thread_state_t,
                                                unsigned int,
                                                mach_vm_offset_t *,
                                                thread_state_t,
                                                unsigned int,
                                                mach_vm_offset_t *,
-                                               int *);
+                                               int *,
+                                               boolean_t);
 
 extern kern_return_t   thread_entrypoint(
                                                thread_t,
 
 extern kern_return_t   thread_entrypoint(
                                                thread_t,
@@ -862,8 +1006,8 @@ extern kern_return_t       thread_entrypoint(
                                                mach_vm_offset_t *); 
 
 extern kern_return_t   thread_userstackdefault(
                                                mach_vm_offset_t *); 
 
 extern kern_return_t   thread_userstackdefault(
-                                               thread_t,
-                                               mach_vm_offset_t *);
+                                               mach_vm_offset_t *,
+                                               boolean_t);
 
 extern kern_return_t   thread_wire_internal(
                                                        host_priv_t             host_priv,
 
 extern kern_return_t   thread_wire_internal(
                                                        host_priv_t             host_priv,
@@ -874,9 +1018,14 @@ extern kern_return_t      thread_wire_internal(
 
 extern kern_return_t   thread_dup(thread_t);
 
 
 extern kern_return_t   thread_dup(thread_t);
 
+extern kern_return_t thread_dup2(thread_t, thread_t);
+
+#if !defined(_SCHED_CALL_T_DEFINED)
+#define _SCHED_CALL_T_DEFINED
 typedef void   (*sched_call_t)(
                                        int                             type,
                                        thread_t                thread);
 typedef void   (*sched_call_t)(
                                        int                             type,
                                        thread_t                thread);
+#endif
 
 #define SCHED_CALL_BLOCK               0x1
 #define SCHED_CALL_UNBLOCK             0x2
 
 #define SCHED_CALL_BLOCK               0x1
 #define SCHED_CALL_UNBLOCK             0x2
@@ -885,6 +1034,14 @@ extern void               thread_sched_call(
                                        thread_t                thread,
                                        sched_call_t    call);
 
                                        thread_t                thread,
                                        sched_call_t    call);
 
+extern sched_call_t    thread_disable_sched_call(
+                                       thread_t                thread,
+                                       sched_call_t    call);
+
+extern void    thread_reenable_sched_call(
+                                       thread_t                thread,
+                                       sched_call_t    call);
+
 extern void            thread_static_param(
                                        thread_t                thread,
                                        boolean_t               state);
 extern void            thread_static_param(
                                        thread_t                thread,
                                        boolean_t               state);
@@ -892,16 +1049,6 @@ extern void               thread_static_param(
 extern boolean_t       thread_is_static_param(
                                        thread_t                thread);
 
 extern boolean_t       thread_is_static_param(
                                        thread_t                thread);
 
-extern kern_return_t   thread_policy_set_internal(
-                                       thread_t                thread,
-                                       thread_policy_flavor_t  flavor,
-                                       thread_policy_t         policy_info,
-                                       mach_msg_type_number_t  count);
-
-extern boolean_t thread_has_qos_policy(thread_t thread);
-
-extern kern_return_t thread_remove_qos_policy(thread_t thread);
-
 extern task_t  get_threadtask(thread_t);
 #define thread_is_64bit(thd)   \
        task_has_64BitAddr(get_threadtask(thd))
 extern task_t  get_threadtask(thread_t);
 #define thread_is_64bit(thd)   \
        task_has_64BitAddr(get_threadtask(thd))
@@ -910,9 +1057,16 @@ extern task_t     get_threadtask(thread_t);
 extern void            *get_bsdthread_info(thread_t);
 extern void            set_bsdthread_info(thread_t, void *);
 extern void            *uthread_alloc(task_t, thread_t, int);
 extern void            *get_bsdthread_info(thread_t);
 extern void            set_bsdthread_info(thread_t, void *);
 extern void            *uthread_alloc(task_t, thread_t, int);
-extern void            uthread_cleanup(task_t, void *, void *); 
+extern void            uthread_cleanup_name(void *uthread);
+extern void            uthread_cleanup(task_t, void *, void *);
 extern void            uthread_zone_free(void *); 
 extern void            uthread_zone_free(void *); 
-extern void            uthread_cred_free(void *); 
+extern void            uthread_cred_free(void *);
+
+extern void            uthread_reset_proc_refcount(void *);
+#if PROC_REF_DEBUG
+extern int             uthread_get_proc_refcount(void *);
+extern int             proc_ref_tracking_disabled;
+#endif
 
 extern boolean_t       thread_should_halt(
                                                thread_t                thread);
 
 extern boolean_t       thread_should_halt(
                                                thread_t                thread);
@@ -922,13 +1076,16 @@ extern boolean_t thread_should_abort(
 
 extern int is_64signalregset(void);
 
 
 extern int is_64signalregset(void);
 
-void act_set_apc(thread_t);
-void act_set_kperf(thread_t);
+extern void act_set_kperf(thread_t);
+extern void act_set_astledger(thread_t thread);
+extern void act_set_astledger_async(thread_t thread);
+extern void act_set_io_telemetry_ast(thread_t);
 
 extern uint32_t dtrace_get_thread_predcache(thread_t);
 extern int64_t dtrace_get_thread_vtime(thread_t);
 extern int64_t dtrace_get_thread_tracing(thread_t);
 extern boolean_t dtrace_get_thread_reentering(thread_t);
 
 extern uint32_t dtrace_get_thread_predcache(thread_t);
 extern int64_t dtrace_get_thread_vtime(thread_t);
 extern int64_t dtrace_get_thread_tracing(thread_t);
 extern boolean_t dtrace_get_thread_reentering(thread_t);
+extern int dtrace_get_thread_last_cpu_id(thread_t);
 extern vm_offset_t dtrace_get_kernel_stack(thread_t);
 extern void dtrace_set_thread_predcache(thread_t, uint32_t);
 extern void dtrace_set_thread_vtime(thread_t, int64_t);
 extern vm_offset_t dtrace_get_kernel_stack(thread_t);
 extern void dtrace_set_thread_predcache(thread_t, uint32_t);
 extern void dtrace_set_thread_vtime(thread_t, int64_t);
@@ -953,15 +1110,66 @@ extern vm_offset_t       kernel_stack_mask;
 extern vm_offset_t     kernel_stack_size;
 extern vm_offset_t     kernel_stack_depth_max;
 
 extern vm_offset_t     kernel_stack_size;
 extern vm_offset_t     kernel_stack_depth_max;
 
-void guard_ast(thread_t thread);
-extern void fd_guard_ast(thread_t thread);
-extern void mach_port_guard_ast(thread_t thread);
-extern void thread_guard_violation(thread_t thread, unsigned type);
-extern void thread_update_io_stats(thread_t thread, int size, int io_flags);
+extern void guard_ast(thread_t);
+extern void fd_guard_ast(thread_t,
+       mach_exception_code_t, mach_exception_subcode_t);
+#if CONFIG_VNGUARD
+extern void vn_guard_ast(thread_t,
+       mach_exception_code_t, mach_exception_subcode_t);
+#endif
+extern void mach_port_guard_ast(thread_t,
+       mach_exception_code_t, mach_exception_subcode_t);
+extern void thread_guard_violation(thread_t,
+       mach_exception_code_t, mach_exception_subcode_t);
+extern void thread_update_io_stats(thread_t, int size, int io_flags);
 
 extern kern_return_t   thread_set_voucher_name(mach_port_name_t name);
 extern kern_return_t thread_get_current_voucher_origin_pid(int32_t *pid);
 
 
 extern kern_return_t   thread_set_voucher_name(mach_port_name_t name);
 extern kern_return_t thread_get_current_voucher_origin_pid(int32_t *pid);
 
+extern void set_thread_rwlock_boost(void);
+extern void clear_thread_rwlock_boost(void);
+
+/*! @function thread_has_thread_name
+    @abstract Checks if a thread has a name.
+    @discussion This function takes one input, a thread, and returns a boolean value indicating if that thread already has a name associated with it.
+    @param th The thread to inspect.
+    @result TRUE if the thread has a name, FALSE otherwise.
+*/
+extern boolean_t thread_has_thread_name(thread_t th);
+
+/*! @function thread_set_thread_name
+    @abstract Set a thread's name.
+    @discussion This function takes two input parameters: a thread to name, and the name to apply to the thread.  The name will be attached to the thread in order to better identify the thread.
+    @param th The thread to be named.
+    @param name The name to apply to the thread.
+*/
+extern void thread_set_thread_name(thread_t th, const char* name);
+
+extern void thread_enable_send_importance(thread_t thread, boolean_t enable);
+
+/* Get a backtrace for a threads kernel or user stack (user_p), with pc and optionally
+ * frame pointer (getfp). Returns bytes added to buffer, and kThreadTruncatedBT in
+ * thread_trace_flags if a user page is not present after kdp_lightweight_fault() is
+ * called.
+ */
+
+extern int                             machine_trace_thread(
+                                                       thread_t thread,
+                                                       char *tracepos,
+                                                       char *tracebound,
+                                                       int nframes,
+                                                       boolean_t user_p,
+                                                       boolean_t getfp,
+                                                       uint32_t *thread_trace_flags);
+
+extern int                             machine_trace_thread64(thread_t thread,
+                                                       char *tracepos,
+                                                       char *tracebound,
+                                                       int nframes,
+                                                       boolean_t user_p,
+                                                       boolean_t getfp,
+                                                       uint32_t *thread_trace_flags);
+
 #endif /* XNU_KERNEL_PRIVATE */
 
 
 #endif /* XNU_KERNEL_PRIVATE */
 
 
@@ -978,11 +1186,17 @@ extern kern_return_t     kernel_thread_start(
                                                        thread_continue_t       continuation,
                                                        void                            *parameter,
                                                        thread_t                        *new_thread);
                                                        thread_continue_t       continuation,
                                                        void                            *parameter,
                                                        thread_t                        *new_thread);
+
 #ifdef KERNEL_PRIVATE
 void thread_set_eager_preempt(thread_t thread);
 void thread_clear_eager_preempt(thread_t thread);
 #ifdef KERNEL_PRIVATE
 void thread_set_eager_preempt(thread_t thread);
 void thread_clear_eager_preempt(thread_t thread);
+void thread_set_honor_qlimit(thread_t thread);
+void thread_clear_honor_qlimit(thread_t thread);
 extern ipc_port_t convert_thread_to_port(thread_t);
 extern ipc_port_t convert_thread_to_port(thread_t);
+extern ipc_port_t convert_thread_inspect_to_port(thread_inspect_t);
+extern boolean_t is_vm_privileged(void);
 extern boolean_t set_vm_privilege(boolean_t);
 extern boolean_t set_vm_privilege(boolean_t);
+extern kern_allocation_name_t thread_set_allocation_name(kern_allocation_name_t new_name);
 #endif /* KERNEL_PRIVATE */
 
 __END_DECLS
 #endif /* KERNEL_PRIVATE */
 
 __END_DECLS
mirror of XNU