]> git.saurik.com Git - apple/xnu.git/blobdiff - osfmk/kern/thread.h
projects / apple / xnu.git / blobdiff
? search:
summary | shortlog | log | commit | commitdiff | tree
raw | inline | side by side
xnu-4570.71.2.tar.gz
[apple/xnu.git] / osfmk / kern / thread.h
diff --git a/osfmk/kern/thread.h b/osfmk/kern/thread.h
index b497fa3facfd3e7999dd50c42d37bc9e5726bf38..ca2217584b3ad5fa91efa71b055004e77bc2238a 100644 (file)
--- a/osfmk/kern/thread.h
+++ b/osfmk/kern/thread.h
@@ -1,5 +1,5 @@
 /*
 /*
- * Copyright (c) 2000-2009 Apple Inc. All rights reserved.
+ * Copyright (c) 2000-2016 Apple Inc. All rights reserved.
  *
  * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
  *
  * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
@@ -94,13 +94,12 @@
 #include <mach/exception_types.h>
 
 #include <kern/kern_types.h>
 #include <mach/exception_types.h>
 
 #include <kern/kern_types.h>
+#include <vm/vm_kern.h>
 
 #include <sys/cdefs.h>
 
 #ifdef MACH_KERNEL_PRIVATE
 
 
 #include <sys/cdefs.h>
 
 #ifdef MACH_KERNEL_PRIVATE
 
-#include <cputypes.h>
-
 #include <mach_assert.h>
 #include <mach_ldebug.h>
 
 #include <mach_assert.h>
 #include <mach_ldebug.h>
 
@@ -108,69 +107,108 @@
 
 #include <mach/port.h>
 #include <kern/cpu_number.h>
 
 #include <mach/port.h>
 #include <kern/cpu_number.h>
+#include <kern/smp.h>
 #include <kern/queue.h>
 #include <kern/timer.h>
 #include <kern/queue.h>
 #include <kern/timer.h>
-#include <kern/lock.h>
+#include <kern/simple_lock.h>
 #include <kern/locks.h>
 #include <kern/sched.h>
 #include <kern/sched_prim.h>
 #include <kern/locks.h>
 #include <kern/sched.h>
 #include <kern/sched_prim.h>
+#include <mach/sfi_class.h>
 #include <kern/thread_call.h>
 #include <kern/thread_call.h>
+#include <kern/thread_group.h>
 #include <kern/timer_call.h>
 #include <kern/task.h>
 #include <kern/exception.h>
 #include <kern/affinity.h>
 #include <kern/timer_call.h>
 #include <kern/task.h>
 #include <kern/exception.h>
 #include <kern/affinity.h>
+#include <kern/debug.h>
+#include <kern/block_hint.h>
+
+#include <kern/waitq.h>
+#include <san/kasan.h>
 
 #include <ipc/ipc_kmsg.h>
 
 #include <machine/cpu_data.h>
 #include <machine/thread.h>
 
 
 #include <ipc/ipc_kmsg.h>
 
 #include <machine/cpu_data.h>
 #include <machine/thread.h>
 
+#if MONOTONIC
+#include <stdatomic.h>
+#include <machine/monotonic.h>
+#endif /* MONOTONIC */
+
+#if CONFIG_EMBEDDED
+/* Taskwatch related. TODO: find this a better home */
+typedef struct task_watcher task_watch_t;
+#endif /* CONFIG_EMBEDDED */
+
 struct thread {
 struct thread {
+
+#if MACH_ASSERT
+#define THREAD_MAGIC 0x1234ABCDDCBA4321ULL
+       /* Ensure nothing uses &thread as a queue entry */
+       uint64_t                thread_magic;
+#endif /* MACH_ASSERT */
+
        /*
         *      NOTE:   The runq field in the thread structure has an unusual
         *      locking protocol.  If its value is PROCESSOR_NULL, then it is
         *      locked by the thread_lock, but if its value is something else
        /*
         *      NOTE:   The runq field in the thread structure has an unusual
         *      locking protocol.  If its value is PROCESSOR_NULL, then it is
         *      locked by the thread_lock, but if its value is something else
-        *      then it is locked by the associated run queue lock.
+        *      then it is locked by the associated run queue lock. It is
+        *      set to PROCESSOR_NULL without holding the thread lock, but the
+        *      transition from PROCESSOR_NULL to non-null must be done
+        *      under the thread lock and the run queue lock.
         *
         *
-        *      When the thread is on a wait queue, these first three fields
-        *      are treated as an unofficial union with a wait_queue_element.
-        *      If you change these, you must change that definition as well
-        *      (kern/wait_queue.h).
+        *      New waitq APIs allow the 'links' and 'runq' fields to be
+        *      anywhere in the thread structure.
         */
         */
-       /* Items examined often, modified infrequently */
-       queue_chain_t   links;                          /* run/wait queue links */
-       processor_t             runq;                           /* run queue assignment */
-       wait_queue_t    wait_queue;                     /* wait queue we are currently on */
-       event64_t               wait_event;                     /* wait queue event */
-       integer_t               options;                        /* options set by thread itself */
-#define TH_OPT_INTMASK         0x03            /* interrupt / abort level */
-#define TH_OPT_VMPRIV          0x04            /* may allocate reserved memory */
-#define TH_OPT_DTRACE          0x08            /* executing under dtrace_probe */
-#define TH_OPT_SYSTEM_CRITICAL 0x10            /* Thread must always be allowed to run - even under heavy load */
-#define TH_OPT_PROC_CPULIMIT   0x20            /* Thread has a task-wide CPU limit applied to it */
-#define TH_OPT_PRVT_CPULIMIT   0x40            /* Thread has a thread-private CPU limit applied to it */
+       union {
+               queue_chain_t           runq_links;     /* run queue links */
+               queue_chain_t           wait_links;     /* wait queue links */
+       };
+
+       processor_t             runq;           /* run queue assignment */
+
+       event64_t               wait_event;     /* wait queue event */
+       struct waitq           *waitq;          /* wait queue this thread is enqueued on */
 
        /* Data updated during assert_wait/thread_wakeup */
 
        /* Data updated during assert_wait/thread_wakeup */
+#if __SMP__
        decl_simple_lock_data(,sched_lock)      /* scheduling lock (thread_lock()) */
        decl_simple_lock_data(,wake_lock)       /* for thread stop / wait (wake_lock()) */
        decl_simple_lock_data(,sched_lock)      /* scheduling lock (thread_lock()) */
        decl_simple_lock_data(,wake_lock)       /* for thread stop / wait (wake_lock()) */
+#endif
+       integer_t               options;                        /* options set by thread itself */
+#define TH_OPT_INTMASK         0x0003          /* interrupt / abort level */
+#define TH_OPT_VMPRIV          0x0004          /* may allocate reserved memory */
+#define TH_OPT_DTRACE          0x0008          /* executing under dtrace_probe */
+#define TH_OPT_SYSTEM_CRITICAL 0x0010          /* Thread must always be allowed to run - even under heavy load */
+#define TH_OPT_PROC_CPULIMIT   0x0020          /* Thread has a task-wide CPU limit applied to it */
+#define TH_OPT_PRVT_CPULIMIT   0x0040          /* Thread has a thread-private CPU limit applied to it */
+#define TH_OPT_IDLE_THREAD     0x0080          /* Thread is a per-processor idle thread */
+#define TH_OPT_GLOBAL_FORCED_IDLE      0x0100  /* Thread performs forced idle for thermal control */
+#define TH_OPT_SCHED_VM_GROUP  0x0200          /* Thread belongs to special scheduler VM group */
+#define TH_OPT_HONOR_QLIMIT    0x0400          /* Thread will honor qlimit while sending mach_msg, regardless of MACH_SEND_ALWAYS */
+#define TH_OPT_SEND_IMPORTANCE 0x0800          /* Thread will allow importance donation from kernel rpc */
+#define TH_OPT_ZONE_GC         0x1000          /* zone_gc() called on this thread */
+
        boolean_t                       wake_active;    /* wake event on stop */
        int                                     at_safe_point;  /* thread_abort_safely allowed */
        ast_t                           reason;                 /* why we blocked */
        boolean_t                       wake_active;    /* wake event on stop */
        int                                     at_safe_point;  /* thread_abort_safely allowed */
        ast_t                           reason;                 /* why we blocked */
-       wait_result_t           wait_result;    /* outcome of wait -
-                                                                                * may be examined by this thread
-                                                                                * WITHOUT locking */
+       uint32_t                        quantum_remaining;
+       wait_result_t                   wait_result;    /* outcome of wait -
+                                                        * may be examined by this thread
+                                                        * WITHOUT locking */
        thread_continue_t       continuation;   /* continue here next dispatch */
        void                            *parameter;             /* continuation parameter */
 
        /* Data updated/used in thread_invoke */
        thread_continue_t       continuation;   /* continue here next dispatch */
        void                            *parameter;             /* continuation parameter */
 
        /* Data updated/used in thread_invoke */
-    struct funnel_lock *funnel_lock;           /* Non-reentrancy funnel */
-    int                                    funnel_state;
-#define TH_FN_OWNED                    0x1                             /* we own the funnel */
-#define TH_FN_REFUNNEL         0x2                             /* re-acquire funnel on dispatch */
-
        vm_offset_t             kernel_stack;           /* current kernel stack */
        vm_offset_t                     reserved_stack;         /* reserved kernel stack */
 
        vm_offset_t             kernel_stack;           /* current kernel stack */
        vm_offset_t                     reserved_stack;         /* reserved kernel stack */
 
+#if KASAN
+       struct kasan_thread_data kasan_data;
+#endif
+
        /* Thread state: */
        int                                     state;
 /*
        /* Thread state: */
        int                                     state;
 /*
@@ -180,20 +218,27 @@ struct thread {
 #define TH_SUSP                        0x02                    /* stopped or requested to stop */
 #define TH_RUN                 0x04                    /* running or on runq */
 #define TH_UNINT               0x08                    /* waiting uninteruptibly */
 #define TH_SUSP                        0x02                    /* stopped or requested to stop */
 #define TH_RUN                 0x04                    /* running or on runq */
 #define TH_UNINT               0x08                    /* waiting uninteruptibly */
-#define        TH_TERMINATE    0x10                    /* halted at termination */
-#define        TH_TERMINATE2   0x20                    /* added to termination queue */
+#define TH_TERMINATE           0x10                    /* halted at termination */
+#define TH_TERMINATE2          0x20                    /* added to termination queue */
 
 #define TH_IDLE                        0x80                    /* idling processor */
 
        /* Scheduling information */
        sched_mode_t                    sched_mode;             /* scheduling mode */
        sched_mode_t                    saved_mode;             /* saved mode during forced mode demotion */
 
 #define TH_IDLE                        0x80                    /* idling processor */
 
        /* Scheduling information */
        sched_mode_t                    sched_mode;             /* scheduling mode */
        sched_mode_t                    saved_mode;             /* saved mode during forced mode demotion */
-       
-       unsigned int                    sched_flags;            /* current flag bits */
-#define TH_SFLAG_FAIRSHARE_TRIPPED     0x0001          /* fairshare scheduling activated */
+
+       /* This thread's contribution to global sched counters */
+       sched_bucket_t                  th_sched_bucket;
+
+       sfi_class_id_t                  sfi_class;              /* SFI class (XXX Updated on CSW/QE/AST) */
+       sfi_class_id_t                  sfi_wait_class; /* Currently in SFI wait for this class, protected by sfi_lock */
+
+
+       uint32_t                        sched_flags;            /* current flag bits */
+/* TH_SFLAG_FAIRSHARE_TRIPPED (unused) 0x0001 */
 #define TH_SFLAG_FAILSAFE              0x0002          /* fail-safe has tripped */
 #define TH_SFLAG_FAILSAFE              0x0002          /* fail-safe has tripped */
-#define TH_SFLAG_THROTTLED             0x0004      /* owner task in throttled state */
-#define TH_SFLAG_DEMOTED_MASK      (TH_SFLAG_THROTTLED | TH_SFLAG_FAILSAFE | TH_SFLAG_FAIRSHARE_TRIPPED)
+#define TH_SFLAG_THROTTLED             0x0004          /* throttled thread forced to timeshare mode (may be applied in addition to failsafe) */
+#define TH_SFLAG_DEMOTED_MASK      (TH_SFLAG_THROTTLED | TH_SFLAG_FAILSAFE)    /* saved_mode contains previous sched_mode */
 
 #define        TH_SFLAG_PROMOTED               0x0008          /* sched pri has been promoted */
 #define TH_SFLAG_ABORT                 0x0010          /* abort interruptible waits */
 
 #define        TH_SFLAG_PROMOTED               0x0008          /* sched pri has been promoted */
 #define TH_SFLAG_ABORT                 0x0010          /* abort interruptible waits */
@@ -202,21 +247,22 @@ struct thread {
 #define        TH_SFLAG_DEPRESS                0x0040          /* normal depress yield */
 #define TH_SFLAG_POLLDEPRESS           0x0080          /* polled depress yield */
 #define TH_SFLAG_DEPRESSED_MASK                (TH_SFLAG_DEPRESS | TH_SFLAG_POLLDEPRESS)
 #define        TH_SFLAG_DEPRESS                0x0040          /* normal depress yield */
 #define TH_SFLAG_POLLDEPRESS           0x0080          /* polled depress yield */
 #define TH_SFLAG_DEPRESSED_MASK                (TH_SFLAG_DEPRESS | TH_SFLAG_POLLDEPRESS)
-#define TH_SFLAG_PRI_UPDATE            0x0100          /* Updating priority */
+/* unused TH_SFLAG_PRI_UPDATE           0x0100 */
 #define TH_SFLAG_EAGERPREEMPT          0x0200          /* Any preemption of this thread should be treated as if AST_URGENT applied */
 #define TH_SFLAG_EAGERPREEMPT          0x0200          /* Any preemption of this thread should be treated as if AST_URGENT applied */
+#define TH_SFLAG_RW_PROMOTED           0x0400          /* sched pri has been promoted due to blocking with RW lock held */
+/* unused TH_SFLAG_THROTTLE_DEMOTED     0x0800 */
+#define TH_SFLAG_WAITQ_PROMOTED                0x1000          /* sched pri promoted from waitq wakeup (generally for IPC receive) */
 
 
-/*
- * A thread can either be completely unthrottled, about to be throttled,
- * throttled (TH_SFLAG_THROTTLED), or about to be unthrottled
- */
-#define        TH_SFLAG_PENDING_THROTTLE_DEMOTION      0x1000  /* Pending sched_mode demotion */
-#define        TH_SFLAG_PENDING_THROTTLE_PROMOTION     0x2000  /* Pending sched_mode promition */
-#define        TH_SFLAG_PENDING_THROTTLE_MASK          (TH_SFLAG_PENDING_THROTTLE_DEMOTION | TH_SFLAG_PENDING_THROTTLE_PROMOTION)
 
 
-       integer_t                       sched_pri;                      /* scheduled (current) priority */
-       integer_t                       priority;                       /* base priority */
-       integer_t                       max_priority;           /* max base priority */
-       integer_t                       task_priority;          /* copy of task base priority */
+#define TH_SFLAG_EXEC_PROMOTED          0x8000         /* sched pri has been promoted since thread is in an exec */
+#define TH_SFLAG_PROMOTED_MASK         (TH_SFLAG_PROMOTED | TH_SFLAG_RW_PROMOTED | TH_SFLAG_WAITQ_PROMOTED | TH_SFLAG_EXEC_PROMOTED)
+
+#define TH_SFLAG_RW_PROMOTED_BIT       (10)    /* 0x400 */
+
+       int16_t                         sched_pri;              /* scheduled (current) priority */
+       int16_t                         base_pri;               /* base priority */
+       int16_t                         max_priority;           /* copy of max base priority */
+       int16_t                         task_priority;          /* copy of task base priority */
 
 #if defined(CONFIG_SCHED_GRRR)
 #if 0
 
 #if defined(CONFIG_SCHED_GRRR)
 #if 0
@@ -224,26 +270,38 @@ struct thread {
 #endif
 #endif
        
 #endif
 #endif
        
-       integer_t                       promotions;                     /* level of promotion */
-       integer_t                       pending_promoter_index;
+       int16_t                         promotions;                     /* level of promotion */
+       int16_t                         pending_promoter_index;
+       _Atomic uint32_t                ref_count;              /* number of references to me */
        void                            *pending_promoter[2];
 
        void                            *pending_promoter[2];
 
+       uint32_t                        rwlock_count;   /* Number of lck_rw_t locks held by thread */
+
        integer_t                       importance;                     /* task-relative importance */
        integer_t                       importance;                     /* task-relative importance */
+       uint32_t                        was_promoted_on_wakeup;
 
 
-                                                                                       /* real-time parameters */
+       /* Priority depression expiration */
+       integer_t                       depress_timer_active;
+       timer_call_data_t       depress_timer;
+                                                                               /* real-time parameters */
        struct {                                                                /* see mach/thread_policy.h */
                uint32_t                        period;
                uint32_t                        computation;
                uint32_t                        constraint;
                boolean_t                       preemptible;
        struct {                                                                /* see mach/thread_policy.h */
                uint32_t                        period;
                uint32_t                        computation;
                uint32_t                        constraint;
                boolean_t                       preemptible;
-
                uint64_t                        deadline;
        }                                       realtime;
 
                uint64_t                        deadline;
        }                                       realtime;
 
-       uint32_t                        was_promoted_on_wakeup;
-       uint32_t                        current_quantum;        /* duration of current quantum */
        uint64_t                        last_run_time;          /* time when thread was switched away from */
        uint64_t                        last_run_time;          /* time when thread was switched away from */
-       uint64_t                        last_quantum_refill_time;       /* time when current_quantum was refilled after expiration */
+       uint64_t                        last_made_runnable_time;        /* time when thread was unblocked or preempted */
+       uint64_t                        last_basepri_change_time;       /* time when thread was last changed in basepri while runnable */
+       uint64_t                        same_pri_latency;
+#define THREAD_NOT_RUNNABLE (~0ULL)
+
+
+#if defined(CONFIG_SCHED_MULTIQ)
+       sched_group_t                   sched_group;
+#endif /* defined(CONFIG_SCHED_MULTIQ) */
 
   /* Data used during setrun/dispatch */
        timer_data_t            system_timer;           /* system mode timer */
 
   /* Data used during setrun/dispatch */
        timer_data_t            system_timer;           /* system mode timer */
@@ -265,17 +323,19 @@ struct thread {
 #endif
        
        /* Statistics and timesharing calculations */
 #endif
        
        /* Statistics and timesharing calculations */
-#if defined(CONFIG_SCHED_TRADITIONAL)
+#if defined(CONFIG_SCHED_TIMESHARE_CORE)
        natural_t                       sched_stamp;    /* last scheduler tick */
        natural_t                       sched_usage;    /* timesharing cpu usage [sched] */
        natural_t                       pri_shift;              /* usage -> priority from pset */
        natural_t                       cpu_usage;              /* instrumented cpu usage [%cpu] */
        natural_t                       cpu_delta;              /* accumulated cpu_usage delta */
        natural_t                       sched_stamp;    /* last scheduler tick */
        natural_t                       sched_usage;    /* timesharing cpu usage [sched] */
        natural_t                       pri_shift;              /* usage -> priority from pset */
        natural_t                       cpu_usage;              /* instrumented cpu usage [%cpu] */
        natural_t                       cpu_delta;              /* accumulated cpu_usage delta */
-#endif
+#endif /* CONFIG_SCHED_TIMESHARE_CORE */
+
        uint32_t                        c_switch;               /* total context switches */
        uint32_t                        p_switch;               /* total processor switches */
        uint32_t                        ps_switch;              /* total pset switches */
 
        uint32_t                        c_switch;               /* total context switches */
        uint32_t                        p_switch;               /* total processor switches */
        uint32_t                        ps_switch;              /* total pset switches */
 
+       integer_t mutex_count;  /* total count of locks held */
        /* Timing data structures */
        int                                     precise_user_kernel_time; /* precise user/kernel enabled for this thread */
        timer_data_t            user_timer;                     /* user mode timer */
        /* Timing data structures */
        int                                     precise_user_kernel_time; /* precise user/kernel enabled for this thread */
        timer_data_t            user_timer;                     /* user mode timer */
@@ -284,15 +344,20 @@ struct thread {
        uint64_t                        vtimer_user_save;       /* saved values for vtimers */
        uint64_t                        vtimer_prof_save;
        uint64_t                        vtimer_rlim_save;
        uint64_t                        vtimer_user_save;       /* saved values for vtimers */
        uint64_t                        vtimer_prof_save;
        uint64_t                        vtimer_rlim_save;
+       uint64_t                        vtimer_qos_save;
+
+       timer_data_t            ptime;                  /* time executing in P mode */
+
+#if CONFIG_SCHED_SFI
+       /* Timing for wait state */
+       uint64_t                wait_sfi_begin_time;    /* start time for thread waiting in SFI */
+#endif
 
        /* Timed wait expiration */
        timer_call_data_t       wait_timer;
        integer_t                       wait_timer_active;
        boolean_t                       wait_timer_is_set;
 
 
        /* Timed wait expiration */
        timer_call_data_t       wait_timer;
        integer_t                       wait_timer_active;
        boolean_t                       wait_timer_is_set;
 
-       /* Priority depression expiration */
-       timer_call_data_t       depress_timer;
-       integer_t                       depress_timer_active;
 
        /*
         * Processor/cache affinity
 
        /*
         * Processor/cache affinity
@@ -301,18 +366,26 @@ struct thread {
        affinity_set_t                  affinity_set;
        queue_chain_t                   affinity_threads;
 
        affinity_set_t                  affinity_set;
        queue_chain_t                   affinity_threads;
 
-       /* Various bits of stashed state */
+       /* Various bits of state to stash across a continuation, exclusive to the current thread block point */
        union {
                struct {
                        mach_msg_return_t       state;          /* receive state */
        union {
                struct {
                        mach_msg_return_t       state;          /* receive state */
+                       mach_port_seqno_t       seqno;          /* seqno of recvd message */
                        ipc_object_t            object;         /* object received on */
                        mach_vm_address_t       msg_addr;       /* receive buffer pointer */
                        ipc_object_t            object;         /* object received on */
                        mach_vm_address_t       msg_addr;       /* receive buffer pointer */
-                       mach_msg_size_t         msize;          /* max size for recvd msg */
+                       mach_msg_size_t         rsize;          /* max size for recvd msg */
+                       mach_msg_size_t         msize;          /* actual size for recvd msg */
                        mach_msg_option_t       option;         /* options for receive */
                        mach_msg_option_t       option;         /* options for receive */
-                       mach_msg_size_t         slist_size;     /* scatter list size */
                        mach_port_name_t        receiver_name;  /* the receive port name */
                        mach_port_name_t        receiver_name;  /* the receive port name */
-                       struct ipc_kmsg         *kmsg;          /* received message */
-                       mach_port_seqno_t       seqno;          /* seqno of recvd message */
+                       struct knote            *knote;         /* knote fired for rcv */
+                       union {
+                               struct ipc_kmsg   *kmsg;        /* received message */
+                               struct ipc_mqueue *peekq;       /* mqueue to peek at */
+                               struct {
+                                       mach_msg_priority_t qos;        /* received message qos */
+                                       mach_msg_priority_t oqos;       /* override qos for message */
+                               } received_qos;
+                       };
                        mach_msg_continue_t     continuation;
                } receive;
                struct {
                        mach_msg_continue_t     continuation;
                } receive;
                struct {
@@ -324,93 +397,120 @@ struct thread {
                } sema;
                struct {
                        int                                     option;         /* switch option */
                } sema;
                struct {
                        int                                     option;         /* switch option */
+                       boolean_t                               reenable_workq_callback;        /* on entry, callbacks were suspended */
                } swtch;
                } swtch;
-               int                                             misc;           /* catch-all for other state */
        } saved;
 
        } saved;
 
+       /* Only user threads can cause guard exceptions, only kernel threads can be thread call threads */
+       union {
+               /* Group and call this thread is working on behalf of */
+               struct {
+                       struct thread_call_group * thc_group;
+                       struct thread_call *       thc_call;                    /* debug only, may be deallocated */
+               } thc_state;
+
+               /* Structure to save information about guard exception */
+               struct {
+                       mach_exception_code_t           code;
+                       mach_exception_subcode_t        subcode;
+               } guard_exc_info;
+       };
+
+       /* Kernel holds on this thread  */
+       int16_t                                         suspend_count;
+       /* User level suspensions */
+       int16_t                                         user_stop_count;
+
        /* IPC data structures */
        /* IPC data structures */
-       struct ipc_kmsg_queue ith_messages;
+#if IMPORTANCE_INHERITANCE
+       natural_t ith_assertions;                       /* assertions pending drop */
+#endif
+       struct ipc_kmsg_queue ith_messages;             /* messages to reap */
        mach_port_t ith_rpc_reply;                      /* reply port for kernel RPCs */
 
        /* Ast/Halt data structures */
        vm_offset_t                                     recover;                /* page fault recover(copyin/out) */
        mach_port_t ith_rpc_reply;                      /* reply port for kernel RPCs */
 
        /* Ast/Halt data structures */
        vm_offset_t                                     recover;                /* page fault recover(copyin/out) */
-       uint32_t                                        ref_count;              /* number of references to me */
 
        queue_chain_t                           threads;                /* global list of all threads */
 
        /* Activation */
                queue_chain_t                   task_threads;
 
 
        queue_chain_t                           threads;                /* global list of all threads */
 
        /* Activation */
                queue_chain_t                   task_threads;
 
-               /*** Machine-dependent state ***/
-               struct machine_thread   machine;
-
                /* Task membership */
                struct task                             *task;
                vm_map_t                                map;
 
                decl_lck_mtx_data(,mutex)
 
                /* Task membership */
                struct task                             *task;
                vm_map_t                                map;
 
                decl_lck_mtx_data(,mutex)
 
-               /* Kernel holds on this thread  */
-               int                                             suspend_count;
-
-               /* User level suspensions */
-               int                                             user_stop_count;
 
                /* Pending thread ast(s) */
                ast_t                                   ast;
 
                /* Miscellaneous bits guarded by mutex */
                uint32_t
 
                /* Pending thread ast(s) */
                ast_t                                   ast;
 
                /* Miscellaneous bits guarded by mutex */
                uint32_t
-                       active:1,                               /* Thread is active and has not been terminated */
-                       started:1,                              /* Thread has been started after creation */
-                       static_param:1,                 /* Disallow policy parameter changes */
+                       active:1,                       /* Thread is active and has not been terminated */
+                       started:1,                      /* Thread has been started after creation */
+                       static_param:1,                 /* Disallow policy parameter changes */
+                       inspection:1,                   /* TRUE when task is being inspected by crash reporter */
+                       policy_reset:1,                 /* Disallow policy parameter changes on terminating threads */
+                       suspend_parked:1,               /* thread parked in thread_suspended */
+                       corpse_dup:1,                   /* TRUE when thread is an inactive duplicate in a corpse */
                        :0;
 
                        :0;
 
-               /* Return Handers */
-               struct ReturnHandler {
-                       struct ReturnHandler    *next;
-                       void            (*handler)(
-                                                       struct ReturnHandler            *rh,
-                                                       struct thread                           *thread);
-               } *handlers, special_handler;
-
                /* Ports associated with this thread */
                struct ipc_port                 *ith_self;              /* not a right, doesn't hold ref */
                struct ipc_port                 *ith_sself;             /* a send right */
                /* Ports associated with this thread */
                struct ipc_port                 *ith_self;              /* not a right, doesn't hold ref */
                struct ipc_port                 *ith_sself;             /* a send right */
-               struct exception_action exc_actions[EXC_TYPES_COUNT];
-
-               /* Owned ulocks (a lock set element) */
-               queue_head_t                    held_ulocks;
+               struct ipc_port                 *ith_special_reply_port; /* ref to special reply port */
+               struct exception_action *exc_actions;
 
 #ifdef MACH_BSD
                void                                    *uthread;
 #endif
 
 #if CONFIG_DTRACE
 
 #ifdef MACH_BSD
                void                                    *uthread;
 #endif
 
 #if CONFIG_DTRACE
+               uint32_t t_dtrace_flags;        /* DTrace thread states */
+#define        TH_DTRACE_EXECSUCCESS   0x01
                uint32_t t_dtrace_predcache;/* DTrace per thread predicate value hint */
                int64_t t_dtrace_tracing;       /* Thread time under dtrace_probe() */
                int64_t t_dtrace_vtime;
 #endif
 
                uint32_t t_dtrace_predcache;/* DTrace per thread predicate value hint */
                int64_t t_dtrace_tracing;       /* Thread time under dtrace_probe() */
                int64_t t_dtrace_vtime;
 #endif
 
-               uint32_t    t_page_creation_count;
                clock_sec_t t_page_creation_time;
                clock_sec_t t_page_creation_time;
+               uint32_t    t_page_creation_count;
+               uint32_t    t_page_creation_throttled;
+#if (DEVELOPMENT || DEBUG)
+               uint64_t    t_page_creation_throttled_hard;
+               uint64_t    t_page_creation_throttled_soft;
+#endif /* DEVELOPMENT || DEBUG */
+
+#ifdef KPERF
+/* The high 7 bits are the number of frames to sample of a user callstack. */
+#define T_KPERF_CALLSTACK_DEPTH_OFFSET     (25)
+#define T_KPERF_SET_CALLSTACK_DEPTH(DEPTH) (((uint32_t)(DEPTH)) << T_KPERF_CALLSTACK_DEPTH_OFFSET)
+#define T_KPERF_GET_CALLSTACK_DEPTH(FLAGS) ((FLAGS) >> T_KPERF_CALLSTACK_DEPTH_OFFSET)
+#endif
 
 
-#define T_CHUD_MARKED           0x01          /* this thread is marked by CHUD */
-#define T_IN_CHUD               0x02          /* this thread is already in a CHUD handler */
-#define THREAD_PMC_FLAG         0x04          /* Bit in "t_chud" signifying PMC interest */    
-#define T_AST_CALLSTACK         0x08          /* Thread scheduled to dump a
-                                              * callstack on its next
-                                              * AST */
-#define T_AST_NAME              0x10          /* Thread scheduled to dump
-                                              * its name on its next
-                                              * AST */
-#define T_NAME_DONE             0x20          /* Thread has previously
-                                              * recorded its name */
+#define T_KPERF_AST_CALLSTACK (1U << 0) /* dump a callstack on thread's next AST */
+#define T_KPERF_AST_DISPATCH  (1U << 1) /* dump a name on thread's next AST */
+#define T_KPC_ALLOC           (1U << 2) /* thread needs a kpc_buf allocated */
+/* only go up to T_KPERF_CALLSTACK_DEPTH_OFFSET - 1 */
 
 
-               uint32_t t_chud;        /* CHUD flags, used for Shark */
-               uint32_t chud_c_switch; /* last dispatch detection */
+#ifdef KPERF
+       uint32_t kperf_flags;
+       uint32_t kperf_pet_gen;  /* last generation of PET that sampled this thread*/
+       uint32_t kperf_c_switch; /* last dispatch detection */
+       uint32_t kperf_pet_cnt;  /* how many times a thread has been sampled by PET */
+#endif
 
 
-               integer_t mutex_count;  /* total count of locks held */
+#ifdef KPC
+       /* accumulated performance counters for this thread */
+       uint64_t *kpc_buf;
+#endif
+
+#if HYPERVISOR
+       /* hypervisor virtual CPU object associated with this thread */
+       void *hv_thread_target;
+#endif /* HYPERVISOR */
 
                uint64_t thread_id;     /*system wide unique thread-id*/
 
 
                uint64_t thread_id;     /*system wide unique thread-id*/
 
@@ -419,44 +519,112 @@ struct thread {
        uint32_t                syscalls_mach;
        ledger_t                t_ledger;
        ledger_t                t_threadledger; /* per thread ledger */
        uint32_t                syscalls_mach;
        ledger_t                t_ledger;
        ledger_t                t_threadledger; /* per thread ledger */
-       struct process_policy ext_appliedstate; /* externally applied actions */
-       struct process_policy ext_policystate;  /* externally defined process policy states*/
-       struct process_policy appliedstate;             /* self applied acions */
-       struct process_policy policystate;              /* process wide policy states */
+       ledger_t                t_bankledger;                /* ledger to charge someone */
+       uint64_t                t_deduct_bank_ledger_time;   /* cpu time to be deducted from bank ledger */
+       uint64_t                t_deduct_bank_ledger_energy; /* energy to be deducted from bank ledger */
+
+#if MONOTONIC
+       struct mt_thread t_monotonic;
+#endif /* MONOTONIC */
+
+       /*** Machine-dependent state ***/
+       struct machine_thread   machine;
+
+       /* policy is protected by the thread mutex */
+       struct thread_requested_policy  requested_policy;
+       struct thread_effective_policy  effective_policy;
+
+       /* usynch override is protected by the task lock, eventually will be thread mutex */
+       struct thread_qos_override {
+               struct thread_qos_override      *override_next;
+               uint32_t        override_contended_resource_count;
+               int16_t         override_qos;
+               int16_t         override_resource_type;
+               user_addr_t     override_resource;
+       } *overrides;
+
+       _Atomic uint32_t kqwl_owning_count;
+       uint32_t        ipc_overrides;
+       uint32_t        sync_ipc_overrides;
+       uint32_t        user_promotions;
+       uint16_t        user_promotion_basepri;
+       _Atomic uint16_t kevent_ast_bits;
+
+       block_hint_t    pending_block_hint;
+       block_hint_t    block_hint;      /* What type of primitive last caused us to block. */
+
+       int     iotier_override; /* atomic operations to set, cleared on ret to user */
+       io_stat_info_t                  thread_io_stats; /* per-thread I/O statistics */
+
 #if CONFIG_EMBEDDED
        task_watch_t *  taskwatch;              /* task watch */
 #if CONFIG_EMBEDDED
        task_watch_t *  taskwatch;              /* task watch */
-       integer_t               saved_importance;               /* saved task-relative importance */
 #endif /* CONFIG_EMBEDDED */
 #endif /* CONFIG_EMBEDDED */
-};
 
 
-#define ith_state              saved.receive.state
-#define ith_object             saved.receive.object
-#define ith_msg_addr                   saved.receive.msg_addr
-#define ith_msize              saved.receive.msize
-#define        ith_option              saved.receive.option
-#define ith_scatter_list_size  saved.receive.slist_size
-#define ith_receiver_name      saved.receive.receiver_name
-#define ith_continuation       saved.receive.continuation
-#define ith_kmsg               saved.receive.kmsg
-#define ith_seqno              saved.receive.seqno
+       uint32_t                        thread_callout_interrupt_wakeups;
+       uint32_t                        thread_callout_platform_idle_wakeups;
+       uint32_t                        thread_timer_wakeups_bin_1;
+       uint32_t                        thread_timer_wakeups_bin_2;
+       uint16_t                        thread_tag;
+       uint16_t                        callout_woken_from_icontext:1,
+                                       callout_woken_from_platform_idle:1,
+                                       callout_woke_thread:1,
+                                       thread_bitfield_unused:13;
+
+       mach_port_name_t                ith_voucher_name;
+       ipc_voucher_t                   ith_voucher;
+#if CONFIG_IOSCHED
+       void                            *decmp_upl;
+#endif /* CONFIG_IOSCHED */
+
+       /* work interval (if any) associated with the thread. Uses thread mutex */
+       struct work_interval            *th_work_interval;
+
+#if    SCHED_TRACE_THREAD_WAKEUPS
+       uintptr_t               thread_wakeup_bt[64];
+#endif
+};
 
 
-#define sth_waitsemaphore      saved.sema.waitsemaphore
-#define sth_signalsemaphore    saved.sema.signalsemaphore
-#define sth_options            saved.sema.options
-#define sth_result             saved.sema.result
-#define sth_continuation       saved.sema.continuation
+#define ith_state           saved.receive.state
+#define ith_object          saved.receive.object
+#define ith_msg_addr        saved.receive.msg_addr
+#define ith_rsize           saved.receive.rsize
+#define ith_msize           saved.receive.msize
+#define        ith_option          saved.receive.option
+#define ith_receiver_name   saved.receive.receiver_name
+#define ith_continuation    saved.receive.continuation
+#define ith_kmsg            saved.receive.kmsg
+#define ith_peekq           saved.receive.peekq
+#define ith_knote           saved.receive.knote
+#define ith_qos             saved.receive.received_qos.qos
+#define ith_qos_override    saved.receive.received_qos.oqos
+#define ith_seqno           saved.receive.seqno
+
+#define sth_waitsemaphore   saved.sema.waitsemaphore
+#define sth_signalsemaphore saved.sema.signalsemaphore
+#define sth_options         saved.sema.options
+#define sth_result          saved.sema.result
+#define sth_continuation    saved.sema.continuation
+
+#define ITH_KNOTE_NULL      ((void *)NULL)
+#define ITH_KNOTE_PSEUDO    ((void *)0xdeadbeef)
+#define ITH_KNOTE_VALID(kn) ((kn) != ITH_KNOTE_NULL && (kn) != ITH_KNOTE_PSEUDO)
+
+#if MACH_ASSERT
+#define assert_thread_magic(thread) assertf((thread)->thread_magic == THREAD_MAGIC, \
+                                            "bad thread magic 0x%llx for thread %p, expected 0x%llx", \
+                                            (thread)->thread_magic, (thread), THREAD_MAGIC)
+#else
+#define assert_thread_magic(thread) do { (void)(thread); } while (0)
+#endif
 
 
-extern void                    thread_bootstrap(void) __attribute__((section("__TEXT, initcode")));
+extern void                    thread_bootstrap(void);
 
 
-extern void                    thread_init(void) __attribute__((section("__TEXT, initcode")));
+extern void                    thread_init(void);
 
 extern void                    thread_daemon_init(void);
 
 #define        thread_reference_internal(thread)       \
 
 extern void                    thread_daemon_init(void);
 
 #define        thread_reference_internal(thread)       \
-                       (void)hw_atomic_add(&(thread)->ref_count, 1)
-
-#define thread_deallocate_internal(thread)     \
-                       hw_atomic_sub(&(thread)->ref_count, 1)
+                       (void)atomic_fetch_add_explicit(&(thread)->ref_count, 1, memory_order_relaxed)
 
 #define thread_reference(thread)                                       \
 MACRO_BEGIN                                                                                    \
 
 #define thread_reference(thread)                                       \
 MACRO_BEGIN                                                                                    \
@@ -467,17 +635,39 @@ MACRO_END
 extern void                    thread_deallocate(
                                                thread_t                thread);
 
 extern void                    thread_deallocate(
                                                thread_t                thread);
 
+extern void                    thread_deallocate_safe(
+                                               thread_t                thread);
+
+extern void                    thread_inspect_deallocate(
+                                               thread_inspect_t        thread);
+
 extern void                    thread_terminate_self(void);
 
 extern kern_return_t   thread_terminate_internal(
                                                        thread_t                thread);
 
 extern void                    thread_terminate_self(void);
 
 extern kern_return_t   thread_terminate_internal(
                                                        thread_t                thread);
 
-extern void                    thread_start_internal(
+extern void                    thread_start(
                                                        thread_t                        thread) __attribute__ ((noinline));
 
                                                        thread_t                        thread) __attribute__ ((noinline));
 
+extern void                    thread_start_in_assert_wait(
+                                                       thread_t                        thread,
+                                                       event_t             event,
+                                                       wait_interrupt_t    interruptible) __attribute__ ((noinline));
+
 extern void                    thread_terminate_enqueue(
                                                thread_t                thread);
 
 extern void                    thread_terminate_enqueue(
                                                thread_t                thread);
 
+extern void                    thread_exception_enqueue(
+                                               task_t          task,
+                                               thread_t        thread,
+                                               exception_type_t etype);
+
+extern void                    thread_copy_resource_info(
+                                               thread_t dst_thread,
+                                               thread_t src_thread);
+
+extern void                    thread_terminate_crashed_threads(void);
+
 extern void                    thread_stack_enqueue(
                                                thread_t                thread);
 
 extern void                    thread_stack_enqueue(
                                                thread_t                thread);
 
@@ -487,7 +677,12 @@ extern void                        thread_hold(
 extern void                    thread_release(
                                                thread_t        thread);
 
 extern void                    thread_release(
                                                thread_t        thread);
 
+extern void                    thread_corpse_continue(void);
 
 
+extern boolean_t               thread_is_active(thread_t thread);
+
+/* Locking for scheduler state, always acquired with interrupts disabled (splsched()) */
+#if __SMP__
 #define        thread_lock_init(th)    simple_lock_init(&(th)->sched_lock, 0)
 #define thread_lock(th)                        simple_lock(&(th)->sched_lock)
 #define thread_unlock(th)              simple_unlock(&(th)->sched_lock)
 #define        thread_lock_init(th)    simple_lock_init(&(th)->sched_lock, 0)
 #define thread_lock(th)                        simple_lock(&(th)->sched_lock)
 #define thread_unlock(th)              simple_unlock(&(th)->sched_lock)
@@ -495,6 +690,15 @@ extern void                        thread_release(
 #define wake_lock_init(th)             simple_lock_init(&(th)->wake_lock, 0)
 #define wake_lock(th)                  simple_lock(&(th)->wake_lock)
 #define wake_unlock(th)                        simple_unlock(&(th)->wake_lock)
 #define wake_lock_init(th)             simple_lock_init(&(th)->wake_lock, 0)
 #define wake_lock(th)                  simple_lock(&(th)->wake_lock)
 #define wake_unlock(th)                        simple_unlock(&(th)->wake_lock)
+#else
+#define thread_lock_init(th)   do { (void)th; } while(0)
+#define thread_lock(th)                        do { (void)th; } while(0)
+#define thread_unlock(th)              do { (void)th; } while(0)
+
+#define wake_lock_init(th)             do { (void)th; } while(0)
+#define wake_lock(th)                  do { (void)th; } while(0)
+#define wake_unlock(th)                        do { (void)th; } while(0)
+#endif
 
 #define thread_should_halt_fast(thread)                (!(thread)->active)
 
 
 #define thread_should_halt_fast(thread)                (!(thread)->active)
 
@@ -516,7 +720,7 @@ extern boolean_t            stack_alloc_try(
 
 extern void                            stack_collect(void);
 
 
 extern void                            stack_collect(void);
 
-extern void                            stack_init(void) __attribute__((section("__TEXT, initcode")));
+extern void                            stack_init(void);
 
 
 extern kern_return_t   thread_info_internal(
 
 
 extern kern_return_t   thread_info_internal(
@@ -525,13 +729,7 @@ extern kern_return_t       thread_info_internal(
                                                        thread_info_t                   thread_info_out,
                                                        mach_msg_type_number_t  *thread_info_count);
 
                                                        thread_info_t                   thread_info_out,
                                                        mach_msg_type_number_t  *thread_info_count);
 
-extern void                            thread_task_priority(
-                                                       thread_t                thread,
-                                                       integer_t               priority,
-                                                       integer_t               max_priority);
 
 
-extern void                            thread_policy_reset(
-                                                       thread_t                thread);
 
 extern kern_return_t   kernel_thread_create(
                                                        thread_continue_t       continuation,
 
 extern kern_return_t   kernel_thread_create(
                                                        thread_continue_t       continuation,
@@ -562,7 +760,8 @@ extern thread_t                     machine_switch_context(
                                                        thread_t                        new_thread);
 
 extern void                            machine_load_context(
                                                        thread_t                        new_thread);
 
 extern void                            machine_load_context(
-                                                       thread_t                thread);
+                                                       thread_t                thread) __attribute__((noreturn));
+
 
 extern kern_return_t   machine_thread_state_initialize(
                                                        thread_t                                thread);
 
 extern kern_return_t   machine_thread_state_initialize(
                                                        thread_t                                thread);
@@ -607,43 +806,34 @@ extern kern_return_t      machine_thread_inherit_taskwide(
                                                        thread_t                thread,
                                                        task_t                  parent_task);
 
                                                        thread_t                thread,
                                                        task_t                  parent_task);
 
-/*
- * XXX Funnel locks XXX
- */
-
-struct funnel_lock {
-       int                     fnl_type;                       /* funnel type */
-       lck_mtx_t       *fnl_mutex;                     /* underlying mutex for the funnel */
-       void *          fnl_mtxholder;          /* thread (last)holdng mutex */
-       void *          fnl_mtxrelease;         /* thread (last)releasing mutex */
-       lck_mtx_t       *fnl_oldmutex;          /* Mutex before collapsing split funnel */
-};
-
-typedef struct ReturnHandler           ReturnHandler;
+extern kern_return_t   machine_thread_set_tsd_base(
+                                                       thread_t                                thread,
+                                                       mach_vm_offset_t                tsd_base);
 
 #define        thread_mtx_lock(thread)                 lck_mtx_lock(&(thread)->mutex)
 #define        thread_mtx_try(thread)                  lck_mtx_try_lock(&(thread)->mutex)
 #define        thread_mtx_unlock(thread)               lck_mtx_unlock(&(thread)->mutex)
 
 
 #define        thread_mtx_lock(thread)                 lck_mtx_lock(&(thread)->mutex)
 #define        thread_mtx_try(thread)                  lck_mtx_try_lock(&(thread)->mutex)
 #define        thread_mtx_unlock(thread)               lck_mtx_unlock(&(thread)->mutex)
 
-extern void                    act_execute_returnhandlers(void);
+extern void thread_apc_ast(thread_t thread);
 
 
-extern void                    install_special_handler(
-                                               thread_t                thread);
-
-extern void                    special_handler(
-                                               ReturnHandler   *rh,
-                                               thread_t                thread);
+extern void thread_update_qos_cpu_time(thread_t thread);
 
 void act_machine_sv_free(thread_t, int);
 
 vm_offset_t                    min_valid_stack_address(void);
 vm_offset_t                    max_valid_stack_address(void);
 
 
 void act_machine_sv_free(thread_t, int);
 
 vm_offset_t                    min_valid_stack_address(void);
 vm_offset_t                    max_valid_stack_address(void);
 
-extern void            funnel_lock(
-                                               struct funnel_lock      *lock);
+static inline uint16_t thread_set_tag_internal(thread_t        thread, uint16_t tag) {
+       return __sync_fetch_and_or(&thread->thread_tag, tag);
+}
+
+static inline uint16_t thread_get_tag_internal(thread_t        thread) {
+       return thread->thread_tag;
+}
+
+
+extern void thread_set_options(uint32_t thopt);
 
 
-extern void            funnel_unlock(
-                                               struct funnel_lock      *lock);
 
 #else  /* MACH_KERNEL_PRIVATE */
 
 
 #else  /* MACH_KERNEL_PRIVATE */
 
@@ -665,28 +855,52 @@ __END_DECLS
 
 __BEGIN_DECLS
 
 
 __BEGIN_DECLS
 
-#if defined(__i386__)
+extern void                    thread_starts_owning_workloop(
+                                               thread_t                thread);
+
+extern void                    thread_ends_owning_workloop(
+                                               thread_t                thread);
 
 
-extern thread_t                kernel_thread(
-                                               task_t          task,
-                                               void            (*start)(void));
+extern uint32_t                thread_owned_workloops_count(
+                                               thread_t                thread);
 
 
-#endif /* defined(__i386__) */
 
 
-extern uint64_t                        thread_tid(
+extern uint64_t                        thread_dispatchqaddr(
                                                thread_t thread);
 
                                                thread_t thread);
 
-extern uint64_t                        thread_dispatchqaddr(
+extern uint64_t                        thread_rettokern_addr(
                                                thread_t thread);
 
 __END_DECLS
 
 #endif /* KERNEL_PRIVATE */
 
                                                thread_t thread);
 
 __END_DECLS
 
 #endif /* KERNEL_PRIVATE */
 
+#ifdef KERNEL
+__BEGIN_DECLS
+
+extern uint64_t                        thread_tid(thread_t thread);
+
+__END_DECLS
+
+#endif  /* KERNEL */
+
 __BEGIN_DECLS
 
 #ifdef XNU_KERNEL_PRIVATE
 
 __BEGIN_DECLS
 
 #ifdef XNU_KERNEL_PRIVATE
 
+/*
+ * Thread tags; for easy identification.
+ */
+#define        THREAD_TAG_MAINTHREAD 0x1
+#define        THREAD_TAG_CALLOUT 0x2
+#define        THREAD_TAG_IOWORKLOOP 0x4
+
+#define        THREAD_TAG_PTHREAD 0x10
+#define        THREAD_TAG_WORKQUEUE 0x20
+
+uint16_t       thread_set_tag(thread_t, uint16_t);
+uint16_t       thread_get_tag(thread_t);
+
 extern kern_return_t    thread_state_initialize(
                                                        thread_t                                thread);
 
 extern kern_return_t    thread_state_initialize(
                                                        thread_t                                thread);
 
@@ -702,22 +916,42 @@ extern kern_return_t      thread_getstatus(
                                                        thread_state_t                  tstate,
                                                        mach_msg_type_number_t  *count);
 
                                                        thread_state_t                  tstate,
                                                        mach_msg_type_number_t  *count);
 
+extern kern_return_t   thread_create_with_continuation(
+                                                       task_t task,
+                                                       thread_t *new_thread,
+                                                       thread_continue_t continuation);
+
+extern kern_return_t thread_create_waiting(task_t               task,
+                                           thread_continue_t    continuation,
+                                           event_t              event,
+                                           thread_t             *new_thread);
+
 extern kern_return_t   thread_create_workq(
                                                        task_t                  task,
                                                        thread_continue_t       thread_return,
                                                        thread_t                *new_thread);
 
 extern kern_return_t   thread_create_workq(
                                                        task_t                  task,
                                                        thread_continue_t       thread_return,
                                                        thread_t                *new_thread);
 
+extern kern_return_t   thread_create_workq_waiting(
+                                                       task_t                  task,
+                                                       thread_continue_t       thread_return,
+                                                       event_t         event,
+                                                       thread_t                *new_thread);
+
 extern void    thread_yield_internal(
        mach_msg_timeout_t      interval);
 
 extern void    thread_yield_internal(
        mach_msg_timeout_t      interval);
 
+extern void    thread_yield_to_preemption(void);
+
 /*
  * Thread-private CPU limits: apply a private CPU limit to this thread only. Available actions are:
  * 
  * 1) Block. Prevent CPU consumption of the thread from exceeding the limit.
  * 2) Exception. Generate a resource consumption exception when the limit is exceeded.
 /*
  * Thread-private CPU limits: apply a private CPU limit to this thread only. Available actions are:
  * 
  * 1) Block. Prevent CPU consumption of the thread from exceeding the limit.
  * 2) Exception. Generate a resource consumption exception when the limit is exceeded.
+ * 3) Disable. Remove any existing CPU limit.
  */
 #define THREAD_CPULIMIT_BLOCK          0x1
 #define THREAD_CPULIMIT_EXCEPTION      0x2
  */
 #define THREAD_CPULIMIT_BLOCK          0x1
 #define THREAD_CPULIMIT_EXCEPTION      0x2
+#define        THREAD_CPULIMIT_DISABLE         0x3
 
 struct _thread_ledger_indices {
        int cpu_time;
 
 struct _thread_ledger_indices {
        int cpu_time;
@@ -725,29 +959,16 @@ struct _thread_ledger_indices {
 
 extern struct _thread_ledger_indices thread_ledgers;
 
 
 extern struct _thread_ledger_indices thread_ledgers;
 
+extern int thread_get_cpulimit(int *action, uint8_t *percentage, uint64_t *interval_ns);
 extern int thread_set_cpulimit(int action, uint8_t percentage, uint64_t interval_ns);
 
 extern int thread_set_cpulimit(int action, uint8_t percentage, uint64_t interval_ns);
 
-typedef struct funnel_lock             funnel_t;
-
-#define THR_FUNNEL_NULL (funnel_t *)0
-
-extern funnel_t                 *funnel_alloc(
-                                               int                     type);
-
-extern void                    funnel_free(
-                                               funnel_t        *lock);
-
-extern funnel_t                *thread_funnel_get(void);
-
-extern boolean_t       thread_funnel_set(
-                                               funnel_t        *lock,
-                                               boolean_t        funneled);
-
 extern void                    thread_read_times(
                                                thread_t                thread,
                                                time_value_t    *user_time,
                                                time_value_t    *system_time);
 
 extern void                    thread_read_times(
                                                thread_t                thread,
                                                time_value_t    *user_time,
                                                time_value_t    *system_time);
 
+extern uint64_t                thread_get_runtime_self(void);
+
 extern void                    thread_setuserstack(
                                                thread_t                thread,
                                                mach_vm_offset_t        user_stack);
 extern void                    thread_setuserstack(
                                                thread_t                thread,
                                                mach_vm_offset_t        user_stack);
@@ -760,6 +981,10 @@ extern void                        thread_setentrypoint(
                                                thread_t                thread,
                                                mach_vm_offset_t        entry);
 
                                                thread_t                thread,
                                                mach_vm_offset_t        entry);
 
+extern kern_return_t   thread_set_tsd_base(
+                                                       thread_t        thread,
+                                                       mach_vm_offset_t tsd_base);
+
 extern kern_return_t   thread_setsinglestep(
                                                thread_t                thread,
                                                int                     on);
 extern kern_return_t   thread_setsinglestep(
                                                thread_t                thread,
                                                int                     on);
@@ -770,7 +995,8 @@ extern kern_return_t        thread_userstack(
                                                thread_state_t,
                                                unsigned int,
                                                mach_vm_offset_t *,
                                                thread_state_t,
                                                unsigned int,
                                                mach_vm_offset_t *,
-                                               int *);
+                                               int *,
+                                               boolean_t);
 
 extern kern_return_t   thread_entrypoint(
                                                thread_t,
 
 extern kern_return_t   thread_entrypoint(
                                                thread_t,
@@ -780,8 +1006,8 @@ extern kern_return_t       thread_entrypoint(
                                                mach_vm_offset_t *); 
 
 extern kern_return_t   thread_userstackdefault(
                                                mach_vm_offset_t *); 
 
 extern kern_return_t   thread_userstackdefault(
-                                               thread_t,
-                                               mach_vm_offset_t *);
+                                               mach_vm_offset_t *,
+                                               boolean_t);
 
 extern kern_return_t   thread_wire_internal(
                                                        host_priv_t             host_priv,
 
 extern kern_return_t   thread_wire_internal(
                                                        host_priv_t             host_priv,
@@ -789,11 +1015,17 @@ extern kern_return_t     thread_wire_internal(
                                                        boolean_t               wired,
                                                        boolean_t               *prev_state);
 
                                                        boolean_t               wired,
                                                        boolean_t               *prev_state);
 
+
 extern kern_return_t   thread_dup(thread_t);
 
 extern kern_return_t   thread_dup(thread_t);
 
+extern kern_return_t thread_dup2(thread_t, thread_t);
+
+#if !defined(_SCHED_CALL_T_DEFINED)
+#define _SCHED_CALL_T_DEFINED
 typedef void   (*sched_call_t)(
                                        int                             type,
                                        thread_t                thread);
 typedef void   (*sched_call_t)(
                                        int                             type,
                                        thread_t                thread);
+#endif
 
 #define SCHED_CALL_BLOCK               0x1
 #define SCHED_CALL_UNBLOCK             0x2
 
 #define SCHED_CALL_BLOCK               0x1
 #define SCHED_CALL_UNBLOCK             0x2
@@ -802,16 +1034,20 @@ extern void              thread_sched_call(
                                        thread_t                thread,
                                        sched_call_t    call);
 
                                        thread_t                thread,
                                        sched_call_t    call);
 
+extern sched_call_t    thread_disable_sched_call(
+                                       thread_t                thread,
+                                       sched_call_t    call);
+
+extern void    thread_reenable_sched_call(
+                                       thread_t                thread,
+                                       sched_call_t    call);
+
 extern void            thread_static_param(
                                        thread_t                thread,
                                        boolean_t               state);
 
 extern void            thread_static_param(
                                        thread_t                thread,
                                        boolean_t               state);
 
-extern kern_return_t   thread_policy_set_internal(
-                                       thread_t                thread,
-                                       thread_policy_flavor_t  flavor,
-                                       thread_policy_t         policy_info,
-                                       mach_msg_type_number_t  count);
-
+extern boolean_t       thread_is_static_param(
+                                       thread_t                thread);
 
 extern task_t  get_threadtask(thread_t);
 #define thread_is_64bit(thd)   \
 
 extern task_t  get_threadtask(thread_t);
 #define thread_is_64bit(thd)   \
@@ -821,9 +1057,16 @@ extern task_t     get_threadtask(thread_t);
 extern void            *get_bsdthread_info(thread_t);
 extern void            set_bsdthread_info(thread_t, void *);
 extern void            *uthread_alloc(task_t, thread_t, int);
 extern void            *get_bsdthread_info(thread_t);
 extern void            set_bsdthread_info(thread_t, void *);
 extern void            *uthread_alloc(task_t, thread_t, int);
-extern void            uthread_cleanup(task_t, void *, void *); 
+extern void            uthread_cleanup_name(void *uthread);
+extern void            uthread_cleanup(task_t, void *, void *);
 extern void            uthread_zone_free(void *); 
 extern void            uthread_zone_free(void *); 
-extern void            uthread_cred_free(void *); 
+extern void            uthread_cred_free(void *);
+
+extern void            uthread_reset_proc_refcount(void *);
+#if PROC_REF_DEBUG
+extern int             uthread_get_proc_refcount(void *);
+extern int             proc_ref_tracking_disabled;
+#endif
 
 extern boolean_t       thread_should_halt(
                                                thread_t                thread);
 
 extern boolean_t       thread_should_halt(
                                                thread_t                thread);
@@ -833,13 +1076,16 @@ extern boolean_t thread_should_abort(
 
 extern int is_64signalregset(void);
 
 
 extern int is_64signalregset(void);
 
-void act_set_apc(thread_t);
-void act_set_kperf(thread_t);
+extern void act_set_kperf(thread_t);
+extern void act_set_astledger(thread_t thread);
+extern void act_set_astledger_async(thread_t thread);
+extern void act_set_io_telemetry_ast(thread_t);
 
 extern uint32_t dtrace_get_thread_predcache(thread_t);
 extern int64_t dtrace_get_thread_vtime(thread_t);
 extern int64_t dtrace_get_thread_tracing(thread_t);
 extern boolean_t dtrace_get_thread_reentering(thread_t);
 
 extern uint32_t dtrace_get_thread_predcache(thread_t);
 extern int64_t dtrace_get_thread_vtime(thread_t);
 extern int64_t dtrace_get_thread_tracing(thread_t);
 extern boolean_t dtrace_get_thread_reentering(thread_t);
+extern int dtrace_get_thread_last_cpu_id(thread_t);
 extern vm_offset_t dtrace_get_kernel_stack(thread_t);
 extern void dtrace_set_thread_predcache(thread_t, uint32_t);
 extern void dtrace_set_thread_vtime(thread_t, int64_t);
 extern vm_offset_t dtrace_get_kernel_stack(thread_t);
 extern void dtrace_set_thread_predcache(thread_t, uint32_t);
 extern void dtrace_set_thread_vtime(thread_t, int64_t);
@@ -847,15 +1093,16 @@ extern void dtrace_set_thread_tracing(thread_t, int64_t);
 extern void dtrace_set_thread_reentering(thread_t, boolean_t);
 extern vm_offset_t dtrace_set_thread_recover(thread_t, vm_offset_t);
 extern void dtrace_thread_bootstrap(void);
 extern void dtrace_set_thread_reentering(thread_t, boolean_t);
 extern vm_offset_t dtrace_set_thread_recover(thread_t, vm_offset_t);
 extern void dtrace_thread_bootstrap(void);
+extern void dtrace_thread_didexec(thread_t);
 
 extern int64_t dtrace_calc_thread_recent_vtime(thread_t);
 
 
 
 extern int64_t dtrace_calc_thread_recent_vtime(thread_t);
 
 
-extern void            thread_set_wq_state32(
+extern kern_return_t   thread_set_wq_state32(
                                              thread_t          thread,
                                              thread_state_t    tstate);
 
                                              thread_t          thread,
                                              thread_state_t    tstate);
 
-extern void            thread_set_wq_state64(
+extern kern_return_t   thread_set_wq_state64(
                                              thread_t          thread,
                                              thread_state_t    tstate);
 
                                              thread_t          thread,
                                              thread_state_t    tstate);
 
@@ -863,8 +1110,69 @@ extern vm_offset_t        kernel_stack_mask;
 extern vm_offset_t     kernel_stack_size;
 extern vm_offset_t     kernel_stack_depth_max;
 
 extern vm_offset_t     kernel_stack_size;
 extern vm_offset_t     kernel_stack_depth_max;
 
+extern void guard_ast(thread_t);
+extern void fd_guard_ast(thread_t,
+       mach_exception_code_t, mach_exception_subcode_t);
+#if CONFIG_VNGUARD
+extern void vn_guard_ast(thread_t,
+       mach_exception_code_t, mach_exception_subcode_t);
+#endif
+extern void mach_port_guard_ast(thread_t,
+       mach_exception_code_t, mach_exception_subcode_t);
+extern void thread_guard_violation(thread_t,
+       mach_exception_code_t, mach_exception_subcode_t);
+extern void thread_update_io_stats(thread_t, int size, int io_flags);
+
+extern kern_return_t   thread_set_voucher_name(mach_port_name_t name);
+extern kern_return_t thread_get_current_voucher_origin_pid(int32_t *pid);
+
+extern void set_thread_rwlock_boost(void);
+extern void clear_thread_rwlock_boost(void);
+
+/*! @function thread_has_thread_name
+    @abstract Checks if a thread has a name.
+    @discussion This function takes one input, a thread, and returns a boolean value indicating if that thread already has a name associated with it.
+    @param th The thread to inspect.
+    @result TRUE if the thread has a name, FALSE otherwise.
+*/
+extern boolean_t thread_has_thread_name(thread_t th);
+
+/*! @function thread_set_thread_name
+    @abstract Set a thread's name.
+    @discussion This function takes two input parameters: a thread to name, and the name to apply to the thread.  The name will be attached to the thread in order to better identify the thread.
+    @param th The thread to be named.
+    @param name The name to apply to the thread.
+*/
+extern void thread_set_thread_name(thread_t th, const char* name);
+
+extern void thread_enable_send_importance(thread_t thread, boolean_t enable);
+
+/* Get a backtrace for a threads kernel or user stack (user_p), with pc and optionally
+ * frame pointer (getfp). Returns bytes added to buffer, and kThreadTruncatedBT in
+ * thread_trace_flags if a user page is not present after kdp_lightweight_fault() is
+ * called.
+ */
+
+extern int                             machine_trace_thread(
+                                                       thread_t thread,
+                                                       char *tracepos,
+                                                       char *tracebound,
+                                                       int nframes,
+                                                       boolean_t user_p,
+                                                       boolean_t getfp,
+                                                       uint32_t *thread_trace_flags);
+
+extern int                             machine_trace_thread64(thread_t thread,
+                                                       char *tracepos,
+                                                       char *tracebound,
+                                                       int nframes,
+                                                       boolean_t user_p,
+                                                       boolean_t getfp,
+                                                       uint32_t *thread_trace_flags);
+
 #endif /* XNU_KERNEL_PRIVATE */
 
 #endif /* XNU_KERNEL_PRIVATE */
 
+
 /*! @function kernel_thread_start
     @abstract Create a kernel thread.
     @discussion This function takes three input parameters, namely reference to the function that the thread should execute, caller specified data and a reference which is used to return the newly created kernel thread. The function returns KERN_SUCCESS on success or an appropriate kernel code type indicating the error. It may be noted that the caller is responsible for explicitly releasing the reference to the created thread when no longer needed. This should be done by calling thread_deallocate(new_thread).
 /*! @function kernel_thread_start
     @abstract Create a kernel thread.
     @discussion This function takes three input parameters, namely reference to the function that the thread should execute, caller specified data and a reference which is used to return the newly created kernel thread. The function returns KERN_SUCCESS on success or an appropriate kernel code type indicating the error. It may be noted that the caller is responsible for explicitly releasing the reference to the created thread when no longer needed. This should be done by calling thread_deallocate(new_thread).
@@ -878,10 +1186,17 @@ extern kern_return_t     kernel_thread_start(
                                                        thread_continue_t       continuation,
                                                        void                            *parameter,
                                                        thread_t                        *new_thread);
                                                        thread_continue_t       continuation,
                                                        void                            *parameter,
                                                        thread_t                        *new_thread);
+
 #ifdef KERNEL_PRIVATE
 void thread_set_eager_preempt(thread_t thread);
 void thread_clear_eager_preempt(thread_t thread);
 #ifdef KERNEL_PRIVATE
 void thread_set_eager_preempt(thread_t thread);
 void thread_clear_eager_preempt(thread_t thread);
+void thread_set_honor_qlimit(thread_t thread);
+void thread_clear_honor_qlimit(thread_t thread);
 extern ipc_port_t convert_thread_to_port(thread_t);
 extern ipc_port_t convert_thread_to_port(thread_t);
+extern ipc_port_t convert_thread_inspect_to_port(thread_inspect_t);
+extern boolean_t is_vm_privileged(void);
+extern boolean_t set_vm_privilege(boolean_t);
+extern kern_allocation_name_t thread_set_allocation_name(kern_allocation_name_t new_name);
 #endif /* KERNEL_PRIVATE */
 
 __END_DECLS
 #endif /* KERNEL_PRIVATE */
 
 __END_DECLS
mirror of XNU