xnu-6153.141.1.tar.gz

[apple/xnu.git] / osfmk / kern / thread.c

diff --git a/osfmk/kern/thread.c b/osfmk/kern/thread.c
index bf0749afccb675cd91e50f88d83575068c70bd90..fde3ef327598d1b3c6fae7e07ebf3ff3e7ef3294 100644 (file)
--- a/osfmk/kern/thread.c
+++ b/osfmk/kern/thread.c
@@ -1,8 +1,8 @@
 /*
 /*

- * Copyright (c) 2000-2015 Apple Inc. All rights reserved.
+ * Copyright (c) 2000-2019 Apple Inc. All rights reserved.

  *
  * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  *
  * @APPLE_OSREFERENCE_LICENSE_HEADER_START@

- * 
+ *

  * This file contains Original Code and/or Modifications of Original Code
  * as defined in and that are subject to the Apple Public Source License
  * Version 2.0 (the 'License'). You may not use this file except in
  * This file contains Original Code and/or Modifications of Original Code
  * as defined in and that are subject to the Apple Public Source License
  * Version 2.0 (the 'License'). You may not use this file except in


@@ -11,10 +11,10 @@
  * unlawful or unlicensed copies of an Apple operating system, or to
  * circumvent, violate, or enable the circumvention or violation of, any
  * terms of an Apple operating system software license agreement.
  * unlawful or unlicensed copies of an Apple operating system, or to
  * circumvent, violate, or enable the circumvention or violation of, any
  * terms of an Apple operating system software license agreement.

- * 
+ *

  * Please obtain a copy of the License at
  * http://www.opensource.apple.com/apsl/ and read it before using this file.
  * Please obtain a copy of the License at
  * http://www.opensource.apple.com/apsl/ and read it before using this file.

- * 
+ *

  * The Original Code and all software distributed under the License are
  * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * The Original Code and all software distributed under the License are
  * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,


@@ -22,34 +22,34 @@
  * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
  * Please see the License for the specific language governing rights and
  * limitations under the License.
  * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
  * Please see the License for the specific language governing rights and
  * limitations under the License.

- * 
+ *

  * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*
  * @OSF_FREE_COPYRIGHT@
  */
  * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*
  * @OSF_FREE_COPYRIGHT@
  */

-/* 
+/*

  * Mach Operating System
  * Copyright (c) 1991,1990,1989,1988,1987 Carnegie Mellon University
  * All Rights Reserved.
  * Mach Operating System
  * Copyright (c) 1991,1990,1989,1988,1987 Carnegie Mellon University
  * All Rights Reserved.

- * 
+ *

  * Permission to use, copy, modify and distribute this software and its
  * documentation is hereby granted, provided that both the copyright
  * notice and this permission notice appear in all copies of the
  * software, derivative works or modified versions, and any portions
  * thereof, and that both notices appear in supporting documentation.
  * Permission to use, copy, modify and distribute this software and its
  * documentation is hereby granted, provided that both the copyright
  * notice and this permission notice appear in all copies of the
  * software, derivative works or modified versions, and any portions
  * thereof, and that both notices appear in supporting documentation.

- * 
+ *

  * CARNEGIE MELLON ALLOWS FREE USE OF THIS SOFTWARE IN ITS "AS IS"
  * CONDITION.  CARNEGIE MELLON DISCLAIMS ANY LIABILITY OF ANY KIND FOR
  * ANY DAMAGES WHATSOEVER RESULTING FROM THE USE OF THIS SOFTWARE.
  * CARNEGIE MELLON ALLOWS FREE USE OF THIS SOFTWARE IN ITS "AS IS"
  * CONDITION.  CARNEGIE MELLON DISCLAIMS ANY LIABILITY OF ANY KIND FOR
  * ANY DAMAGES WHATSOEVER RESULTING FROM THE USE OF THIS SOFTWARE.

- * 
+ *

  * Carnegie Mellon requests users of this software to return to
  * Carnegie Mellon requests users of this software to return to

- * 
+ *

  *  Software Distribution Coordinator  or  Software.Distribution@CS.CMU.EDU
  *  School of Computer Science
  *  Carnegie Mellon University
  *  Pittsburgh PA 15213-3890
  *  Software Distribution Coordinator  or  Software.Distribution@CS.CMU.EDU
  *  School of Computer Science
  *  Carnegie Mellon University
  *  Pittsburgh PA 15213-3890

- * 
+ *

  * any improvements or extensions that they make and grant Carnegie Mellon
  * the rights to redistribute these changes.
  */
  * any improvements or extensions that they make and grant Carnegie Mellon
  * the rights to redistribute these changes.
  */


@@ -112,18 +112,28 @@
 #include <kern/syscall_subr.h>
 #include <kern/task.h>
 #include <kern/thread.h>
 #include <kern/syscall_subr.h>
 #include <kern/task.h>
 #include <kern/thread.h>

+#include <kern/thread_group.h>
+#include <kern/coalition.h>

 #include <kern/host.h>
 #include <kern/zalloc.h>
 #include <kern/assert.h>
 #include <kern/exc_resource.h>
 #include <kern/host.h>
 #include <kern/zalloc.h>
 #include <kern/assert.h>
 #include <kern/exc_resource.h>

+#include <kern/exc_guard.h>

 #include <kern/telemetry.h>
 #include <kern/policy_internal.h>
 #include <kern/telemetry.h>
 #include <kern/policy_internal.h>

+#include <kern/turnstile.h>
+#include <kern/sched_clutch.h>

 
 #include <corpses/task_corpse.h>
 #if KPC
 #include <kern/kpc.h>
 #endif
 
 
 #include <corpses/task_corpse.h>
 #if KPC
 #include <kern/kpc.h>
 #endif
 

+#if MONOTONIC
+#include <kern/monotonic.h>
+#include <machine/monotonic.h>
+#endif /* MONOTONIC */
+

 #include <ipc/ipc_kmsg.h>
 #include <ipc/ipc_port.h>
 #include <bank/bank_types.h>
 #include <ipc/ipc_kmsg.h>
 #include <ipc/ipc_port.h>
 #include <bank/bank_types.h>


@@ -134,6 +144,17 @@
 #include <sys/kdebug.h>
 #include <sys/bsdtask_info.h>
 #include <mach/sdt.h>
 #include <sys/kdebug.h>
 #include <sys/bsdtask_info.h>
 #include <mach/sdt.h>

+#include <san/kasan.h>
+#if CONFIG_KSANCOV
+#include <san/ksancov.h>
+#endif
+
+#include <stdatomic.h>
+
+#if defined(HAS_APPLE_PAC)
+#include <ptrauth.h>
+#include <arm64/proc_reg.h>
+#endif /* defined(HAS_APPLE_PAC) */

 
 /*
  * Exported interfaces
 
 /*
  * Exported interfaces


@@ -145,53 +166,51 @@
 #include <mach/mach_voucher_server.h>
 #include <kern/policy_internal.h>
 
 #include <mach/mach_voucher_server.h>
 #include <kern/policy_internal.h>
 

-static struct zone                     *thread_zone;
-static lck_grp_attr_t          thread_lck_grp_attr;
-lck_attr_t                                     thread_lck_attr;
-lck_grp_t                                      thread_lck_grp;
-
-struct zone                                    *thread_qos_override_zone;
-
-decl_simple_lock_data(static,thread_stack_lock)
-static queue_head_t            thread_stack_queue;
+static struct zone                      *thread_zone;
+static lck_grp_attr_t           thread_lck_grp_attr;
+lck_attr_t                                      thread_lck_attr;
+lck_grp_t                                       thread_lck_grp;

 
 

-decl_simple_lock_data(static,thread_terminate_lock)
-static queue_head_t            thread_terminate_queue;
+struct zone                                     *thread_qos_override_zone;

 
 

-static queue_head_t            crashed_threads_queue;
+static struct mpsc_daemon_queue thread_stack_queue;
+static struct mpsc_daemon_queue thread_terminate_queue;
+static struct mpsc_daemon_queue thread_deallocate_queue;
+static struct mpsc_daemon_queue thread_exception_queue;

 
 

-decl_simple_lock_data(static,thread_exception_lock)
-static queue_head_t            thread_exception_queue;
+decl_simple_lock_data(static, crashed_threads_lock);
+static queue_head_t             crashed_threads_queue;

 
 struct thread_exception_elt {
 
 struct thread_exception_elt {

-       queue_chain_t   elt;
-       task_t          exception_task;
-       thread_t        exception_thread;
+       struct mpsc_queue_chain link;
+       exception_type_t        exception_type;
+       task_t                  exception_task;
+       thread_t                exception_thread;

 };
 
 };
 

-static struct thread   thread_template, init_thread;
-
-static void            sched_call_null(
-                                       int                     type,
-                                       thread_t        thread);
+static struct thread    thread_template, init_thread;
+static void thread_deallocate_enqueue(thread_t thread);
+static void thread_deallocate_complete(thread_t thread);

 
 #ifdef MACH_BSD
 extern void proc_exit(void *);
 extern mach_exception_data_type_t proc_encode_exit_exception_code(void *);
 extern uint64_t get_dispatchqueue_offset_from_proc(void *);
 
 #ifdef MACH_BSD
 extern void proc_exit(void *);
 extern mach_exception_data_type_t proc_encode_exit_exception_code(void *);
 extern uint64_t get_dispatchqueue_offset_from_proc(void *);

+extern uint64_t get_return_to_kernel_offset_from_proc(void *p);

 extern int      proc_selfpid(void);
 extern int      proc_selfpid(void);

+extern void     proc_name(int, char*, int);

 extern char *   proc_name_address(void *p);
 #endif /* MACH_BSD */
 
 extern int disable_exc_resource;
 extern int audio_active;
 extern int debug_task;
 extern char *   proc_name_address(void *p);
 #endif /* MACH_BSD */
 
 extern int disable_exc_resource;
 extern int audio_active;
 extern int debug_task;

-int thread_max = CONFIG_THREAD_MAX;    /* Max number of threads */
+int thread_max = CONFIG_THREAD_MAX;     /* Max number of threads */

 int task_threadmax = CONFIG_THREAD_MAX;
 
 int task_threadmax = CONFIG_THREAD_MAX;
 

-static uint64_t                thread_unique_id = 100;
+static uint64_t         thread_unique_id = 100;

 
 

-struct _thread_ledger_indices thread_ledgers = { -1 };
+struct _thread_ledger_indices thread_ledgers = { .cpu_time = -1 };

 static ledger_template_t thread_ledger_template = NULL;
 static void init_thread_ledgers(void);
 
 static ledger_template_t thread_ledger_template = NULL;
 static void init_thread_ledgers(void);
 


@@ -199,23 +218,35 @@ static void init_thread_ledgers(void);
 void jetsam_on_ledger_cpulimit_exceeded(void);
 #endif
 
 void jetsam_on_ledger_cpulimit_exceeded(void);
 #endif
 

+extern int task_thread_soft_limit;
+extern int exc_via_corpse_forking;
+
+#if DEVELOPMENT || DEBUG
+extern int exc_resource_threads_enabled;
+#endif /* DEVELOPMENT || DEBUG */
+

 /*
  * Level (in terms of percentage of the limit) at which the CPU usage monitor triggers telemetry.
  *
  * (ie when any thread's CPU consumption exceeds 70% of the limit, start taking user
  *  stacktraces, aka micro-stackshots)
  */
 /*
  * Level (in terms of percentage of the limit) at which the CPU usage monitor triggers telemetry.
  *
  * (ie when any thread's CPU consumption exceeds 70% of the limit, start taking user
  *  stacktraces, aka micro-stackshots)
  */

-#define        CPUMON_USTACKSHOTS_TRIGGER_DEFAULT_PCT 70
+#define CPUMON_USTACKSHOTS_TRIGGER_DEFAULT_PCT 70

 
 int cpumon_ustackshots_trigger_pct; /* Percentage. Level at which we start gathering telemetry. */
 void __attribute__((noinline)) SENDING_NOTIFICATION__THIS_THREAD_IS_CONSUMING_TOO_MUCH_CPU(void);
 
 int cpumon_ustackshots_trigger_pct; /* Percentage. Level at which we start gathering telemetry. */
 void __attribute__((noinline)) SENDING_NOTIFICATION__THIS_THREAD_IS_CONSUMING_TOO_MUCH_CPU(void);

+#if DEVELOPMENT || DEBUG
+void __attribute__((noinline)) SENDING_NOTIFICATION__TASK_HAS_TOO_MANY_THREADS(task_t, int);
+#endif /* DEVELOPMENT || DEBUG */

 
 /*
  * The smallest interval over which we support limiting CPU consumption is 1ms
  */
 #define MINIMUM_CPULIMIT_INTERVAL_MS 1
 
 
 /*
  * The smallest interval over which we support limiting CPU consumption is 1ms
  */
 #define MINIMUM_CPULIMIT_INTERVAL_MS 1
 

-void
+os_refgrp_decl(static, thread_refgrp, "thread", NULL);
+
+thread_t

 thread_bootstrap(void)
 {
        /*
 thread_bootstrap(void)
 {
        /*


@@ -228,8 +259,6 @@ thread_bootstrap(void)
 
        thread_template.runq = PROCESSOR_NULL;
 
 
        thread_template.runq = PROCESSOR_NULL;
 

-       thread_template.ref_count = 2;
-

        thread_template.reason = AST_NONE;
        thread_template.at_safe_point = FALSE;
        thread_template.wait_event = NO_EVENT64;
        thread_template.reason = AST_NONE;
        thread_template.at_safe_point = FALSE;
        thread_template.wait_event = NO_EVENT64;


@@ -260,18 +289,17 @@ thread_bootstrap(void)
        thread_template.sched_pri = 0;
        thread_template.max_priority = 0;
        thread_template.task_priority = 0;
        thread_template.sched_pri = 0;
        thread_template.max_priority = 0;
        thread_template.task_priority = 0;

-       thread_template.promotions = 0;
-       thread_template.pending_promoter_index = 0;
-       thread_template.pending_promoter[0] = NULL;
-       thread_template.pending_promoter[1] = NULL;

        thread_template.rwlock_count = 0;
        thread_template.rwlock_count = 0;

+       thread_template.waiting_for_mutex = NULL;

 
 
        thread_template.realtime.deadline = UINT64_MAX;
 
        thread_template.quantum_remaining = 0;
        thread_template.last_run_time = 0;
 
 
        thread_template.realtime.deadline = UINT64_MAX;
 
        thread_template.quantum_remaining = 0;
        thread_template.last_run_time = 0;

-       thread_template.last_made_runnable_time = 0;
+       thread_template.last_made_runnable_time = THREAD_NOT_RUNNABLE;
+       thread_template.last_basepri_change_time = THREAD_NOT_RUNNABLE;
+       thread_template.same_pri_latency = 0;

 
        thread_template.computation_metered = 0;
        thread_template.computation_epoch = 0;
 
        thread_template.computation_metered = 0;
        thread_template.computation_epoch = 0;


@@ -284,13 +312,20 @@ thread_bootstrap(void)
 #endif
        thread_template.c_switch = thread_template.p_switch = thread_template.ps_switch = 0;
 
 #endif
        thread_template.c_switch = thread_template.p_switch = thread_template.ps_switch = 0;
 

+#if MONOTONIC
+       memset(&thread_template.t_monotonic, 0,
+           sizeof(thread_template.t_monotonic));
+#endif /* MONOTONIC */
+

        thread_template.bound_processor = PROCESSOR_NULL;
        thread_template.last_processor = PROCESSOR_NULL;
 
        thread_template.bound_processor = PROCESSOR_NULL;
        thread_template.last_processor = PROCESSOR_NULL;
 

-       thread_template.sched_call = sched_call_null;
+       thread_template.sched_call = NULL;

 
        timer_init(&thread_template.user_timer);
        timer_init(&thread_template.system_timer);
 
        timer_init(&thread_template.user_timer);
        timer_init(&thread_template.system_timer);

+       timer_init(&thread_template.ptime);
+       timer_init(&thread_template.runnable_timer);

        thread_template.user_timer_save = 0;
        thread_template.system_timer_save = 0;
        thread_template.vtimer_user_save = 0;
        thread_template.user_timer_save = 0;
        thread_template.system_timer_save = 0;
        thread_template.vtimer_user_save = 0;


@@ -308,8 +343,11 @@ thread_bootstrap(void)
        thread_template.depress_timer_active = 0;
 
        thread_template.recover = (vm_offset_t)NULL;
        thread_template.depress_timer_active = 0;
 
        thread_template.recover = (vm_offset_t)NULL;

-       
+

        thread_template.map = VM_MAP_NULL;
        thread_template.map = VM_MAP_NULL;

+#if DEVELOPMENT || DEBUG
+       thread_template.pmap_footprint_suspended = FALSE;
+#endif /* DEVELOPMENT || DEBUG */

 
 #if CONFIG_DTRACE
        thread_template.t_dtrace_predcache = 0;
 
 #if CONFIG_DTRACE
        thread_template.t_dtrace_predcache = 0;


@@ -318,7 +356,7 @@ thread_bootstrap(void)
 #endif /* CONFIG_DTRACE */
 
 #if KPERF
 #endif /* CONFIG_DTRACE */
 
 #if KPERF

-       thread_template.kperf_flags = 0;
+       thread_template.kperf_ast = 0;

        thread_template.kperf_pet_gen = 0;
        thread_template.kperf_c_switch = 0;
        thread_template.kperf_pet_cnt = 0;
        thread_template.kperf_pet_gen = 0;
        thread_template.kperf_c_switch = 0;
        thread_template.kperf_pet_cnt = 0;


@@ -341,38 +379,52 @@ thread_bootstrap(void)
        thread_template.t_page_creation_time = 0;
 
        thread_template.affinity_set = NULL;
        thread_template.t_page_creation_time = 0;
 
        thread_template.affinity_set = NULL;

-       
+

        thread_template.syscalls_unix = 0;
        thread_template.syscalls_mach = 0;
 
        thread_template.t_ledger = LEDGER_NULL;
        thread_template.t_threadledger = LEDGER_NULL;
        thread_template.syscalls_unix = 0;
        thread_template.syscalls_mach = 0;
 
        thread_template.t_ledger = LEDGER_NULL;
        thread_template.t_threadledger = LEDGER_NULL;

-#ifdef CONFIG_BANK

        thread_template.t_bankledger = LEDGER_NULL;
        thread_template.t_deduct_bank_ledger_time = 0;
        thread_template.t_bankledger = LEDGER_NULL;
        thread_template.t_deduct_bank_ledger_time = 0;

-#endif

 
        thread_template.requested_policy = (struct thread_requested_policy) {};
        thread_template.effective_policy = (struct thread_effective_policy) {};
 
        bzero(&thread_template.overrides, sizeof(thread_template.overrides));
 
        thread_template.requested_policy = (struct thread_requested_policy) {};
        thread_template.effective_policy = (struct thread_effective_policy) {};
 
        bzero(&thread_template.overrides, sizeof(thread_template.overrides));

+       thread_template.kevent_overrides = 0;

 
        thread_template.iotier_override = THROTTLE_LEVEL_NONE;
        thread_template.thread_io_stats = NULL;
 
        thread_template.iotier_override = THROTTLE_LEVEL_NONE;
        thread_template.thread_io_stats = NULL;

+#if CONFIG_EMBEDDED
+       thread_template.taskwatch = NULL;
+#endif /* CONFIG_EMBEDDED */

        thread_template.thread_callout_interrupt_wakeups = thread_template.thread_callout_platform_idle_wakeups = 0;
 
        thread_template.thread_timer_wakeups_bin_1 = thread_template.thread_timer_wakeups_bin_2 = 0;
        thread_template.callout_woken_from_icontext = thread_template.callout_woken_from_platform_idle = 0;
        thread_template.thread_callout_interrupt_wakeups = thread_template.thread_callout_platform_idle_wakeups = 0;
 
        thread_template.thread_timer_wakeups_bin_1 = thread_template.thread_timer_wakeups_bin_2 = 0;
        thread_template.callout_woken_from_icontext = thread_template.callout_woken_from_platform_idle = 0;

+       thread_template.guard_exc_fatal = 0;

 
        thread_template.thread_tag = 0;
 
        thread_template.ith_voucher_name = MACH_PORT_NULL;
        thread_template.ith_voucher = IPC_VOUCHER_NULL;
 
 
        thread_template.thread_tag = 0;
 
        thread_template.ith_voucher_name = MACH_PORT_NULL;
        thread_template.ith_voucher = IPC_VOUCHER_NULL;
 

-       thread_template.work_interval_id = 0;
+       thread_template.th_work_interval = NULL;

 
 

+       thread_template.decompressions = 0;

        init_thread = thread_template;
        init_thread = thread_template;

-       machine_set_current_thread(&init_thread);
+
+       /* fiddle with init thread to skip asserts in set_sched_pri */
+       init_thread.sched_pri = MAXPRI_KERNEL;
+
+       return &init_thread;
+}
+
+void
+thread_machine_init_template(void)
+{
+       machine_thread_template_init(&thread_template);

 }
 
 extern boolean_t allow_qos_policy_set;
 }
 
 extern boolean_t allow_qos_policy_set;


@@ -381,10 +433,10 @@ void
 thread_init(void)
 {
        thread_zone = zinit(
 thread_init(void)
 {
        thread_zone = zinit(

-                       sizeof(struct thread),
-                       thread_max * sizeof(struct thread),
-                       THREAD_CHUNK * sizeof(struct thread),
-                       "threads");
+               sizeof(struct thread),
+               thread_max * sizeof(struct thread),
+               THREAD_CHUNK * sizeof(struct thread),
+               "threads");

 
        thread_qos_override_zone = zinit(
                sizeof(struct thread_qos_override),
 
        thread_qos_override_zone = zinit(
                sizeof(struct thread_qos_override),


@@ -411,11 +463,11 @@ thread_init(void)
        machine_thread_init();
 
        if (!PE_parse_boot_argn("cpumon_ustackshots_trigger_pct", &cpumon_ustackshots_trigger_pct,
        machine_thread_init();
 
        if (!PE_parse_boot_argn("cpumon_ustackshots_trigger_pct", &cpumon_ustackshots_trigger_pct,

-               sizeof (cpumon_ustackshots_trigger_pct))) {
+           sizeof(cpumon_ustackshots_trigger_pct))) {

                cpumon_ustackshots_trigger_pct = CPUMON_USTACKSHOTS_TRIGGER_DEFAULT_PCT;
        }
 
                cpumon_ustackshots_trigger_pct = CPUMON_USTACKSHOTS_TRIGGER_DEFAULT_PCT;
        }
 

-       PE_parse_boot_argn("-qos-policy-allow", &allow_qos_policy_set, sizeof(allow_qos_policy_set));   
+       PE_parse_boot_argn("-qos-policy-allow", &allow_qos_policy_set, sizeof(allow_qos_policy_set));

 
        init_thread_ledgers();
 }
 
        init_thread_ledgers();
 }


@@ -423,7 +475,7 @@ thread_init(void)
 boolean_t
 thread_is_active(thread_t thread)
 {
 boolean_t
 thread_is_active(thread_t thread)
 {

-       return (thread->active);
+       return thread->active;

 }
 
 void
 }
 
 void


@@ -432,13 +484,20 @@ thread_corpse_continue(void)
        thread_t thread = current_thread();
 
        thread_terminate_internal(thread);
        thread_t thread = current_thread();
 
        thread_terminate_internal(thread);

-       ml_set_interrupts_enabled(FALSE);
-       ast_taken(AST_APC, TRUE);
+
+       /*
+        * Handle the thread termination directly
+        * here instead of returning to userspace.
+        */
+       assert(thread->active == FALSE);
+       thread_ast_clear(thread, AST_APC);
+       thread_apc_ast(thread);

 
        panic("thread_corpse_continue");
        /*NOTREACHED*/
 }
 
 
        panic("thread_corpse_continue");
        /*NOTREACHED*/
 }
 

+__dead2

 static void
 thread_terminate_continue(void)
 {
 static void
 thread_terminate_continue(void)
 {


@@ -452,9 +511,8 @@ thread_terminate_continue(void)
 void
 thread_terminate_self(void)
 {
 void
 thread_terminate_self(void)
 {

-       thread_t                thread = current_thread();
-       task_t                  task;
-       spl_t                   s;
+       thread_t                thread = current_thread();
+       task_t                  task;

        int threadcnt;
 
        pal_thread_terminate_self(thread);
        int threadcnt;
 
        pal_thread_terminate_self(thread);


@@ -467,38 +525,21 @@ thread_terminate_self(void)
 
        thread_mtx_unlock(thread);
 
 
        thread_mtx_unlock(thread);
 

-       s = splsched();
-       thread_lock(thread);
-
-       /*
-        *      Cancel priority depression, wait for concurrent expirations
-        *      on other processors.
-        */
-       if (thread->sched_flags & TH_SFLAG_DEPRESSED_MASK) {
-               thread->sched_flags &= ~TH_SFLAG_DEPRESSED_MASK;
-
-               /* If our priority was low because of a depressed yield, restore it in case we block below */
-               thread_recompute_sched_pri(thread, FALSE);
-
-               if (timer_call_cancel(&thread->depress_timer))
-                       thread->depress_timer_active--;
-       }
-
-       while (thread->depress_timer_active > 0) {
-               thread_unlock(thread);
-               splx(s);
-
-               delay(1);
+       thread_sched_call(thread, NULL);

 
 

-               s = splsched();
-               thread_lock(thread);
-       }
+       spl_t s = splsched();
+       thread_lock(thread);

 
 

-       thread_sched_call(thread, NULL);
+       thread_depress_abort_locked(thread);

 
        thread_unlock(thread);
        splx(s);
 
 
        thread_unlock(thread);
        splx(s);
 

+#if CONFIG_EMBEDDED
+       thead_remove_taskwatch(thread);
+#endif /* CONFIG_EMBEDDED */
+
+       work_interval_thread_terminate(thread);

 
        thread_mtx_lock(thread);
 
 
        thread_mtx_lock(thread);
 


@@ -506,34 +547,46 @@ thread_terminate_self(void)
 
        thread_mtx_unlock(thread);
 
 
        thread_mtx_unlock(thread);
 

+       bank_swap_thread_bank_ledger(thread, NULL);
+
+       if (kdebug_enable && bsd_hasthreadname(thread->uthread)) {
+               char threadname[MAXTHREADNAMESIZE];
+               bsd_getthreadname(thread->uthread, threadname);
+               kernel_debug_string_simple(TRACE_STRING_THREADNAME_PREV, threadname);
+       }
+

        task = thread->task;
        uthread_cleanup(task, thread->uthread, task->bsd_info);
        task = thread->task;
        uthread_cleanup(task, thread->uthread, task->bsd_info);

-       threadcnt = hw_atomic_sub(&task->active_thread_count, 1);

 
 

-       if (task->bsd_info && !task_is_exec_copy(task)) {
+       if (kdebug_enable && task->bsd_info && !task_is_exec_copy(task)) {

                /* trace out pid before we sign off */
                /* trace out pid before we sign off */

-               long    dbg_arg1 = 0;
-
-               kdbg_trace_data(thread->task->bsd_info, &dbg_arg1);
+               long dbg_arg1 = 0;
+               long dbg_arg2 = 0;

 
 

-               KERNEL_DEBUG_CONSTANT(TRACE_DATA_THREAD_TERMINATE_PID | DBG_FUNC_NONE,
-                       dbg_arg1, 0, 0, 0, 0);
+               kdbg_trace_data(thread->task->bsd_info, &dbg_arg1, &dbg_arg2);
+               KDBG_RELEASE(TRACE_DATA_THREAD_TERMINATE_PID, dbg_arg1, dbg_arg2);

        }
 
        }
 

+       /*
+        * After this subtraction, this thread should never access
+        * task->bsd_info unless it got 0 back from the os_atomic_dec.  It
+        * could be racing with other threads to be the last thread in the
+        * process, and the last thread in the process will tear down the proc
+        * structure and zero-out task->bsd_info.
+        */
+       threadcnt = os_atomic_dec(&task->active_thread_count, relaxed);
+

        /*
         * If we are the last thread to terminate and the task is
         * associated with a BSD process, perform BSD process exit.
         */
        if (threadcnt == 0 && task->bsd_info != NULL && !task_is_exec_copy(task)) {
                mach_exception_data_type_t subcode = 0;
        /*
         * If we are the last thread to terminate and the task is
         * associated with a BSD process, perform BSD process exit.
         */
        if (threadcnt == 0 && task->bsd_info != NULL && !task_is_exec_copy(task)) {
                mach_exception_data_type_t subcode = 0;

-               {
+               if (kdebug_enable) {

                        /* since we're the last thread in this process, trace out the command name too */
                        /* since we're the last thread in this process, trace out the command name too */

-                       long    dbg_arg1 = 0, dbg_arg2 = 0, dbg_arg3 = 0, dbg_arg4 = 0;
-
-                       kdbg_trace_string(thread->task->bsd_info, &dbg_arg1, &dbg_arg2, &dbg_arg3, &dbg_arg4);
-
-                       KERNEL_DEBUG_CONSTANT(TRACE_STRING_PROC_EXIT | DBG_FUNC_NONE,
-                               dbg_arg1, dbg_arg2, dbg_arg3, dbg_arg4, 0);
+                       long args[4] = {};
+                       kdbg_trace_string(thread->task->bsd_info, &args[0], &args[1], &args[2], &args[3]);
+                       KDBG_RELEASE(TRACE_STRING_PROC_EXIT, args[0], args[1], args[2], args[3]);

                }
 
                /* Get the exit reason before proc_exit */
                }
 
                /* Get the exit reason before proc_exit */


@@ -545,7 +598,7 @@ thread_terminate_self(void)
                 * last thread for this task.
                 */
                if (task->corpse_info) {
                 * last thread for this task.
                 */
                if (task->corpse_info) {

-                       task_deliver_crash_notification(task, current_thread(), subcode);
+                       task_deliver_crash_notification(task, current_thread(), EXC_RESOURCE, subcode);

                }
        }
 
                }
        }
 


@@ -562,6 +615,33 @@ thread_terminate_self(void)
        s = splsched();
        thread_lock(thread);
 
        s = splsched();
        thread_lock(thread);
 

+       /*
+        * Ensure that the depress timer is no longer enqueued,
+        * so the timer (stored in the thread) can be safely deallocated
+        *
+        * TODO: build timer_call_cancel_wait
+        */
+
+       assert((thread->sched_flags & TH_SFLAG_DEPRESSED_MASK) == 0);
+
+       uint32_t delay_us = 1;
+
+       while (thread->depress_timer_active > 0) {
+               thread_unlock(thread);
+               splx(s);
+
+               delay(delay_us++);
+
+               if (delay_us > USEC_PER_SEC) {
+                       panic("depress timer failed to inactivate!"
+                           "thread: %p depress_timer_active: %d",
+                           thread, thread->depress_timer_active);
+               }
+
+               s = splsched();
+               thread_lock(thread);
+       }
+

        /*
         *      Cancel wait timer, and wait for
         *      concurrent expirations.
        /*
         *      Cancel wait timer, and wait for
         *      concurrent expirations.


@@ -569,15 +649,24 @@ thread_terminate_self(void)
        if (thread->wait_timer_is_set) {
                thread->wait_timer_is_set = FALSE;
 
        if (thread->wait_timer_is_set) {
                thread->wait_timer_is_set = FALSE;
 

-               if (timer_call_cancel(&thread->wait_timer))
+               if (timer_call_cancel(&thread->wait_timer)) {

                        thread->wait_timer_active--;
                        thread->wait_timer_active--;

+               }

        }
 
        }
 

+       delay_us = 1;
+

        while (thread->wait_timer_active > 0) {
                thread_unlock(thread);
                splx(s);
 
        while (thread->wait_timer_active > 0) {
                thread_unlock(thread);
                splx(s);
 

-               delay(1);
+               delay(delay_us++);
+
+               if (delay_us > USEC_PER_SEC) {
+                       panic("wait timer failed to inactivate!"
+                           "thread: %p wait_timer_active: %d",
+                           thread, thread->wait_timer_active);
+               }

 
                s = splsched();
                thread_lock(thread);
 
                s = splsched();
                thread_lock(thread);


@@ -596,10 +685,15 @@ thread_terminate_self(void)
         */
        thread->state |= TH_TERMINATE;
        thread_mark_wait_locked(thread, THREAD_UNINT);
         */
        thread->state |= TH_TERMINATE;
        thread_mark_wait_locked(thread, THREAD_UNINT);

+
+       assert((thread->sched_flags & TH_SFLAG_WAITQ_PROMOTED) == 0);
+       assert((thread->sched_flags & TH_SFLAG_RW_PROMOTED) == 0);
+       assert((thread->sched_flags & TH_SFLAG_EXEC_PROMOTED) == 0);

        assert((thread->sched_flags & TH_SFLAG_PROMOTED) == 0);
        assert((thread->sched_flags & TH_SFLAG_PROMOTED) == 0);

-       assert(thread->promotions == 0);
-       assert(!(thread->sched_flags & TH_SFLAG_WAITQ_PROMOTED));
+       assert(thread->kern_promotion_schedpri == 0);
+       assert(thread->waiting_for_mutex == NULL);

        assert(thread->rwlock_count == 0);
        assert(thread->rwlock_count == 0);

+

        thread_unlock(thread);
        /* splsched */
 
        thread_unlock(thread);
        /* splsched */
 


@@ -607,40 +701,52 @@ thread_terminate_self(void)
        /*NOTREACHED*/
 }
 
        /*NOTREACHED*/
 }
 

-/* Drop a thread refcount that definitely isn't the last one. */
-void
-thread_deallocate_safe(thread_t thread)
+static bool
+thread_ref_release(thread_t thread)

 {
 {

+       if (thread == THREAD_NULL) {
+               return false;
+       }
+

        assert_thread_magic(thread);
 
        assert_thread_magic(thread);
 

-       uint32_t old_refcount = hw_atomic_sub(&(thread)->ref_count, 1) + 1;
+       return os_ref_release(&thread->ref_count) == 0;
+}

 
 

-       if (__improbable(old_refcount <= 1))
-               panic("bad thread refcount: %d", old_refcount);
+/* Drop a thread refcount safely without triggering a zfree */
+void
+thread_deallocate_safe(thread_t thread)
+{
+       if (__improbable(thread_ref_release(thread))) {
+               /* enqueue the thread for thread deallocate deamon to call thread_deallocate_complete */
+               thread_deallocate_enqueue(thread);
+       }

 }
 
 void
 }
 
 void

-thread_deallocate(
-       thread_t                        thread)
+thread_deallocate(thread_t thread)

 {
 {

-       task_t                          task;
+       if (__improbable(thread_ref_release(thread))) {
+               thread_deallocate_complete(thread);
+       }
+}

 
 

-       if (thread == THREAD_NULL)
-               return;
+void
+thread_deallocate_complete(
+       thread_t                        thread)
+{
+       task_t                          task;

 
        assert_thread_magic(thread);
 
        assert_thread_magic(thread);

-       assert(thread->ref_count > 0);

 
 

-       if (__probable(hw_atomic_sub(&(thread)->ref_count, 1) > 0))
-               return;
+       assert(os_ref_get_count(&thread->ref_count) == 0);

 
 

-       if(!(thread->state & TH_TERMINATE2))
+       if (!(thread->state & TH_TERMINATE2)) {

                panic("thread_deallocate: thread not properly terminated\n");
                panic("thread_deallocate: thread not properly terminated\n");

+       }

 
        assert(thread->runq == PROCESSOR_NULL);
 
 
        assert(thread->runq == PROCESSOR_NULL);
 

-       assert(thread->user_promotions == 0);
-

 #if KPC
        kpc_thread_destroy(thread);
 #endif
 #if KPC
        kpc_thread_destroy(thread);
 #endif


@@ -651,28 +757,38 @@ thread_deallocate(
 
        task = thread->task;
 
 
        task = thread->task;
 

-#ifdef MACH_BSD 
+#ifdef MACH_BSD

        {
                void *ut = thread->uthread;
 
                thread->uthread = NULL;
                uthread_zone_free(ut);
        }
        {
                void *ut = thread->uthread;
 
                thread->uthread = NULL;
                uthread_zone_free(ut);
        }

-#endif  /* MACH_BSD */   
+#endif /* MACH_BSD */

 
 

-       if (thread->t_ledger)
+       if (thread->t_ledger) {

                ledger_dereference(thread->t_ledger);
                ledger_dereference(thread->t_ledger);

-       if (thread->t_threadledger)
+       }
+       if (thread->t_threadledger) {

                ledger_dereference(thread->t_threadledger);
                ledger_dereference(thread->t_threadledger);

+       }
+
+       assert(thread->turnstile != TURNSTILE_NULL);
+       if (thread->turnstile) {
+               turnstile_deallocate(thread->turnstile);
+       }

 
 

-       if (IPC_VOUCHER_NULL != thread->ith_voucher)
+       if (IPC_VOUCHER_NULL != thread->ith_voucher) {

                ipc_voucher_release(thread->ith_voucher);
                ipc_voucher_release(thread->ith_voucher);

+       }

 
 

-       if (thread->thread_io_stats)
+       if (thread->thread_io_stats) {

                kfree(thread->thread_io_stats, sizeof(struct io_stat_info));
                kfree(thread->thread_io_stats, sizeof(struct io_stat_info));

+       }

 
 

-       if (thread->kernel_stack != 0)
+       if (thread->kernel_stack != 0) {

                stack_free(thread);
                stack_free(thread);

+       }

 
        lck_mtx_destroy(&thread->mutex, &thread_lck_grp);
        machine_thread_destroy(thread);
 
        lck_mtx_destroy(&thread->mutex, &thread_lck_grp);
        machine_thread_destroy(thread);


@@ -688,70 +804,74 @@ thread_deallocate(
 }
 
 /*
 }
 
 /*

- *     thread_exception_daemon:
+ *     thread_inspect_deallocate:

  *
  *

- *     Deliver EXC_RESOURCE exception
+ *     Drop a thread inspection reference.
+ */
+void
+thread_inspect_deallocate(
+       thread_inspect_t                thread_inspect)
+{
+       return thread_deallocate((thread_t)thread_inspect);
+}
+
+/*
+ *     thread_exception_queue_invoke:
+ *
+ *     Deliver EXC_{RESOURCE,GUARD} exception

  */
 static void
  */
 static void

-thread_exception_daemon(void)
+thread_exception_queue_invoke(mpsc_queue_chain_t elm,
+    __assert_only mpsc_daemon_queue_t dq)

 {
        struct thread_exception_elt *elt;
        task_t task;
        thread_t thread;
 {
        struct thread_exception_elt *elt;
        task_t task;
        thread_t thread;

+       exception_type_t etype;

 
 

-       simple_lock(&thread_exception_lock);
-       while ((elt = (struct thread_exception_elt *)dequeue_head(&thread_exception_queue)) != NULL) {
-               simple_unlock(&thread_exception_lock);
-
-               task = elt->exception_task;
-               thread = elt->exception_thread;
-               assert_thread_magic(thread);
+       assert(dq == &thread_exception_queue);
+       elt = mpsc_queue_element(elm, struct thread_exception_elt, link);

 
 

-               kfree(elt, sizeof(struct thread_exception_elt));
-
-               /* wait for all the threads in the task to terminate */
-               task_lock(task);
-               task_wait_till_threads_terminate_locked(task);
-               task_unlock(task);
-
-               /* Consumes the task ref returned by task_generate_corpse_internal */
-               task_deallocate(task);
-               /* Consumes the thread ref returned by task_generate_corpse_internal */
-               thread_deallocate(thread);
+       etype = elt->exception_type;
+       task = elt->exception_task;
+       thread = elt->exception_thread;
+       assert_thread_magic(thread);

 
 

-               /* Deliver the EXC_RESOURCE notification, also clears the corpse. */
-               task_deliver_crash_notification(task, thread, 0);
+       kfree(elt, sizeof(*elt));

 
 

-               simple_lock(&thread_exception_lock);
-       }
+       /* wait for all the threads in the task to terminate */
+       task_lock(task);
+       task_wait_till_threads_terminate_locked(task);
+       task_unlock(task);

 
 

-       assert_wait((event_t)&thread_exception_queue, THREAD_UNINT);
-       simple_unlock(&thread_exception_lock);
+       /* Consumes the task ref returned by task_generate_corpse_internal */
+       task_deallocate(task);
+       /* Consumes the thread ref returned by task_generate_corpse_internal */
+       thread_deallocate(thread);

 
 

-       thread_block((thread_continue_t)thread_exception_daemon);
+       /* Deliver the notification, also clears the corpse. */
+       task_deliver_crash_notification(task, thread, etype, 0);

 }
 
 /*
  *     thread_exception_enqueue:
  *
 }
 
 /*
  *     thread_exception_enqueue:
  *

- *     Enqueue a corpse port to be delivered an EXC_RESOURCE.
+ *     Enqueue a corpse port to be delivered an EXC_{RESOURCE,GUARD}.

  */
 void
 thread_exception_enqueue(
  */
 void
 thread_exception_enqueue(

-       task_t          task,
-       thread_t        thread)
+       task_t          task,
+       thread_t        thread,
+       exception_type_t etype)

 {
 {

-       struct thread_exception_elt *elt = (struct thread_exception_elt*) kalloc(
-                                               sizeof(struct thread_exception_elt));
-
+       assert(EXC_RESOURCE == etype || EXC_GUARD == etype);
+       struct thread_exception_elt *elt = kalloc(sizeof(*elt));
+       elt->exception_type = etype;

        elt->exception_task = task;
        elt->exception_thread = thread;
 
        elt->exception_task = task;
        elt->exception_thread = thread;
 

-       simple_lock(&thread_exception_lock);
-       enqueue_tail(&thread_exception_queue, (queue_entry_t)elt);
-       simple_unlock(&thread_exception_lock);
-
-       thread_wakeup((event_t)&thread_exception_queue);
+       mpsc_daemon_enqueue(&thread_exception_queue, &elt->link,
+           MPSC_QUEUE_DISABLE_PREEMPTION);

 }
 
 /*
 }
 
 /*


@@ -765,108 +885,113 @@ thread_copy_resource_info(
        thread_t dst_thread,
        thread_t src_thread)
 {
        thread_t dst_thread,
        thread_t src_thread)
 {

-       dst_thread->thread_tag = src_thread->thread_tag;

        dst_thread->c_switch = src_thread->c_switch;
        dst_thread->p_switch = src_thread->p_switch;
        dst_thread->ps_switch = src_thread->ps_switch;
        dst_thread->precise_user_kernel_time = src_thread->precise_user_kernel_time;
        dst_thread->user_timer = src_thread->user_timer;
        dst_thread->user_timer_save = src_thread->user_timer_save;
        dst_thread->c_switch = src_thread->c_switch;
        dst_thread->p_switch = src_thread->p_switch;
        dst_thread->ps_switch = src_thread->ps_switch;
        dst_thread->precise_user_kernel_time = src_thread->precise_user_kernel_time;
        dst_thread->user_timer = src_thread->user_timer;
        dst_thread->user_timer_save = src_thread->user_timer_save;

+       dst_thread->system_timer = src_thread->system_timer;

        dst_thread->system_timer_save = src_thread->system_timer_save;
        dst_thread->system_timer_save = src_thread->system_timer_save;

+       dst_thread->runnable_timer = src_thread->runnable_timer;
+       dst_thread->vtimer_user_save = src_thread->vtimer_user_save;
+       dst_thread->vtimer_prof_save = src_thread->vtimer_prof_save;
+       dst_thread->vtimer_rlim_save = src_thread->vtimer_rlim_save;
+       dst_thread->vtimer_qos_save = src_thread->vtimer_qos_save;

        dst_thread->syscalls_unix = src_thread->syscalls_unix;
        dst_thread->syscalls_mach = src_thread->syscalls_mach;
        ledger_rollup(dst_thread->t_threadledger, src_thread->t_threadledger);
        *dst_thread->thread_io_stats = *src_thread->thread_io_stats;
        dst_thread->syscalls_unix = src_thread->syscalls_unix;
        dst_thread->syscalls_mach = src_thread->syscalls_mach;
        ledger_rollup(dst_thread->t_threadledger, src_thread->t_threadledger);
        *dst_thread->thread_io_stats = *src_thread->thread_io_stats;

-

 }
 
 }
 

-/*
- *     thread_terminate_daemon:
- *
- *     Perform final clean up for terminating threads.
- */

 static void
 static void

-thread_terminate_daemon(void)
+thread_terminate_queue_invoke(mpsc_queue_chain_t e,
+    __assert_only mpsc_daemon_queue_t dq)

 {
 {

-       thread_t        self, thread;
-       task_t          task;
+       thread_t thread = mpsc_queue_element(e, struct thread, mpsc_links);
+       task_t task = thread->task;

 
 

-       self = current_thread();
-       self->options |= TH_OPT_SYSTEM_CRITICAL;
+       assert(dq == &thread_terminate_queue);

 
 

-       (void)splsched();
-       simple_lock(&thread_terminate_lock);
+       task_lock(task);

 
 

-       while ((thread = qe_dequeue_head(&thread_terminate_queue, struct thread, runq_links)) != THREAD_NULL) {
-               assert_thread_magic(thread);
+       /*
+        * if marked for crash reporting, skip reaping.
+        * The corpse delivery thread will clear bit and enqueue
+        * for reaping when done
+        *
+        * Note: the inspection field is set under the task lock
+        *
+        * FIXME[mad]: why enqueue for termination before `inspection` is false ?
+        */
+       if (__improbable(thread->inspection)) {
+               simple_lock(&crashed_threads_lock, &thread_lck_grp);
+               task_unlock(task);

 
 

-               /* 
-                * if marked for crash reporting, skip reaping. 
-                * The corpse delivery thread will clear bit and enqueue 
-                * for reaping when done
-                */
-               if (thread->inspection){
-                       enqueue_tail(&crashed_threads_queue, &thread->runq_links);
-                       continue;
-               }
+               enqueue_tail(&crashed_threads_queue, &thread->runq_links);
+               simple_unlock(&crashed_threads_lock);
+               return;
+       }

 
 

-               simple_unlock(&thread_terminate_lock);
-               (void)spllo();

 
 

-               task = thread->task;
+       task->total_user_time += timer_grab(&thread->user_timer);
+       task->total_ptime += timer_grab(&thread->ptime);
+       task->total_runnable_time += timer_grab(&thread->runnable_timer);
+       if (thread->precise_user_kernel_time) {
+               task->total_system_time += timer_grab(&thread->system_timer);
+       } else {
+               task->total_user_time += timer_grab(&thread->system_timer);
+       }

 
 

-               task_lock(task);
-               task->total_user_time += timer_grab(&thread->user_timer);
-               if (thread->precise_user_kernel_time) {
-                       task->total_system_time += timer_grab(&thread->system_timer);
-               } else {
-                       task->total_user_time += timer_grab(&thread->system_timer);
-               }
+       task->c_switch += thread->c_switch;
+       task->p_switch += thread->p_switch;
+       task->ps_switch += thread->ps_switch;

 
 

-               task->c_switch += thread->c_switch;
-               task->p_switch += thread->p_switch;
-               task->ps_switch += thread->ps_switch;
+       task->syscalls_unix += thread->syscalls_unix;
+       task->syscalls_mach += thread->syscalls_mach;

 
 

-               task->syscalls_unix += thread->syscalls_unix;
-               task->syscalls_mach += thread->syscalls_mach;
+       task->task_timer_wakeups_bin_1 += thread->thread_timer_wakeups_bin_1;
+       task->task_timer_wakeups_bin_2 += thread->thread_timer_wakeups_bin_2;
+       task->task_gpu_ns += ml_gpu_stat(thread);
+       task->task_energy += ml_energy_stat(thread);
+       task->decompressions += thread->decompressions;

 
 

-               task->task_timer_wakeups_bin_1 += thread->thread_timer_wakeups_bin_1;
-               task->task_timer_wakeups_bin_2 += thread->thread_timer_wakeups_bin_2;
-               task->task_gpu_ns += ml_gpu_stat(thread);
-               task->task_energy += ml_energy_stat(thread);
+#if MONOTONIC
+       mt_terminate_update(task, thread);
+#endif /* MONOTONIC */

 
 

-               thread_update_qos_cpu_time(thread);
+       thread_update_qos_cpu_time(thread);

 
 

-               queue_remove(&task->threads, thread, thread_t, task_threads);
-               task->thread_count--;
+       queue_remove(&task->threads, thread, thread_t, task_threads);
+       task->thread_count--;

 
 

-               /* 
-                * If the task is being halted, and there is only one thread
-                * left in the task after this one, then wakeup that thread.
-                */
-               if (task->thread_count == 1 && task->halting)
-                       thread_wakeup((event_t)&task->halting);
+       /*
+        * If the task is being halted, and there is only one thread
+        * left in the task after this one, then wakeup that thread.
+        */
+       if (task->thread_count == 1 && task->halting) {
+               thread_wakeup((event_t)&task->halting);
+       }

 
 

-               task_unlock(task);
+       task_unlock(task);

 
 

-               lck_mtx_lock(&tasks_threads_lock);
-               queue_remove(&threads, thread, thread_t, threads);
-               threads_count--;
-               lck_mtx_unlock(&tasks_threads_lock);
+       lck_mtx_lock(&tasks_threads_lock);
+       queue_remove(&threads, thread, thread_t, threads);
+       threads_count--;
+       lck_mtx_unlock(&tasks_threads_lock);

 
 

-               thread_deallocate(thread);
+       thread_deallocate(thread);
+}

 
 

-               (void)splsched();
-               simple_lock(&thread_terminate_lock);
-       }
+static void
+thread_deallocate_queue_invoke(mpsc_queue_chain_t e,
+    __assert_only mpsc_daemon_queue_t dq)
+{
+       thread_t thread = mpsc_queue_element(e, struct thread, mpsc_links);

 
 

-       assert_wait((event_t)&thread_terminate_queue, THREAD_UNINT);
-       simple_unlock(&thread_terminate_lock);
-       /* splsched */
+       assert(dq == &thread_deallocate_queue);

 
 

-       self->options &= ~TH_OPT_SYSTEM_CRITICAL;
-       thread_block((thread_continue_t)thread_terminate_daemon);
-       /*NOTREACHED*/
+       thread_deallocate_complete(thread);

 }
 
 /*
 }
 
 /*


@@ -878,15 +1003,25 @@ thread_terminate_daemon(void)
  */
 void
 thread_terminate_enqueue(
  */
 void
 thread_terminate_enqueue(

-       thread_t                thread)
+       thread_t                thread)

 {
 {

-       KERNEL_DEBUG_CONSTANT(TRACE_DATA_THREAD_TERMINATE | DBG_FUNC_NONE, thread->thread_id, 0, 0, 0, 0);
+       KDBG_RELEASE(TRACE_DATA_THREAD_TERMINATE, thread->thread_id);

 
 

-       simple_lock(&thread_terminate_lock);
-       enqueue_tail(&thread_terminate_queue, &thread->runq_links);
-       simple_unlock(&thread_terminate_lock);
+       mpsc_daemon_enqueue(&thread_terminate_queue, &thread->mpsc_links,
+           MPSC_QUEUE_DISABLE_PREEMPTION);
+}

 
 

-       thread_wakeup((event_t)&thread_terminate_queue);
+/*
+ *     thread_deallocate_enqueue:
+ *
+ *     Enqueue a thread for final deallocation.
+ */
+static void
+thread_deallocate_enqueue(
+       thread_t                thread)
+{
+       mpsc_daemon_enqueue(&thread_deallocate_queue, &thread->mpsc_links,
+           MPSC_QUEUE_DISABLE_PREEMPTION);

 }
 
 /*
 }
 
 /*


@@ -895,12 +1030,11 @@ thread_terminate_enqueue(
  * who are no longer being inspected.
  */
 void
  * who are no longer being inspected.
  */
 void

-thread_terminate_crashed_threads()
+thread_terminate_crashed_threads(void)

 {
        thread_t th_remove;
 {
        thread_t th_remove;

-       boolean_t should_wake_terminate_queue = FALSE;

 
 

-       simple_lock(&thread_terminate_lock);
+       simple_lock(&crashed_threads_lock, &thread_lck_grp);

        /*
         * loop through the crashed threads queue
         * to put any threads that are not being inspected anymore
        /*
         * loop through the crashed threads queue
         * to put any threads that are not being inspected anymore


@@ -911,57 +1045,39 @@ thread_terminate_crashed_threads()
                assert(th_remove != current_thread());
 
                if (th_remove->inspection == FALSE) {
                assert(th_remove != current_thread());
 
                if (th_remove->inspection == FALSE) {

-                       re_queue_tail(&thread_terminate_queue, &th_remove->runq_links);
-                       should_wake_terminate_queue = TRUE;
+                       remqueue(&th_remove->runq_links);
+                       mpsc_daemon_enqueue(&thread_terminate_queue, &th_remove->mpsc_links,
+                           MPSC_QUEUE_NONE);

                }
        }
 
                }
        }
 

-       simple_unlock(&thread_terminate_lock);
-       if (should_wake_terminate_queue == TRUE) {
-               thread_wakeup((event_t)&thread_terminate_queue);
-       }
+       simple_unlock(&crashed_threads_lock);

 }
 
 /*
 }
 
 /*

- *     thread_stack_daemon:
+ *     thread_stack_queue_invoke:

  *
  *     Perform stack allocation as required due to
  *     invoke failures.
  */
 static void
  *
  *     Perform stack allocation as required due to
  *     invoke failures.
  */
 static void

-thread_stack_daemon(void)
+thread_stack_queue_invoke(mpsc_queue_chain_t elm,
+    __assert_only mpsc_daemon_queue_t dq)

 {
 {

-       thread_t                thread;
-       spl_t                   s;
-
-       s = splsched();
-       simple_lock(&thread_stack_lock);
-
-       while ((thread = qe_dequeue_head(&thread_stack_queue, struct thread, runq_links)) != THREAD_NULL) {
-               assert_thread_magic(thread);
-
-               simple_unlock(&thread_stack_lock);
-               splx(s);
+       thread_t thread = mpsc_queue_element(elm, struct thread, mpsc_links);

 
 

-               /* allocate stack with interrupts enabled so that we can call into VM */
-               stack_alloc(thread);
+       assert(dq == &thread_stack_queue);

 
 

-               KERNEL_DEBUG_CONSTANT(MACHDBG_CODE(DBG_MACH_SCHED,MACH_STACK_WAIT) | DBG_FUNC_END, thread_tid(thread), 0, 0, 0, 0);
-               
-               s = splsched();
-               thread_lock(thread);
-               thread_setrun(thread, SCHED_PREEMPT | SCHED_TAILQ);
-               thread_unlock(thread);
+       /* allocate stack with interrupts enabled so that we can call into VM */
+       stack_alloc(thread);

 
 

-               simple_lock(&thread_stack_lock);
-       }
+       KERNEL_DEBUG_CONSTANT(MACHDBG_CODE(DBG_MACH_SCHED, MACH_STACK_WAIT) | DBG_FUNC_END, thread_tid(thread), 0, 0, 0, 0);

 
 

-       assert_wait((event_t)&thread_stack_queue, THREAD_UNINT);
-       simple_unlock(&thread_stack_lock);
+       spl_t s = splsched();
+       thread_lock(thread);
+       thread_setrun(thread, SCHED_PREEMPT | SCHED_TAILQ);
+       thread_unlock(thread);

        splx(s);
        splx(s);

-
-       thread_block((thread_continue_t)thread_stack_daemon);
-       /*NOTREACHED*/

 }
 
 /*
 }
 
 /*


@@ -973,56 +1089,50 @@ thread_stack_daemon(void)
  */
 void
 thread_stack_enqueue(
  */
 void
 thread_stack_enqueue(

-       thread_t                thread)
+       thread_t                thread)

 {
 {

-       KERNEL_DEBUG_CONSTANT(MACHDBG_CODE(DBG_MACH_SCHED,MACH_STACK_WAIT) | DBG_FUNC_START, thread_tid(thread), 0, 0, 0, 0);
+       KERNEL_DEBUG_CONSTANT(MACHDBG_CODE(DBG_MACH_SCHED, MACH_STACK_WAIT) | DBG_FUNC_START, thread_tid(thread), 0, 0, 0, 0);

        assert_thread_magic(thread);
 
        assert_thread_magic(thread);
 

-       simple_lock(&thread_stack_lock);
-       enqueue_tail(&thread_stack_queue, &thread->runq_links);
-       simple_unlock(&thread_stack_lock);
-
-       thread_wakeup((event_t)&thread_stack_queue);
+       mpsc_daemon_enqueue(&thread_stack_queue, &thread->mpsc_links,
+           MPSC_QUEUE_DISABLE_PREEMPTION);

 }
 
 void
 thread_daemon_init(void)
 {
 }
 
 void
 thread_daemon_init(void)
 {

-       kern_return_t   result;
-       thread_t        thread = NULL;
+       kern_return_t   result;

 
 

-       simple_lock_init(&thread_terminate_lock, 0);
-       queue_init(&thread_terminate_queue);
-       queue_init(&crashed_threads_queue);
+       thread_deallocate_daemon_init();

 
 

-       result = kernel_thread_start_priority((thread_continue_t)thread_terminate_daemon, NULL, MINPRI_KERNEL, &thread);
-       if (result != KERN_SUCCESS)
-               panic("thread_daemon_init: thread_terminate_daemon");
+       thread_deallocate_daemon_register_queue(&thread_terminate_queue,
+           thread_terminate_queue_invoke);

 
 

-       thread_deallocate(thread);
+       thread_deallocate_daemon_register_queue(&thread_deallocate_queue,
+           thread_deallocate_queue_invoke);

 
 

-       simple_lock_init(&thread_stack_lock, 0);
-       queue_init(&thread_stack_queue);
+       simple_lock_init(&crashed_threads_lock, 0);
+       queue_init(&crashed_threads_queue);

 
 

-       result = kernel_thread_start_priority((thread_continue_t)thread_stack_daemon, NULL, BASEPRI_PREEMPT, &thread);
-       if (result != KERN_SUCCESS)
+       result = mpsc_daemon_queue_init_with_thread(&thread_stack_queue,
+           thread_stack_queue_invoke, BASEPRI_PREEMPT_HIGH,
+           "daemon.thread-stack");
+       if (result != KERN_SUCCESS) {

                panic("thread_daemon_init: thread_stack_daemon");
                panic("thread_daemon_init: thread_stack_daemon");

+       }

 
 

-       thread_deallocate(thread);
-
-       simple_lock_init(&thread_exception_lock, 0);
-       queue_init(&thread_exception_queue);
-
-       result = kernel_thread_start_priority((thread_continue_t)thread_exception_daemon, NULL, MINPRI_KERNEL, &thread);
-       if (result != KERN_SUCCESS)
+       result = mpsc_daemon_queue_init_with_thread(&thread_exception_queue,
+           thread_exception_queue_invoke, MINPRI_KERNEL,
+           "daemon.thread-exception");
+       if (result != KERN_SUCCESS) {

                panic("thread_daemon_init: thread_exception_daemon");
                panic("thread_daemon_init: thread_exception_daemon");

-
-       thread_deallocate(thread);
+       }

 }
 
 }
 

-#define TH_OPTION_NONE         0x00
-#define TH_OPTION_NOCRED       0x01
-#define TH_OPTION_NOSUSP       0x02
+#define TH_OPTION_NONE          0x00
+#define TH_OPTION_NOCRED        0x01
+#define TH_OPTION_NOSUSP        0x02
+#define TH_OPTION_WORKQ         0x04

 
 /*
  * Create a new thread.
 
 /*
  * Create a new thread.


@@ -1032,27 +1142,33 @@ thread_daemon_init(void)
  */
 static kern_return_t
 thread_create_internal(
  */
 static kern_return_t
 thread_create_internal(

-       task_t                                  parent_task,
-       integer_t                               priority,
-       thread_continue_t               continuation,
-       int                                             options,
-       thread_t                                *out_thread)
+       task_t                                  parent_task,
+       integer_t                               priority,
+       thread_continue_t               continuation,
+       void                                    *parameter,
+       int                                             options,
+       thread_t                                *out_thread)

 {
 {

-       thread_t                                new_thread;
-       static thread_t                 first_thread;
+       thread_t                                new_thread;
+       static thread_t                 first_thread;

 
        /*
         *      Allocate a thread and initialize static fields
         */
 
        /*
         *      Allocate a thread and initialize static fields
         */

-       if (first_thread == THREAD_NULL)
+       if (first_thread == THREAD_NULL) {

                new_thread = first_thread = current_thread();
                new_thread = first_thread = current_thread();

-       else
+       } else {

                new_thread = (thread_t)zalloc(thread_zone);
                new_thread = (thread_t)zalloc(thread_zone);

-       if (new_thread == THREAD_NULL)
-               return (KERN_RESOURCE_SHORTAGE);
+       }
+       if (new_thread == THREAD_NULL) {
+               return KERN_RESOURCE_SHORTAGE;
+       }

 
 

-       if (new_thread != first_thread)
+       if (new_thread != first_thread) {

                *new_thread = thread_template;
                *new_thread = thread_template;

+       }
+
+       os_ref_init_count(&new_thread->ref_count, &thread_refgrp, 2);

 
 #ifdef MACH_BSD
        new_thread->uthread = uthread_alloc(parent_task, new_thread, (options & TH_OPTION_NOCRED) != 0);
 
 #ifdef MACH_BSD
        new_thread->uthread = uthread_alloc(parent_task, new_thread, (options & TH_OPTION_NOCRED) != 0);


@@ -1062,7 +1178,7 @@ thread_create_internal(
 #endif /* MACH_ASSERT */
 
                zfree(thread_zone, new_thread);
 #endif /* MACH_ASSERT */
 
                zfree(thread_zone, new_thread);

-               return (KERN_RESOURCE_SHORTAGE);
+               return KERN_RESOURCE_SHORTAGE;

        }
 #endif  /* MACH_BSD */
 
        }
 #endif  /* MACH_BSD */
 


@@ -1082,7 +1198,7 @@ thread_create_internal(
 #endif /* MACH_ASSERT */
 
                zfree(thread_zone, new_thread);
 #endif /* MACH_ASSERT */
 
                zfree(thread_zone, new_thread);

-               return (KERN_FAILURE);
+               return KERN_FAILURE;

        }
 
        new_thread->task = parent_task;
        }
 
        new_thread->task = parent_task;


@@ -1095,16 +1211,50 @@ thread_create_internal(
        ipc_thread_init(new_thread);
 
        new_thread->continuation = continuation;
        ipc_thread_init(new_thread);
 
        new_thread->continuation = continuation;

+       new_thread->parameter = parameter;
+       new_thread->inheritor_flags = TURNSTILE_UPDATE_FLAGS_NONE;
+       priority_queue_init(&new_thread->sched_inheritor_queue,
+           PRIORITY_QUEUE_BUILTIN_MAX_HEAP);
+       priority_queue_init(&new_thread->base_inheritor_queue,
+           PRIORITY_QUEUE_BUILTIN_MAX_HEAP);
+#if CONFIG_SCHED_CLUTCH
+       priority_queue_entry_init(&new_thread->sched_clutchpri_link);
+#endif /* CONFIG_SCHED_CLUTCH */

 
        /* Allocate I/O Statistics structure */
        new_thread->thread_io_stats = (io_stat_info_t)kalloc(sizeof(struct io_stat_info));
        assert(new_thread->thread_io_stats != NULL);
        bzero(new_thread->thread_io_stats, sizeof(struct io_stat_info));
 
 
        /* Allocate I/O Statistics structure */
        new_thread->thread_io_stats = (io_stat_info_t)kalloc(sizeof(struct io_stat_info));
        assert(new_thread->thread_io_stats != NULL);
        bzero(new_thread->thread_io_stats, sizeof(struct io_stat_info));
 

+#if KASAN
+       kasan_init_thread(&new_thread->kasan_data);
+#endif
+
+#if CONFIG_KSANCOV
+       new_thread->ksancov_data = NULL;
+#endif
+

 #if CONFIG_IOSCHED
        /* Clear out the I/O Scheduling info for AppleFSCompression */
        new_thread->decmp_upl = NULL;
 #if CONFIG_IOSCHED
        /* Clear out the I/O Scheduling info for AppleFSCompression */
        new_thread->decmp_upl = NULL;

-#endif /* CONFIG_IOSCHED */ 
+#endif /* CONFIG_IOSCHED */
+
+#if DEVELOPMENT || DEBUG
+       task_lock(parent_task);
+       uint16_t thread_limit = parent_task->task_thread_limit;
+       if (exc_resource_threads_enabled &&
+           thread_limit > 0 &&
+           parent_task->thread_count >= thread_limit &&
+           !parent_task->task_has_crossed_thread_limit &&
+           !(parent_task->t_flags & TF_CORPSE)) {
+               int thread_count = parent_task->thread_count;
+               parent_task->task_has_crossed_thread_limit = TRUE;
+               task_unlock(parent_task);
+               SENDING_NOTIFICATION__TASK_HAS_TOO_MANY_THREADS(parent_task, thread_count);
+       } else {
+               task_unlock(parent_task);
+       }
+#endif

 
        lck_mtx_lock(&tasks_threads_lock);
        task_lock(parent_task);
 
        lck_mtx_lock(&tasks_threads_lock);
        task_lock(parent_task);


@@ -1136,9 +1286,12 @@ thread_create_internal(
                lck_mtx_destroy(&new_thread->mutex, &thread_lck_grp);
                machine_thread_destroy(new_thread);
                zfree(thread_zone, new_thread);
                lck_mtx_destroy(&new_thread->mutex, &thread_lck_grp);
                machine_thread_destroy(new_thread);
                zfree(thread_zone, new_thread);

-               return (KERN_FAILURE);
+               return KERN_FAILURE;

        }
 
        }
 

+       /* Protected by the tasks_threads_lock */
+       new_thread->thread_id = ++thread_unique_id;
+

        /* New threads inherit any default state on the task */
        machine_thread_inherit_taskwide(new_thread, parent_task);
 
        /* New threads inherit any default state on the task */
        machine_thread_inherit_taskwide(new_thread, parent_task);
 


@@ -1149,24 +1302,23 @@ thread_create_internal(
                 * This task has a per-thread CPU limit; make sure this new thread
                 * gets its limit set too, before it gets out of the kernel.
                 */
                 * This task has a per-thread CPU limit; make sure this new thread
                 * gets its limit set too, before it gets out of the kernel.
                 */

-               set_astledger(new_thread);
+               act_set_astledger(new_thread);

        }
 
        /* Instantiate a thread ledger. Do not fail thread creation if ledger creation fails. */
        if ((new_thread->t_threadledger = ledger_instantiate(thread_ledger_template,
        }
 
        /* Instantiate a thread ledger. Do not fail thread creation if ledger creation fails. */
        if ((new_thread->t_threadledger = ledger_instantiate(thread_ledger_template,

-                               LEDGER_CREATE_INACTIVE_ENTRIES)) != LEDGER_NULL) {
-
+           LEDGER_CREATE_INACTIVE_ENTRIES)) != LEDGER_NULL) {

                ledger_entry_setactive(new_thread->t_threadledger, thread_ledgers.cpu_time);
        }
 
                ledger_entry_setactive(new_thread->t_threadledger, thread_ledgers.cpu_time);
        }
 

-#ifdef CONFIG_BANK

        new_thread->t_bankledger = LEDGER_NULL;
        new_thread->t_deduct_bank_ledger_time = 0;
        new_thread->t_bankledger = LEDGER_NULL;
        new_thread->t_deduct_bank_ledger_time = 0;

-#endif
+       new_thread->t_deduct_bank_ledger_energy = 0;

 
        new_thread->t_ledger = new_thread->task->ledger;
 
        new_thread->t_ledger = new_thread->task->ledger;

-       if (new_thread->t_ledger)
+       if (new_thread->t_ledger) {

                ledger_reference(new_thread->t_ledger);
                ledger_reference(new_thread->t_ledger);

+       }

 
 #if defined(CONFIG_SCHED_MULTIQ)
        /* Cache the task's sched_group */
 
 #if defined(CONFIG_SCHED_MULTIQ)
        /* Cache the task's sched_group */


@@ -1188,10 +1340,17 @@ thread_create_internal(
        new_thread->max_priority = parent_task->max_priority;
        new_thread->task_priority = parent_task->priority;
 
        new_thread->max_priority = parent_task->max_priority;
        new_thread->task_priority = parent_task->priority;
 

+

        int new_priority = (priority < 0) ? parent_task->priority: priority;
        new_priority = (priority < 0)? parent_task->priority: priority;
        int new_priority = (priority < 0) ? parent_task->priority: priority;
        new_priority = (priority < 0)? parent_task->priority: priority;

-       if (new_priority > new_thread->max_priority)
+       if (new_priority > new_thread->max_priority) {

                new_priority = new_thread->max_priority;
                new_priority = new_thread->max_priority;

+       }
+#if CONFIG_EMBEDDED
+       if (new_priority < MAXPRI_THROTTLE) {
+               new_priority = MAXPRI_THROTTLE;
+       }
+#endif /* CONFIG_EMBEDDED */

 
        new_thread->importance = new_priority - new_thread->task_priority;
 
 
        new_thread->importance = new_priority - new_thread->task_priority;
 


@@ -1199,9 +1358,18 @@ thread_create_internal(
 
 #if defined(CONFIG_SCHED_TIMESHARE_CORE)
        new_thread->sched_stamp = sched_tick;
 
 #if defined(CONFIG_SCHED_TIMESHARE_CORE)
        new_thread->sched_stamp = sched_tick;

+#if CONFIG_SCHED_CLUTCH
+       new_thread->pri_shift = sched_clutch_thread_pri_shift(new_thread, new_thread->th_sched_bucket);
+#else /* CONFIG_SCHED_CLUTCH */

        new_thread->pri_shift = sched_pri_shifts[new_thread->th_sched_bucket];
        new_thread->pri_shift = sched_pri_shifts[new_thread->th_sched_bucket];

+#endif /* CONFIG_SCHED_CLUTCH */

 #endif /* defined(CONFIG_SCHED_TIMESHARE_CORE) */
 
 #endif /* defined(CONFIG_SCHED_TIMESHARE_CORE) */
 

+#if CONFIG_EMBEDDED
+       if (parent_task->max_priority <= MAXPRI_THROTTLE) {
+               sched_thread_mode_demote(new_thread, TH_SFLAG_THROTTLED);
+       }
+#endif /* CONFIG_EMBEDDED */

 
        thread_policy_create(new_thread);
 
 
        thread_policy_create(new_thread);
 


@@ -1210,10 +1378,7 @@ thread_create_internal(
        parent_task->thread_count++;
 
        /* So terminating threads don't need to take the task lock to decrement */
        parent_task->thread_count++;
 
        /* So terminating threads don't need to take the task lock to decrement */

-       hw_atomic_add(&parent_task->active_thread_count, 1);
-
-       /* Protected by the tasks_threads_lock */
-       new_thread->thread_id = ++thread_unique_id;
+       os_atomic_inc(&parent_task->active_thread_count, relaxed);

 
        queue_enter(&threads, new_thread, thread_t, threads);
        threads_count++;
 
        queue_enter(&threads, new_thread, thread_t, threads);
        threads_count++;


@@ -1226,12 +1391,13 @@ thread_create_internal(
                new_thread->inspection = FALSE;
        }
        new_thread->corpse_dup = FALSE;
                new_thread->inspection = FALSE;
        }
        new_thread->corpse_dup = FALSE;

+       new_thread->turnstile = turnstile_alloc();

        *out_thread = new_thread;
 
        *out_thread = new_thread;
 

-       {
-               long    dbg_arg1, dbg_arg2, dbg_arg3, dbg_arg4;
+       if (kdebug_enable) {
+               long args[4] = {};

 
 

-               kdbg_trace_data(parent_task->bsd_info, &dbg_arg2);
+               kdbg_trace_data(parent_task->bsd_info, &args[1], &args[3]);

 
                /*
                 * Starting with 26604425, exec'ing creates a new task/thread.
 
                /*
                 * Starting with 26604425, exec'ing creates a new task/thread.


@@ -1239,92 +1405,94 @@ thread_create_internal(
                 * NEWTHREAD in the current process has two possible meanings:
                 *
                 * 1) Create a new thread for this process.
                 * NEWTHREAD in the current process has two possible meanings:
                 *
                 * 1) Create a new thread for this process.

-                * 2) Create a new thread for the future process this will become in an exec.
+                * 2) Create a new thread for the future process this will become in an
+                * exec.

                 *
                 * To disambiguate these, arg3 will be set to TRUE for case #2.
                 *
                 * The value we need to find (TPF_EXEC_COPY) is stable in the case of a
                 * task exec'ing. The read of t_procflags does not take the proc_lock.
                 */
                 *
                 * To disambiguate these, arg3 will be set to TRUE for case #2.
                 *
                 * The value we need to find (TPF_EXEC_COPY) is stable in the case of a
                 * task exec'ing. The read of t_procflags does not take the proc_lock.
                 */

-               dbg_arg3 = (task_is_exec_copy(parent_task)) ? TRUE : 0;
+               args[2] = task_is_exec_copy(parent_task) ? 1 : 0;

 
 

-               KERNEL_DEBUG_CONSTANT_IST(KDEBUG_TRACE, 
-                       TRACE_DATA_NEWTHREAD | DBG_FUNC_NONE,
-                       (vm_address_t)(uintptr_t)thread_tid(new_thread), dbg_arg2, dbg_arg3, 0, 0);
+               KDBG_RELEASE(TRACE_DATA_NEWTHREAD, (uintptr_t)thread_tid(new_thread),
+                   args[1], args[2], args[3]);

 
 

-               kdbg_trace_string(parent_task->bsd_info,
-                                                       &dbg_arg1, &dbg_arg2, &dbg_arg3, &dbg_arg4);
-
-               KERNEL_DEBUG_CONSTANT_IST(KDEBUG_TRACE, 
-                       TRACE_STRING_NEWTHREAD | DBG_FUNC_NONE,
-                       dbg_arg1, dbg_arg2, dbg_arg3, dbg_arg4, 0);
+               kdbg_trace_string(parent_task->bsd_info, &args[0], &args[1],
+                   &args[2], &args[3]);
+               KDBG_RELEASE(TRACE_STRING_NEWTHREAD, args[0], args[1], args[2],
+                   args[3]);

        }
 
        DTRACE_PROC1(lwp__create, thread_t, *out_thread);
 
        }
 
        DTRACE_PROC1(lwp__create, thread_t, *out_thread);
 

-       return (KERN_SUCCESS);
+       return KERN_SUCCESS;

 }
 
 static kern_return_t
 thread_create_internal2(
 }
 
 static kern_return_t
 thread_create_internal2(

-       task_t                          task,
-       thread_t                        *new_thread,
-       boolean_t                       from_user,
-       thread_continue_t               continuation)
+       task_t                          task,
+       thread_t                        *new_thread,
+       boolean_t                       from_user,
+       thread_continue_t               continuation)

 {
 {

-       kern_return_t           result;
-       thread_t                        thread;
+       kern_return_t           result;
+       thread_t                        thread;

 
 

-       if (task == TASK_NULL || task == kernel_task)
-               return (KERN_INVALID_ARGUMENT);
+       if (task == TASK_NULL || task == kernel_task) {
+               return KERN_INVALID_ARGUMENT;
+       }

 
 

-       result = thread_create_internal(task, -1, continuation, TH_OPTION_NONE, &thread);
-       if (result != KERN_SUCCESS)
-               return (result);
+       result = thread_create_internal(task, -1, continuation, NULL, TH_OPTION_NONE, &thread);
+       if (result != KERN_SUCCESS) {
+               return result;
+       }

 
        thread->user_stop_count = 1;
        thread_hold(thread);
 
        thread->user_stop_count = 1;
        thread_hold(thread);

-       if (task->suspend_count > 0)
+       if (task->suspend_count > 0) {

                thread_hold(thread);
                thread_hold(thread);

+       }

 
 

-       if (from_user)
+       if (from_user) {

                extmod_statistics_incr_thread_create(task);
                extmod_statistics_incr_thread_create(task);

+       }

 
        task_unlock(task);
        lck_mtx_unlock(&tasks_threads_lock);
 
        task_unlock(task);
        lck_mtx_unlock(&tasks_threads_lock);

-       
+

        *new_thread = thread;
 
        *new_thread = thread;
 

-       return (KERN_SUCCESS);
+       return KERN_SUCCESS;

 }
 
 /* No prototype, since task_server.h has the _from_user version if KERNEL_SERVER */
 kern_return_t
 thread_create(
 }
 
 /* No prototype, since task_server.h has the _from_user version if KERNEL_SERVER */
 kern_return_t
 thread_create(

-       task_t                          task,
-       thread_t                        *new_thread);
+       task_t                          task,
+       thread_t                        *new_thread);

 
 kern_return_t
 thread_create(
 
 kern_return_t
 thread_create(

-       task_t                          task,
-       thread_t                        *new_thread)
+       task_t                          task,
+       thread_t                        *new_thread)

 {
        return thread_create_internal2(task, new_thread, FALSE, (thread_continue_t)thread_bootstrap_return);
 }
 
 kern_return_t
 thread_create_from_user(
 {
        return thread_create_internal2(task, new_thread, FALSE, (thread_continue_t)thread_bootstrap_return);
 }
 
 kern_return_t
 thread_create_from_user(

-       task_t                          task,
-       thread_t                        *new_thread)
+       task_t                          task,
+       thread_t                        *new_thread)

 {
        return thread_create_internal2(task, new_thread, TRUE, (thread_continue_t)thread_bootstrap_return);
 }
 
 kern_return_t
 thread_create_with_continuation(
 {
        return thread_create_internal2(task, new_thread, TRUE, (thread_continue_t)thread_bootstrap_return);
 }
 
 kern_return_t
 thread_create_with_continuation(

-       task_t                          task,
-       thread_t                        *new_thread,
-       thread_continue_t               continuation)
+       task_t                          task,
+       thread_t                        *new_thread,
+       thread_continue_t               continuation)

 {
        return thread_create_internal2(task, new_thread, FALSE, continuation);
 }
 {
        return thread_create_internal2(task, new_thread, FALSE, continuation);
 }


@@ -1337,25 +1505,35 @@ thread_create_waiting_internal(
        task_t                  task,
        thread_continue_t       continuation,
        event_t                 event,
        task_t                  task,
        thread_continue_t       continuation,
        event_t                 event,

+       block_hint_t            block_hint,

        int                     options,
        thread_t                *new_thread)
 {
        kern_return_t result;
        thread_t thread;
 
        int                     options,
        thread_t                *new_thread)
 {
        kern_return_t result;
        thread_t thread;
 

-       if (task == TASK_NULL || task == kernel_task)
-               return (KERN_INVALID_ARGUMENT);
+       if (task == TASK_NULL || task == kernel_task) {
+               return KERN_INVALID_ARGUMENT;
+       }

 
 

-       result = thread_create_internal(task, -1, continuation, options, &thread);
-       if (result != KERN_SUCCESS)
-               return (result);
+       result = thread_create_internal(task, -1, continuation, NULL,
+           options, &thread);
+       if (result != KERN_SUCCESS) {
+               return result;
+       }

 
        /* note no user_stop_count or thread_hold here */
 
 
        /* note no user_stop_count or thread_hold here */
 

-       if (task->suspend_count > 0)
+       if (task->suspend_count > 0) {

                thread_hold(thread);
                thread_hold(thread);

+       }

 
        thread_mtx_lock(thread);
 
        thread_mtx_lock(thread);

+       thread_set_pending_block_hint(thread, block_hint);
+       if (options & TH_OPTION_WORKQ) {
+               thread->static_param = true;
+               event = workq_thread_init_and_wq_lock(task, thread);
+       }

        thread_start_in_assert_wait(thread, event, THREAD_INTERRUPTIBLE);
        thread_mtx_unlock(thread);
 
        thread_start_in_assert_wait(thread, event, THREAD_INTERRUPTIBLE);
        thread_mtx_unlock(thread);
 


@@ -1364,7 +1542,7 @@ thread_create_waiting_internal(
 
        *new_thread = thread;
 
 
        *new_thread = thread;
 

-       return (KERN_SUCCESS);
+       return KERN_SUCCESS;

 }
 
 kern_return_t
 }
 
 kern_return_t


@@ -1375,7 +1553,7 @@ thread_create_waiting(
        thread_t                *new_thread)
 {
        return thread_create_waiting_internal(task, continuation, event,
        thread_t                *new_thread)
 {
        return thread_create_waiting_internal(task, continuation, event,

-                                             TH_OPTION_NONE, new_thread);
+                  kThreadWaitNone, TH_OPTION_NONE, new_thread);

 }
 
 
 }
 
 


@@ -1385,45 +1563,58 @@ thread_create_running_internal2(
        int                     flavor,
        thread_state_t          new_state,
        mach_msg_type_number_t  new_state_count,
        int                     flavor,
        thread_state_t          new_state,
        mach_msg_type_number_t  new_state_count,

-       thread_t                                *new_thread,
-       boolean_t                               from_user)
+       thread_t                                *new_thread,
+       boolean_t                               from_user)

 {
        kern_return_t  result;
 {
        kern_return_t  result;

-       thread_t                                thread;
+       thread_t                                thread;

 
 

-       if (task == TASK_NULL || task == kernel_task)
-               return (KERN_INVALID_ARGUMENT);
+       if (task == TASK_NULL || task == kernel_task) {
+               return KERN_INVALID_ARGUMENT;
+       }

 
 

-       result = thread_create_internal(task, -1, (thread_continue_t)thread_bootstrap_return, TH_OPTION_NONE, &thread);
-       if (result != KERN_SUCCESS)
-               return (result);
+       result = thread_create_internal(task, -1,
+           (thread_continue_t)thread_bootstrap_return, NULL,
+           TH_OPTION_NONE, &thread);
+       if (result != KERN_SUCCESS) {
+               return result;
+       }

 
 

-       if (task->suspend_count > 0)
+       if (task->suspend_count > 0) {

                thread_hold(thread);
                thread_hold(thread);

+       }

 
 

-       result = machine_thread_set_state(thread, flavor, new_state, new_state_count);
+       if (from_user) {
+               result = machine_thread_state_convert_from_user(thread, flavor,
+                   new_state, new_state_count);
+       }
+       if (result == KERN_SUCCESS) {
+               result = machine_thread_set_state(thread, flavor, new_state,
+                   new_state_count);
+       }

        if (result != KERN_SUCCESS) {
                task_unlock(task);
                lck_mtx_unlock(&tasks_threads_lock);
 
                thread_terminate(thread);
                thread_deallocate(thread);
        if (result != KERN_SUCCESS) {
                task_unlock(task);
                lck_mtx_unlock(&tasks_threads_lock);
 
                thread_terminate(thread);
                thread_deallocate(thread);

-               return (result);
+               return result;

        }
 
        thread_mtx_lock(thread);
        thread_start(thread);
        thread_mtx_unlock(thread);
 
        }
 
        thread_mtx_lock(thread);
        thread_start(thread);
        thread_mtx_unlock(thread);
 

-       if (from_user)
+       if (from_user) {

                extmod_statistics_incr_thread_create(task);
                extmod_statistics_incr_thread_create(task);

+       }

 
        task_unlock(task);
        lck_mtx_unlock(&tasks_threads_lock);
 
        *new_thread = thread;
 
 
        task_unlock(task);
        lck_mtx_unlock(&tasks_threads_lock);
 
        *new_thread = thread;
 

-       return (result);
+       return result;

 }
 
 /* Prototype, see justification above */
 }
 
 /* Prototype, see justification above */


@@ -1433,7 +1624,7 @@ thread_create_running(
        int                     flavor,
        thread_state_t          new_state,
        mach_msg_type_number_t  new_state_count,
        int                     flavor,
        thread_state_t          new_state,
        mach_msg_type_number_t  new_state_count,

-       thread_t                                *new_thread);
+       thread_t                                *new_thread);

 
 kern_return_t
 thread_create_running(
 
 kern_return_t
 thread_create_running(


@@ -1441,7 +1632,7 @@ thread_create_running(
        int                     flavor,
        thread_state_t          new_state,
        mach_msg_type_number_t  new_state_count,
        int                     flavor,
        thread_state_t          new_state,
        mach_msg_type_number_t  new_state_count,

-       thread_t                                *new_thread)
+       thread_t                                *new_thread)

 {
        return thread_create_running_internal2(
                task, flavor, new_state, new_state_count,
 {
        return thread_create_running_internal2(
                task, flavor, new_state, new_state_count,


@@ -1454,53 +1645,22 @@ thread_create_running_from_user(
        int                     flavor,
        thread_state_t          new_state,
        mach_msg_type_number_t  new_state_count,
        int                     flavor,
        thread_state_t          new_state,
        mach_msg_type_number_t  new_state_count,

-       thread_t                                *new_thread)
+       thread_t                                *new_thread)

 {
        return thread_create_running_internal2(
                task, flavor, new_state, new_state_count,
                new_thread, TRUE);
 }
 
 {
        return thread_create_running_internal2(
                task, flavor, new_state, new_state_count,
                new_thread, TRUE);
 }
 

-kern_return_t
-thread_create_workq(
-       task_t                          task,
-       thread_continue_t               thread_return,
-       thread_t                        *new_thread)
-{
-       kern_return_t           result;
-       thread_t                        thread;
-
-       if (task == TASK_NULL || task == kernel_task)
-               return (KERN_INVALID_ARGUMENT);
-
-       result = thread_create_internal(task, -1, thread_return, TH_OPTION_NOCRED | TH_OPTION_NOSUSP, &thread);
-       if (result != KERN_SUCCESS)
-               return (result);
-
-       thread->user_stop_count = 1;
-       thread_hold(thread);
-       if (task->suspend_count > 0)
-               thread_hold(thread);
-
-       task_unlock(task);
-       lck_mtx_unlock(&tasks_threads_lock);
-       
-       *new_thread = thread;
-
-       return (KERN_SUCCESS);
-}
-

 kern_return_t
 thread_create_workq_waiting(
        task_t              task,
        thread_continue_t   continuation,
 kern_return_t
 thread_create_workq_waiting(
        task_t              task,
        thread_continue_t   continuation,

-       event_t             event,

        thread_t            *new_thread)
 {
        thread_t            *new_thread)
 {

-
-       return thread_create_waiting_internal(task, continuation, event,
-                                             TH_OPTION_NOCRED | TH_OPTION_NOSUSP,
-                                             new_thread);
+       int options = TH_OPTION_NOCRED | TH_OPTION_NOSUSP | TH_OPTION_WORKQ;
+       return thread_create_waiting_internal(task, continuation, NULL,
+                  kThreadWaitParkedWorkQueue, options, new_thread);

 }
 
 /*
 }
 
 /*


@@ -1511,63 +1671,68 @@ thread_create_workq_waiting(
  */
 kern_return_t
 kernel_thread_create(
  */
 kern_return_t
 kernel_thread_create(

-       thread_continue_t       continuation,
-       void                            *parameter,
-       integer_t                       priority,
-       thread_t                        *new_thread)
+       thread_continue_t       continuation,
+       void                            *parameter,
+       integer_t                       priority,
+       thread_t                        *new_thread)

 {
 {

-       kern_return_t           result;
-       thread_t                        thread;
-       task_t                          task = kernel_task;
+       kern_return_t           result;
+       thread_t                        thread;
+       task_t                          task = kernel_task;

 
 

-       result = thread_create_internal(task, priority, continuation, TH_OPTION_NONE, &thread);
-       if (result != KERN_SUCCESS)
-               return (result);
+       result = thread_create_internal(task, priority, continuation, parameter,
+           TH_OPTION_NOCRED | TH_OPTION_NONE, &thread);
+       if (result != KERN_SUCCESS) {
+               return result;
+       }

 
        task_unlock(task);
        lck_mtx_unlock(&tasks_threads_lock);
 
        stack_alloc(thread);
        assert(thread->kernel_stack != 0);
 
        task_unlock(task);
        lck_mtx_unlock(&tasks_threads_lock);
 
        stack_alloc(thread);
        assert(thread->kernel_stack != 0);

+#if CONFIG_EMBEDDED
+       if (priority > BASEPRI_KERNEL)
+#endif

        thread->reserved_stack = thread->kernel_stack;
 
        thread->reserved_stack = thread->kernel_stack;
 

-       thread->parameter = parameter;
-
-if(debug_task & 1)
-       kprintf("kernel_thread_create: thread = %p continuation = %p\n", thread, continuation);
+       if (debug_task & 1) {
+               kprintf("kernel_thread_create: thread = %p continuation = %p\n", thread, continuation);
+       }

        *new_thread = thread;
 
        *new_thread = thread;
 

-       return (result);
+       return result;

 }
 
 kern_return_t
 kernel_thread_start_priority(
 }
 
 kern_return_t
 kernel_thread_start_priority(

-       thread_continue_t       continuation,
-       void                            *parameter,
-       integer_t                       priority,
-       thread_t                        *new_thread)
+       thread_continue_t       continuation,
+       void                            *parameter,
+       integer_t                       priority,
+       thread_t                        *new_thread)

 {
 {

-       kern_return_t   result;
-       thread_t                thread;
+       kern_return_t   result;
+       thread_t                thread;

 
        result = kernel_thread_create(continuation, parameter, priority, &thread);
 
        result = kernel_thread_create(continuation, parameter, priority, &thread);

-       if (result != KERN_SUCCESS)
-               return (result);
+       if (result != KERN_SUCCESS) {
+               return result;
+       }

 
 

-       *new_thread = thread;   
+       *new_thread = thread;

 
        thread_mtx_lock(thread);
        thread_start(thread);
        thread_mtx_unlock(thread);
 
 
        thread_mtx_lock(thread);
        thread_start(thread);
        thread_mtx_unlock(thread);
 

-       return (result);
+       return result;

 }
 
 kern_return_t
 kernel_thread_start(
 }
 
 kern_return_t
 kernel_thread_start(

-       thread_continue_t       continuation,
-       void                            *parameter,
-       thread_t                        *new_thread)
+       thread_continue_t       continuation,
+       void                            *parameter,
+       thread_t                        *new_thread)

 {
        return kernel_thread_start_priority(continuation, parameter, -1, new_thread);
 }
 {
        return kernel_thread_start_priority(continuation, parameter, -1, new_thread);
 }


@@ -1577,18 +1742,19 @@ kernel_thread_start(
 static void
 retrieve_thread_basic_info(thread_t thread, thread_basic_info_t basic_info)
 {
 static void
 retrieve_thread_basic_info(thread_t thread, thread_basic_info_t basic_info)
 {

-       int     state, flags;
+       int     state, flags;

 
        /* fill in info */
 
        thread_read_times(thread, &basic_info->user_time,
 
        /* fill in info */
 
        thread_read_times(thread, &basic_info->user_time,

-                                                               &basic_info->system_time);
+           &basic_info->system_time, NULL);

 
        /*
         *      Update lazy-evaluated scheduler info because someone wants it.
         */
 
        /*
         *      Update lazy-evaluated scheduler info because someone wants it.
         */

-       if (SCHED(can_update_priority)(thread))
+       if (SCHED(can_update_priority)(thread)) {

                SCHED(update_priority)(thread);
                SCHED(update_priority)(thread);

+       }

 
        basic_info->sleep_time = 0;
 
 
        basic_info->sleep_time = 0;
 


@@ -1600,44 +1766,44 @@ retrieve_thread_basic_info(thread_t thread, thread_basic_info_t basic_info)
        basic_info->cpu_usage = 0;
 #if defined(CONFIG_SCHED_TIMESHARE_CORE)
        if (sched_tick_interval) {
        basic_info->cpu_usage = 0;
 #if defined(CONFIG_SCHED_TIMESHARE_CORE)
        if (sched_tick_interval) {

-               basic_info->cpu_usage = (integer_t)(((uint64_t)thread->cpu_usage
-                                                                       * TH_USAGE_SCALE) /     sched_tick_interval);
+               basic_info->cpu_usage = (integer_t)(((uint64_t)thread->cpu_usage
+                   * TH_USAGE_SCALE) /     sched_tick_interval);

                basic_info->cpu_usage = (basic_info->cpu_usage * 3) / 5;
        }
 #endif
 
                basic_info->cpu_usage = (basic_info->cpu_usage * 3) / 5;
        }
 #endif
 

-       if (basic_info->cpu_usage > TH_USAGE_SCALE)
+       if (basic_info->cpu_usage > TH_USAGE_SCALE) {

                basic_info->cpu_usage = TH_USAGE_SCALE;
                basic_info->cpu_usage = TH_USAGE_SCALE;

+       }

 
        basic_info->policy = ((thread->sched_mode == TH_MODE_TIMESHARE)?
 
        basic_info->policy = ((thread->sched_mode == TH_MODE_TIMESHARE)?

-                                                                                       POLICY_TIMESHARE: POLICY_RR);
+           POLICY_TIMESHARE: POLICY_RR);

 
        flags = 0;
 
        flags = 0;

-       if (thread->options & TH_OPT_IDLE_THREAD)
+       if (thread->options & TH_OPT_IDLE_THREAD) {

                flags |= TH_FLAGS_IDLE;
                flags |= TH_FLAGS_IDLE;

+       }

 
        if (thread->options & TH_OPT_GLOBAL_FORCED_IDLE) {
                flags |= TH_FLAGS_GLOBAL_FORCED_IDLE;
        }
 
 
        if (thread->options & TH_OPT_GLOBAL_FORCED_IDLE) {
                flags |= TH_FLAGS_GLOBAL_FORCED_IDLE;
        }
 

-       if (!thread->kernel_stack)
+       if (!thread->kernel_stack) {

                flags |= TH_FLAGS_SWAPPED;
                flags |= TH_FLAGS_SWAPPED;

+       }

 
        state = 0;
 
        state = 0;

-       if (thread->state & TH_TERMINATE)
+       if (thread->state & TH_TERMINATE) {

                state = TH_STATE_HALTED;
                state = TH_STATE_HALTED;

-       else
-       if (thread->state & TH_RUN)
+       } else if (thread->state & TH_RUN) {

                state = TH_STATE_RUNNING;
                state = TH_STATE_RUNNING;

-       else
-       if (thread->state & TH_UNINT)
+       } else if (thread->state & TH_UNINT) {

                state = TH_STATE_UNINTERRUPTIBLE;
                state = TH_STATE_UNINTERRUPTIBLE;

-       else
-       if (thread->state & TH_SUSP)
+       } else if (thread->state & TH_SUSP) {

                state = TH_STATE_STOPPED;
                state = TH_STATE_STOPPED;

-       else
-       if (thread->state & TH_WAIT)
+       } else if (thread->state & TH_WAIT) {

                state = TH_STATE_WAITING;
                state = TH_STATE_WAITING;

+       }

 
        basic_info->run_state = state;
        basic_info->flags = flags;
 
        basic_info->run_state = state;
        basic_info->flags = flags;


@@ -1649,20 +1815,21 @@ retrieve_thread_basic_info(thread_t thread, thread_basic_info_t basic_info)
 
 kern_return_t
 thread_info_internal(
 
 kern_return_t
 thread_info_internal(

-       thread_t                thread,
-       thread_flavor_t                 flavor,
-       thread_info_t                   thread_info_out,        /* ptr to OUT array */
-       mach_msg_type_number_t  *thread_info_count)     /*IN/OUT*/
+       thread_t                thread,
+       thread_flavor_t                 flavor,
+       thread_info_t                   thread_info_out,        /* ptr to OUT array */
+       mach_msg_type_number_t  *thread_info_count)     /*IN/OUT*/

 {
 {

-       spl_t   s;
+       spl_t   s;

 
 

-       if (thread == THREAD_NULL)
-               return (KERN_INVALID_ARGUMENT);
+       if (thread == THREAD_NULL) {
+               return KERN_INVALID_ARGUMENT;
+       }

 
        if (flavor == THREAD_BASIC_INFO) {
 
        if (flavor == THREAD_BASIC_INFO) {

-
-               if (*thread_info_count < THREAD_BASIC_INFO_COUNT)
-                       return (KERN_INVALID_ARGUMENT);
+               if (*thread_info_count < THREAD_BASIC_INFO_COUNT) {
+                       return KERN_INVALID_ARGUMENT;
+               }

 
                s = splsched();
                thread_lock(thread);
 
                s = splsched();
                thread_lock(thread);


@@ -1674,14 +1841,13 @@ thread_info_internal(
 
                *thread_info_count = THREAD_BASIC_INFO_COUNT;
 
 
                *thread_info_count = THREAD_BASIC_INFO_COUNT;
 

-               return (KERN_SUCCESS);
-       }
-       else
-       if (flavor == THREAD_IDENTIFIER_INFO) {
-               thread_identifier_info_t        identifier_info;
+               return KERN_SUCCESS;
+       } else if (flavor == THREAD_IDENTIFIER_INFO) {
+               thread_identifier_info_t        identifier_info;

 
 

-               if (*thread_info_count < THREAD_IDENTIFIER_INFO_COUNT)
-                       return (KERN_INVALID_ARGUMENT);
+               if (*thread_info_count < THREAD_IDENTIFIER_INFO_COUNT) {
+                       return KERN_INVALID_ARGUMENT;
+               }

 
                identifier_info = (thread_identifier_info_t) thread_info_out;
 
 
                identifier_info = (thread_identifier_info_t) thread_info_out;
 


@@ -1695,13 +1861,12 @@ thread_info_internal(
                thread_unlock(thread);
                splx(s);
                return KERN_SUCCESS;
                thread_unlock(thread);
                splx(s);
                return KERN_SUCCESS;

-       }
-       else
-       if (flavor == THREAD_SCHED_TIMESHARE_INFO) {
-               policy_timeshare_info_t         ts_info;
+       } else if (flavor == THREAD_SCHED_TIMESHARE_INFO) {
+               policy_timeshare_info_t         ts_info;

 
 

-               if (*thread_info_count < POLICY_TIMESHARE_INFO_COUNT)
-                       return (KERN_INVALID_ARGUMENT);
+               if (*thread_info_count < POLICY_TIMESHARE_INFO_COUNT) {
+                       return KERN_INVALID_ARGUMENT;
+               }

 
                ts_info = (policy_timeshare_info_t)thread_info_out;
 
 
                ts_info = (policy_timeshare_info_t)thread_info_out;
 


@@ -1711,44 +1876,41 @@ thread_info_internal(
                if (thread->sched_mode != TH_MODE_TIMESHARE) {
                        thread_unlock(thread);
                        splx(s);
                if (thread->sched_mode != TH_MODE_TIMESHARE) {
                        thread_unlock(thread);
                        splx(s);

-                       return (KERN_INVALID_POLICY);
+                       return KERN_INVALID_POLICY;

                }
 
                ts_info->depressed = (thread->sched_flags & TH_SFLAG_DEPRESSED_MASK) != 0;
                if (ts_info->depressed) {
                        ts_info->base_priority = DEPRESSPRI;
                        ts_info->depress_priority = thread->base_pri;
                }
 
                ts_info->depressed = (thread->sched_flags & TH_SFLAG_DEPRESSED_MASK) != 0;
                if (ts_info->depressed) {
                        ts_info->base_priority = DEPRESSPRI;
                        ts_info->depress_priority = thread->base_pri;

-               }
-               else {
+               } else {

                        ts_info->base_priority = thread->base_pri;
                        ts_info->depress_priority = -1;
                }
 
                ts_info->cur_priority = thread->sched_pri;
                        ts_info->base_priority = thread->base_pri;
                        ts_info->depress_priority = -1;
                }
 
                ts_info->cur_priority = thread->sched_pri;

-               ts_info->max_priority = thread->max_priority;
+               ts_info->max_priority = thread->max_priority;

 
                thread_unlock(thread);
                splx(s);
 
                *thread_info_count = POLICY_TIMESHARE_INFO_COUNT;
 
 
                thread_unlock(thread);
                splx(s);
 
                *thread_info_count = POLICY_TIMESHARE_INFO_COUNT;
 

-               return (KERN_SUCCESS);
-       }
-       else
-       if (flavor == THREAD_SCHED_FIFO_INFO) {
-               if (*thread_info_count < POLICY_FIFO_INFO_COUNT)
-                       return (KERN_INVALID_ARGUMENT);
+               return KERN_SUCCESS;
+       } else if (flavor == THREAD_SCHED_FIFO_INFO) {
+               if (*thread_info_count < POLICY_FIFO_INFO_COUNT) {
+                       return KERN_INVALID_ARGUMENT;
+               }

 
 

-               return (KERN_INVALID_POLICY);
-       }
-       else
-       if (flavor == THREAD_SCHED_RR_INFO) {
-               policy_rr_info_t                        rr_info;
+               return KERN_INVALID_POLICY;
+       } else if (flavor == THREAD_SCHED_RR_INFO) {
+               policy_rr_info_t                        rr_info;

                uint32_t quantum_time;
                uint64_t quantum_ns;
 
                uint32_t quantum_time;
                uint64_t quantum_ns;
 

-               if (*thread_info_count < POLICY_RR_INFO_COUNT)
-                       return (KERN_INVALID_ARGUMENT);
+               if (*thread_info_count < POLICY_RR_INFO_COUNT) {
+                       return KERN_INVALID_ARGUMENT;
+               }

 
                rr_info = (policy_rr_info_t) thread_info_out;
 
 
                rr_info = (policy_rr_info_t) thread_info_out;
 


@@ -1759,15 +1921,14 @@ thread_info_internal(
                        thread_unlock(thread);
                        splx(s);
 
                        thread_unlock(thread);
                        splx(s);
 

-                       return (KERN_INVALID_POLICY);
-           }
+                       return KERN_INVALID_POLICY;
+               }

 
                rr_info->depressed = (thread->sched_flags & TH_SFLAG_DEPRESSED_MASK) != 0;
                if (rr_info->depressed) {
                        rr_info->base_priority = DEPRESSPRI;
                        rr_info->depress_priority = thread->base_pri;
 
                rr_info->depressed = (thread->sched_flags & TH_SFLAG_DEPRESSED_MASK) != 0;
                if (rr_info->depressed) {
                        rr_info->base_priority = DEPRESSPRI;
                        rr_info->depress_priority = thread->base_pri;

-               }
-               else {
+               } else {

                        rr_info->base_priority = thread->base_pri;
                        rr_info->depress_priority = -1;
                }
                        rr_info->base_priority = thread->base_pri;
                        rr_info->depress_priority = -1;
                }


@@ -1783,15 +1944,13 @@ thread_info_internal(
 
                *thread_info_count = POLICY_RR_INFO_COUNT;
 
 
                *thread_info_count = POLICY_RR_INFO_COUNT;
 

-               return (KERN_SUCCESS);
-       }
-       else
-       if (flavor == THREAD_EXTENDED_INFO) {
-               thread_basic_info_data_t        basic_info;
-               thread_extended_info_t          extended_info = (thread_extended_info_t) thread_info_out;
+               return KERN_SUCCESS;
+       } else if (flavor == THREAD_EXTENDED_INFO) {
+               thread_basic_info_data_t        basic_info;
+               thread_extended_info_t          extended_info = (thread_extended_info_t) thread_info_out;

 
                if (*thread_info_count < THREAD_EXTENDED_INFO_COUNT) {
 
                if (*thread_info_count < THREAD_EXTENDED_INFO_COUNT) {

-                       return (KERN_INVALID_ARGUMENT);
+                       return KERN_INVALID_ARGUMENT;

                }
 
                s = splsched();
                }
 
                s = splsched();


@@ -1801,8 +1960,8 @@ thread_info_internal(
                 * the PROC_PIDTHREADINFO flavor (which can't be used on corpses)
                 */
                retrieve_thread_basic_info(thread, &basic_info);
                 * the PROC_PIDTHREADINFO flavor (which can't be used on corpses)
                 */
                retrieve_thread_basic_info(thread, &basic_info);

-               extended_info->pth_user_time = ((basic_info.user_time.seconds * (integer_t)NSEC_PER_SEC) + (basic_info.user_time.microseconds * (integer_t)NSEC_PER_USEC));
-               extended_info->pth_system_time = ((basic_info.system_time.seconds * (integer_t)NSEC_PER_SEC) + (basic_info.system_time.microseconds * (integer_t)NSEC_PER_USEC));
+               extended_info->pth_user_time = (((uint64_t)basic_info.user_time.seconds * NSEC_PER_SEC) + ((uint64_t)basic_info.user_time.microseconds * NSEC_PER_USEC));
+               extended_info->pth_system_time = (((uint64_t)basic_info.system_time.seconds * NSEC_PER_SEC) + ((uint64_t)basic_info.system_time.microseconds * NSEC_PER_USEC));

 
                extended_info->pth_cpu_usage = basic_info.cpu_usage;
                extended_info->pth_policy = basic_info.policy;
 
                extended_info->pth_cpu_usage = basic_info.cpu_usage;
                extended_info->pth_policy = basic_info.policy;


@@ -1813,46 +1972,47 @@ thread_info_internal(
                extended_info->pth_priority = thread->base_pri;
                extended_info->pth_maxpriority = thread->max_priority;
 
                extended_info->pth_priority = thread->base_pri;
                extended_info->pth_maxpriority = thread->max_priority;
 

-               bsd_getthreadname(thread->uthread,extended_info->pth_name);
+               bsd_getthreadname(thread->uthread, extended_info->pth_name);

 
                thread_unlock(thread);
                splx(s);
 
                *thread_info_count = THREAD_EXTENDED_INFO_COUNT;
 
 
                thread_unlock(thread);
                splx(s);
 
                *thread_info_count = THREAD_EXTENDED_INFO_COUNT;
 

-               return (KERN_SUCCESS);
-       }
-       else
-       if (flavor == THREAD_DEBUG_INFO_INTERNAL) {
+               return KERN_SUCCESS;
+       } else if (flavor == THREAD_DEBUG_INFO_INTERNAL) {

 #if DEVELOPMENT || DEBUG
                thread_debug_info_internal_t dbg_info;
 #if DEVELOPMENT || DEBUG
                thread_debug_info_internal_t dbg_info;

-               if (*thread_info_count < THREAD_DEBUG_INFO_INTERNAL_COUNT)
-                       return (KERN_NOT_SUPPORTED);
+               if (*thread_info_count < THREAD_DEBUG_INFO_INTERNAL_COUNT) {
+                       return KERN_NOT_SUPPORTED;
+               }

 
 

-               if (thread_info_out == NULL)
-                       return (KERN_INVALID_ARGUMENT);
+               if (thread_info_out == NULL) {
+                       return KERN_INVALID_ARGUMENT;
+               }

 
                dbg_info = (thread_debug_info_internal_t) thread_info_out;
                dbg_info->page_creation_count = thread->t_page_creation_count;
 
                *thread_info_count = THREAD_DEBUG_INFO_INTERNAL_COUNT;
 
                dbg_info = (thread_debug_info_internal_t) thread_info_out;
                dbg_info->page_creation_count = thread->t_page_creation_count;
 
                *thread_info_count = THREAD_DEBUG_INFO_INTERNAL_COUNT;

-               return (KERN_SUCCESS);
+               return KERN_SUCCESS;

 #endif /* DEVELOPMENT || DEBUG */
 #endif /* DEVELOPMENT || DEBUG */

-               return (KERN_NOT_SUPPORTED);
+               return KERN_NOT_SUPPORTED;

        }
 
        }
 

-       return (KERN_INVALID_ARGUMENT);
+       return KERN_INVALID_ARGUMENT;

 }
 
 void
 thread_read_times(
 }
 
 void
 thread_read_times(

-       thread_t                thread,
-       time_value_t    *user_time,
-       time_value_t    *system_time)
+       thread_t                thread,
+       time_value_t    *user_time,
+       time_value_t    *system_time,
+       time_value_t    *runnable_time)

 {
 {

-       clock_sec_t             secs;
-       clock_usec_t    usecs;
-       uint64_t                tval_user, tval_system;
+       clock_sec_t             secs;
+       clock_usec_t    usecs;
+       uint64_t                tval_user, tval_system;

 
        tval_user = timer_grab(&thread->user_timer);
        tval_system = timer_grab(&thread->system_timer);
 
        tval_user = timer_grab(&thread->user_timer);
        tval_system = timer_grab(&thread->system_timer);


@@ -1861,7 +2021,7 @@ thread_read_times(
                absolutetime_to_microtime(tval_user, &secs, &usecs);
                user_time->seconds = (typeof(user_time->seconds))secs;
                user_time->microseconds = usecs;
                absolutetime_to_microtime(tval_user, &secs, &usecs);
                user_time->seconds = (typeof(user_time->seconds))secs;
                user_time->microseconds = usecs;

-               
+

                absolutetime_to_microtime(tval_system, &secs, &usecs);
                system_time->seconds = (typeof(system_time->seconds))secs;
                system_time->microseconds = usecs;
                absolutetime_to_microtime(tval_system, &secs, &usecs);
                system_time->seconds = (typeof(system_time->seconds))secs;
                system_time->microseconds = usecs;


@@ -1875,9 +2035,17 @@ thread_read_times(
                system_time->seconds = 0;
                system_time->microseconds = 0;
        }
                system_time->seconds = 0;
                system_time->microseconds = 0;
        }

+
+       if (runnable_time) {
+               uint64_t tval_runnable = timer_grab(&thread->runnable_timer);
+               absolutetime_to_microtime(tval_runnable, &secs, &usecs);
+               runnable_time->seconds = (typeof(runnable_time->seconds))secs;
+               runnable_time->microseconds = usecs;
+       }

 }
 
 }
 

-uint64_t thread_get_runtime_self(void)
+uint64_t
+thread_get_runtime_self(void)

 {
        boolean_t interrupt_state;
        uint64_t runtime;
 {
        boolean_t interrupt_state;
        uint64_t runtime;


@@ -1889,7 +2057,7 @@ uint64_t thread_get_runtime_self(void)
        /* Not interrupt safe, as the scheduler may otherwise update timer values underneath us */
        interrupt_state = ml_set_interrupts_enabled(FALSE);
        processor = current_processor();
        /* Not interrupt safe, as the scheduler may otherwise update timer values underneath us */
        interrupt_state = ml_set_interrupts_enabled(FALSE);
        processor = current_processor();

-       timer_switch(PROCESSOR_DATA(processor, thread_timer), mach_absolute_time(), PROCESSOR_DATA(processor, thread_timer));
+       timer_update(PROCESSOR_DATA(processor, thread_timer), mach_absolute_time());

        runtime = (timer_grab(&thread->user_timer) + timer_grab(&thread->system_timer));
        ml_set_interrupts_enabled(interrupt_state);
 
        runtime = (timer_grab(&thread->user_timer) + timer_grab(&thread->system_timer));
        ml_set_interrupts_enabled(interrupt_state);
 


@@ -1898,10 +2066,10 @@ uint64_t thread_get_runtime_self(void)
 
 kern_return_t
 thread_assign(
 
 kern_return_t
 thread_assign(

-       __unused thread_t                       thread,
-       __unused processor_set_t        new_pset)
+       __unused thread_t                       thread,
+       __unused processor_set_t        new_pset)

 {
 {

-       return (KERN_FAILURE);
+       return KERN_FAILURE;

 }
 
 /*
 }
 
 /*


@@ -1912,27 +2080,28 @@ thread_assign(
  */
 kern_return_t
 thread_assign_default(
  */
 kern_return_t
 thread_assign_default(

-       thread_t                thread)
+       thread_t                thread)

 {
 {

-       return (thread_assign(thread, &pset0));
+       return thread_assign(thread, &pset0);

 }
 
 /*
  *     thread_get_assignment
  *
  *     Return current assignment for this thread.
 }
 
 /*
  *     thread_get_assignment
  *
  *     Return current assignment for this thread.

- */        
+ */

 kern_return_t
 thread_get_assignment(
 kern_return_t
 thread_get_assignment(

-       thread_t                thread,
-       processor_set_t *pset)
+       thread_t                thread,
+       processor_set_t *pset)

 {
 {

-       if (thread == NULL)
-               return (KERN_INVALID_ARGUMENT);
+       if (thread == NULL) {
+               return KERN_INVALID_ARGUMENT;
+       }

 
        *pset = &pset0;
 
 
        *pset = &pset0;
 

-       return (KERN_SUCCESS);
+       return KERN_SUCCESS;

 }
 
 /*
 }
 
 /*


@@ -1943,31 +2112,34 @@ thread_get_assignment(
  */
 kern_return_t
 thread_wire_internal(
  */
 kern_return_t
 thread_wire_internal(

-       host_priv_t             host_priv,
-       thread_t                thread,
-       boolean_t               wired,
-       boolean_t               *prev_state)
+       host_priv_t             host_priv,
+       thread_t                thread,
+       boolean_t               wired,
+       boolean_t               *prev_state)

 {
 {

-       if (host_priv == NULL || thread != current_thread())
-               return (KERN_INVALID_ARGUMENT);
+       if (host_priv == NULL || thread != current_thread()) {
+               return KERN_INVALID_ARGUMENT;
+       }

 
        assert(host_priv == &realhost);
 
 
        assert(host_priv == &realhost);
 

-       if (prev_state)
-           *prev_state = (thread->options & TH_OPT_VMPRIV) != 0;
-       
-       if (wired) {
-           if (!(thread->options & TH_OPT_VMPRIV)) 
-                   vm_page_free_reserve(1);    /* XXX */
-           thread->options |= TH_OPT_VMPRIV;
+       if (prev_state) {
+               *prev_state = (thread->options & TH_OPT_VMPRIV) != 0;

        }
        }

-       else {
-           if (thread->options & TH_OPT_VMPRIV) 
-                   vm_page_free_reserve(-1);   /* XXX */
-           thread->options &= ~TH_OPT_VMPRIV;
+
+       if (wired) {
+               if (!(thread->options & TH_OPT_VMPRIV)) {
+                       vm_page_free_reserve(1); /* XXX */
+               }
+               thread->options |= TH_OPT_VMPRIV;
+       } else {
+               if (thread->options & TH_OPT_VMPRIV) {
+                       vm_page_free_reserve(-1); /* XXX */
+               }
+               thread->options &= ~TH_OPT_VMPRIV;

        }
 
        }
 

-       return (KERN_SUCCESS);
+       return KERN_SUCCESS;

 }
 
 
 }
 
 


@@ -1978,11 +2150,11 @@ thread_wire_internal(
  */
 kern_return_t
 thread_wire(
  */
 kern_return_t
 thread_wire(

-       host_priv_t     host_priv,
-       thread_t        thread,
-       boolean_t       wired)
+       host_priv_t     host_priv,
+       thread_t        thread,
+       boolean_t       wired)

 {
 {

-    return (thread_wire_internal(host_priv, thread, wired, NULL));
+       return thread_wire_internal(host_priv, thread, wired, NULL);

 }
 
 
 }
 
 


@@ -1997,17 +2169,19 @@ set_vm_privilege(boolean_t privileged)
 {
        boolean_t       was_vmpriv;
 
 {
        boolean_t       was_vmpriv;
 

-       if (current_thread()->options & TH_OPT_VMPRIV)
+       if (current_thread()->options & TH_OPT_VMPRIV) {

                was_vmpriv = TRUE;
                was_vmpriv = TRUE;

-       else
+       } else {

                was_vmpriv = FALSE;
                was_vmpriv = FALSE;

+       }

 
 

-       if (privileged != FALSE)
+       if (privileged != FALSE) {

                current_thread()->options |= TH_OPT_VMPRIV;
                current_thread()->options |= TH_OPT_VMPRIV;

-       else
+       } else {

                current_thread()->options &= ~TH_OPT_VMPRIV;
                current_thread()->options &= ~TH_OPT_VMPRIV;

+       }

 
 

-       return (was_vmpriv);
+       return was_vmpriv;

 }
 
 void
 }
 
 void


@@ -2022,8 +2196,7 @@ clear_thread_rwlock_boost(void)
        thread_t thread = current_thread();
 
        if ((thread->rwlock_count-- == 1) && (thread->sched_flags & TH_SFLAG_RW_PROMOTED)) {
        thread_t thread = current_thread();
 
        if ((thread->rwlock_count-- == 1) && (thread->sched_flags & TH_SFLAG_RW_PROMOTED)) {

-
-               lck_rw_clear_promotion(thread);
+               lck_rw_clear_promotion(thread, 0);

        }
 }
 
        }
 }
 


@@ -2031,20 +2204,39 @@ clear_thread_rwlock_boost(void)
  * XXX assuming current thread only, for now...
  */
 void
  * XXX assuming current thread only, for now...
  */
 void

-thread_guard_violation(thread_t thread, unsigned type)
+thread_guard_violation(thread_t thread,
+    mach_exception_data_type_t code, mach_exception_data_type_t subcode, boolean_t fatal)

 {
        assert(thread == current_thread());
 
 {
        assert(thread == current_thread());
 

-       spl_t s = splsched();
+       /* Don't set up the AST for kernel threads; this check is needed to ensure
+        * that the guard_exc_* fields in the thread structure are set only by the
+        * current thread and therefore, don't require a lock.
+        */
+       if (thread->task == kernel_task) {
+               return;
+       }
+
+       assert(EXC_GUARD_DECODE_GUARD_TYPE(code));
+

        /*
         * Use the saved state area of the thread structure
         * to store all info required to handle the AST when
        /*
         * Use the saved state area of the thread structure
         * to store all info required to handle the AST when

-        * returning to userspace
+        * returning to userspace. It's possible that there is
+        * already a pending guard exception. If it's non-fatal,
+        * it can only be over-written by a fatal exception code.

         */
         */

-       thread->guard_exc_info.type = type;
-       thread_ast_set(thread, AST_GUARD);
-       ast_propagate(thread->ast);
+       if (thread->guard_exc_info.code && (thread->guard_exc_fatal || !fatal)) {
+               return;
+       }

 
 

+       thread->guard_exc_info.code = code;
+       thread->guard_exc_info.subcode = subcode;
+       thread->guard_exc_fatal = fatal ? 1 : 0;
+
+       spl_t s = splsched();
+       thread_ast_set(thread, AST_GUARD);
+       ast_propagate(thread);

        splx(s);
 }
 
        splx(s);
 }
 


@@ -2053,24 +2245,49 @@ thread_guard_violation(thread_t thread, unsigned type)
  *
  *     Handle AST_GUARD for a thread. This routine looks at the
  *     state saved in the thread structure to determine the cause
  *
  *     Handle AST_GUARD for a thread. This routine looks at the
  *     state saved in the thread structure to determine the cause

- *     of this exception. Based on this value, it invokes the 
+ *     of this exception. Based on this value, it invokes the

  *     appropriate routine which determines other exception related
  *     info and raises the exception.
  */
 void
  *     appropriate routine which determines other exception related
  *     info and raises the exception.
  */
 void

-guard_ast(thread_t thread)
-{
-       if (thread->guard_exc_info.type == GUARD_TYPE_MACH_PORT)
-               mach_port_guard_ast(thread);
-       else
-               fd_guard_ast(thread);
+guard_ast(thread_t t)
+{
+       const mach_exception_data_type_t
+           code = t->guard_exc_info.code,
+           subcode = t->guard_exc_info.subcode;
+
+       t->guard_exc_info.code = 0;
+       t->guard_exc_info.subcode = 0;
+       t->guard_exc_fatal = 0;
+
+       switch (EXC_GUARD_DECODE_GUARD_TYPE(code)) {
+       case GUARD_TYPE_NONE:
+               /* lingering AST_GUARD on the processor? */
+               break;
+       case GUARD_TYPE_MACH_PORT:
+               mach_port_guard_ast(t, code, subcode);
+               break;
+       case GUARD_TYPE_FD:
+               fd_guard_ast(t, code, subcode);
+               break;
+#if CONFIG_VNGUARD
+       case GUARD_TYPE_VN:
+               vn_guard_ast(t, code, subcode);
+               break;
+#endif
+       case GUARD_TYPE_VIRT_MEMORY:
+               virt_memory_guard_ast(t, code, subcode);
+               break;
+       default:
+               panic("guard_exc_info %llx %llx", code, subcode);
+       }

 }
 
 static void
 thread_cputime_callback(int warning, __unused const void *arg0, __unused const void *arg1)
 {
        if (warning == LEDGER_WARNING_ROSE_ABOVE) {
 }
 
 static void
 thread_cputime_callback(int warning, __unused const void *arg0, __unused const void *arg1)
 {
        if (warning == LEDGER_WARNING_ROSE_ABOVE) {

-#if CONFIG_TELEMETRY           
+#if CONFIG_TELEMETRY

                /*
                 * This thread is in danger of violating the CPU usage monitor. Enable telemetry
                 * on the entire task so there are micro-stackshots available if and when
                /*
                 * This thread is in danger of violating the CPU usage monitor. Enable telemetry
                 * on the entire task so there are micro-stackshots available if and when


@@ -2100,10 +2317,10 @@ void __attribute__((noinline))
 SENDING_NOTIFICATION__THIS_THREAD_IS_CONSUMING_TOO_MUCH_CPU(void)
 {
        int          pid                = 0;
 SENDING_NOTIFICATION__THIS_THREAD_IS_CONSUMING_TOO_MUCH_CPU(void)
 {
        int          pid                = 0;

-       task_t           task                           = current_task();
+       task_t           task                           = current_task();

        thread_t     thread             = current_thread();
        uint64_t     tid                = thread->thread_id;
        thread_t     thread             = current_thread();
        uint64_t     tid                = thread->thread_id;

-       const char       *procname          = "unknown";
+       const char       *procname          = "unknown";

        time_value_t thread_total_time  = {0, 0};
        time_value_t thread_system_time;
        time_value_t thread_user_time;
        time_value_t thread_total_time  = {0, 0};
        time_value_t thread_system_time;
        time_value_t thread_user_time;


@@ -2113,14 +2330,14 @@ SENDING_NOTIFICATION__THIS_THREAD_IS_CONSUMING_TOO_MUCH_CPU(void)
        uint32_t     interval_sec;
        uint64_t     interval_ns;
        uint64_t     balance_ns;
        uint32_t     interval_sec;
        uint64_t     interval_ns;
        uint64_t     balance_ns;

-       boolean_t        fatal = FALSE;
-       boolean_t        send_exc_resource = TRUE; /* in addition to RESOURCE_NOTIFY */
-       kern_return_t   kr;
+       boolean_t        fatal = FALSE;
+       boolean_t        send_exc_resource = TRUE; /* in addition to RESOURCE_NOTIFY */
+       kern_return_t   kr;

 
 #ifdef EXC_RESOURCE_MONITORS
 
 #ifdef EXC_RESOURCE_MONITORS

-       mach_exception_data_type_t      code[EXCEPTION_CODE_MAX];
+       mach_exception_data_type_t      code[EXCEPTION_CODE_MAX];

 #endif /* EXC_RESOURCE_MONITORS */
 #endif /* EXC_RESOURCE_MONITORS */

-       struct ledger_entry_info        lei;
+       struct ledger_entry_info        lei;

 
        assert(thread->t_threadledger != LEDGER_NULL);
 
 
        assert(thread->t_threadledger != LEDGER_NULL);
 


@@ -2133,15 +2350,17 @@ SENDING_NOTIFICATION__THIS_THREAD_IS_CONSUMING_TOO_MUCH_CPU(void)
                send_exc_resource = TRUE;
        }
        /* Only one thread can be here at a time.  Whichever makes it through
                send_exc_resource = TRUE;
        }
        /* Only one thread can be here at a time.  Whichever makes it through

-          first will successfully suspend the monitor and proceed to send the
-          notification.  Other threads will get an error trying to suspend the
-          monitor and give up on sending the notification.  In the first release,
-          the monitor won't be resumed for a number of seconds, but we may
-          eventually need to handle low-latency resume.
+        *  first will successfully suspend the monitor and proceed to send the
+        *  notification.  Other threads will get an error trying to suspend the
+        *  monitor and give up on sending the notification.  In the first release,
+        *  the monitor won't be resumed for a number of seconds, but we may
+        *  eventually need to handle low-latency resume.

         */
        kr = task_suspend_cpumon(task);
        task_unlock(task);
         */
        kr = task_suspend_cpumon(task);
        task_unlock(task);

-       if (kr == KERN_INVALID_ARGUMENT)        return;
+       if (kr == KERN_INVALID_ARGUMENT) {
+               return;
+       }

 
 #ifdef MACH_BSD
        pid = proc_selfpid();
 
 #ifdef MACH_BSD
        pid = proc_selfpid();


@@ -2154,39 +2373,36 @@ SENDING_NOTIFICATION__THIS_THREAD_IS_CONSUMING_TOO_MUCH_CPU(void)
 
        interval_sec = (uint32_t)(interval_ns / NSEC_PER_SEC);
 
 
        interval_sec = (uint32_t)(interval_ns / NSEC_PER_SEC);
 

-       thread_read_times(thread, &thread_user_time, &thread_system_time);
+       thread_read_times(thread, &thread_user_time, &thread_system_time, NULL);

        time_value_add(&thread_total_time, &thread_user_time);
        time_value_add(&thread_total_time, &thread_system_time);
        ledger_get_entry_info(thread->t_threadledger, thread_ledgers.cpu_time, &lei);
 
        /* credit/debit/balance/limit are in absolute time units;
        time_value_add(&thread_total_time, &thread_user_time);
        time_value_add(&thread_total_time, &thread_system_time);
        ledger_get_entry_info(thread->t_threadledger, thread_ledgers.cpu_time, &lei);
 
        /* credit/debit/balance/limit are in absolute time units;

-          the refill info is in nanoseconds. */
+        *  the refill info is in nanoseconds. */

        absolutetime_to_nanoseconds(lei.lei_balance, &balance_ns);
        if (lei.lei_last_refill > 0) {
        absolutetime_to_nanoseconds(lei.lei_balance, &balance_ns);
        if (lei.lei_last_refill > 0) {

-               usage_percent = (uint32_t)((balance_ns*100ULL) / lei.lei_last_refill);
+               usage_percent = (uint32_t)((balance_ns * 100ULL) / lei.lei_last_refill);

        }
 
        /* TODO: show task total runtime (via TASK_ABSOLUTETIME_INFO)? */
        }
 
        /* TODO: show task total runtime (via TASK_ABSOLUTETIME_INFO)? */

-       printf("process %s[%d] thread %llu caught burning CPU! "
-              "It used more than %d%% CPU over %u seconds "
-              "(actual recent usage: %d%% over ~%llu seconds).  "
-              "Thread lifetime cpu usage %d.%06ds, (%d.%06d user, %d.%06d sys) "
-              "ledger balance: %lld mabs credit: %lld mabs debit: %lld mabs "
-              "limit: %llu mabs period: %llu ns last refill: %llu ns%s.\n",
-              procname, pid, tid,
-              percentage, interval_sec,
-              usage_percent,
-              (lei.lei_last_refill + NSEC_PER_SEC/2) / NSEC_PER_SEC,
-              thread_total_time.seconds, thread_total_time.microseconds,
-              thread_user_time.seconds, thread_user_time.microseconds,
-              thread_system_time.seconds,thread_system_time.microseconds,
-              lei.lei_balance, lei.lei_credit, lei.lei_debit,
-              lei.lei_limit, lei.lei_refill_period, lei.lei_last_refill,
-              (fatal ? " [fatal violation]" : ""));
+       printf("process %s[%d] thread %llu caught burning CPU! It used more than %d%% CPU over %u seconds\n",
+           procname, pid, tid, percentage, interval_sec);
+       printf("  (actual recent usage: %d%% over ~%llu seconds)\n",
+           usage_percent, (lei.lei_last_refill + NSEC_PER_SEC / 2) / NSEC_PER_SEC);
+       printf("  Thread lifetime cpu usage %d.%06ds, (%d.%06d user, %d.%06d sys)\n",
+           thread_total_time.seconds, thread_total_time.microseconds,
+           thread_user_time.seconds, thread_user_time.microseconds,
+           thread_system_time.seconds, thread_system_time.microseconds);
+       printf("  Ledger balance: %lld; mabs credit: %lld; mabs debit: %lld\n",
+           lei.lei_balance, lei.lei_credit, lei.lei_debit);
+       printf("  mabs limit: %llu; mabs period: %llu ns; last refill: %llu ns%s.\n",
+           lei.lei_limit, lei.lei_refill_period, lei.lei_last_refill,
+           (fatal ? " [fatal violation]" : ""));

 
        /*
 
        /*

-          For now, send RESOURCE_NOTIFY in parallel with EXC_RESOURCE.  Once
-          we have logging parity, we will stop sending EXC_RESOURCE (24508922).
+        *  For now, send RESOURCE_NOTIFY in parallel with EXC_RESOURCE.  Once
+        *  we have logging parity, we will stop sending EXC_RESOURCE (24508922).

         */
 
        /* RESOURCE_NOTIFY MIG specifies nanoseconds of CPU time */
         */
 
        /* RESOURCE_NOTIFY MIG specifies nanoseconds of CPU time */


@@ -2194,7 +2410,7 @@ SENDING_NOTIFICATION__THIS_THREAD_IS_CONSUMING_TOO_MUCH_CPU(void)
        absolutetime_to_nanoseconds(lei.lei_limit, &lei.lei_limit);
        trace_resource_violation(RMON_CPUUSAGE_VIOLATED, &lei);
        kr = send_resource_violation(send_cpu_usage_violation, task, &lei,
        absolutetime_to_nanoseconds(lei.lei_limit, &lei.lei_limit);
        trace_resource_violation(RMON_CPUUSAGE_VIOLATED, &lei);
        kr = send_resource_violation(send_cpu_usage_violation, task, &lei,

-                                                                fatal ? kRNFatalLimitFlag : 0);
+           fatal ? kRNFatalLimitFlag : 0);

        if (kr) {
                printf("send_resource_violation(CPU usage, ...): error %#x\n", kr);
        }
        if (kr) {
                printf("send_resource_violation(CPU usage, ...): error %#x\n", kr);
        }


@@ -2203,15 +2419,15 @@ SENDING_NOTIFICATION__THIS_THREAD_IS_CONSUMING_TOO_MUCH_CPU(void)
        if (send_exc_resource) {
                if (disable_exc_resource) {
                        printf("process %s[%d] thread %llu caught burning CPU! "
        if (send_exc_resource) {
                if (disable_exc_resource) {
                        printf("process %s[%d] thread %llu caught burning CPU! "

-                                  "EXC_RESOURCE%s supressed by a boot-arg\n",
-                                  procname, pid, tid, fatal ? " (and termination)" : "");
+                           "EXC_RESOURCE%s supressed by a boot-arg\n",
+                           procname, pid, tid, fatal ? " (and termination)" : "");

                        return;
                }
 
                if (audio_active) {
                        printf("process %s[%d] thread %llu caught burning CPU! "
                        return;
                }
 
                if (audio_active) {
                        printf("process %s[%d] thread %llu caught burning CPU! "

-                          "EXC_RESOURCE & termination supressed due to audio playback\n",
-                                  procname, pid, tid);
+                           "EXC_RESOURCE & termination supressed due to audio playback\n",
+                           procname, pid, tid);

                        return;
                }
        }
                        return;
                }
        }


@@ -2222,7 +2438,7 @@ SENDING_NOTIFICATION__THIS_THREAD_IS_CONSUMING_TOO_MUCH_CPU(void)
                EXC_RESOURCE_ENCODE_TYPE(code[0], RESOURCE_TYPE_CPU);
                if (fatal) {
                        EXC_RESOURCE_ENCODE_FLAVOR(code[0], FLAVOR_CPU_MONITOR_FATAL);
                EXC_RESOURCE_ENCODE_TYPE(code[0], RESOURCE_TYPE_CPU);
                if (fatal) {
                        EXC_RESOURCE_ENCODE_FLAVOR(code[0], FLAVOR_CPU_MONITOR_FATAL);

-               }else {
+               } else {

                        EXC_RESOURCE_ENCODE_FLAVOR(code[0], FLAVOR_CPU_MONITOR);
                }
                EXC_RESOURCE_CPUMONITOR_ENCODE_INTERVAL(code[0], interval_sec);
                        EXC_RESOURCE_ENCODE_FLAVOR(code[0], FLAVOR_CPU_MONITOR);
                }
                EXC_RESOURCE_CPUMONITOR_ENCODE_INTERVAL(code[0], interval_sec);


@@ -2241,30 +2457,79 @@ SENDING_NOTIFICATION__THIS_THREAD_IS_CONSUMING_TOO_MUCH_CPU(void)
        }
 }
 
        }
 }
 

-void thread_update_io_stats(thread_t thread, int size, int io_flags)
+#if DEVELOPMENT || DEBUG
+void __attribute__((noinline))
+SENDING_NOTIFICATION__TASK_HAS_TOO_MANY_THREADS(task_t task, int thread_count)
+{
+       mach_exception_data_type_t code[EXCEPTION_CODE_MAX] = {0};
+       int pid = task_pid(task);
+       char procname[MAXCOMLEN + 1] = "unknown";
+
+       if (pid == 1) {
+               /*
+                * Cannot suspend launchd
+                */
+               return;
+       }
+
+       proc_name(pid, procname, sizeof(procname));
+
+       if (disable_exc_resource) {
+               printf("process %s[%d] crossed thread count high watermark (%d), EXC_RESOURCE "
+                   "supressed by a boot-arg. \n", procname, pid, thread_count);
+               return;
+       }
+
+       if (audio_active) {
+               printf("process %s[%d] crossed thread count high watermark (%d), EXC_RESOURCE "
+                   "supressed due to audio playback.\n", procname, pid, thread_count);
+               return;
+       }
+
+       if (exc_via_corpse_forking == 0) {
+               printf("process %s[%d] crossed thread count high watermark (%d), EXC_RESOURCE "
+                   "supressed due to corpse forking being disabled.\n", procname, pid,
+                   thread_count);
+               return;
+       }
+
+       printf("process %s[%d] crossed thread count high watermark (%d), sending "
+           "EXC_RESOURCE\n", procname, pid, thread_count);
+
+       EXC_RESOURCE_ENCODE_TYPE(code[0], RESOURCE_TYPE_THREADS);
+       EXC_RESOURCE_ENCODE_FLAVOR(code[0], FLAVOR_THREADS_HIGH_WATERMARK);
+       EXC_RESOURCE_THREADS_ENCODE_THREADS(code[0], thread_count);
+
+       task_enqueue_exception_with_corpse(task, EXC_RESOURCE, code, EXCEPTION_CODE_MAX, NULL);
+}
+#endif /* DEVELOPMENT || DEBUG */
+
+void
+thread_update_io_stats(thread_t thread, int size, int io_flags)

 {
        int io_tier;
 
 {
        int io_tier;
 

-       if (thread->thread_io_stats == NULL || thread->task->task_io_stats == NULL)
+       if (thread->thread_io_stats == NULL || thread->task->task_io_stats == NULL) {

                return;
                return;

+       }

 
        if (io_flags & DKIO_READ) {
                UPDATE_IO_STATS(thread->thread_io_stats->disk_reads, size);
                UPDATE_IO_STATS_ATOMIC(thread->task->task_io_stats->disk_reads, size);
        }
 
        if (io_flags & DKIO_READ) {
                UPDATE_IO_STATS(thread->thread_io_stats->disk_reads, size);
                UPDATE_IO_STATS_ATOMIC(thread->task->task_io_stats->disk_reads, size);
        }

-       
+

        if (io_flags & DKIO_META) {
                UPDATE_IO_STATS(thread->thread_io_stats->metadata, size);
                UPDATE_IO_STATS_ATOMIC(thread->task->task_io_stats->metadata, size);
        }
        if (io_flags & DKIO_META) {
                UPDATE_IO_STATS(thread->thread_io_stats->metadata, size);
                UPDATE_IO_STATS_ATOMIC(thread->task->task_io_stats->metadata, size);
        }

-       
+

        if (io_flags & DKIO_PAGING) {
                UPDATE_IO_STATS(thread->thread_io_stats->paging, size);
                UPDATE_IO_STATS_ATOMIC(thread->task->task_io_stats->paging, size);
        }
 
        io_tier = ((io_flags & DKIO_TIER_MASK) >> DKIO_TIER_SHIFT);
        if (io_flags & DKIO_PAGING) {
                UPDATE_IO_STATS(thread->thread_io_stats->paging, size);
                UPDATE_IO_STATS_ATOMIC(thread->task->task_io_stats->paging, size);
        }
 
        io_tier = ((io_flags & DKIO_TIER_MASK) >> DKIO_TIER_SHIFT);

-       assert (io_tier < IO_NUM_PRIORITIES);
+       assert(io_tier < IO_NUM_PRIORITIES);

 
        UPDATE_IO_STATS(thread->thread_io_stats->io_priority[io_tier], size);
        UPDATE_IO_STATS_ATOMIC(thread->task->task_io_stats->io_priority[io_tier], size);
 
        UPDATE_IO_STATS(thread->thread_io_stats->io_priority[io_tier], size);
        UPDATE_IO_STATS_ATOMIC(thread->task->task_io_stats->io_priority[io_tier], size);


@@ -2280,25 +2545,28 @@ void thread_update_io_stats(thread_t thread, int size, int io_flags)
 }
 
 static void
 }
 
 static void

-init_thread_ledgers(void) {
+init_thread_ledgers(void)
+{

        ledger_template_t t;
        int idx;
        ledger_template_t t;
        int idx;

-       
+

        assert(thread_ledger_template == NULL);
 
        assert(thread_ledger_template == NULL);
 

-       if ((t = ledger_template_create("Per-thread ledger")) == NULL)
+       if ((t = ledger_template_create("Per-thread ledger")) == NULL) {

                panic("couldn't create thread ledger template");
                panic("couldn't create thread ledger template");

+       }

 
        if ((idx = ledger_entry_add(t, "cpu_time", "sched", "ns")) < 0) {
                panic("couldn't create cpu_time entry for thread ledger template");
        }
 
        if (ledger_set_callback(t, idx, thread_cputime_callback, NULL, NULL) < 0) {
 
        if ((idx = ledger_entry_add(t, "cpu_time", "sched", "ns")) < 0) {
                panic("couldn't create cpu_time entry for thread ledger template");
        }
 
        if (ledger_set_callback(t, idx, thread_cputime_callback, NULL, NULL) < 0) {

-               panic("couldn't set thread ledger callback for cpu_time entry");
+               panic("couldn't set thread ledger callback for cpu_time entry");

        }
 
        thread_ledgers.cpu_time = idx;
        }
 
        thread_ledgers.cpu_time = idx;

-       
+
+       ledger_template_complete(t);

        thread_ledger_template = t;
 }
 
        thread_ledger_template = t;
 }
 


@@ -2308,9 +2576,9 @@ init_thread_ledgers(void) {
 int
 thread_get_cpulimit(int *action, uint8_t *percentage, uint64_t *interval_ns)
 {
 int
 thread_get_cpulimit(int *action, uint8_t *percentage, uint64_t *interval_ns)
 {

-       int64_t         abstime = 0;
-       uint64_t        limittime = 0;
-       thread_t        thread = current_thread();
+       int64_t         abstime = 0;
+       uint64_t        limittime = 0;
+       thread_t        thread = current_thread();

 
        *percentage  = 0;
        *interval_ns = 0;
 
        *percentage  = 0;
        *interval_ns = 0;


@@ -2321,7 +2589,7 @@ thread_get_cpulimit(int *action, uint8_t *percentage, uint64_t *interval_ns)
                 * This thread has no per-thread ledger, so it can't possibly
                 * have a CPU limit applied.
                 */
                 * This thread has no per-thread ledger, so it can't possibly
                 * have a CPU limit applied.
                 */

-               return (KERN_SUCCESS);
+               return KERN_SUCCESS;

        }
 
        ledger_get_period(thread->t_threadledger, thread_ledgers.cpu_time, interval_ns);
        }
 
        ledger_get_period(thread->t_threadledger, thread_ledgers.cpu_time, interval_ns);


@@ -2332,7 +2600,7 @@ thread_get_cpulimit(int *action, uint8_t *percentage, uint64_t *interval_ns)
                 * This thread's CPU time ledger has no period or limit; so it
                 * doesn't have a CPU limit applied.
                 */
                 * This thread's CPU time ledger has no period or limit; so it
                 * doesn't have a CPU limit applied.
                 */

-                return (KERN_SUCCESS);
+               return KERN_SUCCESS;

        }
 
        /*
        }
 
        /*


@@ -2354,7 +2622,7 @@ thread_get_cpulimit(int *action, uint8_t *percentage, uint64_t *interval_ns)
                *action = THREAD_CPULIMIT_DISABLE;
        }
 
                *action = THREAD_CPULIMIT_DISABLE;
        }
 

-       return (KERN_SUCCESS);
+       return KERN_SUCCESS;

 }
 
 /*
 }
 
 /*


@@ -2365,10 +2633,10 @@ thread_get_cpulimit(int *action, uint8_t *percentage, uint64_t *interval_ns)
 int
 thread_set_cpulimit(int action, uint8_t percentage, uint64_t interval_ns)
 {
 int
 thread_set_cpulimit(int action, uint8_t percentage, uint64_t interval_ns)
 {

-       thread_t        thread = current_thread(); 
-       ledger_t        l;
-       uint64_t        limittime = 0;
-       uint64_t        abstime = 0;
+       thread_t        thread = current_thread();
+       ledger_t        l;
+       uint64_t        limittime = 0;
+       uint64_t        abstime = 0;

 
        assert(percentage <= 100);
 
 
        assert(percentage <= 100);
 


@@ -2383,20 +2651,21 @@ thread_set_cpulimit(int action, uint8_t percentage, uint64_t interval_ns)
                        thread->options &= ~(TH_OPT_PROC_CPULIMIT | TH_OPT_PRVT_CPULIMIT);
                }
 
                        thread->options &= ~(TH_OPT_PROC_CPULIMIT | TH_OPT_PRVT_CPULIMIT);
                }
 

-               return (0);
+               return 0;

        }
 
        if (interval_ns < MINIMUM_CPULIMIT_INTERVAL_MS * NSEC_PER_MSEC) {
        }
 
        if (interval_ns < MINIMUM_CPULIMIT_INTERVAL_MS * NSEC_PER_MSEC) {

-               return (KERN_INVALID_ARGUMENT);
+               return KERN_INVALID_ARGUMENT;

        }
 
        }
 

-       l = thread->t_threadledger;
+       l = thread->t_threadledger;

        if (l == LEDGER_NULL) {
                /*
                 * This thread doesn't yet have a per-thread ledger; so create one with the CPU time entry active.
                 */
        if (l == LEDGER_NULL) {
                /*
                 * This thread doesn't yet have a per-thread ledger; so create one with the CPU time entry active.
                 */

-               if ((l = ledger_instantiate(thread_ledger_template, LEDGER_CREATE_INACTIVE_ENTRIES)) == LEDGER_NULL)
-                       return (KERN_RESOURCE_SHORTAGE);
+               if ((l = ledger_instantiate(thread_ledger_template, LEDGER_CREATE_INACTIVE_ENTRIES)) == LEDGER_NULL) {
+                       return KERN_RESOURCE_SHORTAGE;
+               }

 
                /*
                 * We are the first to create this thread's ledger, so only activate our entry.
 
                /*
                 * We are the first to create this thread's ledger, so only activate our entry.


@@ -2410,7 +2679,7 @@ thread_set_cpulimit(int action, uint8_t percentage, uint64_t interval_ns)
         * Calculate the amount of CPU time that the thread needs to consume in order to hit the limit.
         */
        limittime = (interval_ns * percentage) / 100;
         * Calculate the amount of CPU time that the thread needs to consume in order to hit the limit.
         */
        limittime = (interval_ns * percentage) / 100;

-       nanoseconds_to_absolutetime(limittime, &abstime); 
+       nanoseconds_to_absolutetime(limittime, &abstime);

        ledger_set_limit(l, thread_ledgers.cpu_time, abstime, cpumon_ustackshots_trigger_pct);
        /*
         * Refill the thread's allotted CPU time every interval_ns nanoseconds.
        ledger_set_limit(l, thread_ledgers.cpu_time, abstime, cpumon_ustackshots_trigger_pct);
        /*
         * Refill the thread's allotted CPU time every interval_ns nanoseconds.


@@ -2438,7 +2707,7 @@ thread_set_cpulimit(int action, uint8_t percentage, uint64_t interval_ns)
                 * We deliberately override any CPU limit imposed by a task-wide limit (eg
                 * CPU usage monitor).
                 */
                 * We deliberately override any CPU limit imposed by a task-wide limit (eg
                 * CPU usage monitor).
                 */

-               thread->options &= ~TH_OPT_PROC_CPULIMIT;               
+               thread->options &= ~TH_OPT_PROC_CPULIMIT;

 
                thread->options |= TH_OPT_PRVT_CPULIMIT;
                /* The per-thread ledger template by default has a callback for CPU time */
 
                thread->options |= TH_OPT_PRVT_CPULIMIT;
                /* The per-thread ledger template by default has a callback for CPU time */


@@ -2446,138 +2715,148 @@ thread_set_cpulimit(int action, uint8_t percentage, uint64_t interval_ns)
                ledger_set_action(l, thread_ledgers.cpu_time, LEDGER_ACTION_BLOCK);
        }
 
                ledger_set_action(l, thread_ledgers.cpu_time, LEDGER_ACTION_BLOCK);
        }
 

-       return (0);
-}
-
-static void
-sched_call_null(
-__unused       int                     type,
-__unused       thread_t        thread)
-{
-       return;
+       return 0;

 }
 
 void
 thread_sched_call(
 }
 
 void
 thread_sched_call(

-       thread_t                thread,
-       sched_call_t    call)
+       thread_t                thread,
+       sched_call_t    call)

 {
 {

-       thread->sched_call = (call != NULL)? call: sched_call_null;
+       assert((thread->state & TH_WAIT_REPORT) == 0);
+       thread->sched_call = call;

 }
 
 }
 

-sched_call_t
-thread_disable_sched_call(
-       thread_t                thread,
-       sched_call_t    call)
+uint64_t
+thread_tid(
+       thread_t        thread)

 {
 {

-       if (call) {
-               spl_t s = splsched();
-               thread_lock(thread);
-               if (thread->sched_call == call) {
-                       thread->sched_call = sched_call_null;
-               } else {
-                       call = NULL;
-               }
-               thread_unlock(thread);
-               splx(s);
-       }
-       return call;
+       return thread != THREAD_NULL? thread->thread_id: 0;

 }
 
 }
 

-void
-thread_reenable_sched_call(
-       thread_t                thread,
-       sched_call_t    call)
+uint16_t
+thread_set_tag(thread_t th, uint16_t tag)

 {
 {

-       if (call) {
-               spl_t s = splsched();
-               thread_lock(thread);
-               thread_sched_call(thread, call);
-               thread_unlock(thread);
-               splx(s);
-       }
+       return thread_set_tag_internal(th, tag);

 }
 
 }
 

-void
-thread_static_param(
-       thread_t                thread,
-       boolean_t               state)
+uint16_t
+thread_get_tag(thread_t th)

 {
 {

-       thread_mtx_lock(thread);
-       thread->static_param = state;
-       thread_mtx_unlock(thread);
+       return thread_get_tag_internal(th);

 }
 
 uint64_t
 }
 
 uint64_t

-thread_tid(
-       thread_t        thread)
+thread_last_run_time(thread_t th)

 {
 {

-       return (thread != THREAD_NULL? thread->thread_id: 0);
-}
-
-uint16_t       thread_set_tag(thread_t th, uint16_t tag) {
-       return thread_set_tag_internal(th, tag);
-}
-uint16_t       thread_get_tag(thread_t th) {
-       return thread_get_tag_internal(th);
+       return th->last_run_time;

 }
 
 uint64_t
 thread_dispatchqaddr(
 }
 
 uint64_t
 thread_dispatchqaddr(

-       thread_t                thread)
+       thread_t                thread)

 {
 {

-       uint64_t        dispatchqueue_addr;
-       uint64_t        thread_handle;
+       uint64_t        dispatchqueue_addr;
+       uint64_t        thread_handle;

 
 

-       if (thread == THREAD_NULL)
+       if (thread == THREAD_NULL) {

                return 0;
                return 0;

+       }

 
        thread_handle = thread->machine.cthread_self;
 
        thread_handle = thread->machine.cthread_self;

-       if (thread_handle == 0)
+       if (thread_handle == 0) {

                return 0;
                return 0;

-       
-       if (thread->inspection == TRUE)
+       }
+
+       if (thread->inspection == TRUE) {

                dispatchqueue_addr = thread_handle + get_task_dispatchqueue_offset(thread->task);
                dispatchqueue_addr = thread_handle + get_task_dispatchqueue_offset(thread->task);

-       else if (thread->task->bsd_info)
+       } else if (thread->task->bsd_info) {

                dispatchqueue_addr = thread_handle + get_dispatchqueue_offset_from_proc(thread->task->bsd_info);
                dispatchqueue_addr = thread_handle + get_dispatchqueue_offset_from_proc(thread->task->bsd_info);

-       else
+       } else {

                dispatchqueue_addr = 0;
                dispatchqueue_addr = 0;

+       }

 
        return dispatchqueue_addr;
 }
 
 
        return dispatchqueue_addr;
 }
 

+uint64_t
+thread_rettokern_addr(
+       thread_t                thread)
+{
+       uint64_t        rettokern_addr;
+       uint64_t        rettokern_offset;
+       uint64_t        thread_handle;
+
+       if (thread == THREAD_NULL) {
+               return 0;
+       }
+
+       thread_handle = thread->machine.cthread_self;
+       if (thread_handle == 0) {
+               return 0;
+       }
+
+       if (thread->task->bsd_info) {
+               rettokern_offset = get_return_to_kernel_offset_from_proc(thread->task->bsd_info);
+
+               /* Return 0 if return to kernel offset is not initialized. */
+               if (rettokern_offset == 0) {
+                       rettokern_addr = 0;
+               } else {
+                       rettokern_addr = thread_handle + rettokern_offset;
+               }
+       } else {
+               rettokern_addr = 0;
+       }
+
+       return rettokern_addr;
+}
+

 /*
  * Export routines to other components for things that are done as macros
  * within the osfmk component.
  */
 
 /*
  * Export routines to other components for things that are done as macros
  * within the osfmk component.
  */
 

+#undef thread_mtx_lock
+void thread_mtx_lock(thread_t thread);
+void
+thread_mtx_lock(thread_t thread)
+{
+       lck_mtx_lock(&thread->mutex);
+}
+
+#undef thread_mtx_unlock
+void thread_mtx_unlock(thread_t thread);
+void
+thread_mtx_unlock(thread_t thread)
+{
+       lck_mtx_unlock(&thread->mutex);
+}
+

 #undef thread_reference
 void thread_reference(thread_t thread);
 void
 thread_reference(
 #undef thread_reference
 void thread_reference(thread_t thread);
 void
 thread_reference(

-       thread_t        thread)
+       thread_t        thread)

 {
 {

-       if (thread != THREAD_NULL)
+       if (thread != THREAD_NULL) {

                thread_reference_internal(thread);
                thread_reference_internal(thread);

+       }

 }
 
 #undef thread_should_halt
 
 boolean_t
 thread_should_halt(
 }
 
 #undef thread_should_halt
 
 boolean_t
 thread_should_halt(

-       thread_t                th)
+       thread_t                th)

 {
 {

-       return (thread_should_halt_fast(th));
+       return thread_should_halt_fast(th);

 }
 
 /*
  * thread_set_voucher_name - reset the voucher port name bound to this thread
  *
  * Conditions:  nothing locked
 }
 
 /*
  * thread_set_voucher_name - reset the voucher port name bound to this thread
  *
  * Conditions:  nothing locked

- *
- *     If we already converted the previous name to a cached voucher
- *     reference, then we discard that reference here.  The next lookup
- *     will cache it again.

  */
 
 kern_return_t
  */
 
 kern_return_t


@@ -2586,75 +2865,70 @@ thread_set_voucher_name(mach_port_name_t voucher_name)
        thread_t thread = current_thread();
        ipc_voucher_t new_voucher = IPC_VOUCHER_NULL;
        ipc_voucher_t voucher;
        thread_t thread = current_thread();
        ipc_voucher_t new_voucher = IPC_VOUCHER_NULL;
        ipc_voucher_t voucher;

-#ifdef CONFIG_BANK

        ledger_t bankledger = NULL;
        ledger_t bankledger = NULL;

-#endif
+       struct thread_group *banktg = NULL;
+       uint32_t persona_id = 0;

 
 

-       if (MACH_PORT_DEAD == voucher_name)
+       if (MACH_PORT_DEAD == voucher_name) {

                return KERN_INVALID_RIGHT;
                return KERN_INVALID_RIGHT;

+       }

 
        /*
         * agressively convert to voucher reference
         */
        if (MACH_PORT_VALID(voucher_name)) {
                new_voucher = convert_port_name_to_voucher(voucher_name);
 
        /*
         * agressively convert to voucher reference
         */
        if (MACH_PORT_VALID(voucher_name)) {
                new_voucher = convert_port_name_to_voucher(voucher_name);

-               if (IPC_VOUCHER_NULL == new_voucher)
+               if (IPC_VOUCHER_NULL == new_voucher) {

                        return KERN_INVALID_ARGUMENT;
                        return KERN_INVALID_ARGUMENT;

+               }

        }
        }

-#ifdef CONFIG_BANK
-       bankledger = bank_get_voucher_ledger(new_voucher);
-#endif
+       bank_get_bank_ledger_thread_group_and_persona(new_voucher, &bankledger, &banktg, &persona_id);

 
        thread_mtx_lock(thread);
        voucher = thread->ith_voucher;
        thread->ith_voucher_name = voucher_name;
        thread->ith_voucher = new_voucher;
 
        thread_mtx_lock(thread);
        voucher = thread->ith_voucher;
        thread->ith_voucher_name = voucher_name;
        thread->ith_voucher = new_voucher;

-#ifdef CONFIG_BANK
-       bank_swap_thread_bank_ledger(thread, bankledger);
-#endif

        thread_mtx_unlock(thread);
 
        thread_mtx_unlock(thread);
 

+       bank_swap_thread_bank_ledger(thread, bankledger);
+

        KERNEL_DEBUG_CONSTANT_IST(KDEBUG_TRACE,
        KERNEL_DEBUG_CONSTANT_IST(KDEBUG_TRACE,

-                                 MACHDBG_CODE(DBG_MACH_IPC,MACH_THREAD_SET_VOUCHER) | DBG_FUNC_NONE,
-                                 (uintptr_t)thread_tid(thread),
-                                 (uintptr_t)voucher_name,
-                                 VM_KERNEL_ADDRPERM((uintptr_t)new_voucher),
-                                 1, 0);
+           MACHDBG_CODE(DBG_MACH_IPC, MACH_THREAD_SET_VOUCHER) | DBG_FUNC_NONE,
+           (uintptr_t)thread_tid(thread),
+           (uintptr_t)voucher_name,
+           VM_KERNEL_ADDRPERM((uintptr_t)new_voucher),
+           persona_id, 0);

 
 

-       if (IPC_VOUCHER_NULL != voucher)
+       if (IPC_VOUCHER_NULL != voucher) {

                ipc_voucher_release(voucher);
                ipc_voucher_release(voucher);

+       }

 
        return KERN_SUCCESS;
 }
 
 
        return KERN_SUCCESS;
 }
 

-/* 
+/*

  *  thread_get_mach_voucher - return a voucher reference for the specified thread voucher
  *
  *  Conditions:  nothing locked
  *
  *  thread_get_mach_voucher - return a voucher reference for the specified thread voucher
  *
  *  Conditions:  nothing locked
  *

- *  A reference to the voucher may be lazily pending, if someone set the voucher name
- *  but nobody has done a lookup yet.  In that case, we'll have to do the equivalent
- *  lookup here.
- *
- *  NOTE:      At the moment, there is no distinction between the current and effective
+ *  NOTE:       At the moment, there is no distinction between the current and effective

  *             vouchers because we only set them at the thread level currently.
  */
  *             vouchers because we only set them at the thread level currently.
  */

-kern_return_t 
+kern_return_t

 thread_get_mach_voucher(
 thread_get_mach_voucher(

-       thread_act_t            thread,
+       thread_act_t            thread,

        mach_voucher_selector_t __unused which,
        mach_voucher_selector_t __unused which,

-       ipc_voucher_t           *voucherp)
+       ipc_voucher_t           *voucherp)

 {
 {

-       ipc_voucher_t           voucher;
-       mach_port_name_t        voucher_name;
+       ipc_voucher_t           voucher;

 
 

-       if (THREAD_NULL == thread)
+       if (THREAD_NULL == thread) {

                return KERN_INVALID_ARGUMENT;
                return KERN_INVALID_ARGUMENT;

+       }

 
        thread_mtx_lock(thread);
        voucher = thread->ith_voucher;
 
 
        thread_mtx_lock(thread);
        voucher = thread->ith_voucher;
 

-       /* if already cached, just return a ref */

        if (IPC_VOUCHER_NULL != voucher) {
                ipc_voucher_reference(voucher);
                thread_mtx_unlock(thread);
        if (IPC_VOUCHER_NULL != voucher) {
                ipc_voucher_reference(voucher);
                thread_mtx_unlock(thread);


@@ -2662,181 +2936,88 @@ thread_get_mach_voucher(
                return KERN_SUCCESS;
        }
 
                return KERN_SUCCESS;
        }
 

-       voucher_name = thread->ith_voucher_name;
-
-       /* convert the name to a port, then voucher reference */
-       if (MACH_PORT_VALID(voucher_name)) {
-               ipc_port_t port;
-
-               if (KERN_SUCCESS !=
-                   ipc_object_copyin(thread->task->itk_space, voucher_name,
-                                     MACH_MSG_TYPE_COPY_SEND, (ipc_object_t *)&port)) {
-                       thread->ith_voucher_name = MACH_PORT_NULL;
-                       thread_mtx_unlock(thread);
-                       *voucherp = IPC_VOUCHER_NULL;
-                       return KERN_SUCCESS;
-               }
-
-               /* convert to a voucher ref to return, and cache a ref on thread */
-               voucher = convert_port_to_voucher(port);
-               ipc_voucher_reference(voucher);
-               thread->ith_voucher = voucher;
-               thread_mtx_unlock(thread);
-
-               KERNEL_DEBUG_CONSTANT_IST(KDEBUG_TRACE,
-                                         MACHDBG_CODE(DBG_MACH_IPC,MACH_THREAD_SET_VOUCHER) | DBG_FUNC_NONE,
-                                         (uintptr_t)thread_tid(thread),
-                                         (uintptr_t)port,
-                                         VM_KERNEL_ADDRPERM((uintptr_t)voucher),
-                                         2, 0);
-
-
-               ipc_port_release_send(port);
-       } else
-               thread_mtx_unlock(thread);
+       thread_mtx_unlock(thread);

 
 

-       *voucherp = voucher;
+       *voucherp = IPC_VOUCHER_NULL;

        return KERN_SUCCESS;
 }
 
        return KERN_SUCCESS;
 }
 

-/* 
+/*

  *  thread_set_mach_voucher - set a voucher reference for the specified thread voucher
  *
  *  Conditions: callers holds a reference on the voucher.
  *             nothing locked.
  *
  *  thread_set_mach_voucher - set a voucher reference for the specified thread voucher
  *
  *  Conditions: callers holds a reference on the voucher.
  *             nothing locked.
  *

- *  We grab another reference to the voucher and bind it to the thread.  Any lazy
- *  binding is erased.  The old voucher reference associated with the thread is
+ *  We grab another reference to the voucher and bind it to the thread.
+ *  The old voucher reference associated with the thread is

  *  discarded.
  */
  *  discarded.
  */

-kern_return_t 
+kern_return_t

 thread_set_mach_voucher(
 thread_set_mach_voucher(

-       thread_t                thread,
-       ipc_voucher_t           voucher)
+       thread_t                thread,
+       ipc_voucher_t           voucher)

 {
        ipc_voucher_t old_voucher;
 {
        ipc_voucher_t old_voucher;

-#ifdef CONFIG_BANK

        ledger_t bankledger = NULL;
        ledger_t bankledger = NULL;

-#endif
+       struct thread_group *banktg = NULL;
+       uint32_t persona_id = 0;

 
 

-       if (THREAD_NULL == thread)
+       if (THREAD_NULL == thread) {

                return KERN_INVALID_ARGUMENT;
                return KERN_INVALID_ARGUMENT;

+       }

 
 

-       if (thread != current_thread() || thread->started)
+       if (thread != current_thread() && thread->started) {

                return KERN_INVALID_ARGUMENT;
                return KERN_INVALID_ARGUMENT;

-
+       }

 
        ipc_voucher_reference(voucher);
 
        ipc_voucher_reference(voucher);

-#ifdef CONFIG_BANK
-       bankledger = bank_get_voucher_ledger(voucher);
-#endif
+       bank_get_bank_ledger_thread_group_and_persona(voucher, &bankledger, &banktg, &persona_id);
+

        thread_mtx_lock(thread);
        old_voucher = thread->ith_voucher;
        thread->ith_voucher = voucher;
        thread->ith_voucher_name = MACH_PORT_NULL;
        thread_mtx_lock(thread);
        old_voucher = thread->ith_voucher;
        thread->ith_voucher = voucher;
        thread->ith_voucher_name = MACH_PORT_NULL;

-#ifdef CONFIG_BANK
-       bank_swap_thread_bank_ledger(thread, bankledger);
-#endif

        thread_mtx_unlock(thread);
 
        thread_mtx_unlock(thread);
 

+       bank_swap_thread_bank_ledger(thread, bankledger);
+

        KERNEL_DEBUG_CONSTANT_IST(KDEBUG_TRACE,
        KERNEL_DEBUG_CONSTANT_IST(KDEBUG_TRACE,

-                                 MACHDBG_CODE(DBG_MACH_IPC,MACH_THREAD_SET_VOUCHER) | DBG_FUNC_NONE,
-                                 (uintptr_t)thread_tid(thread),
-                                 (uintptr_t)MACH_PORT_NULL,
-                                 VM_KERNEL_ADDRPERM((uintptr_t)voucher),
-                                 3, 0);
+           MACHDBG_CODE(DBG_MACH_IPC, MACH_THREAD_SET_VOUCHER) | DBG_FUNC_NONE,
+           (uintptr_t)thread_tid(thread),
+           (uintptr_t)MACH_PORT_NULL,
+           VM_KERNEL_ADDRPERM((uintptr_t)voucher),
+           persona_id, 0);

 
        ipc_voucher_release(old_voucher);
 
        return KERN_SUCCESS;
 }
 
 
        ipc_voucher_release(old_voucher);
 
        return KERN_SUCCESS;
 }
 

-/* 
+/*

  *  thread_swap_mach_voucher - swap a voucher reference for the specified thread voucher
  *
  *  Conditions: callers holds a reference on the new and presumed old voucher(s).
  *             nothing locked.
  *
  *  thread_swap_mach_voucher - swap a voucher reference for the specified thread voucher
  *
  *  Conditions: callers holds a reference on the new and presumed old voucher(s).
  *             nothing locked.
  *

- *  If the old voucher is still the same as passed in, replace it with new voucher
- *  and discard the old (and the reference passed in).  Otherwise, discard the new
- *  and return an updated old voucher.
+ *  This function is no longer supported.

  */
 kern_return_t
 thread_swap_mach_voucher(
  */
 kern_return_t
 thread_swap_mach_voucher(

-       thread_t                thread,
-       ipc_voucher_t           new_voucher,
-       ipc_voucher_t           *in_out_old_voucher)
+       __unused thread_t               thread,
+       __unused ipc_voucher_t          new_voucher,
+       ipc_voucher_t                   *in_out_old_voucher)

 {
 {

-       mach_port_name_t old_voucher_name;
-       ipc_voucher_t old_voucher;
-#ifdef CONFIG_BANK
-       ledger_t bankledger = NULL;
-#endif
-
-       if (THREAD_NULL == thread)
-               return KERN_INVALID_TASK;
-
-       if (thread != current_thread() || thread->started)
-               return KERN_INVALID_ARGUMENT;
-
-#ifdef CONFIG_BANK
-       bankledger = bank_get_voucher_ledger(new_voucher);
-#endif
-
-       thread_mtx_lock(thread);
-
-       old_voucher = thread->ith_voucher;
-
-       if (IPC_VOUCHER_NULL == old_voucher) {
-               old_voucher_name = thread->ith_voucher_name;
-
-               /* perform lazy binding if needed */
-               if (MACH_PORT_VALID(old_voucher_name)) {
-                       old_voucher = convert_port_name_to_voucher(old_voucher_name);
-                       thread->ith_voucher_name = MACH_PORT_NULL;
-                       thread->ith_voucher = old_voucher;
-
-                       KERNEL_DEBUG_CONSTANT_IST(KDEBUG_TRACE,
-                                                 MACHDBG_CODE(DBG_MACH_IPC,MACH_THREAD_SET_VOUCHER) | DBG_FUNC_NONE,
-                                                 (uintptr_t)thread_tid(thread),
-                                                 (uintptr_t)old_voucher_name,
-                                                 VM_KERNEL_ADDRPERM((uintptr_t)old_voucher),
-                                                 4, 0);
-
-               }
-       }
-
-       /* swap in new voucher, if old voucher matches the one supplied */
-       if (old_voucher == *in_out_old_voucher) {
-               ipc_voucher_reference(new_voucher);
-               thread->ith_voucher = new_voucher;
-               thread->ith_voucher_name = MACH_PORT_NULL;
-#ifdef CONFIG_BANK
-               bank_swap_thread_bank_ledger(thread, bankledger);
-#endif
-               thread_mtx_unlock(thread);
-
-               KERNEL_DEBUG_CONSTANT_IST(KDEBUG_TRACE,
-                                         MACHDBG_CODE(DBG_MACH_IPC,MACH_THREAD_SET_VOUCHER) | DBG_FUNC_NONE,
-                                         (uintptr_t)thread_tid(thread),
-                                         (uintptr_t)MACH_PORT_NULL,
-                                         VM_KERNEL_ADDRPERM((uintptr_t)new_voucher),
-                                         5, 0);
-
-               ipc_voucher_release(old_voucher);
-
-               *in_out_old_voucher = IPC_VOUCHER_NULL;
-               return KERN_SUCCESS;
-       }
-
-       /* Otherwise, just return old voucher reference */
-       ipc_voucher_reference(old_voucher);
-       thread_mtx_unlock(thread);
-       *in_out_old_voucher = old_voucher;
-       return KERN_SUCCESS;
+       /*
+        * Currently this function is only called from a MIG generated
+        * routine which doesn't release the reference on the voucher
+        * addressed by in_out_old_voucher. To avoid leaking this reference,
+        * a call to release it has been added here.
+        */
+       ipc_voucher_release(*in_out_old_voucher);
+       return KERN_NOT_SUPPORTED;

 }
 
 }
 

-/* 
+/*

  *  thread_get_current_voucher_origin_pid - get the pid of the originator of the current voucher.
  */
 kern_return_t
  *  thread_get_current_voucher_origin_pid - get the pid of the originator of the current voucher.
  */
 kern_return_t


@@ -2846,19 +3027,20 @@ thread_get_current_voucher_origin_pid(
        uint32_t buf_size;
        kern_return_t kr;
        thread_t thread = current_thread();
        uint32_t buf_size;
        kern_return_t kr;
        thread_t thread = current_thread();

-       
+

        buf_size = sizeof(*pid);
        kr = mach_voucher_attr_command(thread->ith_voucher,
        buf_size = sizeof(*pid);
        kr = mach_voucher_attr_command(thread->ith_voucher,

-               MACH_VOUCHER_ATTR_KEY_BANK,
-               BANK_ORIGINATOR_PID,
-               NULL,
-               0,
-               (mach_voucher_attr_content_t)pid,
-               &buf_size);
+           MACH_VOUCHER_ATTR_KEY_BANK,
+           BANK_ORIGINATOR_PID,
+           NULL,
+           0,
+           (mach_voucher_attr_content_t)pid,
+           &buf_size);

 
        return kr;
 }
 
 
        return kr;
 }
 

+

 boolean_t
 thread_has_thread_name(thread_t th)
 {
 boolean_t
 thread_has_thread_name(thread_t th)
 {


@@ -2881,35 +3063,109 @@ thread_set_thread_name(thread_t th, const char* name)
        }
 }
 
        }
 }
 

+void
+thread_set_honor_qlimit(thread_t thread)
+{
+       thread->options |= TH_OPT_HONOR_QLIMIT;
+}
+
+void
+thread_clear_honor_qlimit(thread_t thread)
+{
+       thread->options &= (~TH_OPT_HONOR_QLIMIT);
+}
+

 /*
  * thread_enable_send_importance - set/clear the SEND_IMPORTANCE thread option bit.
  */
 /*
  * thread_enable_send_importance - set/clear the SEND_IMPORTANCE thread option bit.
  */

-void thread_enable_send_importance(thread_t thread, boolean_t enable)
+void
+thread_enable_send_importance(thread_t thread, boolean_t enable)

 {
 {

-       if (enable == TRUE)
+       if (enable == TRUE) {

                thread->options |= TH_OPT_SEND_IMPORTANCE;
                thread->options |= TH_OPT_SEND_IMPORTANCE;

-       else
+       } else {

                thread->options &= ~TH_OPT_SEND_IMPORTANCE;
                thread->options &= ~TH_OPT_SEND_IMPORTANCE;

+       }
+}
+
+/*
+ * thread_set_allocation_name - .
+ */
+
+kern_allocation_name_t
+thread_set_allocation_name(kern_allocation_name_t new_name)
+{
+       kern_allocation_name_t ret;
+       thread_kernel_state_t kstate = thread_get_kernel_state(current_thread());
+       ret = kstate->allocation_name;
+       // fifo
+       if (!new_name || !kstate->allocation_name) {
+               kstate->allocation_name = new_name;
+       }
+       return ret;
+}
+
+void *
+thread_iokit_tls_get(uint32_t index)
+{
+       assert(index < THREAD_SAVE_IOKIT_TLS_COUNT);
+       return current_thread()->saved.iokit.tls[index];
+}
+
+void
+thread_iokit_tls_set(uint32_t index, void * data)
+{
+       assert(index < THREAD_SAVE_IOKIT_TLS_COUNT);
+       current_thread()->saved.iokit.tls[index] = data;
+}
+
+uint64_t
+thread_get_last_wait_duration(thread_t thread)
+{
+       return thread->last_made_runnable_time - thread->last_run_time;
+}
+
+integer_t
+thread_kern_get_pri(thread_t thr)
+{
+       return thr->base_pri;
+}
+
+void
+thread_kern_set_pri(thread_t thr, integer_t pri)
+{
+       sched_set_kernel_thread_priority(thr, pri);
+}
+
+integer_t
+thread_kern_get_kernel_maxpri(void)
+{
+       return MAXPRI_KERNEL;

 }
 
 #if CONFIG_DTRACE
 }
 
 #if CONFIG_DTRACE

-uint32_t dtrace_get_thread_predcache(thread_t thread)
+uint32_t
+dtrace_get_thread_predcache(thread_t thread)

 {
 {

-       if (thread != THREAD_NULL)
+       if (thread != THREAD_NULL) {

                return thread->t_dtrace_predcache;
                return thread->t_dtrace_predcache;

-       else
+       } else {

                return 0;
                return 0;

+       }

 }
 
 }
 

-int64_t dtrace_get_thread_vtime(thread_t thread)
+int64_t
+dtrace_get_thread_vtime(thread_t thread)

 {
 {

-       if (thread != THREAD_NULL)
+       if (thread != THREAD_NULL) {

                return thread->t_dtrace_vtime;
                return thread->t_dtrace_vtime;

-       else
+       } else {

                return 0;
                return 0;

+       }

 }
 
 }
 

-int dtrace_get_thread_last_cpu_id(thread_t thread)
+int
+dtrace_get_thread_last_cpu_id(thread_t thread)

 {
        if ((thread != THREAD_NULL) && (thread->last_processor != PROCESSOR_NULL)) {
                return thread->last_processor->cpu_id;
 {
        if ((thread != THREAD_NULL) && (thread->last_processor != PROCESSOR_NULL)) {
                return thread->last_processor->cpu_id;


@@ -2918,74 +3174,103 @@ int dtrace_get_thread_last_cpu_id(thread_t thread)
        }
 }
 
        }
 }
 

-int64_t dtrace_get_thread_tracing(thread_t thread)
+int64_t
+dtrace_get_thread_tracing(thread_t thread)

 {
 {

-       if (thread != THREAD_NULL)
+       if (thread != THREAD_NULL) {

                return thread->t_dtrace_tracing;
                return thread->t_dtrace_tracing;

-       else
+       } else {

                return 0;
                return 0;

+       }

 }
 
 }
 

-boolean_t dtrace_get_thread_reentering(thread_t thread)
+uint16_t
+dtrace_get_thread_inprobe(thread_t thread)

 {
 {

-       if (thread != THREAD_NULL)
-               return (thread->options & TH_OPT_DTRACE) ? TRUE : FALSE;
-       else
+       if (thread != THREAD_NULL) {
+               return thread->t_dtrace_inprobe;
+       } else {

                return 0;
                return 0;

+       }

 }
 
 }
 

-vm_offset_t dtrace_get_kernel_stack(thread_t thread)
+vm_offset_t
+dtrace_get_kernel_stack(thread_t thread)

 {
 {

-       if (thread != THREAD_NULL)
+       if (thread != THREAD_NULL) {

                return thread->kernel_stack;
                return thread->kernel_stack;

-       else
+       } else {

                return 0;
                return 0;

+       }
+}
+
+#if KASAN
+struct kasan_thread_data *
+kasan_get_thread_data(thread_t thread)
+{
+       return &thread->kasan_data;
+}
+#endif
+
+#if CONFIG_KSANCOV
+void **
+__sanitizer_get_thread_data(thread_t thread)
+{
+       return &thread->ksancov_data;

 }
 }

+#endif

 
 

-int64_t dtrace_calc_thread_recent_vtime(thread_t thread)
+int64_t
+dtrace_calc_thread_recent_vtime(thread_t thread)

 {
        if (thread != THREAD_NULL) {
                processor_t             processor = current_processor();
 {
        if (thread != THREAD_NULL) {
                processor_t             processor = current_processor();

-               uint64_t                                abstime = mach_absolute_time();
-               timer_t                                 timer;
+               uint64_t                                abstime = mach_absolute_time();
+               timer_t                                 timer;

 
                timer = PROCESSOR_DATA(processor, thread_timer);
 
                return timer_grab(&(thread->system_timer)) + timer_grab(&(thread->user_timer)) +
 
                timer = PROCESSOR_DATA(processor, thread_timer);
 
                return timer_grab(&(thread->system_timer)) + timer_grab(&(thread->user_timer)) +

-                               (abstime - timer->tstamp); /* XXX need interrupts off to prevent missed time? */
-       } else
+                      (abstime - timer->tstamp);          /* XXX need interrupts off to prevent missed time? */
+       } else {

                return 0;
                return 0;

+       }

 }
 
 }
 

-void dtrace_set_thread_predcache(thread_t thread, uint32_t predcache)
+void
+dtrace_set_thread_predcache(thread_t thread, uint32_t predcache)

 {
 {

-       if (thread != THREAD_NULL)
+       if (thread != THREAD_NULL) {

                thread->t_dtrace_predcache = predcache;
                thread->t_dtrace_predcache = predcache;

+       }

 }
 
 }
 

-void dtrace_set_thread_vtime(thread_t thread, int64_t vtime)
+void
+dtrace_set_thread_vtime(thread_t thread, int64_t vtime)

 {
 {

-       if (thread != THREAD_NULL)
+       if (thread != THREAD_NULL) {

                thread->t_dtrace_vtime = vtime;
                thread->t_dtrace_vtime = vtime;

+       }

 }
 
 }
 

-void dtrace_set_thread_tracing(thread_t thread, int64_t accum)
+void
+dtrace_set_thread_tracing(thread_t thread, int64_t accum)

 {
 {

-       if (thread != THREAD_NULL)
+       if (thread != THREAD_NULL) {

                thread->t_dtrace_tracing = accum;
                thread->t_dtrace_tracing = accum;

+       }

 }
 
 }
 

-void dtrace_set_thread_reentering(thread_t thread, boolean_t vbool)
+void
+dtrace_set_thread_inprobe(thread_t thread, uint16_t inprobe)

 {
        if (thread != THREAD_NULL) {
 {
        if (thread != THREAD_NULL) {

-               if (vbool)
-                       thread->options |= TH_OPT_DTRACE;
-               else
-                       thread->options &= (~TH_OPT_DTRACE);
+               thread->t_dtrace_inprobe = inprobe;

        }
 }
 
        }
 }
 

-vm_offset_t dtrace_set_thread_recover(thread_t thread, vm_offset_t recover)
+vm_offset_t
+dtrace_set_thread_recover(thread_t thread, vm_offset_t recover)

 {
        vm_offset_t prev = 0;
 
 {
        vm_offset_t prev = 0;
 


@@ -2996,7 +3281,21 @@ vm_offset_t dtrace_set_thread_recover(thread_t thread, vm_offset_t recover)
        return prev;
 }
 
        return prev;
 }
 

-void dtrace_thread_bootstrap(void)
+vm_offset_t
+dtrace_sign_and_set_thread_recover(thread_t thread, vm_offset_t recover)
+{
+#if defined(HAS_APPLE_PAC)
+       return dtrace_set_thread_recover(thread,
+                  (vm_address_t)ptrauth_sign_unauthenticated((void *)recover,
+                  ptrauth_key_function_pointer,
+                  ptrauth_blend_discriminator(&thread->recover, PAC_DISCRIMINATOR_RECOVER)));
+#else /* defined(HAS_APPLE_PAC) */
+       return dtrace_set_thread_recover(thread, recover);
+#endif /* defined(HAS_APPLE_PAC) */
+}
+
+void
+dtrace_thread_bootstrap(void)

 {
        task_t task = current_task();
 
 {
        task_t task = current_task();
 


@@ -3005,13 +3304,12 @@ void dtrace_thread_bootstrap(void)
                if (thread->t_dtrace_flags & TH_DTRACE_EXECSUCCESS) {
                        thread->t_dtrace_flags &= ~TH_DTRACE_EXECSUCCESS;
                        DTRACE_PROC(exec__success);
                if (thread->t_dtrace_flags & TH_DTRACE_EXECSUCCESS) {
                        thread->t_dtrace_flags &= ~TH_DTRACE_EXECSUCCESS;
                        DTRACE_PROC(exec__success);

-                       KDBG(BSDDBG_CODE(DBG_BSD_PROC,BSD_PROC_EXEC),
-                            task_pid(task));
+                       KDBG(BSDDBG_CODE(DBG_BSD_PROC, BSD_PROC_EXEC),
+                           task_pid(task));

                }
                DTRACE_PROC(start);
        }
        DTRACE_PROC(lwp__start);
                }
                DTRACE_PROC(start);
        }
        DTRACE_PROC(lwp__start);

-

 }
 
 void
 }
 
 void