X-Git-Url: https://git.saurik.com/apple/xnu.git/blobdiff_plain/6601e61aa18bf4f09af135ff61fc7f4771d23b06..d9a64523371fa019c4575bb400cbbc3a50ac9903:/osfmk/kern/sched_prim.c

diff --git a/osfmk/kern/sched_prim.c b/osfmk/kern/sched_prim.c
index aaece698a..751b57417 100644
--- a/osfmk/kern/sched_prim.c
+++ b/osfmk/kern/sched_prim.c
@@ -1,23 +1,29 @@
 /*
- * Copyright (c) 2000-2005 Apple Computer, Inc. All rights reserved.
+ * Copyright (c) 2000-2016 Apple Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*
  * @OSF_FREE_COPYRIGHT@
@@ -59,50 +65,87 @@
  */
 
 #include <debug.h>
-#include <mach_kdb.h>
-
-#include <ddb/db_output.h>
 
 #include <mach/mach_types.h>
 #include <mach/machine.h>
 #include <mach/policy.h>
 #include <mach/sync_policy.h>
+#include <mach/thread_act.h>
 
 #include <machine/machine_routines.h>
 #include <machine/sched_param.h>
+#include <machine/machine_cpu.h>
+#include <machine/machlimits.h>
+#include <machine/atomic.h>
+
+#include <machine/commpage.h>
 
 #include <kern/kern_types.h>
+#include <kern/backtrace.h>
 #include <kern/clock.h>
 #include <kern/counters.h>
 #include <kern/cpu_number.h>
 #include <kern/cpu_data.h>
+#include <kern/smp.h>
 #include <kern/debug.h>
-#include <kern/lock.h>
 #include <kern/macro_help.h>
 #include <kern/machine.h>
 #include <kern/misc_protos.h>
+#if MONOTONIC
+#include <kern/monotonic.h>
+#endif /* MONOTONIC */
 #include <kern/processor.h>
 #include <kern/queue.h>
 #include <kern/sched.h>
 #include <kern/sched_prim.h>
+#include <kern/sfi.h>
 #include <kern/syscall_subr.h>
 #include <kern/task.h>
 #include <kern/thread.h>
-#include <kern/wait_queue.h>
+#include <kern/ledger.h>
+#include <kern/timer_queue.h>
+#include <kern/waitq.h>
+#include <kern/policy_internal.h>
+#include <kern/cpu_quiesce.h>
 
 #include <vm/pmap.h>
 #include <vm/vm_kern.h>
 #include <vm/vm_map.h>
+#include <vm/vm_pageout.h>
+
+#include <mach/sdt.h>
+#include <mach/mach_host.h>
+#include <mach/host_info.h>
 
 #include <sys/kdebug.h>
+#include <kperf/kperf.h>
+#include <kern/kpc.h>
+#include <san/kasan.h>
+#include <kern/pms.h>
+#include <kern/host.h>
+#include <stdatomic.h>
+
+int rt_runq_count(processor_set_t pset)
+{
+    return atomic_load_explicit(&SCHED(rt_runq)(pset)->count, memory_order_relaxed);
+}
 
-#ifdef __ppc__
-#include <ppc/pms.h>
-#endif
+void rt_runq_count_incr(processor_set_t pset)
+{
+    atomic_fetch_add_explicit(&SCHED(rt_runq)(pset)->count, 1, memory_order_relaxed);
+}
+
+void rt_runq_count_decr(processor_set_t pset)
+{
+    atomic_fetch_sub_explicit(&SCHED(rt_runq)(pset)->count, 1, memory_order_relaxed);
+}
 
 #define		DEFAULT_PREEMPTION_RATE		100		/* (1/s) */
 int			default_preemption_rate = DEFAULT_PREEMPTION_RATE;
 
+#define		DEFAULT_BG_PREEMPTION_RATE	400		/* (1/s) */
+int			default_bg_preemption_rate = DEFAULT_BG_PREEMPTION_RATE;
+
 #define		MAX_UNSAFE_QUANTA			800
 int			max_unsafe_quanta = MAX_UNSAFE_QUANTA;
 
@@ -112,112 +155,260 @@ int			max_poll_quanta = MAX_POLL_QUANTA;
 #define		SCHED_POLL_YIELD_SHIFT		4		/* 1/16 */
 int			sched_poll_yield_shift = SCHED_POLL_YIELD_SHIFT;
 
-uint64_t	max_unsafe_computation;
-uint32_t	sched_safe_duration;
 uint64_t	max_poll_computation;
 
+uint64_t	max_unsafe_computation;
+uint64_t	sched_safe_duration;
+
+#if defined(CONFIG_SCHED_TIMESHARE_CORE)
+
 uint32_t	std_quantum;
 uint32_t	min_std_quantum;
+uint32_t	bg_quantum;
 
 uint32_t	std_quantum_us;
+uint32_t	bg_quantum_us;
+
+#endif /* CONFIG_SCHED_TIMESHARE_CORE */
+
+uint32_t	thread_depress_time;
+uint32_t	default_timeshare_computation;
+uint32_t	default_timeshare_constraint;
 
 uint32_t	max_rt_quantum;
 uint32_t	min_rt_quantum;
 
-uint32_t	sched_cswtime;
+#if defined(CONFIG_SCHED_TIMESHARE_CORE)
+
+unsigned		sched_tick;
+uint32_t		sched_tick_interval;
+
+/* Timeshare load calculation interval (15ms) */
+uint32_t		sched_load_compute_interval_us = 15000; 
+uint64_t		sched_load_compute_interval_abs;
+static _Atomic uint64_t	sched_load_compute_deadline;
+
+uint32_t	sched_pri_shifts[TH_BUCKET_MAX];
+uint32_t	sched_fixed_shift;
+
+uint32_t	sched_decay_usage_age_factor = 1; /* accelerate 5/8^n usage aging */
+
+/* Allow foreground to decay past default to resolve inversions */
+#define DEFAULT_DECAY_BAND_LIMIT ((BASEPRI_FOREGROUND - BASEPRI_DEFAULT) + 2)
+int 		sched_pri_decay_band_limit = DEFAULT_DECAY_BAND_LIMIT;
 
-static uint32_t		delay_idle_limit, delay_idle_spin;
-static processor_t	delay_idle(
-						processor_t		processor,
-						thread_t		self);
+/* Defaults for timer deadline profiling */
+#define TIMER_DEADLINE_TRACKING_BIN_1_DEFAULT 2000000 /* Timers with deadlines <=
+							* 2ms */
+#define TIMER_DEADLINE_TRACKING_BIN_2_DEFAULT 5000000 /* Timers with deadlines
+							  <= 5ms */
 
-unsigned	sched_tick;
-uint32_t	sched_tick_interval;
+uint64_t timer_deadline_tracking_bin_1;
+uint64_t timer_deadline_tracking_bin_2;
 
-uint32_t	sched_pri_shift;
+#endif /* CONFIG_SCHED_TIMESHARE_CORE */
+
+thread_t sched_maintenance_thread;
+
+#if __arm__ || __arm64__
+/* interrupts disabled lock to guard recommended cores state */
+decl_simple_lock_data(static,sched_recommended_cores_lock);
+static void sched_recommended_cores_maintenance(void);
+static void sched_update_recommended_cores(uint32_t recommended_cores);
+
+uint64_t    perfcontrol_failsafe_starvation_threshold;
+extern char *proc_name_address(struct proc *p);
+
+#endif /* __arm__ || __arm64__ */
+
+uint64_t	sched_one_second_interval;
 
 /* Forwards */
-void		wait_queues_init(void);
 
-static void		load_shift_init(void);
+#if defined(CONFIG_SCHED_TIMESHARE_CORE)
+
+static void load_shift_init(void);
+static void preempt_pri_init(void);
+
+#endif /* CONFIG_SCHED_TIMESHARE_CORE */
+
+#if CONFIG_SCHED_IDLE_IN_PLACE
+static thread_t	thread_select_idle(
+					thread_t			thread,
+					processor_t			processor);
+#endif
 
-static thread_t	choose_thread(
-					processor_set_t		pset,
+thread_t	processor_idle(
+					thread_t			thread,
 					processor_t			processor);
 
-static void		thread_update_scan(void);
+ast_t
+csw_check_locked(	processor_t		processor,
+					processor_set_t	pset,
+					ast_t			check_reason);
+
+static void processor_setrun(
+				 processor_t			processor,
+				 thread_t			thread,
+				 integer_t			options);
+
+static void
+sched_realtime_timebase_init(void);
+
+static void
+sched_timer_deadline_tracking_init(void);
 
 #if	DEBUG
-static
-boolean_t	thread_runnable(
-				thread_t		thread);
+extern int debug_task;
+#define TLOG(a, fmt, args...) if(debug_task & a) kprintf(fmt, ## args)
+#else
+#define TLOG(a, fmt, args...) do {} while (0)
+#endif
 
-#endif	/*DEBUG*/
+static processor_t
+thread_bind_internal(
+	thread_t		thread,
+	processor_t		processor);
 
+static void
+sched_vm_group_maintenance(void);
 
-/*
- *	State machine
- *
- * states are combinations of:
- *  R	running
- *  W	waiting (or on wait queue)
- *  N	non-interruptible
- *  O	swapped out
- *  I	being swapped in
- *
- * init	action 
- *	assert_wait thread_block    clear_wait 		swapout	swapin
- *
- * R	RW, RWN	    R;   setrun	    -	       		-
- * RN	RWN	    RN;  setrun	    -	       		-
- *
- * RW		    W		    R	       		-
- * RWN		    WN		    RN	       		-
- *
- * W				    R;   setrun		WO
- * WN				    RN;  setrun		-
- *
- * RO				    -			-	R
- *
- */
+#if defined(CONFIG_SCHED_TIMESHARE_CORE)
+int8_t		sched_load_shifts[NRQS];
+bitmap_t	sched_preempt_pri[BITMAP_LEN(NRQS)];
+#endif /* CONFIG_SCHED_TIMESHARE_CORE */
+
+const struct sched_dispatch_table *sched_current_dispatch = NULL;
 
 /*
- *	Waiting protocols and implementation:
- *
- *	Each thread may be waiting for exactly one event; this event
- *	is set using assert_wait().  That thread may be awakened either
- *	by performing a thread_wakeup_prim() on its event,
- *	or by directly waking that thread up with clear_wait().
+ * Statically allocate a buffer to hold the longest possible
+ * scheduler description string, as currently implemented.
+ * bsd/kern/kern_sysctl.c has a corresponding definition in bsd/
+ * to export to userspace via sysctl(3). If either version
+ * changes, update the other.
  *
- *	The implementation of wait events uses a hash table.  Each
- *	bucket is queue of threads having the same hash function
- *	value; the chain for the queue (linked list) is the run queue
- *	field.  [It is not possible to be waiting and runnable at the
- *	same time.]
- *
- *	Locks on both the thread and on the hash buckets govern the
- *	wait event field and the queue chain field.  Because wakeup
- *	operations only have the event as an argument, the event hash
- *	bucket must be locked before any thread.
- *
- *	Scheduling operations may also occur at interrupt level; therefore,
- *	interrupts below splsched() must be prevented when holding
- *	thread or hash bucket locks.
- *
- *	The wait event hash table declarations are as follows:
+ * Note that in addition to being an upper bound on the strings
+ * in the kernel, it's also an exact parameter to PE_get_default(),
+ * which interrogates the device tree on some platforms. That
+ * API requires the caller know the exact size of the device tree
+ * property, so we need both a legacy size (32) and the current size
+ * (48) to deal with old and new device trees. The device tree property
+ * is similarly padded to a fixed size so that the same kernel image
+ * can run on multiple devices with different schedulers configured
+ * in the device tree.
  */
+char sched_string[SCHED_STRING_MAX_LENGTH];
 
-#define NUMQUEUES	59
+uint32_t sched_debug_flags = SCHED_DEBUG_FLAG_CHOOSE_PROCESSOR_TRACEPOINTS;
 
-struct wait_queue wait_queues[NUMQUEUES];
+/* Global flag which indicates whether Background Stepper Context is enabled */
+static int cpu_throttle_enabled = 1;
 
-#define wait_hash(event) \
-	((((int)(event) < 0)? ~(int)(event): (int)(event)) % NUMQUEUES)
+#if DEBUG
 
-int8_t		sched_load_shifts[NRQS];
+/* Since using the indirect function dispatch table has a negative impact on 
+ * context switch performance, only allow DEBUG kernels to use that mechanism.
+ */
+static void
+sched_init_override(void)
+{
+	char sched_arg[SCHED_STRING_MAX_LENGTH] = { '\0' };
+
+	/* Check for runtime selection of the scheduler algorithm */
+	if (!PE_parse_boot_argn("sched", sched_arg, sizeof (sched_arg))) {
+		sched_arg[0] = '\0';
+	}
+	if (strlen(sched_arg) > 0) {
+		if (0) {
+			/* Allow pattern below */
+#if defined(CONFIG_SCHED_TRADITIONAL)
+		} else if (0 == strcmp(sched_arg, sched_traditional_dispatch.sched_name)) {
+			sched_current_dispatch = &sched_traditional_dispatch;
+		} else if (0 == strcmp(sched_arg, sched_traditional_with_pset_runqueue_dispatch.sched_name)) {
+			sched_current_dispatch = &sched_traditional_with_pset_runqueue_dispatch;
+#endif
+#if defined(CONFIG_SCHED_MULTIQ)
+		} else if (0 == strcmp(sched_arg, sched_multiq_dispatch.sched_name)) {
+			sched_current_dispatch = &sched_multiq_dispatch;
+		} else if (0 == strcmp(sched_arg, sched_dualq_dispatch.sched_name)) {
+			sched_current_dispatch = &sched_dualq_dispatch;
+#endif
+		} else {
+#if defined(CONFIG_SCHED_TRADITIONAL)
+			printf("Unrecognized scheduler algorithm: %s\n", sched_arg);
+			printf("Scheduler: Using instead: %s\n", sched_traditional_with_pset_runqueue_dispatch.sched_name);
+			sched_current_dispatch = &sched_traditional_with_pset_runqueue_dispatch;
+#else
+			panic("Unrecognized scheduler algorithm: %s", sched_arg);
+#endif
+		}
+		kprintf("Scheduler: Runtime selection of %s\n", SCHED(sched_name));
+	} else {
+#if   defined(CONFIG_SCHED_MULTIQ)
+		sched_current_dispatch = &sched_dualq_dispatch;
+#elif defined(CONFIG_SCHED_TRADITIONAL)
+		sched_current_dispatch = &sched_traditional_with_pset_runqueue_dispatch;
+#else
+#error No default scheduler implementation
+#endif
+		kprintf("Scheduler: Default of %s\n", SCHED(sched_name));
+	}
+}
+
+#endif /* DEBUG */
 
 void
 sched_init(void)
+{
+#if DEBUG
+	sched_init_override();
+#else /* DEBUG */
+	kprintf("Scheduler: Default of %s\n", SCHED(sched_name));
+#endif /* DEBUG */
+
+	if (!PE_parse_boot_argn("sched_pri_decay_limit", &sched_pri_decay_band_limit, sizeof(sched_pri_decay_band_limit))) {
+		/* No boot-args, check in device tree */
+		if (!PE_get_default("kern.sched_pri_decay_limit",
+							&sched_pri_decay_band_limit,
+							sizeof(sched_pri_decay_band_limit))) {
+			/* Allow decay all the way to normal limits */
+			sched_pri_decay_band_limit = DEFAULT_DECAY_BAND_LIMIT;
+		}
+	}
+
+	kprintf("Setting scheduler priority decay band limit %d\n", sched_pri_decay_band_limit);
+	
+	if (PE_parse_boot_argn("sched_debug", &sched_debug_flags, sizeof(sched_debug_flags))) {
+		kprintf("Scheduler: Debug flags 0x%08x\n", sched_debug_flags);
+	}
+	strlcpy(sched_string, SCHED(sched_name), sizeof(sched_string));
+
+	cpu_quiescent_counter_init();
+
+	SCHED(init)();
+	SCHED(rt_init)(&pset0);
+	sched_timer_deadline_tracking_init();
+
+	SCHED(pset_init)(&pset0);
+	SCHED(processor_init)(master_processor);
+}
+
+void
+sched_timebase_init(void)
+{
+	uint64_t	abstime;
+	
+	clock_interval_to_absolutetime_interval(1, NSEC_PER_SEC, &abstime);
+	sched_one_second_interval = abstime;
+	
+	SCHED(timebase_init)();
+	sched_realtime_timebase_init();
+}
+
+#if defined(CONFIG_SCHED_TIMESHARE_CORE)
+
+void
+sched_timeshare_init(void)
 {
 	/*
 	 * Calculate the timeslicing quantum
@@ -229,18 +420,19 @@ sched_init(void)
 
 	printf("standard timeslicing quantum is %d us\n", std_quantum_us);
 
-	sched_safe_duration = (2 * max_unsafe_quanta / default_preemption_rate) *
-											(1 << SCHED_TICK_SHIFT);
+	if (default_bg_preemption_rate < 1)
+		default_bg_preemption_rate = DEFAULT_BG_PREEMPTION_RATE;
+	bg_quantum_us = (1000 * 1000) / default_bg_preemption_rate;
+
+	printf("standard background quantum is %d us\n", bg_quantum_us);
 
-	wait_queues_init();
 	load_shift_init();
-	pset_init(&default_pset);
+	preempt_pri_init();
 	sched_tick = 0;
-	ast_init();
 }
 
 void
-sched_timebase_init(void)
+sched_timeshare_timebase_init(void)
 {
 	uint64_t	abstime;
 	uint32_t	shift;
@@ -249,30 +441,29 @@ sched_timebase_init(void)
 	clock_interval_to_absolutetime_interval(
 							std_quantum_us, NSEC_PER_USEC, &abstime);
 	assert((abstime >> 32) == 0 && (uint32_t)abstime != 0);
-	std_quantum = abstime;
+	std_quantum = (uint32_t)abstime;
 
 	/* smallest remaining quantum (250 us) */
 	clock_interval_to_absolutetime_interval(250, NSEC_PER_USEC, &abstime);
 	assert((abstime >> 32) == 0 && (uint32_t)abstime != 0);
-	min_std_quantum = abstime;
+	min_std_quantum = (uint32_t)abstime;
 
-	/* smallest rt computaton (50 us) */
-	clock_interval_to_absolutetime_interval(50, NSEC_PER_USEC, &abstime);
-	assert((abstime >> 32) == 0 && (uint32_t)abstime != 0);
-	min_rt_quantum = abstime;
-
-	/* maximum rt computation (50 ms) */
+	/* quantum for background tasks */
 	clock_interval_to_absolutetime_interval(
-							50, 1000*NSEC_PER_USEC, &abstime);
+							bg_quantum_us, NSEC_PER_USEC, &abstime);
 	assert((abstime >> 32) == 0 && (uint32_t)abstime != 0);
-	max_rt_quantum = abstime;
+	bg_quantum = (uint32_t)abstime;
 
 	/* scheduler tick interval */
 	clock_interval_to_absolutetime_interval(USEC_PER_SEC >> SCHED_TICK_SHIFT,
 													NSEC_PER_USEC, &abstime);
 	assert((abstime >> 32) == 0 && (uint32_t)abstime != 0);
-	sched_tick_interval = abstime;
+	sched_tick_interval = (uint32_t)abstime;
 
+	/* timeshare load calculation interval & deadline initialization */
+	clock_interval_to_absolutetime_interval(sched_load_compute_interval_us, NSEC_PER_USEC, &sched_load_compute_interval_abs);
+	sched_load_compute_deadline = sched_load_compute_interval_abs;	
+	
 	/*
 	 * Compute conversion factor from usage to
 	 * timesharing priorities with 5/8 ** n aging.
@@ -280,31 +471,100 @@ sched_timebase_init(void)
 	abstime = (abstime * 5) / 3;
 	for (shift = 0; abstime > BASEPRI_DEFAULT; ++shift)
 		abstime >>= 1;
-	sched_pri_shift = shift;
+	sched_fixed_shift = shift;
+
+	for (uint32_t i = 0 ; i < TH_BUCKET_MAX ; i++)
+		sched_pri_shifts[i] = INT8_MAX;
+
+	max_unsafe_computation = ((uint64_t)max_unsafe_quanta) * std_quantum;
+	sched_safe_duration = 2 * ((uint64_t)max_unsafe_quanta) * std_quantum;
+
+	max_poll_computation = ((uint64_t)max_poll_quanta) * std_quantum;
+	thread_depress_time = 1 * std_quantum;
+	default_timeshare_computation = std_quantum / 2;
+	default_timeshare_constraint = std_quantum;
+
+#if __arm__ || __arm64__
+	       perfcontrol_failsafe_starvation_threshold = (2 * sched_tick_interval);
+#endif /* __arm__ || __arm64__ */
+}
+
+#endif /* CONFIG_SCHED_TIMESHARE_CORE */
+
+void
+pset_rt_init(processor_set_t pset)
+{
+	rt_lock_init(pset);
+
+	pset->rt_runq.count = 0;
+	queue_init(&pset->rt_runq.queue);
+	memset(&pset->rt_runq.runq_stats, 0, sizeof pset->rt_runq.runq_stats);
+}
+
+rt_queue_t
+sched_rtglobal_runq(processor_set_t pset)
+{
+	(void)pset;
+
+	return &pset0.rt_runq;
+}
+
+void
+sched_rtglobal_init(processor_set_t pset)
+{
+	if (pset == &pset0) {
+		return pset_rt_init(pset);
+	}
+
+	/* Only pset0 rt_runq is used, so make it easy to detect
+	 * buggy accesses to others.
+	 */
+	memset(&pset->rt_runq, 0xfd, sizeof pset->rt_runq);
+}
+
+void
+sched_rtglobal_queue_shutdown(processor_t processor)
+{
+	(void)processor;
+}
 
-	max_unsafe_computation = max_unsafe_quanta * std_quantum;
-	max_poll_computation = max_poll_quanta * std_quantum;
+static void
+sched_realtime_timebase_init(void)
+{
+	uint64_t abstime;
 
-	/* delay idle constant(s) (60, 1 us) */
-	clock_interval_to_absolutetime_interval(60, NSEC_PER_USEC, &abstime);
+	/* smallest rt computaton (50 us) */
+	clock_interval_to_absolutetime_interval(50, NSEC_PER_USEC, &abstime);
 	assert((abstime >> 32) == 0 && (uint32_t)abstime != 0);
-	delay_idle_limit = abstime;
+	min_rt_quantum = (uint32_t)abstime;
 
-	clock_interval_to_absolutetime_interval(1, NSEC_PER_USEC, &abstime);
+	/* maximum rt computation (50 ms) */
+	clock_interval_to_absolutetime_interval(
+		50, 1000*NSEC_PER_USEC, &abstime);
 	assert((abstime >> 32) == 0 && (uint32_t)abstime != 0);
-	delay_idle_spin = abstime;
+	max_rt_quantum = (uint32_t)abstime;
+
 }
 
 void
-wait_queues_init(void)
+sched_check_spill(processor_set_t pset, thread_t thread)
 {
-	register int	i;
+	(void)pset;
+	(void)thread;
 
-	for (i = 0; i < NUMQUEUES; i++) {
-		wait_queue_init(&wait_queues[i], SYNC_POLICY_FIFO);
-	}
+	return;
 }
 
+bool
+sched_thread_should_yield(processor_t processor, thread_t thread)
+{
+	(void)thread;
+
+	return (!SCHED(processor_queue_empty)(processor) || rt_runq_count(processor->processor_set) > 0);
+}
+
+#if defined(CONFIG_SCHED_TIMESHARE_CORE)
+
 /*
  * Set up values for timeshare
  * loading factors.
@@ -315,14 +575,59 @@ load_shift_init(void)
 	int8_t		k, *p = sched_load_shifts;
 	uint32_t	i, j;
 
+	uint32_t	sched_decay_penalty = 1;
+
+	if (PE_parse_boot_argn("sched_decay_penalty", &sched_decay_penalty, sizeof (sched_decay_penalty))) {
+		kprintf("Overriding scheduler decay penalty %u\n", sched_decay_penalty);
+	}
+
+	if (PE_parse_boot_argn("sched_decay_usage_age_factor", &sched_decay_usage_age_factor, sizeof (sched_decay_usage_age_factor))) {
+		kprintf("Overriding scheduler decay usage age factor %u\n", sched_decay_usage_age_factor);
+	}
+
+	if (sched_decay_penalty == 0) {
+		/*
+		 * There is no penalty for timeshare threads for using too much
+		 * CPU, so set all load shifts to INT8_MIN. Even under high load,
+		 * sched_pri_shift will be >INT8_MAX, and there will be no
+		 * penalty applied to threads (nor will sched_usage be updated per
+		 * thread).
+		 */
+		for (i = 0; i < NRQS; i++) {
+			sched_load_shifts[i] = INT8_MIN;
+		}
+
+		return;
+	}
+
 	*p++ = INT8_MIN; *p++ = 0;
 
-	for (i = j = 2, k = 1; i < NRQS; ++k) {
-		for (j <<= 1; i < j; ++i)
+	/*
+	 * For a given system load "i", the per-thread priority
+	 * penalty per quantum of CPU usage is ~2^k priority
+	 * levels. "sched_decay_penalty" can cause more
+	 * array entries to be filled with smaller "k" values
+	 */
+	for (i = 2, j = 1 << sched_decay_penalty, k = 1; i < NRQS; ++k) {
+		for (j <<= 1; (i < j) && (i < NRQS); ++i)
 			*p++ = k;
 	}
 }
 
+static void
+preempt_pri_init(void)
+{
+	bitmap_t *p = sched_preempt_pri;
+
+	for (int i = BASEPRI_FOREGROUND; i < MINPRI_KERNEL; ++i)
+		bitmap_set(p, i);
+
+	for (int i = BASEPRI_PREEMPT; i <= MAXPRI; ++i)
+		bitmap_set(p, i);
+}
+
+#endif /* CONFIG_SCHED_TIMESHARE_CORE */
+
 /*
  *	Thread wait timer expiration.
  */
@@ -334,6 +639,8 @@ thread_timer_expire(
 	thread_t		thread = p0;
 	spl_t			s;
 
+	assert_thread_magic(thread);
+
 	s = splsched();
 	thread_lock(thread);
 	if (--thread->wait_timer_active == 0) {
@@ -346,92 +653,34 @@ thread_timer_expire(
 	splx(s);
 }
 
-/*
- *	thread_set_timer:
- *
- *	Set a timer for the current thread, if the thread
- *	is ready to wait.  Must be called between assert_wait()
- *	and thread_block().
- */
-void
-thread_set_timer(
-	uint32_t		interval,
-	uint32_t		scale_factor)
-{
-	thread_t		thread = current_thread();
-	uint64_t		deadline;
-	spl_t			s;
-
-	s = splsched();
-	thread_lock(thread);
-	if ((thread->state & TH_WAIT) != 0) {
-		clock_interval_to_deadline(interval, scale_factor, &deadline);
-		if (!timer_call_enter(&thread->wait_timer, deadline))
-			thread->wait_timer_active++;
-		thread->wait_timer_is_set = TRUE;
-	}
-	thread_unlock(thread);
-	splx(s);
-}
-
-void
-thread_set_timer_deadline(
-	uint64_t		deadline)
-{
-	thread_t		thread = current_thread();
-	spl_t			s;
-
-	s = splsched();
-	thread_lock(thread);
-	if ((thread->state & TH_WAIT) != 0) {
-		if (!timer_call_enter(&thread->wait_timer, deadline))
-			thread->wait_timer_active++;
-		thread->wait_timer_is_set = TRUE;
-	}
-	thread_unlock(thread);
-	splx(s);
-}
-
-void
-thread_cancel_timer(void)
-{
-	thread_t		thread = current_thread();
-	spl_t			s;
-
-	s = splsched();
-	thread_lock(thread);
-	if (thread->wait_timer_is_set) {
-		if (timer_call_cancel(&thread->wait_timer))
-			thread->wait_timer_active--;
-		thread->wait_timer_is_set = FALSE;
-	}
-	thread_unlock(thread);
-	splx(s);
-}
-
 /*
  *	thread_unblock:
  *
  *	Unblock thread on wake up.
  *
- *	Returns TRUE if the thread is still running.
+ *	Returns TRUE if the thread should now be placed on the runqueue.
  *
  *	Thread must be locked.
+ *
+ *	Called at splsched().
  */
 boolean_t
 thread_unblock(
 	thread_t		thread,
 	wait_result_t	wresult)
 {
-	boolean_t		result = FALSE;
+	boolean_t		ready_for_runq = FALSE;
+	thread_t		cthread = current_thread();
+	uint32_t		new_run_count;
+	int				old_thread_state;
 
 	/*
-	 * Set wait_result.
+	 *	Set wait_result.
 	 */
 	thread->wait_result = wresult;
 
 	/*
-	 * Cancel pending wait timer.
+	 *	Cancel pending wait timer.
 	 */
 	if (thread->wait_timer_is_set) {
 		if (timer_call_cancel(&thread->wait_timer))
@@ -440,49 +689,133 @@ thread_unblock(
 	}
 
 	/*
-	 * Update scheduling state.
+	 *	Update scheduling state: not waiting,
+	 *	set running.
 	 */
-	thread->state &= ~(TH_WAIT|TH_UNINT);
+	old_thread_state = thread->state;
+	thread->state = (old_thread_state | TH_RUN) &
+			~(TH_WAIT|TH_UNINT|TH_WAIT_REPORT);
+
+	if ((old_thread_state & TH_RUN) == 0) {
+		uint64_t ctime = mach_approximate_time();
+		thread->last_made_runnable_time = thread->last_basepri_change_time = ctime;
+		timer_start(&thread->runnable_timer, ctime);
 
-	if (!(thread->state & TH_RUN)) {
-		thread->state |= TH_RUN;
+		ready_for_runq = TRUE;
 
+		if (old_thread_state & TH_WAIT_REPORT) {
+			(*thread->sched_call)(SCHED_CALL_UNBLOCK, thread);
+		}
+
+		/* Update the runnable thread count */
+		new_run_count = sched_run_incr(thread);
+	} else {
 		/*
-		 * Mark unblocked if call out.
+		 * Either the thread is idling in place on another processor,
+		 * or it hasn't finished context switching yet.
 		 */
-		if (thread->options & TH_OPT_CALLOUT)
-			call_thread_unblock();
+#if CONFIG_SCHED_IDLE_IN_PLACE
+		if (thread->state & TH_IDLE) {
+			processor_t		processor = thread->last_processor;
 
+			if (processor != current_processor())
+				machine_signal_idle(processor);
+		}
+#else
+		assert((thread->state & TH_IDLE) == 0);
+#endif
 		/*
-		 * Update pset run counts.
+		 * The run count is only dropped after the context switch completes
+		 * and the thread is still waiting, so we should not run_incr here
 		 */
-		pset_run_incr(thread->processor_set);
-		if (thread->sched_mode & TH_MODE_TIMESHARE)
-			pset_share_incr(thread->processor_set);
+		new_run_count = sched_run_buckets[TH_BUCKET_RUN];
 	}
-	else
-		result = TRUE;
+
 
 	/*
 	 * Calculate deadline for real-time threads.
 	 */
-	if (thread->sched_mode & TH_MODE_REALTIME) {
-		thread->realtime.deadline = mach_absolute_time();
-		thread->realtime.deadline += thread->realtime.constraint;
+	if (thread->sched_mode == TH_MODE_REALTIME) {
+		uint64_t ctime;
+
+		ctime = mach_absolute_time();
+		thread->realtime.deadline = thread->realtime.constraint + ctime;
 	}
 
 	/*
 	 * Clear old quantum, fail-safe computation, etc.
 	 */
-	thread->current_quantum = 0;
+	thread->quantum_remaining = 0;
 	thread->computation_metered = 0;
 	thread->reason = AST_NONE;
+	thread->block_hint = kThreadWaitNone;
+
+	/* Obtain power-relevant interrupt and "platform-idle exit" statistics.
+	 * We also account for "double hop" thread signaling via
+	 * the thread callout infrastructure.
+	 * DRK: consider removing the callout wakeup counters in the future
+	 * they're present for verification at the moment.
+	 */
+	boolean_t aticontext, pidle;
+	ml_get_power_state(&aticontext, &pidle);
+
+	if (__improbable(aticontext && !(thread_get_tag_internal(thread) & THREAD_TAG_CALLOUT))) {
+		DTRACE_SCHED2(iwakeup, struct thread *, thread, struct proc *, thread->task->bsd_info);
+
+		uint64_t ttd = PROCESSOR_DATA(current_processor(), timer_call_ttd);
+
+		if (ttd) {
+			if (ttd <= timer_deadline_tracking_bin_1)
+				thread->thread_timer_wakeups_bin_1++;
+			else
+				if (ttd <= timer_deadline_tracking_bin_2)
+					thread->thread_timer_wakeups_bin_2++;
+		}
+
+		ledger_credit_thread(thread, thread->t_ledger,
+		                     task_ledgers.interrupt_wakeups, 1);
+		if (pidle) {
+			ledger_credit_thread(thread, thread->t_ledger,
+			                     task_ledgers.platform_idle_wakeups, 1);
+		}
+
+	} else if (thread_get_tag_internal(cthread) & THREAD_TAG_CALLOUT) {
+		/* TODO: what about an interrupt that does a wake taken on a callout thread? */
+		if (cthread->callout_woken_from_icontext) {
+			ledger_credit_thread(thread, thread->t_ledger,
+			                     task_ledgers.interrupt_wakeups, 1);
+			thread->thread_callout_interrupt_wakeups++;
+
+			if (cthread->callout_woken_from_platform_idle) {
+				ledger_credit_thread(thread, thread->t_ledger,
+				                     task_ledgers.platform_idle_wakeups, 1);
+				thread->thread_callout_platform_idle_wakeups++;
+			}
+
+			cthread->callout_woke_thread = TRUE;
+		}
+	}
+
+	if (thread_get_tag_internal(thread) & THREAD_TAG_CALLOUT) {
+		thread->callout_woken_from_icontext = aticontext;
+		thread->callout_woken_from_platform_idle = pidle;
+		thread->callout_woke_thread = FALSE;
+	}
 
-	KERNEL_DEBUG_CONSTANT(
+#if KPERF
+	if (ready_for_runq) {
+		kperf_make_runnable(thread, aticontext);
+	}
+#endif /* KPERF */
+
+	KERNEL_DEBUG_CONSTANT_IST(KDEBUG_TRACE,
 		MACHDBG_CODE(DBG_MACH_SCHED,MACH_MAKE_RUNNABLE) | DBG_FUNC_NONE,
-					(int)thread, (int)thread->sched_pri, 0, 0, 0);
+		(uintptr_t)thread_tid(thread), thread->sched_pri, thread->wait_result,
+		sched_run_buckets[TH_BUCKET_RUN], 0);
 
-	return (result);
+	DTRACE_SCHED2(wakeup, struct thread *, thread, struct proc *, thread->task->bsd_info);
+
+	return (ready_for_runq);
 }
 
 /*
@@ -492,27 +825,36 @@ thread_unblock(
  *	Conditions:
  *		thread lock held, IPC locks may be held.
  *		thread must have been pulled from wait queue under same lock hold.
- *  Returns:
+ *		thread must have been waiting
+ *	Returns:
  *		KERN_SUCCESS - Thread was set running
- *		KERN_NOT_WAITING - Thread was not waiting
+ *
+ * TODO: This should return void
  */
 kern_return_t
 thread_go(
-	thread_t		thread,
-	wait_result_t	wresult)
+          thread_t        thread,
+          wait_result_t   wresult)
 {
+	assert_thread_magic(thread);
+
 	assert(thread->at_safe_point == FALSE);
 	assert(thread->wait_event == NO_EVENT64);
-	assert(thread->wait_queue == WAIT_QUEUE_NULL);
+	assert(thread->waitq == NULL);
+
+	assert(!(thread->state & (TH_TERMINATE|TH_TERMINATE2)));
+	assert(thread->state & TH_WAIT);
 
-	if ((thread->state & (TH_WAIT|TH_TERMINATE)) == TH_WAIT) {
-		if (!thread_unblock(thread, wresult))
-			thread_setrun(thread, SCHED_PREEMPT | SCHED_TAILQ);
 
-		return (KERN_SUCCESS);
+	if (thread_unblock(thread, wresult)) {
+#if	SCHED_TRACE_THREAD_WAKEUPS
+		backtrace(&thread->thread_wakeup_bt[0],
+		    (sizeof(thread->thread_wakeup_bt)/sizeof(uintptr_t)));
+#endif
+		thread_setrun(thread, SCHED_PREEMPT | SCHED_TAILQ);
 	}
 
-	return (KERN_NOT_WAITING);
+	return (KERN_SUCCESS);
 }
 
 /*
@@ -528,9 +870,12 @@ __private_extern__
 wait_result_t
 thread_mark_wait_locked(
 	thread_t			thread,
-	wait_interrupt_t 	interruptible)
+	wait_interrupt_t	interruptible_orig)
 {
-	boolean_t		at_safe_point;
+	boolean_t			at_safe_point;
+	wait_interrupt_t	interruptible = interruptible_orig;
+
+	assert(!(thread->state & (TH_WAIT|TH_IDLE|TH_UNINT|TH_TERMINATE2|TH_WAIT_REPORT)));
 
 	/*
 	 *	The thread may have certain types of interrupts/aborts masked
@@ -538,22 +883,48 @@ thread_mark_wait_locked(
 	 *	are OK, we have to honor mask settings (outer-scoped code may
 	 *	not be able to handle aborts at the moment).
 	 */
+	interruptible &= TH_OPT_INTMASK;
 	if (interruptible > (thread->options & TH_OPT_INTMASK))
 		interruptible = thread->options & TH_OPT_INTMASK;
 
 	at_safe_point = (interruptible == THREAD_ABORTSAFE);
 
 	if (	interruptible == THREAD_UNINT			||
-			!(thread->state & TH_ABORT)				||
+			!(thread->sched_flags & TH_SFLAG_ABORT)	||
 			(!at_safe_point &&
-			 (thread->state & TH_ABORT_SAFELY))		) {
-		thread->state |= (interruptible) ? TH_WAIT : (TH_WAIT | TH_UNINT);
+				(thread->sched_flags & TH_SFLAG_ABORTSAFELY))) {
+
+		if ( !(thread->state & TH_TERMINATE))
+			DTRACE_SCHED(sleep);
+
+		int state_bits = TH_WAIT;
+		if (!interruptible) {
+			state_bits |= TH_UNINT;
+		}
+		if (thread->sched_call) {
+			wait_interrupt_t mask = THREAD_WAIT_NOREPORT_USER;
+			if (is_kerneltask(thread->task)) {
+				mask = THREAD_WAIT_NOREPORT_KERNEL;
+			}
+			if ((interruptible_orig & mask) == 0) {
+				state_bits |= TH_WAIT_REPORT;
+			}
+		}
+		thread->state |= state_bits;
 		thread->at_safe_point = at_safe_point;
+
+		/* TODO: pass this through assert_wait instead, have
+		 * assert_wait just take a struct as an argument */
+		assert(!thread->block_hint);
+		thread->block_hint = thread->pending_block_hint;
+		thread->pending_block_hint = kThreadWaitNone;
+
 		return (thread->wait_result = THREAD_WAITING);
+	} else {
+		if (thread->sched_flags & TH_SFLAG_ABORTSAFELY)
+			thread->sched_flags &= ~TH_SFLAG_ABORTED_MASK;
 	}
-	else
-	if (thread->state & TH_ABORT_SAFELY)
-		thread->state &= ~(TH_ABORT|TH_ABORT_SAFELY);
+	thread->pending_block_hint = kThreadWaitNone;
 
 	return (thread->wait_result = THREAD_INTERRUPTED);
 }
@@ -585,46 +956,38 @@ thread_interrupt_level(
 }
 
 /*
- * Check to see if an assert wait is possible, without actually doing one.
- * This is used by debug code in locks and elsewhere to verify that it is
- * always OK to block when trying to take a blocking lock (since waiting
- * for the actual assert_wait to catch the case may make it hard to detect
- * this case.
+ *	assert_wait:
+ *
+ *	Assert that the current thread is about to go to
+ *	sleep until the specified event occurs.
  */
-boolean_t
-assert_wait_possible(void)
+wait_result_t
+assert_wait(
+	event_t				event,
+	wait_interrupt_t	interruptible)
 {
+	if (__improbable(event == NO_EVENT))
+		panic("%s() called with NO_EVENT", __func__);
 
-	thread_t thread;
-
-#if	DEBUG
-	if(debug_mode) return TRUE;		/* Always succeed in debug mode */
-#endif
-	
-	thread = current_thread();
+	KERNEL_DEBUG_CONSTANT_IST(KDEBUG_TRACE,
+		MACHDBG_CODE(DBG_MACH_SCHED, MACH_WAIT)|DBG_FUNC_NONE,
+		VM_KERNEL_UNSLIDE_OR_PERM(event), 0, 0, 0, 0);
 
-	return (thread == NULL || wait_queue_assert_possible(thread));
+	struct waitq *waitq;
+	waitq = global_eventq(event);
+	return waitq_assert_wait64(waitq, CAST_EVENT64_T(event), interruptible, TIMEOUT_WAIT_FOREVER);
 }
 
 /*
- *	assert_wait:
+ *	assert_wait_queue:
  *
- *	Assert that the current thread is about to go to
- *	sleep until the specified event occurs.
+ *	Return the global waitq for the specified event
  */
-wait_result_t
-assert_wait(
-	event_t				event,
-	wait_interrupt_t	interruptible)
+struct waitq *
+assert_wait_queue(
+	event_t				event)
 {
-	register wait_queue_t	wq;
-	register int		index;
-
-	assert(event != NO_EVENT);
-
-	index = wait_hash(event);
-	wq = &wait_queues[index];
-	return wait_queue_assert_wait(wq, event, interruptible, 0);
+	return global_eventq(event);
 }
 
 wait_result_t
@@ -636,191 +999,195 @@ assert_wait_timeout(
 {
 	thread_t			thread = current_thread();
 	wait_result_t		wresult;
-	wait_queue_t		wqueue;
 	uint64_t			deadline;
 	spl_t				s;
 
-	assert(event != NO_EVENT);
-	wqueue = &wait_queues[wait_hash(event)];
+	if (__improbable(event == NO_EVENT))
+		panic("%s() called with NO_EVENT", __func__);
+
+	struct waitq *waitq;
+	waitq = global_eventq(event);
 
 	s = splsched();
-	wait_queue_lock(wqueue);
-	thread_lock(thread);
+	waitq_lock(waitq);
 
 	clock_interval_to_deadline(interval, scale_factor, &deadline);
-	wresult = wait_queue_assert_wait64_locked(wqueue, (uint32_t)event,
-													interruptible, deadline, thread);
 
-	thread_unlock(thread);
-	wait_queue_unlock(wqueue);
-	splx(s);
+	KERNEL_DEBUG_CONSTANT_IST(KDEBUG_TRACE,
+				  MACHDBG_CODE(DBG_MACH_SCHED, MACH_WAIT)|DBG_FUNC_NONE,
+				  VM_KERNEL_UNSLIDE_OR_PERM(event), interruptible, deadline, 0, 0);
+
+	wresult = waitq_assert_wait64_locked(waitq, CAST_EVENT64_T(event),
+					     interruptible,
+					     TIMEOUT_URGENCY_SYS_NORMAL,
+					     deadline, TIMEOUT_NO_LEEWAY,
+					     thread);
 
-	return (wresult);
+	waitq_unlock(waitq);
+	splx(s);
+	return wresult;
 }
 
 wait_result_t
-assert_wait_deadline(
+assert_wait_timeout_with_leeway(
 	event_t				event,
 	wait_interrupt_t	interruptible,
-	uint64_t			deadline)
+	wait_timeout_urgency_t	urgency,
+	uint32_t			interval,
+	uint32_t			leeway,
+	uint32_t			scale_factor)
 {
 	thread_t			thread = current_thread();
 	wait_result_t		wresult;
-	wait_queue_t		wqueue;
+	uint64_t			deadline;
+	uint64_t			abstime;
+	uint64_t			slop;
+	uint64_t			now;
 	spl_t				s;
 
-	assert(event != NO_EVENT);
-	wqueue = &wait_queues[wait_hash(event)];
+	if (__improbable(event == NO_EVENT))
+		panic("%s() called with NO_EVENT", __func__);
+
+	now = mach_absolute_time();
+	clock_interval_to_absolutetime_interval(interval, scale_factor, &abstime);
+	deadline = now + abstime;
+
+	clock_interval_to_absolutetime_interval(leeway, scale_factor, &slop);
+
+	struct waitq *waitq;
+	waitq = global_eventq(event);
 
 	s = splsched();
-	wait_queue_lock(wqueue);
-	thread_lock(thread);
+	waitq_lock(waitq);
 
-	wresult = wait_queue_assert_wait64_locked(wqueue, (uint32_t)event,
-													interruptible, deadline, thread);
+	KERNEL_DEBUG_CONSTANT_IST(KDEBUG_TRACE,
+				  MACHDBG_CODE(DBG_MACH_SCHED, MACH_WAIT)|DBG_FUNC_NONE,
+				  VM_KERNEL_UNSLIDE_OR_PERM(event), interruptible, deadline, 0, 0);
 
-	thread_unlock(thread);
-	wait_queue_unlock(wqueue);
-	splx(s);
+	wresult = waitq_assert_wait64_locked(waitq, CAST_EVENT64_T(event),
+					     interruptible,
+					     urgency, deadline, slop,
+					     thread);
 
-	return (wresult);
+	waitq_unlock(waitq);
+	splx(s);
+	return wresult;
 }
 
-/*
- *	thread_sleep_fast_usimple_lock:
- *
- *	Cause the current thread to wait until the specified event
- *	occurs.  The specified simple_lock is unlocked before releasing
- *	the cpu and re-acquired as part of waking up.
- *
- *	This is the simple lock sleep interface for components that use a
- *	faster version of simple_lock() than is provided by usimple_lock().
- */
-__private_extern__ wait_result_t
-thread_sleep_fast_usimple_lock(
-	event_t			event,
-	simple_lock_t		lock,
-	wait_interrupt_t	interruptible)
+wait_result_t
+assert_wait_deadline(
+	event_t				event,
+	wait_interrupt_t	interruptible,
+	uint64_t			deadline)
 {
-	wait_result_t res;
+	thread_t			thread = current_thread();
+	wait_result_t		wresult;
+	spl_t				s;
 
-	res = assert_wait(event, interruptible);
-	if (res == THREAD_WAITING) {
-		simple_unlock(lock);
-		res = thread_block(THREAD_CONTINUE_NULL);
-		simple_lock(lock);
-	}
-	return res;
-}
+	if (__improbable(event == NO_EVENT))
+		panic("%s() called with NO_EVENT", __func__);
 
+	struct waitq *waitq;
+	waitq = global_eventq(event);
 
-/*
- *	thread_sleep_usimple_lock:
- *
- *	Cause the current thread to wait until the specified event
- *	occurs.  The specified usimple_lock is unlocked before releasing
- *	the cpu and re-acquired as part of waking up.
- *
- *	This is the simple lock sleep interface for components where
- *	simple_lock() is defined in terms of usimple_lock().
- */
-wait_result_t
-thread_sleep_usimple_lock(
-	event_t			event,
-	usimple_lock_t		lock,
-	wait_interrupt_t	interruptible)
-{
-	wait_result_t res;
+	s = splsched();
+	waitq_lock(waitq);
 
-	res = assert_wait(event, interruptible);
-	if (res == THREAD_WAITING) {
-		usimple_unlock(lock);
-		res = thread_block(THREAD_CONTINUE_NULL);
-		usimple_lock(lock);
-	}
-	return res;
+	KERNEL_DEBUG_CONSTANT_IST(KDEBUG_TRACE,
+				  MACHDBG_CODE(DBG_MACH_SCHED, MACH_WAIT)|DBG_FUNC_NONE,
+				  VM_KERNEL_UNSLIDE_OR_PERM(event), interruptible, deadline, 0, 0);
+
+	wresult = waitq_assert_wait64_locked(waitq, CAST_EVENT64_T(event),
+					     interruptible,
+					     TIMEOUT_URGENCY_SYS_NORMAL, deadline,
+					     TIMEOUT_NO_LEEWAY, thread);
+	waitq_unlock(waitq);
+	splx(s);
+	return wresult;
 }
 
-/*
- *	thread_sleep_mutex:
- *
- *	Cause the current thread to wait until the specified event
- *	occurs.  The specified mutex is unlocked before releasing
- *	the cpu. The mutex will be re-acquired before returning.
- *
- *	JMM - Add hint to make sure mutex is available before rousting
- */
 wait_result_t
-thread_sleep_mutex(
-	event_t			event,
-	mutex_t			*mutex,
-	wait_interrupt_t interruptible)
+assert_wait_deadline_with_leeway(
+	event_t				event,
+	wait_interrupt_t	interruptible,
+	wait_timeout_urgency_t	urgency,
+	uint64_t			deadline,
+	uint64_t			leeway)
 {
-	wait_result_t	res;
+	thread_t			thread = current_thread();
+	wait_result_t		wresult;
+	spl_t				s;
 
-	res = assert_wait(event, interruptible);
-	if (res == THREAD_WAITING) {
-		mutex_unlock(mutex);
-		res = thread_block(THREAD_CONTINUE_NULL);
-		mutex_lock(mutex);
-	}
-	return res;
-}
-  
-/*
- *	thread_sleep_mutex_deadline:
- *
- *	Cause the current thread to wait until the specified event
- *	(or deadline) occurs.  The specified mutex is unlocked before
- *	releasing the cpu. The mutex will be re-acquired before returning.
- */
-wait_result_t
-thread_sleep_mutex_deadline(
-	event_t			event,
-	mutex_t			*mutex,
-	uint64_t		deadline,
-	wait_interrupt_t interruptible)
-{
-	wait_result_t	res;
+	if (__improbable(event == NO_EVENT))
+		panic("%s() called with NO_EVENT", __func__);
 
-	res = assert_wait_deadline(event, interruptible, deadline);
-	if (res == THREAD_WAITING) {
-		mutex_unlock(mutex);
-		res = thread_block(THREAD_CONTINUE_NULL);
-		mutex_lock(mutex);
-	}
-	return res;
+	struct waitq *waitq;
+	waitq = global_eventq(event);
+
+	s = splsched();
+	waitq_lock(waitq);
+
+	KERNEL_DEBUG_CONSTANT_IST(KDEBUG_TRACE,
+				  MACHDBG_CODE(DBG_MACH_SCHED, MACH_WAIT)|DBG_FUNC_NONE,
+				  VM_KERNEL_UNSLIDE_OR_PERM(event), interruptible, deadline, 0, 0);
+
+	wresult = waitq_assert_wait64_locked(waitq, CAST_EVENT64_T(event),
+					     interruptible,
+					     urgency, deadline, leeway,
+					     thread);
+	waitq_unlock(waitq);
+	splx(s);
+	return wresult;
 }
 
 /*
- *	thread_sleep_lock_write:
+ * thread_isoncpu:
  *
- *	Cause the current thread to wait until the specified event
- *	occurs.  The specified (write) lock is unlocked before releasing
- *	the cpu. The (write) lock will be re-acquired before returning.
+ * Return TRUE if a thread is running on a processor such that an AST
+ * is needed to pull it out of userspace execution, or if executing in
+ * the kernel, bring to a context switch boundary that would cause
+ * thread state to be serialized in the thread PCB.
+ * 
+ * Thread locked, returns the same way. While locked, fields
+ * like "state" cannot change. "runq" can change only from set to unset.
  */
-wait_result_t
-thread_sleep_lock_write(
-	event_t			event,
-	lock_t			*lock,
-	wait_interrupt_t interruptible)
+static inline boolean_t
+thread_isoncpu(thread_t thread)
 {
-	wait_result_t	res;
+	/* Not running or runnable */
+	if (!(thread->state & TH_RUN))
+		return (FALSE);
 
-	res = assert_wait(event, interruptible);
-	if (res == THREAD_WAITING) {
-		lock_write_done(lock);
-		res = thread_block(THREAD_CONTINUE_NULL);
-		lock_write(lock);
-	}
-	return res;
+	/* Waiting on a runqueue, not currently running */
+	/* TODO: This is invalid - it can get dequeued without thread lock, but not context switched. */
+	if (thread->runq != PROCESSOR_NULL)
+		return (FALSE);
+
+	/*
+	 * Thread does not have a stack yet
+	 * It could be on the stack alloc queue or preparing to be invoked
+	 */
+	if (!thread->kernel_stack)
+		return (FALSE);
+
+	/*
+	 * Thread must be running on a processor, or
+	 * about to run, or just did run. In all these
+	 * cases, an AST to the processor is needed
+	 * to guarantee that the thread is kicked out
+	 * of userspace and the processor has
+	 * context switched (and saved register state).
+	 */
+	return (TRUE);
 }
 
 /*
  * thread_stop:
  *
  * Force a preemption point for a thread and wait
- * for it to stop running.  Arbitrates access among
+ * for it to stop running on a CPU. If a stronger
+ * guarantee is requested, wait until no longer
+ * runnable. Arbitrates access among
  * multiple stop requests. (released by unstop)
  *
  * The thread must enter a wait state and stop via a
@@ -830,16 +1197,20 @@ thread_sleep_lock_write(
  */
 boolean_t
 thread_stop(
-	thread_t		thread)
+	thread_t		thread,
+	boolean_t	until_not_runnable)
 {
 	wait_result_t	wresult;
-	spl_t			s;
+	spl_t			s = splsched();
+	boolean_t		oncpu;
 
-	s = splsched();
 	wake_lock(thread);
+	thread_lock(thread);
 
 	while (thread->state & TH_SUSP) {
 		thread->wake_active = TRUE;
+		thread_unlock(thread);
+
 		wresult = assert_wait(&thread->wake_active, THREAD_ABORTSAFE);
 		wake_unlock(thread);
 		splx(s);
@@ -852,21 +1223,24 @@ thread_stop(
 
 		s = splsched();
 		wake_lock(thread);
+		thread_lock(thread);
 	}
 
-	thread_lock(thread);
 	thread->state |= TH_SUSP;
 
-	while (thread->state & TH_RUN) {
-		processor_t		processor = thread->last_processor;
-
-		if (	processor != PROCESSOR_NULL					&&
-				processor->state == PROCESSOR_RUNNING		&&
-				processor->active_thread == thread			)
+	while ((oncpu = thread_isoncpu(thread)) ||
+		   (until_not_runnable && (thread->state & TH_RUN))) {
+		processor_t		processor;
+		
+		if (oncpu) {
+			assert(thread->state & TH_RUN);
+			processor = thread->chosen_processor;
 			cause_ast_check(processor);
-		thread_unlock(thread);
+		}
 
 		thread->wake_active = TRUE;
+		thread_unlock(thread);
+
 		wresult = assert_wait(&thread->wake_active, THREAD_ABORTSAFE);
 		wake_unlock(thread);
 		splx(s);
@@ -887,6 +1261,13 @@ thread_stop(
 	thread_unlock(thread);
 	wake_unlock(thread);
 	splx(s);
+	
+	/*
+	 * We return with the thread unlocked. To prevent it from
+	 * transitioning to a runnable state (or from TH_RUN to
+	 * being on the CPU), the caller must ensure the thread
+	 * is stopped via an external means (such as an AST)
+	 */
 
 	return (TRUE);
 }
@@ -908,23 +1289,19 @@ thread_unstop(
 	wake_lock(thread);
 	thread_lock(thread);
 
-	if ((thread->state & (TH_RUN|TH_WAIT|TH_SUSP)) == TH_SUSP) {
-		thread->state &= ~TH_SUSP;
-		thread_unblock(thread, THREAD_AWAKENED);
+	assert((thread->state & (TH_RUN|TH_WAIT|TH_SUSP)) != TH_SUSP);
 
-		thread_setrun(thread, SCHED_PREEMPT | SCHED_TAILQ);
-	}
-	else
 	if (thread->state & TH_SUSP) {
 		thread->state &= ~TH_SUSP;
 
 		if (thread->wake_active) {
 			thread->wake_active = FALSE;
 			thread_unlock(thread);
+
+			thread_wakeup(&thread->wake_active);
 			wake_unlock(thread);
 			splx(s);
 
-			thread_wakeup(&thread->wake_active);
 			return;
 		}
 	}
@@ -942,24 +1319,36 @@ thread_unstop(
  */
 void
 thread_wait(
-	thread_t		thread)
+	thread_t	thread,
+	boolean_t	until_not_runnable)
 {
 	wait_result_t	wresult;
-	spl_t			s = splsched();
+	boolean_t 	oncpu;
+	processor_t	processor;
+	spl_t		s = splsched();
 
 	wake_lock(thread);
 	thread_lock(thread);
 
-	while (thread->state & TH_RUN) {
-		processor_t		processor = thread->last_processor;
+	/*
+	 * Wait until not running on a CPU.  If stronger requirement
+	 * desired, wait until not runnable.  Assumption: if thread is
+	 * on CPU, then TH_RUN is set, so we're not waiting in any case
+	 * where the original, pure "TH_RUN" check would have let us 
+	 * finish.
+	 */
+	while ((oncpu = thread_isoncpu(thread)) ||
+			(until_not_runnable && (thread->state & TH_RUN))) {
 
-		if (	processor != PROCESSOR_NULL					&&
-				processor->state == PROCESSOR_RUNNING		&&
-				processor->active_thread == thread			)
+		if (oncpu) {
+			assert(thread->state & TH_RUN);
+			processor = thread->chosen_processor;
 			cause_ast_check(processor);
-		thread_unlock(thread);
+		}
 
 		thread->wake_active = TRUE;
+		thread_unlock(thread);
+
 		wresult = assert_wait(&thread->wake_active, THREAD_UNINT);
 		wake_unlock(thread);
 		splx(s);
@@ -998,35 +1387,35 @@ clear_wait_internal(
 	thread_t		thread,
 	wait_result_t	wresult)
 {
-	wait_queue_t	wq = thread->wait_queue;
-	int				i = LockTimeOut;
-
+	uint32_t	i = LockTimeOutUsec;
+	struct waitq *waitq = thread->waitq;
+	
 	do {
 		if (wresult == THREAD_INTERRUPTED && (thread->state & TH_UNINT))
 			return (KERN_FAILURE);
 
-		if (wq != WAIT_QUEUE_NULL) {
-			if (wait_queue_lock_try(wq)) {
-				wait_queue_pull_thread_locked(wq, thread, TRUE);
-				/* wait queue unlocked, thread still locked */
-			}
-			else {
+		if (waitq != NULL) {
+			if (!waitq_pull_thread_locked(waitq, thread)) {
 				thread_unlock(thread);
 				delay(1);
-
+				if (i > 0 && !machine_timeout_suspended())
+					i--;
 				thread_lock(thread);
-				if (wq != thread->wait_queue)
-					return (KERN_NOT_WAITING);
-
+				if (waitq != thread->waitq)
+					return KERN_NOT_WAITING;
 				continue;
 			}
 		}
 
-		return (thread_go(thread, wresult));
-	} while (--i > 0);
+		/* TODO: Can we instead assert TH_TERMINATE is not set?  */
+		if ((thread->state & (TH_WAIT|TH_TERMINATE)) == TH_WAIT)
+			return (thread_go(thread, wresult));
+		else
+			return (KERN_NOT_WAITING);
+	} while (i > 0);
 
-	panic("clear_wait_internal: deadlock: thread=0x%x, wq=0x%x, cpu=%d\n",
-		  thread, wq, cpu_number());
+	panic("clear_wait_internal: deadlock: thread=%p, wq=%p, cpu=%d\n",
+		  thread, waitq, cpu_number());
 
 	return (KERN_FAILURE);
 }
@@ -1068,25 +1457,82 @@ clear_wait(
  */
 kern_return_t
 thread_wakeup_prim(
-	event_t			event,
-	boolean_t		one_thread,
-	wait_result_t	result)
+                   event_t          event,
+                   boolean_t        one_thread,
+                   wait_result_t    result)
 {
-	register wait_queue_t	wq;
-	register int			index;
+	if (__improbable(event == NO_EVENT))
+		panic("%s() called with NO_EVENT", __func__);
+
+	struct waitq *wq = global_eventq(event);
 
-	index = wait_hash(event);
-	wq = &wait_queues[index];
 	if (one_thread)
-	    return (wait_queue_wakeup_one(wq, event, result));
+		return waitq_wakeup64_one(wq, CAST_EVENT64_T(event), result, WAITQ_ALL_PRIORITIES);
 	else
-	    return (wait_queue_wakeup_all(wq, event, result));
+		return waitq_wakeup64_all(wq, CAST_EVENT64_T(event), result, WAITQ_ALL_PRIORITIES);
+}
+
+/*
+ * Wakeup a specified thread if and only if it's waiting for this event
+ */
+kern_return_t
+thread_wakeup_thread(
+                     event_t         event,
+                     thread_t        thread)
+{
+	if (__improbable(event == NO_EVENT))
+		panic("%s() called with NO_EVENT", __func__);
+
+	if (__improbable(thread == THREAD_NULL))
+		panic("%s() called with THREAD_NULL", __func__);
+
+	struct waitq *wq = global_eventq(event);
+
+	return waitq_wakeup64_thread(wq, CAST_EVENT64_T(event), thread, THREAD_AWAKENED);
+}
+
+/*
+ * Wakeup a thread waiting on an event and promote it to a priority.
+ *
+ * Requires woken thread to un-promote itself when done.
+ */
+kern_return_t
+thread_wakeup_one_with_pri(
+                           event_t      event,
+                           int          priority)
+{
+	if (__improbable(event == NO_EVENT))
+		panic("%s() called with NO_EVENT", __func__);
+
+	struct waitq *wq = global_eventq(event);
+
+	return waitq_wakeup64_one(wq, CAST_EVENT64_T(event), THREAD_AWAKENED, priority);
+}
+
+/*
+ * Wakeup a thread waiting on an event,
+ * promote it to a priority,
+ * and return a reference to the woken thread.
+ *
+ * Requires woken thread to un-promote itself when done.
+ */
+thread_t
+thread_wakeup_identify(event_t  event,
+                       int      priority)
+{
+	if (__improbable(event == NO_EVENT))
+		panic("%s() called with NO_EVENT", __func__);
+
+	struct waitq *wq = global_eventq(event);
+
+	return waitq_wakeup64_identify(wq, CAST_EVENT64_T(event), THREAD_AWAKENED, priority);
 }
 
 /*
  *	thread_bind:
  *
- *	Force a thread to execute on the specified processor.
+ *	Force the current thread to execute on the specified processor.
+ *	Takes effect after the next thread_block().
  *
  *	Returns the previous binding.  PROCESSOR_NULL means
  *	not bound.
@@ -1095,1906 +1541,4335 @@ thread_wakeup_prim(
  */
 processor_t
 thread_bind(
-	register thread_t	thread,
-	processor_t			processor)
+	processor_t		processor)
 {
+	thread_t		self = current_thread();
 	processor_t		prev;
-	run_queue_t		runq = RUN_QUEUE_NULL;
 	spl_t			s;
 
 	s = splsched();
-	thread_lock(thread);
-	prev = thread->bound_processor;
-	if (prev != PROCESSOR_NULL)
-		runq = run_queue_remove(thread);
+	thread_lock(self);
 
-	thread->bound_processor = processor;
+	prev = thread_bind_internal(self, processor);
 
-	if (runq != RUN_QUEUE_NULL)
-		thread_setrun(thread, SCHED_PREEMPT | SCHED_TAILQ);
-	thread_unlock(thread);
+	thread_unlock(self);
 	splx(s);
 
 	return (prev);
 }
 
-struct {
-	uint32_t	idle_pset_last,
-				idle_pset_any,
-				idle_bound;
+/*
+ * thread_bind_internal:
+ *
+ * If the specified thread is not the current thread, and it is currently
+ * running on another CPU, a remote AST must be sent to that CPU to cause
+ * the thread to migrate to its bound processor. Otherwise, the migration
+ * will occur at the next quantum expiration or blocking point.
+ *
+ * When the thread is the current thread, and explicit thread_block() should
+ * be used to force the current processor to context switch away and
+ * let the thread migrate to the bound processor.
+ *
+ * Thread must be locked, and at splsched.
+ */
+
+static processor_t
+thread_bind_internal(
+	thread_t		thread,
+	processor_t		processor)
+{
+	processor_t		prev;
+
+	/* <rdar://problem/15102234> */
+	assert(thread->sched_pri < BASEPRI_RTQUEUES);
+	/* A thread can't be bound if it's sitting on a (potentially incorrect) runqueue */
+	assert(thread->runq == PROCESSOR_NULL);
 
-	uint32_t	pset_self,
-				pset_last,
-				pset_other,
-				bound_self,
-				bound_other;
+	KERNEL_DEBUG_CONSTANT(MACHDBG_CODE(DBG_MACH_SCHED, MACH_THREAD_BIND), thread_tid(thread), processor ? (uintptr_t)processor->cpu_id : (uintptr_t)-1, 0, 0, 0);
 
-	uint32_t	realtime_self,
-				realtime_last,
-				realtime_other;
+	prev = thread->bound_processor;
+	thread->bound_processor = processor;
 
-	uint32_t	missed_realtime,
-				missed_other;
-} dispatch_counts;
+	return (prev);
+}
 
 /*
- *	Select a thread for the current processor to run.
+ * thread_vm_bind_group_add:
  *
- *	May select the current thread, which must be locked.
+ * The "VM bind group" is a special mechanism to mark a collection
+ * of threads from the VM subsystem that, in general, should be scheduled
+ * with only one CPU of parallelism. To accomplish this, we initially
+ * bind all the threads to the master processor, which has the effect
+ * that only one of the threads in the group can execute at once, including
+ * preempting threads in the group that are a lower priority. Future
+ * mechanisms may use more dynamic mechanisms to prevent the collection
+ * of VM threads from using more CPU time than desired.
+ *
+ * The current implementation can result in priority inversions where
+ * compute-bound priority 95 or realtime threads that happen to have
+ * landed on the master processor prevent the VM threads from running.
+ * When this situation is detected, we unbind the threads for one
+ * scheduler tick to allow the scheduler to run the threads an
+ * additional CPUs, before restoring the binding (assuming high latency
+ * is no longer a problem).
  */
-thread_t
-thread_select(
-	register processor_t	processor)
-{
-	register thread_t		thread;
-	processor_set_t			pset;
-	boolean_t				other_runnable;
 
-	/*
-	 *	Check for other non-idle runnable threads.
-	 */
-	pset = processor->processor_set;
-	thread = processor->active_thread;
-
-	/* Update the thread's priority */
-	if (thread->sched_stamp != sched_tick)
-		update_priority(thread);
-
-	processor->current_pri = thread->sched_pri;
-
-	simple_lock(&pset->sched_lock);
-
-	other_runnable = processor->runq.count > 0 || pset->runq.count > 0;
-
-	if (	thread->state == TH_RUN							&&
-			thread->processor_set == pset					&&
-			(thread->bound_processor == PROCESSOR_NULL	||
-			 thread->bound_processor == processor)				) {
-		if (	thread->sched_pri >= BASEPRI_RTQUEUES	&&
-						first_timeslice(processor)			) {
-			if (pset->runq.highq >= BASEPRI_RTQUEUES) {
-				register run_queue_t	runq = &pset->runq;
-				register queue_t		q;
-
-				q = runq->queues + runq->highq;
-				if (((thread_t)q->next)->realtime.deadline <
-												processor->deadline) {
-					thread = (thread_t)q->next;
-					((queue_entry_t)thread)->next->prev = q;
-					q->next = ((queue_entry_t)thread)->next;
-					thread->runq = RUN_QUEUE_NULL;
-					assert(thread->sched_mode & TH_MODE_PREEMPT);
-					runq->count--; runq->urgency--;
-					if (queue_empty(q)) {
-						if (runq->highq != IDLEPRI)
-							clrbit(MAXPRI - runq->highq, runq->bitmap);
-						runq->highq = MAXPRI - ffsbit(runq->bitmap);
-					}
-				}
-			}
+/*
+ * The current max is provisioned for:
+ * vm_compressor_swap_trigger_thread (92)
+ * 2 x vm_pageout_iothread_internal (92) when vm_restricted_to_single_processor==TRUE
+ * vm_pageout_continue (92)
+ * memorystatus_thread (95)
+ */
+#define MAX_VM_BIND_GROUP_COUNT (5)
+decl_simple_lock_data(static,sched_vm_group_list_lock);
+static thread_t sched_vm_group_thread_list[MAX_VM_BIND_GROUP_COUNT];
+static int sched_vm_group_thread_count;
+static boolean_t sched_vm_group_temporarily_unbound = FALSE;
 
-			processor->deadline = thread->realtime.deadline;
+void
+thread_vm_bind_group_add(void)
+{
+	thread_t self = current_thread();
 
-			simple_unlock(&pset->sched_lock);
+	thread_reference_internal(self);
+	self->options |= TH_OPT_SCHED_VM_GROUP;
 
-			return (thread);
-		}
+	simple_lock(&sched_vm_group_list_lock);
+	assert(sched_vm_group_thread_count < MAX_VM_BIND_GROUP_COUNT);
+	sched_vm_group_thread_list[sched_vm_group_thread_count++] = self;
+	simple_unlock(&sched_vm_group_list_lock);
 
-		if (	(!other_runnable							||
-				 (processor->runq.highq < thread->sched_pri		&&
-				  pset->runq.highq < thread->sched_pri))			) {
+	thread_bind(master_processor);
 
-			/* I am the highest priority runnable (non-idle) thread */
+	/* Switch to bound processor if not already there */
+	thread_block(THREAD_CONTINUE_NULL);
+}
 
-			processor->deadline = UINT64_MAX;
+static void
+sched_vm_group_maintenance(void)
+{
+	uint64_t ctime = mach_absolute_time();
+	uint64_t longtime = ctime - sched_tick_interval;
+	int i;
+	spl_t s;
+	boolean_t high_latency_observed = FALSE;
+	boolean_t runnable_and_not_on_runq_observed = FALSE;
+	boolean_t bind_target_changed = FALSE;
+	processor_t bind_target = PROCESSOR_NULL;
+
+	/* Make sure nobody attempts to add new threads while we are enumerating them */
+	simple_lock(&sched_vm_group_list_lock);
+
+	s = splsched();
 
-			simple_unlock(&pset->sched_lock);
+	for (i=0; i < sched_vm_group_thread_count; i++) {
+		thread_t thread = sched_vm_group_thread_list[i];
+		assert(thread != THREAD_NULL);
+		thread_lock(thread);
+		if ((thread->state & (TH_RUN|TH_WAIT)) == TH_RUN) {
+			if (thread->runq != PROCESSOR_NULL && thread->last_made_runnable_time < longtime) {
+				high_latency_observed = TRUE;
+			} else if (thread->runq == PROCESSOR_NULL) {
+				/* There are some cases where a thread be transitiong that also fall into this case */
+				runnable_and_not_on_runq_observed = TRUE;
+			}
+		}
+		thread_unlock(thread);
 
-			return (thread);
+		if (high_latency_observed && runnable_and_not_on_runq_observed) {
+			/* All the things we are looking for are true, stop looking */
+			break;
 		}
 	}
 
-	if (other_runnable)
-		thread = choose_thread(pset, processor);
-	else {
+	splx(s);
+
+	if (sched_vm_group_temporarily_unbound) {
+		/* If we turned off binding, make sure everything is OK before rebinding */
+		if (!high_latency_observed) {
+			/* rebind */
+			bind_target_changed = TRUE;
+			bind_target = master_processor;
+			sched_vm_group_temporarily_unbound = FALSE; /* might be reset to TRUE if change cannot be completed */
+		}
+	} else {
 		/*
-		 *	Nothing is runnable, so set this processor idle if it
-		 *	was running.  Return its idle thread.
+		 * Check if we're in a bad state, which is defined by high
+		 * latency with no core currently executing a thread. If a
+		 * single thread is making progress on a CPU, that means the
+		 * binding concept to reduce parallelism is working as
+		 * designed.
 		 */
-		if (processor->state == PROCESSOR_RUNNING) {
-			remqueue(&pset->active_queue, (queue_entry_t)processor);
-			processor->state = PROCESSOR_IDLE;
-
-			enqueue_tail(&pset->idle_queue, (queue_entry_t)processor);
-			pset->idle_count++;
+		if (high_latency_observed && !runnable_and_not_on_runq_observed) {
+			/* unbind */
+			bind_target_changed = TRUE;
+			bind_target = PROCESSOR_NULL;
+			sched_vm_group_temporarily_unbound = TRUE;
 		}
+	}
 
-		processor->deadline = UINT64_MAX;
+	if (bind_target_changed) {
+		s = splsched();
+		for (i=0; i < sched_vm_group_thread_count; i++) {
+			thread_t thread = sched_vm_group_thread_list[i];
+			boolean_t removed;
+			assert(thread != THREAD_NULL);
 
-		thread = processor->idle_thread;
-	}
+			thread_lock(thread);
+			removed = thread_run_queue_remove(thread);
+			if (removed || ((thread->state & (TH_RUN | TH_WAIT)) == TH_WAIT)) {
+				thread_bind_internal(thread, bind_target);
+			} else {
+				/*
+				 * Thread was in the middle of being context-switched-to,
+				 * or was in the process of blocking. To avoid switching the bind
+				 * state out mid-flight, defer the change if possible.
+				 */
+				if (bind_target == PROCESSOR_NULL) {
+					thread_bind_internal(thread, bind_target);
+				} else {
+					sched_vm_group_temporarily_unbound = TRUE; /* next pass will try again */
+				}
+			}
 
-	simple_unlock(&pset->sched_lock);
+			if (removed) {
+				thread_run_queue_reinsert(thread, SCHED_PREEMPT | SCHED_TAILQ);
+			}
+			thread_unlock(thread);
+		}
+		splx(s);
+	}
 
-	return (thread);
+	simple_unlock(&sched_vm_group_list_lock);
 }
 
-/*
- *	Perform a context switch and start executing the new thread.
- *
- *	Returns FALSE on failure, and the thread is re-dispatched.
- *
- *	Called at splsched.
+/* Invoked prior to idle entry to determine if, on SMT capable processors, an SMT
+ * rebalancing opportunity exists when a core is (instantaneously) idle, but
+ * other SMT-capable cores may be over-committed. TODO: some possible negatives:
+ * IPI thrash if this core does not remain idle following the load balancing ASTs
+ * Idle "thrash", when IPI issue is followed by idle entry/core power down
+ * followed by a wakeup shortly thereafter.
  */
 
-#define funnel_release_check(thread, debug)				\
-MACRO_BEGIN												\
-	if ((thread)->funnel_state & TH_FN_OWNED) {			\
-		(thread)->funnel_state = TH_FN_REFUNNEL;		\
-		KERNEL_DEBUG(0x603242c | DBG_FUNC_NONE,			\
-			(thread)->funnel_lock, (debug), 0, 0, 0);	\
-		funnel_unlock((thread)->funnel_lock);			\
-	}													\
-MACRO_END
-
-#define funnel_refunnel_check(thread, debug)				\
-MACRO_BEGIN													\
-	if ((thread)->funnel_state & TH_FN_REFUNNEL) {			\
-		kern_return_t	result = (thread)->wait_result;		\
-															\
-		(thread)->funnel_state = 0;							\
-		KERNEL_DEBUG(0x6032428 | DBG_FUNC_NONE,				\
-			(thread)->funnel_lock, (debug), 0, 0, 0);		\
-		funnel_lock((thread)->funnel_lock);					\
-		KERNEL_DEBUG(0x6032430 | DBG_FUNC_NONE,				\
-			(thread)->funnel_lock, (debug), 0, 0, 0);		\
-		(thread)->funnel_state = TH_FN_OWNED;				\
-		(thread)->wait_result = result;						\
-	}														\
-MACRO_END
+#if (DEVELOPMENT || DEBUG)
+int sched_smt_balance = 1;
+#endif
 
-boolean_t
-thread_invoke(
-	register thread_t	old_thread,
-	register thread_t	new_thread,
-	ast_t				reason)
-{
-	thread_continue_t	new_cont, continuation = old_thread->continuation;
-	void				*new_param, *parameter = old_thread->parameter;
-	processor_t			processor;
-	thread_t			prev_thread;
+#if __SMP__
+/* Invoked with pset locked, returns with pset unlocked */
+void
+sched_SMT_balance(processor_t cprocessor, processor_set_t cpset) {
+	processor_t ast_processor = NULL;
 
-	if (get_preemption_level() != 0)
-		panic("thread_invoke: preemption_level %d\n",
-								get_preemption_level());
+#if (DEVELOPMENT || DEBUG)
+	if (__improbable(sched_smt_balance == 0))
+		goto smt_balance_exit;
+#endif
+	
+	assert(cprocessor == current_processor());
+	if (cprocessor->is_SMT == FALSE)
+		goto smt_balance_exit;
 
-	assert(old_thread == current_thread());
+	processor_t sib_processor = cprocessor->processor_secondary ? cprocessor->processor_secondary : cprocessor->processor_primary;
 
-	/*
-	 * Mark thread interruptible.
+	/* Determine if both this processor and its sibling are idle,
+	 * indicating an SMT rebalancing opportunity.
 	 */
-	thread_lock(new_thread);
-	new_thread->state &= ~TH_UNINT;
+	if (sib_processor->state != PROCESSOR_IDLE)
+		goto smt_balance_exit;
+
+	processor_t sprocessor;
+
+	sched_ipi_type_t ipi_type = SCHED_IPI_NONE;
+	uint64_t running_secondary_map = (cpset->cpu_state_map[PROCESSOR_RUNNING] &
+	                                  ~cpset->primary_map);
+	for (int cpuid = lsb_first(running_secondary_map); cpuid >= 0; cpuid = lsb_next(running_secondary_map, cpuid)) {
+		sprocessor = processor_array[cpuid];
+		if ((sprocessor->processor_primary->state == PROCESSOR_RUNNING) &&
+		    (sprocessor->current_pri < BASEPRI_RTQUEUES)) {
+
+			ipi_type = sched_ipi_action(sprocessor, NULL, false, SCHED_IPI_EVENT_SMT_REBAL);
+			if (ipi_type != SCHED_IPI_NONE) {
+				assert(sprocessor != cprocessor);
+				ast_processor = sprocessor;
+				break;
+			}
+		}
+	}
 
-	assert(thread_runnable(new_thread));
+smt_balance_exit:
+	pset_unlock(cpset);
 
-	/*
-	 * Allow time constraint threads to hang onto
-	 * a stack.
-	 */
-	if (	(old_thread->sched_mode & TH_MODE_REALTIME)		&&
-					!old_thread->reserved_stack				) {
-		old_thread->reserved_stack = old_thread->kernel_stack;
+	if (ast_processor) {
+		KERNEL_DEBUG_CONSTANT(MACHDBG_CODE(DBG_MACH_SCHED, MACH_SCHED_SMT_BALANCE), ast_processor->cpu_id, ast_processor->state, ast_processor->processor_primary->state, 0, 0);
+		sched_ipi_perform(ast_processor, ipi_type);
 	}
+}
+#else
+/* Invoked with pset locked, returns with pset unlocked */
+void
+sched_SMT_balance(__unused processor_t cprocessor, processor_set_t cpset)
+{
+	pset_unlock(cpset);
+}
+#endif /* __SMP__ */
+
+static processor_t choose_processor_for_realtime_thread(processor_set_t pset);
+static bool all_available_primaries_are_running_realtime_threads(processor_set_t pset);
+int sched_allow_rt_smt = 1;
+
+/*
+ *	thread_select:
+ *
+ *	Select a new thread for the current processor to execute.
+ *
+ *	May select the current thread, which must be locked.
+ */
+static thread_t
+thread_select(thread_t          thread,
+              processor_t       processor,
+              ast_t            *reason)
+{
+	processor_set_t		pset = processor->processor_set;
+	thread_t			new_thread = THREAD_NULL;
 
-	if (continuation != NULL) {
-		if (!new_thread->kernel_stack) {
-			/*
-			 * If the old thread is using a privileged stack,
-			 * check to see whether we can exchange it with
-			 * that of the new thread.
-			 */
-			if (	old_thread->kernel_stack == old_thread->reserved_stack	&&
-							!new_thread->reserved_stack)
-				goto need_stack;
-
+	assert(processor == current_processor());
+	assert((thread->state & (TH_RUN|TH_TERMINATE2)) == TH_RUN);
+
+	do {
+		/*
+		 *	Update the priority.
+		 */
+		if (SCHED(can_update_priority)(thread))
+			SCHED(update_priority)(thread);
+		
+		processor_state_update_from_thread(processor, thread);
+
+		pset_lock(pset);
+
+		assert(processor->state != PROCESSOR_OFF_LINE);
+
+		if (!processor->is_recommended) {
 			/*
-			 * Context switch by performing a stack handoff.
+			 * The performance controller has provided a hint to not dispatch more threads,
+			 * unless they are bound to us (and thus we are the only option
+			 */
+			if (!SCHED(processor_bound_count)(processor)) {
+				goto idle;
+			}
+		} else if (processor->processor_primary != processor) {
+			/*
+			 * Should this secondary SMT processor attempt to find work? For pset runqueue systems,
+			 * we should look for work only under the same conditions that choose_processor()
+			 * would have assigned work, which is when all primary processors have been assigned work.
+			 *
+			 * An exception is that bound threads are dispatched to a processor without going through
+			 * choose_processor(), so in those cases we should continue trying to dequeue work.
 			 */
-			new_cont = new_thread->continuation;
-			new_thread->continuation = NULL;
-			new_param = new_thread->parameter;
-			new_thread->parameter = NULL;
+			if (!SCHED(processor_bound_count)(processor)) {
+				if ((pset->recommended_bitmask & pset->primary_map & pset->cpu_state_map[PROCESSOR_IDLE]) != 0) {
+					goto idle;
+				}
+				
+				/* There are no idle primaries */
 
-			processor = current_processor();
-			processor->active_thread = new_thread;
-			processor->current_pri = new_thread->sched_pri;
-			new_thread->last_processor = processor;
-			ast_context(new_thread);
-			thread_unlock(new_thread);
-		
-			current_task()->csw++;
+				if (processor->processor_primary->current_pri >= BASEPRI_RTQUEUES) {
+					bool secondary_can_run_realtime_thread = sched_allow_rt_smt && rt_runq_count(pset) && all_available_primaries_are_running_realtime_threads(pset);
+					if (!secondary_can_run_realtime_thread) {
+						goto idle;
+					}
+				}
+			}
+		}
 
-			old_thread->reason = reason;
+		/*
+		 *	Test to see if the current thread should continue
+		 *	to run on this processor.  Must not be attempting to wait, and not
+		 *	bound to a different processor, nor be in the wrong
+		 *	processor set, nor be forced to context switch by TH_SUSP.
+		 *
+		 *	Note that there are never any RT threads in the regular runqueue.
+		 *
+		 *	This code is very insanely tricky.
+		 */
 
-			processor->last_dispatch = mach_absolute_time();
-			timer_event((uint32_t)processor->last_dispatch,
-										&new_thread->system_timer);
-	   
-			thread_done(old_thread, new_thread, processor);
+		/* i.e. not waiting, not TH_SUSP'ed */
+		boolean_t still_running = ((thread->state & (TH_TERMINATE|TH_IDLE|TH_WAIT|TH_RUN|TH_SUSP)) == TH_RUN);
+
+		/*
+		 * Threads running on SMT processors are forced to context switch. Don't rebalance realtime threads.
+		 * TODO: This should check if it's worth it to rebalance, i.e. 'are there any idle primary processors'
+		 */
+		boolean_t needs_smt_rebalance = (thread->sched_pri < BASEPRI_RTQUEUES && processor->processor_primary != processor);
 
-			machine_stack_handoff(old_thread, new_thread);
+		boolean_t affinity_mismatch   = (thread->affinity_set != AFFINITY_SET_NULL && thread->affinity_set->aset_pset != pset);
 
-			thread_begin(new_thread, processor);
+		boolean_t bound_elsewhere     = (thread->bound_processor != PROCESSOR_NULL && thread->bound_processor != processor);
 
+		boolean_t avoid_processor     = (SCHED(avoid_processor_enabled) && SCHED(thread_avoid_processor)(processor, thread));
+
+		if (still_running && !needs_smt_rebalance && !affinity_mismatch && !bound_elsewhere && !avoid_processor) {
 			/*
-			 * Now dispatch the old thread.
+			 * This thread is eligible to keep running on this processor.
+			 *
+			 * RT threads with un-expired quantum stay on processor,
+			 * unless there's a valid RT thread with an earlier deadline.
 			 */
-			thread_dispatch(old_thread);
+			if (thread->sched_pri >= BASEPRI_RTQUEUES && processor->first_timeslice) {
+				if (rt_runq_count(pset) > 0) {
+
+					rt_lock_lock(pset);
+					
+					if (rt_runq_count(pset) > 0) {
+			
+					    thread_t next_rt = qe_queue_first(&SCHED(rt_runq)(pset)->queue, struct thread, runq_links);
+
+					    if (next_rt->realtime.deadline < processor->deadline &&
+						(next_rt->bound_processor == PROCESSOR_NULL ||
+						next_rt->bound_processor == processor)) {
+						    /* The next RT thread is better, so pick it off the runqueue. */
+						    goto pick_new_rt_thread;
+					    }
+					}
 
-			counter_always(c_thread_invoke_hits++);
+					rt_lock_unlock(pset);
+				}
 
-			funnel_refunnel_check(new_thread, 2);
-			(void) spllo();
+				/* This is still the best RT thread to run. */
+				processor->deadline = thread->realtime.deadline;
 
-			assert(new_cont);
-			call_continuation(new_cont, new_param, new_thread->wait_result);
-			/*NOTREACHED*/
-		}
-		else
-		if (new_thread == old_thread) {
-			/* same thread but with continuation */
-			counter(++c_thread_invoke_same);
-			thread_unlock(new_thread);
+				sched_update_pset_load_average(pset);
 
-			funnel_refunnel_check(new_thread, 3);
-			(void) spllo();
+				processor_t next_rt_processor = PROCESSOR_NULL;
+				sched_ipi_type_t next_rt_ipi_type = SCHED_IPI_NONE;
 
-			call_continuation(continuation, parameter, new_thread->wait_result);
-			/*NOTREACHED*/
-		}
-	}
-	else {
-		/*
-		 * Check that the new thread has a stack
-		 */
-		if (!new_thread->kernel_stack) {
-need_stack:
-			if (!stack_alloc_try(new_thread)) {
-				counter_always(c_thread_invoke_misses++);
-				thread_unlock(new_thread);
-				thread_stack_enqueue(new_thread);
-				return (FALSE);
+				if (rt_runq_count(pset) > 0) {
+					next_rt_processor = choose_processor_for_realtime_thread(pset);
+					if (next_rt_processor) {
+						next_rt_ipi_type = sched_ipi_action(next_rt_processor, NULL, false, SCHED_IPI_EVENT_PREEMPT);
+					}
+				}
+				pset_unlock(pset);
+
+				if (next_rt_processor) {
+					sched_ipi_perform(next_rt_processor, next_rt_ipi_type);
+				}
+
+				return (thread);
 			}
+
+			if ((rt_runq_count(pset) == 0) &&
+			    SCHED(processor_queue_has_priority)(processor, thread->sched_pri, TRUE) == FALSE) {
+				/* This thread is still the highest priority runnable (non-idle) thread */
+				processor->deadline = UINT64_MAX;
+
+				sched_update_pset_load_average(pset);
+				pset_unlock(pset);
+
+				return (thread);
+			}
+		} else {
+			/*
+			 * This processor must context switch.
+			 * If it's due to a rebalance, we should aggressively find this thread a new home.
+			 */
+			if (needs_smt_rebalance || affinity_mismatch || bound_elsewhere || avoid_processor)
+			    *reason |= AST_REBALANCE;
 		}
-		else
-		if (new_thread == old_thread) {
-			counter(++c_thread_invoke_same);
-			thread_unlock(new_thread);
-			return (TRUE);
-		}
-	}
 
-	/*
-	 * Context switch by full context save.
-	 */
-	processor = current_processor();
-	processor->active_thread = new_thread;
-	processor->current_pri = new_thread->sched_pri;
-	new_thread->last_processor = processor;
-	ast_context(new_thread);
-	assert(thread_runnable(new_thread));
-	thread_unlock(new_thread);
+		/* OK, so we're not going to run the current thread. Look at the RT queue. */
+		if (rt_runq_count(pset) > 0) {
 
-	counter_always(c_thread_invoke_csw++);
-	current_task()->csw++;
+			rt_lock_lock(pset);
 
-	assert(old_thread->runq == RUN_QUEUE_NULL);
-	old_thread->reason = reason;
+			if (rt_runq_count(pset) > 0) {
+				thread_t next_rt = qe_queue_first(&SCHED(rt_runq)(pset)->queue, struct thread, runq_links);
 
-	processor->last_dispatch = mach_absolute_time();
-	timer_event((uint32_t)processor->last_dispatch, &new_thread->system_timer);
+				if (__probable((next_rt->bound_processor == PROCESSOR_NULL ||
+						(next_rt->bound_processor == processor)))) {
+pick_new_rt_thread:
+					new_thread = qe_dequeue_head(&SCHED(rt_runq)(pset)->queue, struct thread, runq_links);
 
-	thread_done(old_thread, new_thread, processor);
+					new_thread->runq = PROCESSOR_NULL;
+					SCHED_STATS_RUNQ_CHANGE(&SCHED(rt_runq)(pset)->runq_stats, rt_runq_count(pset));
+					rt_runq_count_decr(pset);
 
-	/*
-	 * This is where we actually switch register context,
-	 * and address space if required.  Control will not
-	 * return here immediately.
-	 */
-	prev_thread = machine_switch_context(old_thread, continuation, new_thread);
+					processor->deadline = new_thread->realtime.deadline;
+					processor_state_update_from_thread(processor, new_thread);
 
-	/*
-	 * We are still old_thread, possibly on a different processor,
-	 * and new_thread is now stale.
-	 */
-	thread_begin(old_thread, old_thread->last_processor);
+					rt_lock_unlock(pset);
+					sched_update_pset_load_average(pset);
 
-	/*
-	 * Now dispatch the thread which resumed us.
-	 */
-	thread_dispatch(prev_thread);
+					processor_t ast_processor = PROCESSOR_NULL;
+					processor_t next_rt_processor = PROCESSOR_NULL;
+					sched_ipi_type_t ipi_type = SCHED_IPI_NONE;
+					sched_ipi_type_t next_rt_ipi_type = SCHED_IPI_NONE;
 
-	if (continuation) {
-		funnel_refunnel_check(old_thread, 3);
-		(void) spllo();
+					if (processor->processor_secondary != NULL) {
+						processor_t sprocessor = processor->processor_secondary;
+						if ((sprocessor->state == PROCESSOR_RUNNING) || (sprocessor->state == PROCESSOR_DISPATCHING)) {
+							ipi_type = sched_ipi_action(sprocessor, NULL, false, SCHED_IPI_EVENT_SMT_REBAL);
+							ast_processor = sprocessor;
+						}
+					}
+					if (rt_runq_count(pset) > 0) {
+						next_rt_processor = choose_processor_for_realtime_thread(pset);
+						if (next_rt_processor) {
+							next_rt_ipi_type = sched_ipi_action(next_rt_processor, NULL, false, SCHED_IPI_EVENT_PREEMPT);
+						}
+					}
+					pset_unlock(pset);
 
-		call_continuation(continuation, parameter, old_thread->wait_result);
-		/*NOTREACHED*/
-	}
+					if (ast_processor) {
+						sched_ipi_perform(ast_processor, ipi_type);
+					}
 
-	return (TRUE);
-}
+					if (next_rt_processor) {
+						sched_ipi_perform(next_rt_processor, next_rt_ipi_type);
+					}
 
-/*
- *	thread_done:
- *
- *	Perform calculations for thread
- *	finishing execution on the current processor.
- *
- *	Called at splsched.
- */
-void
-thread_done(
-	thread_t			old_thread,
-	thread_t			new_thread,
-	processor_t			processor)
-{
-	if (!(old_thread->state & TH_IDLE)) {
-		/*
-		 * Compute remainder of current quantum.
-		 */
-		if (	first_timeslice(processor)							&&
-				processor->quantum_end > processor->last_dispatch		)
-			old_thread->current_quantum =
-					(processor->quantum_end - processor->last_dispatch);
-		else
-			old_thread->current_quantum = 0;
+					return (new_thread);
+				}
+			}
+
+			rt_lock_unlock(pset);
+		}
+
+		processor->deadline = UINT64_MAX;
+
+		/* No RT threads, so let's look at the regular threads. */
+		if ((new_thread = SCHED(choose_thread)(processor, MINPRI, *reason)) != THREAD_NULL) {
+			sched_update_pset_load_average(pset);
+			processor_state_update_from_thread(processor, new_thread);
+			pset_unlock(pset);
+			return (new_thread);
+		}
 
-		if (old_thread->sched_mode & TH_MODE_REALTIME) {
+#if __SMP__
+		if (SCHED(steal_thread_enabled)) {
 			/*
-			 * Cancel the deadline if the thread has
-			 * consumed the entire quantum.
+			 * No runnable threads, attempt to steal
+			 * from other processors. Returns with pset lock dropped.
 			 */
-			if (old_thread->current_quantum == 0) {
-				old_thread->realtime.deadline = UINT64_MAX;
-				old_thread->reason |= AST_QUANTUM;
+
+			if ((new_thread = SCHED(steal_thread)(pset)) != THREAD_NULL) {
+				return (new_thread);
 			}
-		}
-		else {
+
 			/*
-			 * For non-realtime threads treat a tiny
-			 * remaining quantum as an expired quantum
-			 * but include what's left next time.
+			 * If other threads have appeared, shortcut
+			 * around again.
 			 */
-			if (old_thread->current_quantum < min_std_quantum) {
-				old_thread->reason |= AST_QUANTUM;
-				old_thread->current_quantum += std_quantum;
-			}
+			if (!SCHED(processor_queue_empty)(processor) || rt_runq_count(pset) > 0)
+				continue;
+
+			pset_lock(pset);
 		}
+#endif
 
+	idle:
 		/*
-		 * If we are doing a direct handoff then
-		 * give the remainder of our quantum to
-		 * the next thread.
+		 *	Nothing is runnable, so set this processor idle if it
+		 *	was running.
 		 */
-		if ((old_thread->reason & (AST_HANDOFF|AST_QUANTUM)) == AST_HANDOFF) {
-			new_thread->current_quantum = old_thread->current_quantum;
-			old_thread->reason |= AST_QUANTUM;
-			old_thread->current_quantum = 0;
+		if (processor->state == PROCESSOR_RUNNING) {
+			pset_update_processor_state(pset, processor, PROCESSOR_IDLE);
 		}
 
-		old_thread->last_switch = processor->last_dispatch;
+#if __SMP__
+		/* Invoked with pset locked, returns with pset unlocked */
+		SCHED(processor_balance)(processor, pset);
+#else
+		pset_unlock(pset);
+#endif
 
-		old_thread->computation_metered +=
-				(old_thread->last_switch - old_thread->computation_epoch);
-	}
-}
+#if CONFIG_SCHED_IDLE_IN_PLACE
+		/*
+		 *	Choose idle thread if fast idle is not possible.
+		 */
+		if (processor->processor_primary != processor)
+			return (processor->idle_thread);
+
+		if ((thread->state & (TH_IDLE|TH_TERMINATE|TH_SUSP)) || !(thread->state & TH_WAIT) || thread->wake_active || thread->sched_pri >= BASEPRI_RTQUEUES)
+			return (processor->idle_thread);
 
-/*
- *	thread_begin:
- *
- *	Set up for thread beginning execution on
- *	the current processor.
- *
- *	Called at splsched.
- */
-void
-thread_begin(
-	thread_t			thread,
-	processor_t			processor)
-{
-	if (!(thread->state & TH_IDLE)) {
 		/*
-		 * Give the thread a new quantum
-		 * if none remaining.
+		 *	Perform idling activities directly without a
+		 *	context switch.  Return dispatched thread,
+		 *	else check again for a runnable thread.
 		 */
-		if (thread->current_quantum == 0)
-			thread_quantum_init(thread);
+		new_thread = thread_select_idle(thread, processor);
 
+#else /* !CONFIG_SCHED_IDLE_IN_PLACE */
+		
 		/*
-		 * Set up quantum timer and timeslice.
+		 * Do a full context switch to idle so that the current
+		 * thread can start running on another processor without
+		 * waiting for the fast-idled processor to wake up.
 		 */
-		processor->quantum_end =
-				(processor->last_dispatch + thread->current_quantum);
-		timer_call_enter1(&processor->quantum_timer,
-								thread, processor->quantum_end);
+		new_thread = processor->idle_thread;
 
-		processor_timeslice_setup(processor, thread);
+#endif /* !CONFIG_SCHED_IDLE_IN_PLACE */
 
-		thread->last_switch = processor->last_dispatch;
+	} while (new_thread == THREAD_NULL);
 
-		thread->computation_epoch = thread->last_switch;
-	}
-	else {
-		timer_call_cancel(&processor->quantum_timer);
-		processor->timeslice = 1;
-	}
+	return (new_thread);
 }
 
+#if CONFIG_SCHED_IDLE_IN_PLACE
 /*
- *	thread_dispatch:
+ *	thread_select_idle:
  *
- *	Handle previous thread at context switch.  Re-dispatch
- *	if still running, otherwise update run state and perform
- *	special actions.
+ *	Idle the processor using the current thread context.
  *
- *	Called at splsched.
+ *	Called with thread locked, then dropped and relocked.
  */
-void
-thread_dispatch(
-	register thread_t	thread)
+static thread_t
+thread_select_idle(
+	thread_t		thread,
+	processor_t		processor)
 {
+	thread_t		new_thread;
+	uint64_t		arg1, arg2;
+	int			urgency;
+
+	sched_run_decr(thread);
+
+	thread->state |= TH_IDLE;
+	processor_state_update_idle(procssor);
+
+	/* Reload precise timing global policy to thread-local policy */
+	thread->precise_user_kernel_time = use_precise_user_kernel_time(thread);
+	
+	thread_unlock(thread);
+
 	/*
-	 *	If blocked at a continuation, discard
-	 *	the stack.
+	 *	Switch execution timing to processor idle thread.
 	 */
-#ifndef i386
-    if (thread->continuation != NULL && thread->kernel_stack)
-		stack_free(thread);
+	processor->last_dispatch = mach_absolute_time();
+
+#ifdef CONFIG_MACH_APPROXIMATE_TIME
+	commpage_update_mach_approximate_time(processor->last_dispatch);
 #endif
 
-	if (!(thread->state & TH_IDLE)) {
-		wake_lock(thread);
-		thread_lock(thread);
+	thread->last_run_time = processor->last_dispatch;
+	processor_timer_switch_thread(processor->last_dispatch,
+			&processor->idle_thread->system_timer);
+	PROCESSOR_DATA(processor, kernel_timer) = &processor->idle_thread->system_timer;
 
-		if (!(thread->state & TH_WAIT)) {
-			/*
-			 *	Still running.
-			 */
-			if (thread->reason & AST_QUANTUM)
-				thread_setrun(thread, SCHED_TAILQ);
-			else
-			if (thread->reason & AST_PREEMPT)
-				thread_setrun(thread, SCHED_HEADQ);
-			else
-				thread_setrun(thread, SCHED_PREEMPT | SCHED_TAILQ);
 
-			thread->reason = AST_NONE;
+	/*
+	 *	Cancel the quantum timer while idling.
+	 */
+	timer_call_quantum_timer_cancel(&processor->quantum_timer);
+	processor->first_timeslice = FALSE;
 
-			thread_unlock(thread);
-			wake_unlock(thread);
-		}
-		else {
-			boolean_t		wake;
+	if (thread->sched_call) {
+		(*thread->sched_call)(SCHED_CALL_BLOCK, thread);
+	}
 
-			/*
-			 *	Waiting.
-			 */
-			thread->state &= ~TH_RUN;
+	thread_tell_urgency(THREAD_URGENCY_NONE, 0, 0, 0, NULL);
 
-			wake = thread->wake_active;
-			thread->wake_active = FALSE;
+	/*
+	 *	Enable interrupts and perform idling activities.  No
+	 *	preemption due to TH_IDLE being set.
+	 */
+	spllo(); new_thread = processor_idle(thread, processor);
+
+	/*
+	 *	Return at splsched.
+	 */
+	if (thread->sched_call) {
+		(*thread->sched_call)(SCHED_CALL_UNBLOCK, thread);
+	}
 
-			if (thread->sched_mode & TH_MODE_TIMESHARE)
-				pset_share_decr(thread->processor_set);
-			pset_run_decr(thread->processor_set);
+	thread_lock(thread);
 
-			thread_unlock(thread);
-			wake_unlock(thread);
+	/*
+	 *	If awakened, switch to thread timer and start a new quantum.
+	 *	Otherwise skip; we will context switch to another thread or return here.
+	 */
+	if (!(thread->state & TH_WAIT)) {
+		uint64_t time_now = processor->last_dispatch = mach_absolute_time();
+		processor_timer_switch_thread(time_now, &thread->system_timer);
+		timer_update(&thread->runnable_timer, time_now);
+		PROCESSOR_DATA(processor, kernel_timer) = &thread->system_timer;
+		thread_quantum_init(thread);
+		processor->quantum_end = time_now + thread->quantum_remaining;
+		timer_call_quantum_timer_enter(&processor->quantum_timer,
+			thread, processor->quantum_end, time_now);
+		processor->first_timeslice = TRUE;
+
+		thread->computation_epoch = time_now;
+	}
 
-			if (thread->options & TH_OPT_CALLOUT)
-				call_thread_block();
+	thread->state &= ~TH_IDLE;
 
-			if (wake)
-				thread_wakeup((event_t)&thread->wake_active);
+	urgency = thread_get_urgency(thread, &arg1, &arg2);
 
-			if (thread->state & TH_TERMINATE)
-				thread_terminate_enqueue(thread);
-		}
-	}
+	thread_tell_urgency(urgency, arg1, arg2, 0, new_thread);
+
+	sched_run_incr(thread);
+
+	return (new_thread);
 }
+#endif /* CONFIG_SCHED_IDLE_IN_PLACE */
 
 /*
- *	thread_block_reason:
+ * thread_invoke
  *
- *	Forces a reschedule, blocking the caller if a wait
- *	has been asserted.
+ * Called at splsched with neither thread locked.
  *
- *	If a continuation is specified, then thread_invoke will
- *	attempt to discard the thread's kernel stack.  When the
- *	thread resumes, it will execute the continuation function
- *	on a new kernel stack.
+ * Perform a context switch and start executing the new thread.
+ *
+ * Returns FALSE when the context switch didn't happen.
+ * The reference to the new thread is still consumed.
+ *
+ * "self" is what is currently running on the processor,
+ * "thread" is the new thread to context switch to
+ * (which may be the same thread in some cases)
  */
-counter(mach_counter_t  c_thread_block_calls = 0;)
- 
-wait_result_t
-thread_block_reason(
-	thread_continue_t	continuation,
-	void				*parameter,
+static boolean_t
+thread_invoke(
+	thread_t			self,
+	thread_t			thread,
 	ast_t				reason)
 {
-	register thread_t		self = current_thread();
-	register processor_t	processor;
-	register thread_t		new_thread;
-	spl_t					s;
+	if (__improbable(get_preemption_level() != 0)) {
+		int pl = get_preemption_level();
+		panic("thread_invoke: preemption_level %d, possible cause: %s",
+		    pl, (pl < 0 ? "unlocking an unlocked mutex or spinlock" :
+			"blocking while holding a spinlock, or within interrupt context"));
+	}
 
-	counter(++c_thread_block_calls);
+	thread_continue_t       continuation = self->continuation;
+	void                    *parameter   = self->parameter;
+	processor_t             processor;
 
-	s = splsched();
+	uint64_t                ctime = mach_absolute_time();
 
-	if (!(reason & AST_PREEMPT))
-		funnel_release_check(self, 2);
+#ifdef CONFIG_MACH_APPROXIMATE_TIME
+	commpage_update_mach_approximate_time(ctime);
+#endif
 
-	processor = current_processor();
+#if defined(CONFIG_SCHED_TIMESHARE_CORE)
+	if ((thread->state & TH_IDLE) == 0)
+		sched_timeshare_consider_maintenance(ctime);
+#endif
 
-	/*
-	 * Delay switching to the idle thread under certain conditions.
-	 */
-	if (s != FALSE && (self->state & (TH_IDLE|TH_TERMINATE|TH_WAIT)) == TH_WAIT) {
-		if (	processor->processor_set->processor_count > 1	&&
-				processor->processor_set->runq.count == 0		&&
-				processor->runq.count == 0						)
-			processor = delay_idle(processor, self);
-	}
+#if MONOTONIC
+	mt_sched_update(self);
+#endif /* MONOTONIC */
 
-	/* If we're explicitly yielding, force a subsequent quantum */
-	if (reason & AST_YIELD)
-		processor->timeslice = 0;
+	assert_thread_magic(self);
+	assert(self == current_thread());
+	assert(self->runq == PROCESSOR_NULL);
+	assert((self->state & (TH_RUN|TH_TERMINATE2)) == TH_RUN);
 
-	/* We're handling all scheduling AST's */
-	ast_off(AST_SCHEDULING);
+	thread_lock(thread);
 
-	self->continuation = continuation;
-	self->parameter = parameter;
+	assert_thread_magic(thread);
+	assert((thread->state & (TH_RUN|TH_WAIT|TH_UNINT|TH_TERMINATE|TH_TERMINATE2)) == TH_RUN);
+	assert(thread->bound_processor == PROCESSOR_NULL || thread->bound_processor == current_processor());
+	assert(thread->runq == PROCESSOR_NULL);
 
-	thread_lock(self);
-	new_thread = thread_select(processor);
-	assert(new_thread && thread_runnable(new_thread));
-	thread_unlock(self);
-	while (!thread_invoke(self, new_thread, reason)) {
-		thread_lock(self);
-		new_thread = thread_select(processor);
-		assert(new_thread && thread_runnable(new_thread));
-		thread_unlock(self);
-	}
+	/* Reload precise timing global policy to thread-local policy */
+	thread->precise_user_kernel_time = use_precise_user_kernel_time(thread);
 
-	funnel_refunnel_check(self, 5);
-	splx(s);
+	/* Update SFI class based on other factors */
+	thread->sfi_class = sfi_thread_classify(thread);
+	   
+	/* Update the same_pri_latency for the thread (used by perfcontrol callouts) */
+	thread->same_pri_latency = ctime - thread->last_basepri_change_time;
+	/* 
+	 * In case a base_pri update happened between the timestamp and 
+	 * taking the thread lock 
+	 */
+	if (ctime <= thread->last_basepri_change_time)
+		thread->same_pri_latency = ctime - thread->last_made_runnable_time;
 
-	return (self->wait_result);
-}
+	/* Allow realtime threads to hang onto a stack. */
+	if ((self->sched_mode == TH_MODE_REALTIME) && !self->reserved_stack)
+		self->reserved_stack = self->kernel_stack;
 
-/*
- *	thread_block:
- *
- *	Block the current thread if a wait has been asserted.
- */
-wait_result_t
-thread_block(
-	thread_continue_t	continuation)
-{
-	return thread_block_reason(continuation, NULL, AST_NONE);
-}
+    /* Prepare for spin debugging */
+#if INTERRUPT_MASKED_DEBUG
+	ml_spin_debug_clear(thread);
+#endif
 
-wait_result_t
+	if (continuation != NULL) {
+		if (!thread->kernel_stack) {
+			/*
+			 * If we are using a privileged stack,
+			 * check to see whether we can exchange it with
+			 * that of the other thread.
+			 */
+			if (self->kernel_stack == self->reserved_stack && !thread->reserved_stack)
+				goto need_stack;
+
+			/*
+			 * Context switch by performing a stack handoff.
+			 */
+			continuation = thread->continuation;
+			parameter = thread->parameter;
+
+			processor = current_processor();
+			processor->active_thread = thread;
+			processor_state_update_from_thread(processor, thread);
+
+			if (thread->last_processor != processor && thread->last_processor != NULL) {
+				if (thread->last_processor->processor_set != processor->processor_set)
+					thread->ps_switch++;
+				thread->p_switch++;
+			}
+			thread->last_processor = processor;
+			thread->c_switch++;
+			ast_context(thread);
+
+			thread_unlock(thread);
+
+			self->reason = reason;
+
+			processor->last_dispatch = ctime;
+			self->last_run_time = ctime;
+			processor_timer_switch_thread(ctime, &thread->system_timer);
+			timer_update(&thread->runnable_timer, ctime);
+			PROCESSOR_DATA(processor, kernel_timer) = &thread->system_timer;
+
+			/*
+			 * Since non-precise user/kernel time doesn't update the state timer
+			 * during privilege transitions, synthesize an event now.
+			 */
+			if (!thread->precise_user_kernel_time) {
+				timer_update(PROCESSOR_DATA(processor, current_state), ctime);
+			}
+
+			KERNEL_DEBUG_CONSTANT_IST(KDEBUG_TRACE,
+				MACHDBG_CODE(DBG_MACH_SCHED, MACH_STACK_HANDOFF)|DBG_FUNC_NONE,
+				self->reason, (uintptr_t)thread_tid(thread), self->sched_pri, thread->sched_pri, 0);
+
+			if ((thread->chosen_processor != processor) && (thread->chosen_processor != PROCESSOR_NULL)) {
+				SCHED_DEBUG_CHOOSE_PROCESSOR_KERNEL_DEBUG_CONSTANT(MACHDBG_CODE(DBG_MACH_SCHED, MACH_MOVED)|DBG_FUNC_NONE,
+						(uintptr_t)thread_tid(thread), (uintptr_t)thread->chosen_processor->cpu_id, 0, 0, 0);
+			}
+
+			DTRACE_SCHED2(off__cpu, struct thread *, thread, struct proc *, thread->task->bsd_info);
+
+			SCHED_STATS_CSW(processor, self->reason, self->sched_pri, thread->sched_pri);
+
+#if KPERF
+			kperf_off_cpu(self);
+#endif /* KPERF */
+
+			TLOG(1, "thread_invoke: calling stack_handoff\n");
+			stack_handoff(self, thread);
+
+			/* 'self' is now off core */
+			assert(thread == current_thread_volatile());
+
+			DTRACE_SCHED(on__cpu);
+
+#if KPERF
+			kperf_on_cpu(thread, continuation, NULL);
+#endif /* KPERF */
+
+			thread_dispatch(self, thread);
+
+#if KASAN
+			/* Old thread's stack has been moved to the new thread, so explicitly
+			 * unpoison it. */
+			kasan_unpoison_stack(thread->kernel_stack, kernel_stack_size);
+#endif
+
+			thread->continuation = thread->parameter = NULL;
+
+			counter(c_thread_invoke_hits++);
+
+			assert(continuation);
+			call_continuation(continuation, parameter, thread->wait_result, TRUE);
+			/*NOTREACHED*/
+		}
+		else if (thread == self) {
+			/* same thread but with continuation */
+			ast_context(self);
+			counter(++c_thread_invoke_same);
+
+			thread_unlock(self);
+
+#if KPERF
+			kperf_on_cpu(thread, continuation, NULL);
+#endif /* KPERF */
+
+			KERNEL_DEBUG_CONSTANT_IST(KDEBUG_TRACE,
+				MACHDBG_CODE(DBG_MACH_SCHED,MACH_SCHED) | DBG_FUNC_NONE,
+				self->reason, (uintptr_t)thread_tid(thread), self->sched_pri, thread->sched_pri, 0);
+
+#if KASAN
+			/* stack handoff to self - no thread_dispatch(), so clear the stack
+			 * and free the fakestack directly */
+			kasan_fakestack_drop(self);
+			kasan_fakestack_gc(self);
+			kasan_unpoison_stack(self->kernel_stack, kernel_stack_size);
+#endif
+
+			self->continuation = self->parameter = NULL;
+
+			call_continuation(continuation, parameter, self->wait_result, TRUE);
+			/*NOTREACHED*/
+		}
+	} else {
+		/*
+		 * Check that the other thread has a stack
+		 */
+		if (!thread->kernel_stack) {
+need_stack:
+			if (!stack_alloc_try(thread)) {
+				counter(c_thread_invoke_misses++);
+				thread_unlock(thread);
+				thread_stack_enqueue(thread);
+				return (FALSE);
+			}
+		} else if (thread == self) {
+			ast_context(self);
+			counter(++c_thread_invoke_same);
+			thread_unlock(self);
+
+			KERNEL_DEBUG_CONSTANT_IST(KDEBUG_TRACE,
+				MACHDBG_CODE(DBG_MACH_SCHED,MACH_SCHED) | DBG_FUNC_NONE,
+				self->reason, (uintptr_t)thread_tid(thread), self->sched_pri, thread->sched_pri, 0);
+
+			return (TRUE);
+		}
+	}
+
+	/*
+	 * Context switch by full context save.
+	 */
+	processor = current_processor();
+	processor->active_thread = thread;
+	processor_state_update_from_thread(processor, thread);
+	
+	if (thread->last_processor != processor && thread->last_processor != NULL) {
+		if (thread->last_processor->processor_set != processor->processor_set)
+			thread->ps_switch++;
+		thread->p_switch++;
+	}
+	thread->last_processor = processor;
+	thread->c_switch++;
+	ast_context(thread);
+
+	thread_unlock(thread);
+
+	counter(c_thread_invoke_csw++);
+
+	self->reason = reason;
+
+	processor->last_dispatch = ctime;
+	self->last_run_time = ctime;
+	processor_timer_switch_thread(ctime, &thread->system_timer);
+	timer_update(&thread->runnable_timer, ctime);
+	PROCESSOR_DATA(processor, kernel_timer) = &thread->system_timer;
+
+	/*
+	 * Since non-precise user/kernel time doesn't update the state timer
+	 * during privilege transitions, synthesize an event now.
+	 */
+	if (!thread->precise_user_kernel_time) {
+		timer_update(PROCESSOR_DATA(processor, current_state), ctime);
+	}
+
+	KERNEL_DEBUG_CONSTANT_IST(KDEBUG_TRACE,
+		MACHDBG_CODE(DBG_MACH_SCHED,MACH_SCHED) | DBG_FUNC_NONE,
+		self->reason, (uintptr_t)thread_tid(thread), self->sched_pri, thread->sched_pri, 0);
+
+	if ((thread->chosen_processor != processor) && (thread->chosen_processor != NULL)) {
+		SCHED_DEBUG_CHOOSE_PROCESSOR_KERNEL_DEBUG_CONSTANT(MACHDBG_CODE(DBG_MACH_SCHED, MACH_MOVED)|DBG_FUNC_NONE,
+				(uintptr_t)thread_tid(thread), (uintptr_t)thread->chosen_processor->cpu_id, 0, 0, 0);
+	}
+
+	DTRACE_SCHED2(off__cpu, struct thread *, thread, struct proc *, thread->task->bsd_info);
+
+	SCHED_STATS_CSW(processor, self->reason, self->sched_pri, thread->sched_pri);
+
+#if KPERF
+	kperf_off_cpu(self);
+#endif /* KPERF */
+
+	/*
+	 * This is where we actually switch register context,
+	 * and address space if required.  We will next run
+	 * as a result of a subsequent context switch.
+	 *
+	 * Once registers are switched and the processor is running "thread",
+	 * the stack variables and non-volatile registers will contain whatever
+	 * was there the last time that thread blocked. No local variables should
+	 * be used after this point, except for the special case of "thread", which
+	 * the platform layer returns as the previous thread running on the processor
+	 * via the function call ABI as a return register, and "self", which may have
+	 * been stored on the stack or a non-volatile register, but a stale idea of
+	 * what was on the CPU is newly-accurate because that thread is again
+	 * running on the CPU.
+	 */
+	assert(continuation == self->continuation);
+	thread = machine_switch_context(self, continuation, thread);
+	assert(self == current_thread_volatile());
+	TLOG(1,"thread_invoke: returning machine_switch_context: self %p continuation %p thread %p\n", self, continuation, thread);
+
+	DTRACE_SCHED(on__cpu);
+
+#if KPERF
+	kperf_on_cpu(self, NULL, __builtin_frame_address(0));
+#endif /* KPERF */
+
+	/*
+	 * We have been resumed and are set to run.
+	 */
+	thread_dispatch(thread, self);
+
+	if (continuation) {
+		self->continuation = self->parameter = NULL;
+
+		call_continuation(continuation, parameter, self->wait_result, TRUE);
+		/*NOTREACHED*/
+	}
+
+	return (TRUE);
+}
+
+#if defined(CONFIG_SCHED_DEFERRED_AST)
+/*
+ *	pset_cancel_deferred_dispatch:
+ *
+ *	Cancels all ASTs that we can cancel for the given processor set
+ *	if the current processor is running the last runnable thread in the
+ *	system.
+ *
+ *	This function assumes the current thread is runnable.  This must
+ *	be called with the pset unlocked.
+ */
+static void
+pset_cancel_deferred_dispatch(
+	processor_set_t		pset,
+	processor_t		processor)
+{
+	processor_t		active_processor = NULL;
+	uint32_t		sampled_sched_run_count;
+
+	pset_lock(pset);
+	sampled_sched_run_count = (volatile uint32_t) sched_run_buckets[TH_BUCKET_RUN];
+
+	/*
+	 * If we have emptied the run queue, and our current thread is runnable, we
+	 * should tell any processors that are still DISPATCHING that they will
+	 * probably not have any work to do.  In the event that there are no
+	 * pending signals that we can cancel, this is also uninteresting.
+	 *
+	 * In the unlikely event that another thread becomes runnable while we are
+	 * doing this (sched_run_count is atomically updated, not guarded), the
+	 * codepath making it runnable SHOULD (a dangerous word) need the pset lock
+	 * in order to dispatch it to a processor in our pset.  So, the other
+	 * codepath will wait while we squash all cancelable ASTs, get the pset
+	 * lock, and then dispatch the freshly runnable thread.  So this should be
+	 * correct (we won't accidentally have a runnable thread that hasn't been
+	 * dispatched to an idle processor), if not ideal (we may be restarting the
+	 * dispatch process, which could have some overhead).
+	 */
+
+	if ((sampled_sched_run_count == 1) && (pset->pending_deferred_AST_cpu_mask)) {
+		uint64_t dispatching_map = (pset->cpu_state_map[PROCESSOR_DISPATCHING] &
+		                            pset->pending_deferred_AST_cpu_mask &
+		                            ~pset->pending_AST_cpu_mask);
+		for (int cpuid = lsb_first(dispatching_map); cpuid >= 0; cpuid = lsb_next(dispatching_map, cpuid)) {
+			active_processor = processor_array[cpuid];
+			/*
+			 * If a processor is DISPATCHING, it could be because of
+			 * a cancelable signal.
+			 *
+			 * IF the processor is not our
+			 * current processor (the current processor should not
+			 * be DISPATCHING, so this is a bit paranoid), AND there
+			 * is a cancelable signal pending on the processor, AND
+			 * there is no non-cancelable signal pending (as there is
+			 * no point trying to backtrack on bringing the processor
+			 * up if a signal we cannot cancel is outstanding), THEN
+			 * it should make sense to roll back the processor state
+			 * to the IDLE state.
+			 *
+			 * If the racey nature of this approach (as the signal
+			 * will be arbitrated by hardware, and can fire as we
+			 * roll back state) results in the core responding
+			 * despite being pushed back to the IDLE state, it
+			 * should be no different than if the core took some
+			 * interrupt while IDLE.
+			 */
+			if (active_processor != processor) {
+				/*
+				 * Squash all of the processor state back to some
+				 * reasonable facsimile of PROCESSOR_IDLE.
+				 */
+
+				assert(active_processor->next_thread == THREAD_NULL);
+				processor_state_update_idle(active_processor);
+				active_processor->deadline = UINT64_MAX;
+				pset_update_processor_state(pset, active_processor, PROCESSOR_IDLE);
+				bit_clear(pset->pending_deferred_AST_cpu_mask, active_processor->cpu_id);
+				machine_signal_idle_cancel(active_processor);
+			}
+
+		}
+	}
+
+	pset_unlock(pset);
+}
+#else
+/* We don't support deferred ASTs; everything is candycanes and sunshine. */
+#endif
+
+static void
+thread_csw_callout(
+	thread_t	    old,
+	thread_t	    new,
+	uint64_t	    timestamp)
+{
+	perfcontrol_event event = (new->state & TH_IDLE) ? IDLE : CONTEXT_SWITCH;
+	uint64_t same_pri_latency = (new->state & TH_IDLE) ? 0 : new->same_pri_latency;
+	machine_switch_perfcontrol_context(event, timestamp, 0, 
+		same_pri_latency, old, new);
+}
+
+
+/*
+ *	thread_dispatch:
+ *
+ *	Handle threads at context switch.  Re-dispatch other thread
+ *	if still running, otherwise update run state and perform
+ *	special actions.  Update quantum for other thread and begin
+ *	the quantum for ourselves.
+ *
+ *      "thread" is the old thread that we have switched away from.
+ *      "self" is the new current thread that we have context switched to
+ *
+ *	Called at splsched.
+ */
+void
+thread_dispatch(
+	thread_t		thread,
+	thread_t		self)
+{
+	processor_t		processor = self->last_processor;
+
+	assert(processor == current_processor());
+	assert(self == current_thread_volatile());
+	assert(thread != self);
+
+	if (thread != THREAD_NULL) {
+		/* 
+		 * Do the perfcontrol callout for context switch. 
+		 * The reason we do this here is:
+		 * - thread_dispatch() is called from various places that are not 
+		 *   the direct context switch path for eg. processor shutdown etc.
+		 *   So adding the callout here covers all those cases.
+		 * - We want this callout as early as possible to be close 
+		 *   to the timestamp taken in thread_invoke()
+		 * - We want to avoid holding the thread lock while doing the 
+		 *   callout
+		 * - We do not want to callout if "thread" is NULL.
+		 */
+		thread_csw_callout(thread, self, processor->last_dispatch);	
+
+#if KASAN
+		if (thread->continuation != NULL) {
+			/*
+			 * Thread has a continuation and the normal stack is going away.
+			 * Unpoison the stack and mark all fakestack objects as unused.
+			 */
+			kasan_fakestack_drop(thread);
+			if (thread->kernel_stack) {
+				kasan_unpoison_stack(thread->kernel_stack, kernel_stack_size);
+			}
+		}
+
+		/*
+		 * Free all unused fakestack objects.
+		 */
+		kasan_fakestack_gc(thread);
+#endif
+
+		/*
+		 *	If blocked at a continuation, discard
+		 *	the stack.
+		 */
+		if (thread->continuation != NULL && thread->kernel_stack != 0)
+			stack_free(thread);
+
+		if (thread->state & TH_IDLE) {
+			KERNEL_DEBUG_CONSTANT_IST(KDEBUG_TRACE,
+			        MACHDBG_CODE(DBG_MACH_SCHED,MACH_DISPATCH) | DBG_FUNC_NONE,
+			        (uintptr_t)thread_tid(thread), 0, thread->state,
+			        sched_run_buckets[TH_BUCKET_RUN], 0);
+		} else {
+			int64_t consumed;
+			int64_t remainder = 0;
+
+			if (processor->quantum_end > processor->last_dispatch)
+				remainder = processor->quantum_end -
+				    processor->last_dispatch;
+
+			consumed = thread->quantum_remaining - remainder;
+
+			if ((thread->reason & AST_LEDGER) == 0) {
+				/*
+				 * Bill CPU time to both the task and
+				 * the individual thread.
+				 */
+				ledger_credit_thread(thread, thread->t_ledger,
+				                     task_ledgers.cpu_time, consumed);
+				ledger_credit_thread(thread, thread->t_threadledger,
+				                     thread_ledgers.cpu_time, consumed);
+				if (thread->t_bankledger) {
+					ledger_credit_thread(thread, thread->t_bankledger,
+					                     bank_ledgers.cpu_time,
+					                     (consumed - thread->t_deduct_bank_ledger_time));
+				}
+				thread->t_deduct_bank_ledger_time = 0;
+			}
+
+			wake_lock(thread);
+			thread_lock(thread);
+
+			/*
+			 * Apply a priority floor if the thread holds a kernel resource
+			 * Do this before checking starting_pri to avoid overpenalizing
+			 * repeated rwlock blockers.
+			 */
+			if (__improbable(thread->rwlock_count != 0))
+				lck_rw_set_promotion_locked(thread);
+
+			boolean_t keep_quantum = processor->first_timeslice;
+
+			/*
+			 * Treat a thread which has dropped priority since it got on core
+			 * as having expired its quantum.
+			 */
+			if (processor->starting_pri > thread->sched_pri)
+				keep_quantum = FALSE;
+
+			/* Compute remainder of current quantum. */
+			if (keep_quantum &&
+			    processor->quantum_end > processor->last_dispatch)
+				thread->quantum_remaining = (uint32_t)remainder;
+			else
+				thread->quantum_remaining = 0;
+
+			if (thread->sched_mode == TH_MODE_REALTIME) {
+				/*
+				 *	Cancel the deadline if the thread has
+				 *	consumed the entire quantum.
+				 */
+				if (thread->quantum_remaining == 0) {
+					thread->realtime.deadline = UINT64_MAX;
+				}
+			} else {
+#if defined(CONFIG_SCHED_TIMESHARE_CORE)
+				/*
+				 *	For non-realtime threads treat a tiny
+				 *	remaining quantum as an expired quantum
+				 *	but include what's left next time.
+				 */
+				if (thread->quantum_remaining < min_std_quantum) {
+					thread->reason |= AST_QUANTUM;
+					thread->quantum_remaining += SCHED(initial_quantum_size)(thread);
+				}
+#endif /* CONFIG_SCHED_TIMESHARE_CORE */
+			}
+
+			/*
+			 *	If we are doing a direct handoff then
+			 *	take the remainder of the quantum.
+			 */
+			if ((thread->reason & (AST_HANDOFF|AST_QUANTUM)) == AST_HANDOFF) {
+				self->quantum_remaining = thread->quantum_remaining;
+				thread->reason |= AST_QUANTUM;
+				thread->quantum_remaining = 0;
+			} else {
+#if defined(CONFIG_SCHED_MULTIQ)
+				if (SCHED(sched_groups_enabled) &&
+				    thread->sched_group == self->sched_group) {
+					KERNEL_DEBUG_CONSTANT_IST(KDEBUG_TRACE,
+					    MACHDBG_CODE(DBG_MACH_SCHED, MACH_QUANTUM_HANDOFF),
+					    self->reason, (uintptr_t)thread_tid(thread),
+					    self->quantum_remaining, thread->quantum_remaining, 0);
+
+					self->quantum_remaining = thread->quantum_remaining;
+					thread->quantum_remaining = 0;
+					/* Don't set AST_QUANTUM here - old thread might still want to preempt someone else */
+				}
+#endif /* defined(CONFIG_SCHED_MULTIQ) */
+			}
+
+			thread->computation_metered += (processor->last_dispatch - thread->computation_epoch);
+
+			if (!(thread->state & TH_WAIT)) {
+				/*
+				 *	Still runnable.
+				 */
+				thread->last_made_runnable_time = thread->last_basepri_change_time = processor->last_dispatch;
+
+				machine_thread_going_off_core(thread, FALSE, processor->last_dispatch);
+
+				ast_t reason = thread->reason;
+				sched_options_t options = SCHED_NONE;
+
+				if (reason & AST_REBALANCE) {
+					options |= SCHED_REBALANCE;
+					if (reason & AST_QUANTUM) {
+						/*
+						 * Having gone to the trouble of forcing this thread off a less preferred core,
+						 * we should force the preferable core to reschedule immediately to give this
+						 * thread a chance to run instead of just sitting on the run queue where
+						 * it may just be stolen back by the idle core we just forced it off.
+						 * But only do this at the end of a quantum to prevent cascading effects.
+						 */
+						options |= SCHED_PREEMPT;
+					}
+				}
+
+				if (reason & AST_QUANTUM)
+					options |= SCHED_TAILQ;
+				else if (reason & AST_PREEMPT)
+					options |= SCHED_HEADQ;
+				else
+					options |= (SCHED_PREEMPT | SCHED_TAILQ);
+
+				thread_setrun(thread, options);
+
+				KERNEL_DEBUG_CONSTANT_IST(KDEBUG_TRACE,
+				        MACHDBG_CODE(DBG_MACH_SCHED,MACH_DISPATCH) | DBG_FUNC_NONE,
+				        (uintptr_t)thread_tid(thread), thread->reason, thread->state,
+				        sched_run_buckets[TH_BUCKET_RUN], 0);
+
+				if (thread->wake_active) {
+					thread->wake_active = FALSE;
+					thread_unlock(thread);
+
+					thread_wakeup(&thread->wake_active);
+				} else {
+					thread_unlock(thread);
+				}
+
+				wake_unlock(thread);
+			} else {
+				/*
+				 *	Waiting.
+				 */
+				boolean_t should_terminate = FALSE;
+				uint32_t new_run_count;
+				int thread_state = thread->state;
+
+				/* Only the first call to thread_dispatch
+				 * after explicit termination should add
+				 * the thread to the termination queue
+				 */
+				if ((thread_state & (TH_TERMINATE|TH_TERMINATE2)) == TH_TERMINATE) {
+					should_terminate = TRUE;
+					thread_state |= TH_TERMINATE2;
+				}
+
+				timer_stop(&thread->runnable_timer, processor->last_dispatch);
+
+				thread_state &= ~TH_RUN;
+				thread->state = thread_state;
+
+				thread->last_made_runnable_time = thread->last_basepri_change_time = THREAD_NOT_RUNNABLE;
+				thread->chosen_processor = PROCESSOR_NULL;
+
+				new_run_count = sched_run_decr(thread);
+
+#if CONFIG_SCHED_SFI
+				if (thread->reason & AST_SFI) {
+					thread->wait_sfi_begin_time = processor->last_dispatch;
+				}
+#endif
+
+				machine_thread_going_off_core(thread, should_terminate, processor->last_dispatch);
+
+				KERNEL_DEBUG_CONSTANT_IST(KDEBUG_TRACE,
+				        MACHDBG_CODE(DBG_MACH_SCHED,MACH_DISPATCH) | DBG_FUNC_NONE,
+				        (uintptr_t)thread_tid(thread), thread->reason, thread_state,
+				        new_run_count, 0);
+
+				if (thread_state & TH_WAIT_REPORT) {
+					(*thread->sched_call)(SCHED_CALL_BLOCK, thread);
+				}
+
+				if (thread->wake_active) {
+					thread->wake_active = FALSE;
+					thread_unlock(thread);
+
+					thread_wakeup(&thread->wake_active);
+				} else {
+					thread_unlock(thread);
+				}
+
+				wake_unlock(thread);
+
+				if (should_terminate)
+					thread_terminate_enqueue(thread);
+			}
+		}
+	}
+
+	int urgency = THREAD_URGENCY_NONE;
+	uint64_t latency = 0;
+
+	/* Update (new) current thread and reprogram quantum timer */
+	thread_lock(self);
+	
+	if (!(self->state & TH_IDLE)) {
+		uint64_t        arg1, arg2;
+
+#if CONFIG_SCHED_SFI
+		ast_t			new_ast;
+
+		new_ast = sfi_thread_needs_ast(self, NULL);
+
+		if (new_ast != AST_NONE) {
+			ast_on(new_ast);
+		}
+#endif
+
+		assertf(processor->last_dispatch >= self->last_made_runnable_time,
+		        "Non-monotonic time? dispatch at 0x%llx, runnable at 0x%llx",
+		        processor->last_dispatch, self->last_made_runnable_time);
+
+		assert(self->last_made_runnable_time <= self->last_basepri_change_time);
+
+		latency = processor->last_dispatch - self->last_made_runnable_time;
+		assert(latency >= self->same_pri_latency);
+
+		urgency = thread_get_urgency(self, &arg1, &arg2);
+
+		thread_tell_urgency(urgency, arg1, arg2, latency, self);
+
+		/*
+		 *	Get a new quantum if none remaining.
+		 */
+		if (self->quantum_remaining == 0) {
+			thread_quantum_init(self);
+		}
+
+		/*
+		 *	Set up quantum timer and timeslice.
+		 */
+		processor->quantum_end = processor->last_dispatch + self->quantum_remaining;
+		timer_call_quantum_timer_enter(&processor->quantum_timer, self,
+			processor->quantum_end, processor->last_dispatch);
+
+		processor->first_timeslice = TRUE;
+	} else {
+		timer_call_quantum_timer_cancel(&processor->quantum_timer);
+		processor->first_timeslice = FALSE;
+
+		thread_tell_urgency(THREAD_URGENCY_NONE, 0, 0, 0, self);
+	}
+
+	assert(self->block_hint == kThreadWaitNone);
+	self->computation_epoch = processor->last_dispatch;
+	self->reason = AST_NONE;
+	processor->starting_pri = self->sched_pri;
+
+	thread_unlock(self);
+
+	machine_thread_going_on_core(self, urgency, latency, self->same_pri_latency,
+	                             processor->last_dispatch);
+
+#if defined(CONFIG_SCHED_DEFERRED_AST)
+	/*
+	 * TODO: Can we state that redispatching our old thread is also
+	 * uninteresting?
+	 */
+	if ((((volatile uint32_t)sched_run_buckets[TH_BUCKET_RUN]) == 1) &&
+	    !(self->state & TH_IDLE)) {
+		pset_cancel_deferred_dispatch(processor->processor_set, processor);
+	}
+#endif
+}
+
+/*
+ *	thread_block_reason:
+ *
+ *	Forces a reschedule, blocking the caller if a wait
+ *	has been asserted.
+ *
+ *	If a continuation is specified, then thread_invoke will
+ *	attempt to discard the thread's kernel stack.  When the
+ *	thread resumes, it will execute the continuation function
+ *	on a new kernel stack.
+ */
+counter(mach_counter_t  c_thread_block_calls = 0;)
+ 
+wait_result_t
+thread_block_reason(
+	thread_continue_t	continuation,
+	void				*parameter,
+	ast_t				reason)
+{
+	thread_t        self = current_thread();
+	processor_t     processor;
+	thread_t        new_thread;
+	spl_t           s;
+
+	counter(++c_thread_block_calls);
+
+	s = splsched();
+
+	processor = current_processor();
+
+	/* If we're explicitly yielding, force a subsequent quantum */
+	if (reason & AST_YIELD)
+		processor->first_timeslice = FALSE;
+
+	/* We're handling all scheduling AST's */
+	ast_off(AST_SCHEDULING);
+
+#if PROC_REF_DEBUG
+	if ((continuation != NULL) && (self->task != kernel_task)) {
+		if (uthread_get_proc_refcount(self->uthread) != 0) {
+			panic("thread_block_reason with continuation uthread %p with uu_proc_refcount != 0", self->uthread);
+		}
+	}
+#endif
+
+	self->continuation = continuation;
+	self->parameter = parameter;
+
+	if (self->state & ~(TH_RUN | TH_IDLE)) {
+		KERNEL_DEBUG_CONSTANT_IST(KDEBUG_TRACE,
+			MACHDBG_CODE(DBG_MACH_SCHED,MACH_BLOCK), 
+			reason, VM_KERNEL_UNSLIDE(continuation), 0, 0, 0);
+	}
+
+	do {
+		thread_lock(self);
+		new_thread = thread_select(self, processor, &reason);
+		thread_unlock(self);
+	} while (!thread_invoke(self, new_thread, reason));
+
+	splx(s);
+
+	return (self->wait_result);
+}
+
+/*
+ *	thread_block:
+ *
+ *	Block the current thread if a wait has been asserted.
+ */
+wait_result_t
+thread_block(
+	thread_continue_t	continuation)
+{
+	return thread_block_reason(continuation, NULL, AST_NONE);
+}
+
+wait_result_t
 thread_block_parameter(
 	thread_continue_t	continuation,
 	void				*parameter)
 {
-	return thread_block_reason(continuation, parameter, AST_NONE);
+	return thread_block_reason(continuation, parameter, AST_NONE);
+}
+
+/*
+ *	thread_run:
+ *
+ *	Switch directly from the current thread to the
+ *	new thread, handing off our quantum if appropriate.
+ *
+ *	New thread must be runnable, and not on a run queue.
+ *
+ *	Called at splsched.
+ */
+int
+thread_run(
+	thread_t			self,
+	thread_continue_t	continuation,
+	void				*parameter,
+	thread_t			new_thread)
+{
+	ast_t reason = AST_HANDOFF;
+
+	self->continuation = continuation;
+	self->parameter = parameter;
+
+	while (!thread_invoke(self, new_thread, reason)) {
+		/* the handoff failed, so we have to fall back to the normal block path */
+		processor_t processor = current_processor();
+
+		reason = AST_NONE;
+
+		thread_lock(self);
+		new_thread = thread_select(self, processor, &reason);
+		thread_unlock(self);
+	}
+
+	return (self->wait_result);
+}
+
+/*
+ *	thread_continue:
+ *
+ *	Called at splsched when a thread first receives
+ *	a new stack after a continuation.
+ */
+void
+thread_continue(
+	thread_t	thread)
+{
+	thread_t                self = current_thread();
+	thread_continue_t       continuation;
+	void                    *parameter;
+
+	DTRACE_SCHED(on__cpu);
+
+	continuation = self->continuation;
+	parameter = self->parameter;
+
+#if KPERF
+	kperf_on_cpu(self, continuation, NULL);
+#endif
+
+	thread_dispatch(thread, self);
+
+	self->continuation = self->parameter = NULL;
+
+#if INTERRUPT_MASKED_DEBUG
+    /* Reset interrupt-masked spin debugging timeout */
+    ml_spin_debug_clear(self);
+#endif
+
+	TLOG(1, "thread_continue: calling call_continuation\n");
+	
+	boolean_t enable_interrupts = thread != THREAD_NULL;
+	call_continuation(continuation, parameter, self->wait_result, enable_interrupts);
+	/*NOTREACHED*/
+}
+
+void
+thread_quantum_init(thread_t thread)
+{
+	if (thread->sched_mode == TH_MODE_REALTIME) {
+		thread->quantum_remaining = thread->realtime.computation;
+	} else {
+		thread->quantum_remaining = SCHED(initial_quantum_size)(thread);
+	}
+}
+
+uint32_t
+sched_timeshare_initial_quantum_size(thread_t thread)
+{
+	if ((thread != THREAD_NULL) && thread->th_sched_bucket == TH_BUCKET_SHARE_BG)
+		return bg_quantum;
+	else
+		return std_quantum;
+}
+
+/*
+ *	run_queue_init:
+ *
+ *	Initialize a run queue before first use.
+ */
+void
+run_queue_init(
+	run_queue_t		rq)
+{
+	rq->highq = NOPRI;
+	for (u_int i = 0; i < BITMAP_LEN(NRQS); i++)
+		rq->bitmap[i] = 0;
+	rq->urgency = rq->count = 0;
+	for (int i = 0; i < NRQS; i++)
+		queue_init(&rq->queues[i]);
+}
+
+/*
+ *	run_queue_dequeue:
+ *
+ *	Perform a dequeue operation on a run queue,
+ *	and return the resulting thread.
+ *
+ *	The run queue must be locked (see thread_run_queue_remove()
+ *	for more info), and not empty.
+ */
+thread_t
+run_queue_dequeue(
+                  run_queue_t   rq,
+                  integer_t     options)
+{
+	thread_t    thread;
+	queue_t     queue = &rq->queues[rq->highq];
+
+	if (options & SCHED_HEADQ) {
+		thread = qe_dequeue_head(queue, struct thread, runq_links);
+	} else {
+		thread = qe_dequeue_tail(queue, struct thread, runq_links);
+	}
+
+	assert(thread != THREAD_NULL);
+	assert_thread_magic(thread);
+
+	thread->runq = PROCESSOR_NULL;
+	SCHED_STATS_RUNQ_CHANGE(&rq->runq_stats, rq->count);
+	rq->count--;
+	if (SCHED(priority_is_urgent)(rq->highq)) {
+		rq->urgency--; assert(rq->urgency >= 0);
+	}
+	if (queue_empty(queue)) {
+		bitmap_clear(rq->bitmap, rq->highq);
+		rq->highq = bitmap_first(rq->bitmap, NRQS);
+	}
+
+	return thread;
+}
+
+/*
+ *	run_queue_enqueue:
+ *
+ *	Perform a enqueue operation on a run queue.
+ *
+ *	The run queue must be locked (see thread_run_queue_remove()
+ *	for more info).
+ */
+boolean_t
+run_queue_enqueue(
+                  run_queue_t   rq,
+                  thread_t      thread,
+                  integer_t     options)
+{
+	queue_t     queue = &rq->queues[thread->sched_pri];
+	boolean_t   result = FALSE;
+
+	assert_thread_magic(thread);
+
+	if (queue_empty(queue)) {
+		enqueue_tail(queue, &thread->runq_links);
+
+		rq_bitmap_set(rq->bitmap, thread->sched_pri);
+		if (thread->sched_pri > rq->highq) {
+			rq->highq = thread->sched_pri;
+			result = TRUE;
+		}
+	} else {
+		if (options & SCHED_TAILQ)
+			enqueue_tail(queue, &thread->runq_links);
+		else
+			enqueue_head(queue, &thread->runq_links);
+	}
+	if (SCHED(priority_is_urgent)(thread->sched_pri))
+		rq->urgency++;
+	SCHED_STATS_RUNQ_CHANGE(&rq->runq_stats, rq->count);
+	rq->count++;
+
+	return (result);
+}
+
+/*
+ *	run_queue_remove:
+ *
+ *	Remove a specific thread from a runqueue.
+ *
+ *	The run queue must be locked.
+ */
+void
+run_queue_remove(
+                 run_queue_t    rq,
+                 thread_t       thread)
+{
+	assert(thread->runq != PROCESSOR_NULL);
+	assert_thread_magic(thread);
+
+	remqueue(&thread->runq_links);
+	SCHED_STATS_RUNQ_CHANGE(&rq->runq_stats, rq->count);
+	rq->count--;
+	if (SCHED(priority_is_urgent)(thread->sched_pri)) {
+		rq->urgency--; assert(rq->urgency >= 0);
+	}
+
+	if (queue_empty(&rq->queues[thread->sched_pri])) {
+		/* update run queue status */
+		bitmap_clear(rq->bitmap, thread->sched_pri);
+		rq->highq = bitmap_first(rq->bitmap, NRQS);
+	}
+
+	thread->runq = PROCESSOR_NULL;
+}
+
+/* Assumes RT lock is not held, and acquires splsched/rt_lock itself */
+void
+sched_rtglobal_runq_scan(sched_update_scan_context_t scan_context)
+{
+	spl_t		s;
+	thread_t	thread;
+
+	processor_set_t pset = &pset0;
+
+	s = splsched();
+	rt_lock_lock(pset);
+
+	qe_foreach_element_safe(thread, &pset->rt_runq.queue, runq_links) {
+		if (thread->last_made_runnable_time < scan_context->earliest_rt_make_runnable_time) {
+			scan_context->earliest_rt_make_runnable_time = thread->last_made_runnable_time;
+		}
+	}
+
+	rt_lock_unlock(pset);
+	splx(s);
+}
+
+int64_t
+sched_rtglobal_runq_count_sum(void)
+{
+	return pset0.rt_runq.runq_stats.count_sum;
+}
+
+/*
+ *	realtime_queue_insert:
+ *
+ *	Enqueue a thread for realtime execution.
+ */
+static boolean_t
+realtime_queue_insert(processor_t processor, processor_set_t pset, thread_t thread)
+{
+	queue_t     queue       = &SCHED(rt_runq)(pset)->queue;
+	uint64_t    deadline    = thread->realtime.deadline;
+	boolean_t   preempt     = FALSE;
+
+	rt_lock_lock(pset);
+
+	if (queue_empty(queue)) {
+		enqueue_tail(queue, &thread->runq_links);
+		preempt = TRUE;
+	} else {
+		/* Insert into rt_runq in thread deadline order */
+		queue_entry_t iter;
+		qe_foreach(iter, queue) {
+			thread_t iter_thread = qe_element(iter, struct thread, runq_links);
+			assert_thread_magic(iter_thread);
+
+			if (deadline < iter_thread->realtime.deadline) {
+				if (iter == queue_first(queue))
+					preempt = TRUE;
+				insque(&thread->runq_links, queue_prev(iter));
+				break;
+			} else if (iter == queue_last(queue)) {
+				enqueue_tail(queue, &thread->runq_links);
+				break;
+			}
+		}
+	}
+
+	thread->runq = processor;
+	SCHED_STATS_RUNQ_CHANGE(&SCHED(rt_runq)(pset)->runq_stats, rt_runq_count(pset));
+	rt_runq_count_incr(pset);
+
+	rt_lock_unlock(pset);
+
+	return (preempt);
+}
+
+/*
+ *	realtime_setrun:
+ *
+ *	Dispatch a thread for realtime execution.
+ *
+ *	Thread must be locked.  Associated pset must
+ *	be locked, and is returned unlocked.
+ */
+static void
+realtime_setrun(
+	processor_t			processor,
+	thread_t			thread)
+{
+	processor_set_t pset = processor->processor_set;
+	pset_assert_locked(pset);
+	ast_t preempt;
+
+	sched_ipi_type_t ipi_type = SCHED_IPI_NONE;
+
+	thread->chosen_processor = processor;
+
+	/* <rdar://problem/15102234> */
+	assert(thread->bound_processor == PROCESSOR_NULL);
+
+	/*
+	 *	Dispatch directly onto idle processor.
+	 */
+	if ( (thread->bound_processor == processor)
+		&& processor->state == PROCESSOR_IDLE) {
+
+		processor->next_thread = thread;
+		processor_state_update_from_thread(processor, thread);
+		processor->deadline = thread->realtime.deadline;
+		pset_update_processor_state(pset, processor, PROCESSOR_DISPATCHING);
+
+		ipi_type = sched_ipi_action(processor, thread, true, SCHED_IPI_EVENT_BOUND_THR);
+		pset_unlock(pset);
+		sched_ipi_perform(processor, ipi_type);
+		return;
+	}
+
+	if (processor->current_pri < BASEPRI_RTQUEUES)
+		preempt = (AST_PREEMPT | AST_URGENT);
+	else if (thread->realtime.deadline < processor->deadline)
+		preempt = (AST_PREEMPT | AST_URGENT);
+	else
+		preempt = AST_NONE;
+
+	realtime_queue_insert(processor, pset, thread);
+
+	ipi_type = SCHED_IPI_NONE;
+	if (preempt != AST_NONE) {
+		if (processor->state == PROCESSOR_IDLE) {
+			processor->next_thread = THREAD_NULL;
+			processor_state_update_from_thread(processor, thread);
+			processor->deadline = thread->realtime.deadline;
+			pset_update_processor_state(pset, processor, PROCESSOR_DISPATCHING);
+			if (processor == current_processor()) {
+				ast_on(preempt);
+			} else {
+				ipi_type = sched_ipi_action(processor, thread, true, SCHED_IPI_EVENT_PREEMPT);
+			}
+		} else if (processor->state == PROCESSOR_DISPATCHING) {
+			if ((processor->next_thread == THREAD_NULL) && ((processor->current_pri < thread->sched_pri) || (processor->deadline > thread->realtime.deadline))) {
+				processor_state_update_from_thread(processor, thread);
+				processor->deadline = thread->realtime.deadline;
+			}
+		} else {
+			if (processor == current_processor()) {
+				ast_on(preempt);
+			} else {
+				ipi_type = sched_ipi_action(processor, thread, false, SCHED_IPI_EVENT_PREEMPT);
+			}
+		}
+	} else {
+		/* Selected processor was too busy, just keep thread enqueued and let other processors drain it naturally. */
+	}
+
+	pset_unlock(pset);
+	sched_ipi_perform(processor, ipi_type);
+}
+
+
+sched_ipi_type_t sched_ipi_deferred_policy(processor_set_t pset, processor_t dst, 
+	__unused sched_ipi_event_t event)
+{
+#if defined(CONFIG_SCHED_DEFERRED_AST)
+    if (!bit_test(pset->pending_deferred_AST_cpu_mask, dst->cpu_id)) {
+        return SCHED_IPI_DEFERRED;
+    }
+#else /* CONFIG_SCHED_DEFERRED_AST */
+    panic("Request for deferred IPI on an unsupported platform; pset: %p CPU: %d", pset, dst->cpu_id);
+#endif /* CONFIG_SCHED_DEFERRED_AST */
+    return SCHED_IPI_NONE;
+}
+
+sched_ipi_type_t sched_ipi_action(processor_t dst, thread_t thread, boolean_t dst_idle, sched_ipi_event_t event)
+{
+    sched_ipi_type_t ipi_type = SCHED_IPI_NONE;
+    assert(dst != NULL);
+
+    processor_set_t pset = dst->processor_set;
+    if (current_processor() == dst) {
+        return SCHED_IPI_NONE;
+    }
+
+    if (bit_test(pset->pending_AST_cpu_mask, dst->cpu_id)) {
+        return SCHED_IPI_NONE;
+    }
+
+    ipi_type = SCHED(ipi_policy)(dst, thread, dst_idle, event);
+    switch(ipi_type) {
+	case SCHED_IPI_NONE:
+	    return SCHED_IPI_NONE;
+#if defined(CONFIG_SCHED_DEFERRED_AST)	
+	case SCHED_IPI_DEFERRED:
+	    bit_set(pset->pending_deferred_AST_cpu_mask, dst->cpu_id);
+	    break;
+#endif /* CONFIG_SCHED_DEFERRED_AST */
+	default:
+	    bit_set(pset->pending_AST_cpu_mask, dst->cpu_id);
+	    break;
+    }
+    return ipi_type;
+}
+
+sched_ipi_type_t sched_ipi_policy(processor_t dst, thread_t thread, boolean_t dst_idle, sched_ipi_event_t event)
+{
+    sched_ipi_type_t ipi_type = SCHED_IPI_NONE;
+    boolean_t deferred_ipi_supported = false;
+    processor_set_t pset = dst->processor_set;
+
+#if defined(CONFIG_SCHED_DEFERRED_AST)
+    deferred_ipi_supported = true;
+#endif /* CONFIG_SCHED_DEFERRED_AST */
+
+    switch(event) {
+	case SCHED_IPI_EVENT_SPILL:
+	case SCHED_IPI_EVENT_SMT_REBAL:
+	case SCHED_IPI_EVENT_REBALANCE:
+	case SCHED_IPI_EVENT_BOUND_THR:
+	    /* 
+	     * The spill, SMT rebalance, rebalance and the bound thread 
+	     * scenarios use immediate IPIs always.
+	     */
+	    ipi_type = dst_idle ? SCHED_IPI_IDLE : SCHED_IPI_IMMEDIATE;
+	    break;
+	case SCHED_IPI_EVENT_PREEMPT:
+	    /* In the preemption case, use immediate IPIs for RT threads */
+	    if (thread && (thread->sched_pri >= BASEPRI_RTQUEUES)) {
+            ipi_type = dst_idle ? SCHED_IPI_IDLE : SCHED_IPI_IMMEDIATE;
+            break;
+	    }
+
+	    /* 
+	     * For Non-RT threads preemption,
+	     * If the core is active, use immediate IPIs.
+	     * If the core is idle, use deferred IPIs if supported; otherwise immediate IPI.
+	     */
+	    if (deferred_ipi_supported && dst_idle) {
+            return sched_ipi_deferred_policy(pset, dst, event);
+	    }
+	    ipi_type = dst_idle ? SCHED_IPI_IDLE : SCHED_IPI_IMMEDIATE;
+	    break;
+	default:
+	    panic("Unrecognized scheduler IPI event type %d", event);
+    }
+    assert(ipi_type != SCHED_IPI_NONE);
+    return ipi_type;
+}
+
+void sched_ipi_perform(processor_t dst, sched_ipi_type_t ipi)
+{
+    switch (ipi) {
+	case SCHED_IPI_NONE:
+	    break;
+	case SCHED_IPI_IDLE:
+	    machine_signal_idle(dst);
+	    break;
+	case SCHED_IPI_IMMEDIATE:
+	    cause_ast_check(dst);
+	    break;
+	case SCHED_IPI_DEFERRED:
+	    machine_signal_idle_deferred(dst);
+	    break;
+	default:
+	    panic("Unrecognized scheduler IPI type: %d", ipi);
+    }
+}
+
+#if defined(CONFIG_SCHED_TIMESHARE_CORE)
+
+boolean_t
+priority_is_urgent(int priority)
+{
+	return bitmap_test(sched_preempt_pri, priority) ? TRUE : FALSE;
+}
+
+#endif /* CONFIG_SCHED_TIMESHARE_CORE */
+
+/*
+ *	processor_setrun:
+ *
+ *	Dispatch a thread for execution on a
+ *	processor.
+ *
+ *	Thread must be locked.  Associated pset must
+ *	be locked, and is returned unlocked.
+ */
+static void
+processor_setrun(
+	processor_t			processor,
+	thread_t			thread,
+	integer_t			options)
+{
+	processor_set_t pset = processor->processor_set;
+	pset_assert_locked(pset);
+	ast_t preempt;
+	enum { eExitIdle, eInterruptRunning, eDoNothing } ipi_action = eDoNothing;
+
+	sched_ipi_type_t ipi_type = SCHED_IPI_NONE;
+
+	thread->chosen_processor = processor;
+
+	/*
+	 *	Dispatch directly onto idle processor.
+	 */
+	if ( (SCHED(direct_dispatch_to_idle_processors) ||
+		  thread->bound_processor == processor)
+		&& processor->state == PROCESSOR_IDLE) {
+
+		processor->next_thread = thread;
+		processor_state_update_from_thread(processor, thread);
+		processor->deadline = UINT64_MAX;
+		pset_update_processor_state(pset, processor, PROCESSOR_DISPATCHING);
+
+		ipi_type = sched_ipi_action(processor, thread, true, SCHED_IPI_EVENT_BOUND_THR);
+		pset_unlock(pset);
+		sched_ipi_perform(processor, ipi_type);
+		return;
+	}
+
+	/*
+	 *	Set preemption mode.
+	 */
+#if defined(CONFIG_SCHED_DEFERRED_AST)
+	/* TODO: Do we need to care about urgency (see rdar://problem/20136239)? */
+#endif
+	if (SCHED(priority_is_urgent)(thread->sched_pri) && thread->sched_pri > processor->current_pri)
+		preempt = (AST_PREEMPT | AST_URGENT);
+	else if(processor->active_thread && thread_eager_preemption(processor->active_thread))
+		preempt = (AST_PREEMPT | AST_URGENT);
+	else if ((thread->sched_mode == TH_MODE_TIMESHARE) && (thread->sched_pri < thread->base_pri)) {
+		if(SCHED(priority_is_urgent)(thread->base_pri) && thread->sched_pri > processor->current_pri) {
+			preempt = (options & SCHED_PREEMPT)? AST_PREEMPT: AST_NONE;
+		} else {
+			preempt = AST_NONE;
+		}
+	} else
+		preempt = (options & SCHED_PREEMPT)? AST_PREEMPT: AST_NONE;
+
+	if ((options & (SCHED_PREEMPT|SCHED_REBALANCE)) == (SCHED_PREEMPT|SCHED_REBALANCE)) {
+		/*
+		 * Having gone to the trouble of forcing this thread off a less preferred core,
+		 * we should force the preferable core to reschedule immediately to give this
+		 * thread a chance to run instead of just sitting on the run queue where
+		 * it may just be stolen back by the idle core we just forced it off.
+		 */
+		preempt |= AST_PREEMPT;
+	}
+
+	SCHED(processor_enqueue)(processor, thread, options);
+	sched_update_pset_load_average(pset);
+
+	if (preempt != AST_NONE) {
+		if (processor->state == PROCESSOR_IDLE) {
+			processor->next_thread = THREAD_NULL;
+			processor_state_update_from_thread(processor, thread);
+			processor->deadline = UINT64_MAX;
+			pset_update_processor_state(pset, processor, PROCESSOR_DISPATCHING);
+			ipi_action = eExitIdle;
+		} else if ( processor->state == PROCESSOR_DISPATCHING) {
+			if ((processor->next_thread == THREAD_NULL) && (processor->current_pri < thread->sched_pri)) {
+				processor_state_update_from_thread(processor, thread);
+				processor->deadline = UINT64_MAX;
+			}
+		} else if (	(processor->state == PROCESSOR_RUNNING		||
+				 processor->state == PROCESSOR_SHUTDOWN)		&&
+				(thread->sched_pri >= processor->current_pri)) {
+			ipi_action = eInterruptRunning;
+		}
+	} else {
+		/*
+		 * New thread is not important enough to preempt what is running, but
+		 * special processor states may need special handling
+		 */
+		if (processor->state == PROCESSOR_SHUTDOWN		&&
+			thread->sched_pri >= processor->current_pri	) {
+			ipi_action = eInterruptRunning;
+		} else if (processor->state == PROCESSOR_IDLE) {
+
+			processor->next_thread = THREAD_NULL;
+			processor_state_update_from_thread(processor, thread);
+			processor->deadline = UINT64_MAX;
+			pset_update_processor_state(pset, processor, PROCESSOR_DISPATCHING);
+
+			ipi_action = eExitIdle;
+		}
+	}
+
+	if (ipi_action != eDoNothing) {
+        if (processor == current_processor()) {
+            if (csw_check_locked(processor, pset, AST_NONE) != AST_NONE)
+                ast_on(preempt);
+	    } else {
+            sched_ipi_event_t event = (options & SCHED_REBALANCE) ? SCHED_IPI_EVENT_REBALANCE : SCHED_IPI_EVENT_PREEMPT;
+            ipi_type = sched_ipi_action(processor, thread, (ipi_action == eExitIdle), event);
+	    }
+	}
+	pset_unlock(pset);
+	sched_ipi_perform(processor, ipi_type);
+}
+
+/*
+ *	choose_next_pset:
+ *
+ *	Return the next sibling pset containing
+ *	available processors.
+ *
+ *	Returns the original pset if none other is
+ *	suitable.
+ */
+static processor_set_t
+choose_next_pset(
+	processor_set_t		pset)
+{
+	processor_set_t		nset = pset;
+
+	do {
+		nset = next_pset(nset);
+	} while (nset->online_processor_count < 1 && nset != pset);
+
+	return (nset);
+}
+
+/*
+ *	choose_processor:
+ *
+ *	Choose a processor for the thread, beginning at
+ *	the pset.  Accepts an optional processor hint in
+ *	the pset.
+ *
+ *	Returns a processor, possibly from a different pset.
+ *
+ *	The thread must be locked.  The pset must be locked,
+ *	and the resulting pset is locked on return.
+ */
+processor_t
+choose_processor(
+	processor_set_t         starting_pset,
+	processor_t             processor,
+	thread_t                thread)
+{
+	processor_set_t pset = starting_pset;
+	processor_set_t nset;
+
+	assert(thread->sched_pri <= BASEPRI_RTQUEUES);
+
+	/*
+	 * Prefer the hinted processor, when appropriate.
+	 */
+
+	/* Fold last processor hint from secondary processor to its primary */
+	if (processor != PROCESSOR_NULL) {
+		processor = processor->processor_primary;
+	}
+
+	/*
+	 * Only consult platform layer if pset is active, which
+	 * it may not be in some cases when a multi-set system
+	 * is going to sleep.
+	 */
+	if (pset->online_processor_count) {
+		if ((processor == PROCESSOR_NULL) || (processor->processor_set == pset && processor->state == PROCESSOR_IDLE)) {
+			processor_t mc_processor = machine_choose_processor(pset, processor);
+			if (mc_processor != PROCESSOR_NULL)
+				processor = mc_processor->processor_primary;
+		}
+	}
+
+	/*
+	 * At this point, we may have a processor hint, and we may have
+	 * an initial starting pset. If the hint is not in the pset, or
+	 * if the hint is for a processor in an invalid state, discard
+	 * the hint.
+	 */
+	if (processor != PROCESSOR_NULL) {
+		if (processor->processor_set != pset) {
+			processor = PROCESSOR_NULL;
+		} else if (!processor->is_recommended) {
+			processor = PROCESSOR_NULL;
+		} else {
+			switch (processor->state) {
+				case PROCESSOR_START:
+				case PROCESSOR_SHUTDOWN:
+				case PROCESSOR_OFF_LINE:
+					/*
+					 * Hint is for a processor that cannot support running new threads.
+					 */
+					processor = PROCESSOR_NULL;
+					break;
+				case PROCESSOR_IDLE:
+					/*
+					 * Hint is for an idle processor. Assume it is no worse than any other
+					 * idle processor. The platform layer had an opportunity to provide
+					 * the "least cost idle" processor above.
+					 */
+					return (processor);
+				case PROCESSOR_RUNNING:
+				case PROCESSOR_DISPATCHING:
+					/*
+					 * Hint is for an active CPU. This fast-path allows
+					 * realtime threads to preempt non-realtime threads
+					 * to regain their previous executing processor.
+					 */
+					if ((thread->sched_pri >= BASEPRI_RTQUEUES) &&
+						(processor->current_pri < BASEPRI_RTQUEUES))
+						return (processor);
+
+					/* Otherwise, use hint as part of search below */
+					break;
+				default:
+					processor = PROCESSOR_NULL;
+					break;
+			}
+		}
+	}
+
+	/*
+	 * Iterate through the processor sets to locate
+	 * an appropriate processor. Seed results with
+	 * a last-processor hint, if available, so that
+	 * a search must find something strictly better
+	 * to replace it.
+	 *
+	 * A primary/secondary pair of SMT processors are
+	 * "unpaired" if the primary is busy but its
+	 * corresponding secondary is idle (so the physical
+	 * core has full use of its resources).
+	 */
+
+	integer_t lowest_priority = MAXPRI + 1;
+	integer_t lowest_secondary_priority = MAXPRI + 1;
+	integer_t lowest_unpaired_primary_priority = MAXPRI + 1;
+	integer_t lowest_count = INT_MAX;
+	uint64_t  furthest_deadline = 1;
+	processor_t lp_processor = PROCESSOR_NULL;
+	processor_t lp_unpaired_primary_processor = PROCESSOR_NULL;
+	processor_t lp_unpaired_secondary_processor = PROCESSOR_NULL;
+	processor_t lp_paired_secondary_processor = PROCESSOR_NULL;
+	processor_t lc_processor = PROCESSOR_NULL;
+	processor_t fd_processor = PROCESSOR_NULL;
+
+	if (processor != PROCESSOR_NULL) {
+		/* All other states should be enumerated above. */
+		assert(processor->state == PROCESSOR_RUNNING || processor->state == PROCESSOR_DISPATCHING);
+
+		lowest_priority = processor->current_pri;
+		lp_processor = processor;
+
+		if (processor->current_pri >= BASEPRI_RTQUEUES) {
+			furthest_deadline = processor->deadline;
+			fd_processor = processor;
+		}
+
+		lowest_count = SCHED(processor_runq_count)(processor);
+		lc_processor = processor;
+	}
+
+	do {
+		/*
+		 * Choose an idle processor, in pset traversal order
+		 */
+
+		uint64_t idle_primary_map = (pset->cpu_state_map[PROCESSOR_IDLE] &
+		                             pset->primary_map &
+		                             pset->recommended_bitmask &
+		                             ~pset->pending_AST_cpu_mask);
+
+		int cpuid = lsb_first(idle_primary_map);
+		if (cpuid >= 0) {
+			processor = processor_array[cpuid];
+			return processor;
+		}
+
+		/*
+		 * Otherwise, enumerate active and idle processors to find primary candidates
+		 * with lower priority/etc.
+		 */
+
+		uint64_t active_map = ((pset->cpu_state_map[PROCESSOR_RUNNING] | pset->cpu_state_map[PROCESSOR_DISPATCHING]) &
+		                             pset->recommended_bitmask &
+		                             ~pset->pending_AST_cpu_mask);
+		active_map = bit_ror64(active_map, (pset->last_chosen + 1));
+		for (int rotid = lsb_first(active_map); rotid >= 0; rotid = lsb_next(active_map, rotid)) {
+			cpuid = ((rotid + pset->last_chosen + 1) & 63);
+			processor = processor_array[cpuid];
+
+			integer_t cpri = processor->current_pri;
+			if (processor->processor_primary != processor) {
+				if (cpri < lowest_secondary_priority) {
+					lowest_secondary_priority = cpri;
+					lp_paired_secondary_processor = processor;
+				}
+			} else {
+				if (cpri < lowest_priority) {
+					lowest_priority = cpri;
+					lp_processor = processor;
+				}
+			}
+
+			if ((cpri >= BASEPRI_RTQUEUES) && (processor->deadline > furthest_deadline)) {
+				furthest_deadline = processor->deadline;
+				fd_processor = processor;
+			}
+
+			integer_t ccount = SCHED(processor_runq_count)(processor);
+			if (ccount < lowest_count) {
+				lowest_count = ccount;
+				lc_processor = processor;
+			}
+		}
+
+		/*
+		 * For SMT configs, these idle secondary processors must have active primary. Otherwise
+		 * the idle primary would have short-circuited the loop above
+		 */
+		uint64_t idle_secondary_map = (pset->cpu_state_map[PROCESSOR_IDLE] &
+					       ~pset->primary_map &
+					       pset->recommended_bitmask &
+					       ~pset->pending_AST_cpu_mask);
+
+		for (cpuid = lsb_first(idle_secondary_map); cpuid >= 0; cpuid = lsb_next(idle_secondary_map, cpuid)) {
+			processor = processor_array[cpuid];
+
+			processor_t cprimary = processor->processor_primary;
+
+			if (!cprimary->is_recommended) {
+				continue;
+			}
+			if (bit_test(pset->pending_AST_cpu_mask, cprimary->cpu_id)) {
+				continue;
+			}
+
+			/* If the primary processor is offline or starting up, it's not a candidate for this path */
+			if (cprimary->state == PROCESSOR_RUNNING || cprimary->state == PROCESSOR_DISPATCHING) {
+				integer_t primary_pri = cprimary->current_pri;
+
+				if (primary_pri < lowest_unpaired_primary_priority) {
+					lowest_unpaired_primary_priority = primary_pri;
+					lp_unpaired_primary_processor = cprimary;
+					lp_unpaired_secondary_processor = processor;
+				}
+			}
+		}
+
+
+		if (thread->sched_pri >= BASEPRI_RTQUEUES) {
+
+			/*
+			 * For realtime threads, the most important aspect is
+			 * scheduling latency, so we attempt to assign threads
+			 * to good preemption candidates (assuming an idle primary
+			 * processor was not available above).
+			 */
+
+			if (thread->sched_pri > lowest_unpaired_primary_priority) {
+				pset->last_chosen = lp_unpaired_primary_processor->cpu_id;
+				return lp_unpaired_primary_processor;
+			}
+			if (thread->sched_pri > lowest_priority) {
+				pset->last_chosen = lp_processor->cpu_id;
+				return lp_processor;
+			}
+			if (sched_allow_rt_smt && (thread->sched_pri > lowest_secondary_priority)) {
+				pset->last_chosen = lp_paired_secondary_processor->cpu_id;
+				return lp_paired_secondary_processor;
+			}
+			if (thread->realtime.deadline < furthest_deadline)
+				return fd_processor;
+
+			/*
+			 * If all primary and secondary CPUs are busy with realtime
+			 * threads with deadlines earlier than us, move on to next
+			 * pset.
+			 */
+		}
+		else {
+
+			if (thread->sched_pri > lowest_unpaired_primary_priority) {
+				pset->last_chosen = lp_unpaired_primary_processor->cpu_id;
+				return lp_unpaired_primary_processor;
+			}
+			if (thread->sched_pri > lowest_priority) {
+				pset->last_chosen = lp_processor->cpu_id;
+				return lp_processor;
+			}
+
+			/*
+			 * If all primary processor in this pset are running a higher
+			 * priority thread, move on to next pset. Only when we have
+			 * exhausted this search do we fall back to other heuristics.
+			 */
+		}
+
+		/*
+		 * Move onto the next processor set.
+		 */
+		nset = next_pset(pset);
+
+		if (nset != starting_pset) {
+			pset_unlock(pset);
+
+			pset = nset;
+			pset_lock(pset);
+		}
+	} while (nset != starting_pset);
+
+	/*
+	 * Make sure that we pick a running processor,
+	 * and that the correct processor set is locked.
+	 * Since we may have unlock the candidate processor's
+	 * pset, it may have changed state.
+	 *
+	 * All primary processors are running a higher priority
+	 * thread, so the only options left are enqueuing on
+	 * the secondary processor that would perturb the least priority
+	 * primary, or the least busy primary.
+	 */
+	do {
+
+		/* lowest_priority is evaluated in the main loops above */
+		if (lp_unpaired_secondary_processor != PROCESSOR_NULL) {
+			processor = lp_unpaired_secondary_processor;
+			lp_unpaired_secondary_processor = PROCESSOR_NULL;
+		} else if (lp_paired_secondary_processor != PROCESSOR_NULL) {
+			processor = lp_paired_secondary_processor;
+			lp_paired_secondary_processor = PROCESSOR_NULL;
+		} else if (lc_processor != PROCESSOR_NULL) {
+			processor = lc_processor;
+			lc_processor = PROCESSOR_NULL;
+		} else {
+			/*
+			 * All processors are executing higher
+			 * priority threads, and the lowest_count
+			 * candidate was not usable
+			 */
+			processor = master_processor;
+		}
+
+		/*
+		 * Check that the correct processor set is
+		 * returned locked.
+		 */
+		if (pset != processor->processor_set) {
+			pset_unlock(pset);
+			pset = processor->processor_set;
+			pset_lock(pset);
+		}
+
+		/*
+		 * We must verify that the chosen processor is still available.
+		 * master_processor is an exception, since we may need to preempt
+		 * a running thread on it during processor shutdown (for sleep),
+		 * and that thread needs to be enqueued on its runqueue to run
+		 * when the processor is restarted.
+		 */
+		if (processor != master_processor && (processor->state == PROCESSOR_SHUTDOWN || processor->state == PROCESSOR_OFF_LINE))
+			processor = PROCESSOR_NULL;
+
+	} while (processor == PROCESSOR_NULL);
+
+	pset->last_chosen = processor->cpu_id;
+	return processor;
+}
+
+/*
+ *	thread_setrun:
+ *
+ *	Dispatch thread for execution, onto an idle
+ *	processor or run queue, and signal a preemption
+ *	as appropriate.
+ *
+ *	Thread must be locked.
+ */
+void
+thread_setrun(
+	thread_t			thread,
+	integer_t			options)
+{
+	processor_t			processor;
+	processor_set_t		pset;
+
+	assert((thread->state & (TH_RUN|TH_WAIT|TH_UNINT|TH_TERMINATE|TH_TERMINATE2)) == TH_RUN);
+	assert(thread->runq == PROCESSOR_NULL);
+
+	/*
+	 *	Update priority if needed.
+	 */
+	if (SCHED(can_update_priority)(thread))
+		SCHED(update_priority)(thread);
+
+	thread->sfi_class = sfi_thread_classify(thread);
+
+	assert(thread->runq == PROCESSOR_NULL);
+
+#if __SMP__
+	if (thread->bound_processor == PROCESSOR_NULL) {
+		/*
+		 *	Unbound case.
+		 */
+		if (thread->affinity_set != AFFINITY_SET_NULL) {
+			/*
+			 * Use affinity set policy hint.
+			 */
+			pset = thread->affinity_set->aset_pset;
+			pset_lock(pset);
+
+			processor = SCHED(choose_processor)(pset, PROCESSOR_NULL, thread);
+			pset = processor->processor_set;
+
+			SCHED_DEBUG_CHOOSE_PROCESSOR_KERNEL_DEBUG_CONSTANT(MACHDBG_CODE(DBG_MACH_SCHED, MACH_SCHED_CHOOSE_PROCESSOR)|DBG_FUNC_NONE,
+									  (uintptr_t)thread_tid(thread), (uintptr_t)-1, processor->cpu_id, processor->state, 0);
+		} else if (thread->last_processor != PROCESSOR_NULL) {
+			/*
+			 *	Simple (last processor) affinity case.
+			 */
+			processor = thread->last_processor;
+			pset = processor->processor_set;
+			pset_lock(pset);
+			processor = SCHED(choose_processor)(pset, processor, thread);
+			pset = processor->processor_set;
+
+			SCHED_DEBUG_CHOOSE_PROCESSOR_KERNEL_DEBUG_CONSTANT(MACHDBG_CODE(DBG_MACH_SCHED, MACH_SCHED_CHOOSE_PROCESSOR)|DBG_FUNC_NONE,
+								  (uintptr_t)thread_tid(thread), thread->last_processor->cpu_id, processor->cpu_id, processor->state, 0);
+		} else {
+			/*
+			 *	No Affinity case:
+			 *
+			 *	Utilitize a per task hint to spread threads
+			 *	among the available processor sets.
+			 */
+			task_t		task = thread->task;
+
+			pset = task->pset_hint;
+			if (pset == PROCESSOR_SET_NULL)
+				pset = current_processor()->processor_set;
+
+			pset = choose_next_pset(pset);
+			pset_lock(pset);
+
+			processor = SCHED(choose_processor)(pset, PROCESSOR_NULL, thread);
+			pset = processor->processor_set;
+			task->pset_hint = pset;
+
+			SCHED_DEBUG_CHOOSE_PROCESSOR_KERNEL_DEBUG_CONSTANT(MACHDBG_CODE(DBG_MACH_SCHED, MACH_SCHED_CHOOSE_PROCESSOR)|DBG_FUNC_NONE,
+									  (uintptr_t)thread_tid(thread), (uintptr_t)-1, processor->cpu_id, processor->state, 0);
+		}
+	} else {
+		/*
+		 *	Bound case:
+		 *
+		 *	Unconditionally dispatch on the processor.
+		 */
+		processor = thread->bound_processor;
+		pset = processor->processor_set;
+		pset_lock(pset);
+
+		SCHED_DEBUG_CHOOSE_PROCESSOR_KERNEL_DEBUG_CONSTANT(MACHDBG_CODE(DBG_MACH_SCHED, MACH_SCHED_CHOOSE_PROCESSOR)|DBG_FUNC_NONE,
+							  (uintptr_t)thread_tid(thread), (uintptr_t)-2, processor->cpu_id, processor->state, 0);
+	}
+#else /* !__SMP__ */
+	/* Only one processor to choose */
+	assert(thread->bound_processor == PROCESSOR_NULL || thread->bound_processor == master_processor);
+	processor = master_processor;
+	pset = processor->processor_set;
+	pset_lock(pset);
+#endif /* !__SMP__ */
+
+	/*
+	 *	Dispatch the thread on the chosen processor.
+	 *	TODO: This should be based on sched_mode, not sched_pri
+	 */
+	if (thread->sched_pri >= BASEPRI_RTQUEUES) {
+		realtime_setrun(processor, thread);
+	} else {
+		processor_setrun(processor, thread, options);
+	}
+	/* pset is now unlocked */
+	if (thread->bound_processor == PROCESSOR_NULL) {
+		SCHED(check_spill)(pset, thread);
+	}
+}
+
+processor_set_t
+task_choose_pset(
+	task_t		task)
+{
+	processor_set_t		pset = task->pset_hint;
+
+	if (pset != PROCESSOR_SET_NULL)
+		pset = choose_next_pset(pset);
+
+	return (pset);
+}
+
+/*
+ *	Check for a preemption point in
+ *	the current context.
+ *
+ *	Called at splsched with thread locked.
+ */
+ast_t
+csw_check(
+	processor_t		processor,
+	ast_t			check_reason)
+{
+	processor_set_t	pset = processor->processor_set;
+	ast_t			result;
+
+	pset_lock(pset);
+
+	/* If we were sent a remote AST and interrupted a running processor, acknowledge it here with pset lock held */
+	bit_clear(pset->pending_AST_cpu_mask, processor->cpu_id);
+
+	result = csw_check_locked(processor, pset, check_reason);
+
+	pset_unlock(pset);
+
+	return result;
+}
+
+/*
+ * Check for preemption at splsched with
+ * pset and thread locked
+ */
+ast_t
+csw_check_locked(
+	processor_t		processor,
+	processor_set_t		pset,
+	ast_t			check_reason)
+{
+	ast_t			result;
+	thread_t		thread = processor->active_thread;
+
+	if (processor->first_timeslice) {
+		if (rt_runq_count(pset) > 0)
+			return (check_reason | AST_PREEMPT | AST_URGENT);
+	}
+	else {
+		if (rt_runq_count(pset) > 0) {
+			if (BASEPRI_RTQUEUES > processor->current_pri)
+				return (check_reason | AST_PREEMPT | AST_URGENT);
+			else
+				return (check_reason | AST_PREEMPT);
+		}
+	}
+
+#if __SMP__
+	/*
+	 * If the current thread is running on a processor that is no longer recommended,
+	 * urgently preempt it, at which point thread_select() should
+	 * try to idle the processor and re-dispatch the thread to a recommended processor.
+	 */
+	if (!processor->is_recommended) {
+		return (check_reason | AST_PREEMPT | AST_URGENT);
+	}
+#endif
+
+	result = SCHED(processor_csw_check)(processor);
+	if (result != AST_NONE)
+		return (check_reason | result | (thread_eager_preemption(thread) ? AST_URGENT : AST_NONE));
+
+#if __SMP__
+	/*
+	 * Same for avoid-processor
+	 *
+	 * TODO: Should these set AST_REBALANCE?
+	 */
+	if (SCHED(avoid_processor_enabled) && SCHED(thread_avoid_processor)(processor, thread)) {
+		return (check_reason | AST_PREEMPT);
+	}
+
+	/*
+	 * Even though we could continue executing on this processor, a
+	 * secondary SMT core should try to shed load to another primary core.
+	 *
+	 * TODO: Should this do the same check that thread_select does? i.e.
+	 * if no bound threads target this processor, and idle primaries exist, preempt
+	 * The case of RT threads existing is already taken care of above
+	 */
+
+	if (processor->current_pri < BASEPRI_RTQUEUES &&
+	    processor->processor_primary != processor)
+		return (check_reason | AST_PREEMPT);
+#endif
+
+	if (thread->state & TH_SUSP)
+		return (check_reason | AST_PREEMPT);
+
+#if CONFIG_SCHED_SFI
+	/*
+	 * Current thread may not need to be preempted, but maybe needs
+	 * an SFI wait?
+	 */
+	result = sfi_thread_needs_ast(thread, NULL);
+	if (result != AST_NONE)
+		return (check_reason | result);
+#endif
+
+	return (AST_NONE);
+}
+
+/*
+ *	set_sched_pri:
+ *
+ *	Set the scheduled priority of the specified thread.
+ *
+ *	This may cause the thread to change queues.
+ *
+ *	Thread must be locked.
+ */
+void
+set_sched_pri(
+              thread_t        thread,
+              int             new_priority,
+              set_sched_pri_options_t options)
+{
+	thread_t cthread = current_thread();
+	boolean_t is_current_thread = (thread == cthread) ? TRUE : FALSE;
+	int curgency, nurgency;
+	uint64_t urgency_param1, urgency_param2;
+	boolean_t removed_from_runq = FALSE;
+
+	bool lazy_update = ((options & SETPRI_LAZY) == SETPRI_LAZY);
+
+	int old_priority = thread->sched_pri;
+
+	/* If we're already at this priority, no need to mess with the runqueue */
+	if (new_priority == old_priority)
+		return;
+
+	if (is_current_thread) {
+		assert(thread->runq == PROCESSOR_NULL);
+		curgency = thread_get_urgency(thread, &urgency_param1, &urgency_param2);
+	} else {
+		removed_from_runq = thread_run_queue_remove(thread);
+	}
+
+	thread->sched_pri = new_priority;
+
+	KERNEL_DEBUG_CONSTANT(MACHDBG_CODE(DBG_MACH_SCHED, MACH_SCHED_CHANGE_PRIORITY),
+	                      (uintptr_t)thread_tid(thread),
+	                      thread->base_pri,
+	                      thread->sched_pri,
+	                      thread->sched_usage,
+	                      0);
+
+	if (is_current_thread) {
+		nurgency = thread_get_urgency(thread, &urgency_param1, &urgency_param2);
+		/*
+		 * set_sched_pri doesn't alter RT params. We expect direct base priority/QoS
+		 * class alterations from user space to occur relatively infrequently, hence
+		 * those are lazily handled. QoS classes have distinct priority bands, and QoS
+		 * inheritance is expected to involve priority changes.
+		 */
+		uint64_t ctime = mach_approximate_time();
+		if (nurgency != curgency) {
+			thread_tell_urgency(nurgency, urgency_param1, urgency_param2, 0, thread);
+		}
+		machine_thread_going_on_core(thread, nurgency, 0, 0, ctime);
+	}
+
+	if (removed_from_runq)
+		thread_run_queue_reinsert(thread, SCHED_PREEMPT | SCHED_TAILQ);
+	else if (thread->state & TH_RUN) {
+		processor_t processor = thread->last_processor;
+
+		if (is_current_thread) {
+			processor_state_update_from_thread(processor, thread);
+
+			/*
+			 * When dropping in priority, check if the thread no longer belongs on core.
+			 * If a thread raises its own priority, don't aggressively rebalance it.
+			 * <rdar://problem/31699165>
+			 */
+			if (!lazy_update && new_priority < old_priority) {
+				ast_t preempt;
+
+				if ((preempt = csw_check(processor, AST_NONE)) != AST_NONE)
+					ast_on(preempt);
+			}
+		} else if (!lazy_update && processor != PROCESSOR_NULL &&
+		           processor != current_processor() && processor->active_thread == thread) {
+			cause_ast_check(processor);
+		}
+	}
+}
+
+/*
+ * thread_run_queue_remove_for_handoff
+ *
+ * Pull a thread or its (recursive) push target out of the runqueue
+ * so that it is ready for thread_run()
+ *
+ * Called at splsched
+ *
+ * Returns the thread that was pulled or THREAD_NULL if no thread could be pulled.
+ * This may be different than the thread that was passed in.
+ */
+thread_t
+thread_run_queue_remove_for_handoff(thread_t thread) {
+
+	thread_t pulled_thread = THREAD_NULL;
+
+	thread_lock(thread);
+
+	/*
+	 * Check that the thread is not bound
+	 * to a different processor, and that realtime
+	 * is not involved.
+	 *
+	 * Next, pull it off its run queue.  If it
+	 * doesn't come, it's not eligible.
+	 */
+
+	processor_t processor = current_processor();
+	if (processor->current_pri < BASEPRI_RTQUEUES && thread->sched_pri < BASEPRI_RTQUEUES &&
+	    (thread->bound_processor == PROCESSOR_NULL || thread->bound_processor == processor)) {
+
+			if (thread_run_queue_remove(thread))
+				pulled_thread = thread;
+	}
+
+	thread_unlock(thread);
+
+	return pulled_thread;
+}
+
+/*
+ *	thread_run_queue_remove:
+ *
+ *	Remove a thread from its current run queue and
+ *	return TRUE if successful.
+ *
+ *	Thread must be locked.
+ *
+ *	If thread->runq is PROCESSOR_NULL, the thread will not re-enter the
+ *	run queues because the caller locked the thread.  Otherwise
+ *	the thread is on a run queue, but could be chosen for dispatch
+ *	and removed by another processor under a different lock, which
+ *	will set thread->runq to PROCESSOR_NULL.
+ *
+ *	Hence the thread select path must not rely on anything that could
+ *	be changed under the thread lock after calling this function,
+ *	most importantly thread->sched_pri.
+ */
+boolean_t
+thread_run_queue_remove(
+                        thread_t        thread)
+{
+	boolean_t removed = FALSE;
+	processor_t processor = thread->runq;
+
+	if ((thread->state & (TH_RUN|TH_WAIT)) == TH_WAIT) {
+		/* Thread isn't runnable */
+		assert(thread->runq == PROCESSOR_NULL);
+		return FALSE;
+	}
+
+	if (processor == PROCESSOR_NULL) {
+		/*
+		 * The thread is either not on the runq,
+		 * or is in the midst of being removed from the runq.
+		 *
+		 * runq is set to NULL under the pset lock, not the thread
+		 * lock, so the thread may still be in the process of being dequeued
+		 * from the runq. It will wait in invoke for the thread lock to be
+		 * dropped.
+		 */
+
+		return FALSE;
+	}
+
+	if (thread->sched_pri < BASEPRI_RTQUEUES) {
+		return SCHED(processor_queue_remove)(processor, thread);
+	}
+
+	processor_set_t pset = processor->processor_set;
+
+	rt_lock_lock(pset);
+
+	if (thread->runq != PROCESSOR_NULL) {
+		/*
+		 *	Thread is on the RT run queue and we have a lock on
+		 *	that run queue.
+		 */
+
+		remqueue(&thread->runq_links);
+		SCHED_STATS_RUNQ_CHANGE(&SCHED(rt_runq)(pset)->runq_stats, rt_runq_count(pset));
+		rt_runq_count_decr(pset);
+
+		thread->runq = PROCESSOR_NULL;
+
+		removed = TRUE;
+	}
+
+	rt_lock_unlock(pset);
+
+	return (removed);
 }
 
 /*
- *	thread_run:
+ * Put the thread back where it goes after a thread_run_queue_remove
  *
- *	Switch directly from the current thread to the
- *	new thread, handing off our quantum if appropriate.
- *
- *	New thread must be runnable, and not on a run queue.
+ * Thread must have been removed under the same thread lock hold
  *
- *	Called at splsched.
+ * thread locked, at splsched
  */
+void
+thread_run_queue_reinsert(thread_t thread, integer_t options)
+{
+	assert(thread->runq == PROCESSOR_NULL);
+	assert(thread->state & (TH_RUN));
+
+	thread_setrun(thread, options);
+}
+
+void
+sys_override_cpu_throttle(boolean_t enable_override)
+{
+	if (enable_override)
+		cpu_throttle_enabled = 0;
+	else 
+		cpu_throttle_enabled = 1;
+}
+
 int
-thread_run(
-	thread_t			self,
-	thread_continue_t	continuation,
-	void				*parameter,
-	thread_t			new_thread)
+thread_get_urgency(thread_t thread, uint64_t *arg1, uint64_t *arg2)
 {
-	ast_t		handoff = AST_HANDOFF;
+	if (thread == NULL || (thread->state & TH_IDLE)) {
+		*arg1 = 0;
+		*arg2 = 0;
+
+		return (THREAD_URGENCY_NONE);
+	} else if (thread->sched_mode == TH_MODE_REALTIME) {
+		*arg1 = thread->realtime.period;
+		*arg2 = thread->realtime.deadline;
+
+		return (THREAD_URGENCY_REAL_TIME);
+	} else if (cpu_throttle_enabled &&
+		   ((thread->sched_pri <= MAXPRI_THROTTLE) && (thread->base_pri <= MAXPRI_THROTTLE)))  {
+		/*
+		 * Background urgency applied when thread priority is MAXPRI_THROTTLE or lower and thread is not promoted
+		 */
+		*arg1 = thread->sched_pri;
+		*arg2 = thread->base_pri;
 
-	funnel_release_check(self, 3);
+		return (THREAD_URGENCY_BACKGROUND);
+	} else {
+		/* For otherwise unclassified threads, report throughput QoS
+		 * parameters
+		 */
+		*arg1 = proc_get_effective_thread_policy(thread, TASK_POLICY_THROUGH_QOS);
+		*arg2 = proc_get_effective_task_policy(thread->task, TASK_POLICY_THROUGH_QOS);
 
-	self->continuation = continuation;
-	self->parameter = parameter;
+		return (THREAD_URGENCY_NORMAL);
+	}
+}
+
+perfcontrol_class_t
+thread_get_perfcontrol_class(thread_t thread)
+{
+    /* Special case handling */
+    if (thread->state & TH_IDLE)
+        return PERFCONTROL_CLASS_IDLE;
+    if (thread->task == kernel_task)
+        return PERFCONTROL_CLASS_KERNEL;
+    if (thread->sched_mode == TH_MODE_REALTIME)
+        return PERFCONTROL_CLASS_REALTIME;
+
+    /* perfcontrol_class based on base_pri */
+    if (thread->base_pri <= MAXPRI_THROTTLE)
+        return PERFCONTROL_CLASS_BACKGROUND;
+    else if (thread->base_pri <= BASEPRI_UTILITY)
+        return PERFCONTROL_CLASS_UTILITY;
+    else if (thread->base_pri <= BASEPRI_DEFAULT)
+        return PERFCONTROL_CLASS_NONUI;
+    else if (thread->base_pri <= BASEPRI_FOREGROUND)
+        return PERFCONTROL_CLASS_UI;
+    else
+        return PERFCONTROL_CLASS_ABOVEUI;
+}
+
+/*
+ *	This is the processor idle loop, which just looks for other threads
+ *	to execute.  Processor idle threads invoke this without supplying a
+ *	current thread to idle without an asserted wait state.
+ *
+ *	Returns a the next thread to execute if dispatched directly.
+ */
 
-	while (!thread_invoke(self, new_thread, handoff)) {
-		register processor_t		processor = current_processor();
+#if 0
+#define IDLE_KERNEL_DEBUG_CONSTANT(...) KERNEL_DEBUG_CONSTANT(__VA_ARGS__)
+#else
+#define IDLE_KERNEL_DEBUG_CONSTANT(...) do { } while(0)
+#endif
 
-		thread_lock(self);
-		new_thread = thread_select(processor);
-		thread_unlock(self);
-		handoff = AST_NONE;
+thread_t
+processor_idle(
+	thread_t			thread,
+	processor_t			processor)
+{
+	processor_set_t		pset = processor->processor_set;
+	thread_t			new_thread;
+	int					state;
+	(void)splsched();
+
+	KERNEL_DEBUG_CONSTANT_IST(KDEBUG_TRACE,
+		MACHDBG_CODE(DBG_MACH_SCHED,MACH_IDLE) | DBG_FUNC_START, 
+		(uintptr_t)thread_tid(thread), 0, 0, 0, 0);
+
+	SCHED_STATS_CPU_IDLE_START(processor);
+
+	uint64_t ctime = mach_absolute_time();
+
+	timer_switch(&PROCESSOR_DATA(processor, system_state), ctime, &PROCESSOR_DATA(processor, idle_state));
+	PROCESSOR_DATA(processor, current_state) = &PROCESSOR_DATA(processor, idle_state);
+
+	cpu_quiescent_counter_leave(ctime);
+
+	while (1) {
+		/*
+		 * Ensure that updates to my processor and pset state,
+		 * made by the IPI source processor before sending the IPI,
+		 * are visible on this processor now (even though we don't
+		 * take the pset lock yet).
+		 */
+		atomic_thread_fence(memory_order_acquire);
+
+		if (processor->state != PROCESSOR_IDLE)
+			break;
+		if (bit_test(pset->pending_AST_cpu_mask, processor->cpu_id))
+			break;
+#if defined(CONFIG_SCHED_DEFERRED_AST)
+		if (bit_test(pset->pending_deferred_AST_cpu_mask, processor->cpu_id))
+			break;
+#endif
+		if (processor->is_recommended && (processor->processor_primary == processor)) {
+			if (rt_runq_count(pset))
+				break;
+		} else {
+			if (SCHED(processor_bound_count)(processor))
+				break;
+		}
+
+#if CONFIG_SCHED_IDLE_IN_PLACE
+		if (thread != THREAD_NULL) {
+			/* Did idle-in-place thread wake up */
+			if ((thread->state & (TH_WAIT|TH_SUSP)) != TH_WAIT || thread->wake_active)
+				break;
+		}
+#endif
+
+		IDLE_KERNEL_DEBUG_CONSTANT(
+			MACHDBG_CODE(DBG_MACH_SCHED,MACH_IDLE) | DBG_FUNC_NONE, (uintptr_t)thread_tid(thread), rt_runq_count(pset), SCHED(processor_runq_count)(processor), -1, 0);
+
+		machine_track_platform_idle(TRUE);
+
+		machine_idle();
+
+		machine_track_platform_idle(FALSE);
+
+		(void)splsched();
+
+		/* 
+		 * Check if we should call sched_timeshare_consider_maintenance() here.  
+		 * The CPU was woken out of idle due to an interrupt and we should do the 
+		 * call only if the processor is still idle. If the processor is non-idle, 
+		 * the threads running on the processor would do the call as part of 
+		 * context swithing.
+		 */
+		if (processor->state == PROCESSOR_IDLE) {
+			sched_timeshare_consider_maintenance(mach_absolute_time());
+		}
+
+		IDLE_KERNEL_DEBUG_CONSTANT(
+			MACHDBG_CODE(DBG_MACH_SCHED,MACH_IDLE) | DBG_FUNC_NONE, (uintptr_t)thread_tid(thread), rt_runq_count(pset), SCHED(processor_runq_count)(processor), -2, 0);
+
+		if (!SCHED(processor_queue_empty)(processor)) {
+			/* Secondary SMT processors respond to directed wakeups
+			 * exclusively. Some platforms induce 'spurious' SMT wakeups.
+			 */
+			if (processor->processor_primary == processor)
+					break;
+		}
 	}
 
-	funnel_refunnel_check(self, 6);
+	ctime = mach_absolute_time();
 
-	return (self->wait_result);
+	timer_switch(&PROCESSOR_DATA(processor, idle_state), ctime, &PROCESSOR_DATA(processor, system_state));
+	PROCESSOR_DATA(processor, current_state) = &PROCESSOR_DATA(processor, system_state);
+
+	cpu_quiescent_counter_join(ctime);
+
+	pset_lock(pset);
+
+	/* If we were sent a remote AST and came out of idle, acknowledge it here with pset lock held */
+	bit_clear(pset->pending_AST_cpu_mask, processor->cpu_id);
+#if defined(CONFIG_SCHED_DEFERRED_AST)
+	bit_clear(pset->pending_deferred_AST_cpu_mask, processor->cpu_id);
+#endif
+
+	state = processor->state;
+	if (state == PROCESSOR_DISPATCHING) {
+		/*
+		 *	Commmon case -- cpu dispatched.
+		 */
+		new_thread = processor->next_thread;
+		processor->next_thread = THREAD_NULL;
+		pset_update_processor_state(pset, processor, PROCESSOR_RUNNING);
+
+		if ((new_thread != THREAD_NULL) && (SCHED(processor_queue_has_priority)(processor, new_thread->sched_pri, FALSE)					||
+											(rt_runq_count(pset) > 0))	) {
+			/* Something higher priority has popped up on the runqueue - redispatch this thread elsewhere */
+			processor_state_update_idle(processor);
+			processor->deadline = UINT64_MAX;
+
+			pset_unlock(pset);
+
+			thread_lock(new_thread);
+			KERNEL_DEBUG_CONSTANT(MACHDBG_CODE(DBG_MACH_SCHED, MACH_REDISPATCH), (uintptr_t)thread_tid(new_thread), new_thread->sched_pri, rt_runq_count(pset), 0, 0);
+			thread_setrun(new_thread, SCHED_HEADQ);
+			thread_unlock(new_thread);
+
+			KERNEL_DEBUG_CONSTANT_IST(KDEBUG_TRACE,
+				MACHDBG_CODE(DBG_MACH_SCHED,MACH_IDLE) | DBG_FUNC_END, 
+				(uintptr_t)thread_tid(thread), state, 0, 0, 0);
+
+			return (THREAD_NULL);
+		}
+
+		sched_update_pset_load_average(pset);
+
+		pset_unlock(pset);
+
+		KERNEL_DEBUG_CONSTANT_IST(KDEBUG_TRACE,
+			MACHDBG_CODE(DBG_MACH_SCHED,MACH_IDLE) | DBG_FUNC_END, 
+			(uintptr_t)thread_tid(thread), state, (uintptr_t)thread_tid(new_thread), 0, 0);
+
+		return (new_thread);
+
+	} else if (state == PROCESSOR_IDLE) {
+		pset_update_processor_state(pset, processor, PROCESSOR_RUNNING);
+		processor_state_update_idle(processor);
+		processor->deadline = UINT64_MAX;
+
+	} else if (state == PROCESSOR_SHUTDOWN) {
+		/*
+		 *	Going off-line.  Force a
+		 *	reschedule.
+		 */
+		if ((new_thread = processor->next_thread) != THREAD_NULL) {
+			processor->next_thread = THREAD_NULL;
+			processor_state_update_idle(processor);
+			processor->deadline = UINT64_MAX;
+
+			pset_unlock(pset);
+
+			thread_lock(new_thread);
+			thread_setrun(new_thread, SCHED_HEADQ);
+			thread_unlock(new_thread);
+
+			KERNEL_DEBUG_CONSTANT_IST(KDEBUG_TRACE,
+				MACHDBG_CODE(DBG_MACH_SCHED,MACH_IDLE) | DBG_FUNC_END, 
+				(uintptr_t)thread_tid(thread), state, 0, 0, 0);
+		
+			return (THREAD_NULL);
+		}
+	}
+
+	pset_unlock(pset);
+
+	KERNEL_DEBUG_CONSTANT_IST(KDEBUG_TRACE,
+		MACHDBG_CODE(DBG_MACH_SCHED,MACH_IDLE) | DBG_FUNC_END, 
+		(uintptr_t)thread_tid(thread), state, 0, 0, 0);
+		
+	return (THREAD_NULL);
 }
 
 /*
- *	thread_continue:
+ *	Each processor has a dedicated thread which
+ *	executes the idle loop when there is no suitable
+ *	previous context.
+ */
+void
+idle_thread(void)
+{
+	processor_t		processor = current_processor();
+	thread_t		new_thread;
+
+	new_thread = processor_idle(THREAD_NULL, processor);
+	if (new_thread != THREAD_NULL) {
+		thread_run(processor->idle_thread, (thread_continue_t)idle_thread, NULL, new_thread);
+		/*NOTREACHED*/
+	}
+
+	thread_block((thread_continue_t)idle_thread);
+	/*NOTREACHED*/
+}
+
+kern_return_t
+idle_thread_create(
+	processor_t		processor)
+{
+	kern_return_t	result;
+	thread_t		thread;
+	spl_t			s;
+	char			name[MAXTHREADNAMESIZE];
+
+	result = kernel_thread_create((thread_continue_t)idle_thread, NULL, MAXPRI_KERNEL, &thread);
+	if (result != KERN_SUCCESS)
+		return (result);
+
+	snprintf(name, sizeof(name), "idle #%d", processor->cpu_id);
+	thread_set_thread_name(thread, name);
+
+	s = splsched();
+	thread_lock(thread);
+	thread->bound_processor = processor;
+	processor->idle_thread = thread;
+	thread->sched_pri = thread->base_pri = IDLEPRI;
+	thread->state = (TH_RUN | TH_IDLE);
+	thread->options |= TH_OPT_IDLE_THREAD;
+	thread_unlock(thread);
+	splx(s);
+
+	thread_deallocate(thread);
+
+	return (KERN_SUCCESS);
+}
+
+/*
+ * sched_startup:
  *
- *	Called at splsched when a thread first receives
- *	a new stack after a continuation.
+ * Kicks off scheduler services.
+ *
+ * Called at splsched.
  */
 void
-thread_continue(
-	register thread_t	old_thread)
+sched_startup(void)
 {
-	register thread_t			self = current_thread();
-	register thread_continue_t	continuation;
-	register void				*parameter;
-	
-	continuation = self->continuation;
-	self->continuation = NULL;
-	parameter = self->parameter;
-	self->parameter = NULL;
+	kern_return_t	result;
+	thread_t		thread;
+
+	simple_lock_init(&sched_vm_group_list_lock, 0);
+
+#if __arm__ || __arm64__
+	simple_lock_init(&sched_recommended_cores_lock, 0);
+#endif /* __arm__ || __arm64__ */
+
+	result = kernel_thread_start_priority((thread_continue_t)sched_init_thread,
+	    (void *)SCHED(maintenance_continuation), MAXPRI_KERNEL, &thread);
+	if (result != KERN_SUCCESS)
+		panic("sched_startup");
+
+	thread_deallocate(thread);
+
+	assert_thread_magic(thread);
+
+	/*
+	 * Yield to the sched_init_thread once, to
+	 * initialize our own thread after being switched
+	 * back to.
+	 *
+	 * The current thread is the only other thread
+	 * active at this point.
+	 */
+	thread_block(THREAD_CONTINUE_NULL);
+}
+
+#if __arm64__
+static _Atomic uint64_t sched_perfcontrol_callback_deadline;
+#endif /* __arm64__ */
+
+
+#if defined(CONFIG_SCHED_TIMESHARE_CORE)
+
+static volatile uint64_t 		sched_maintenance_deadline;
+static uint64_t				sched_tick_last_abstime;
+static uint64_t				sched_tick_delta;
+uint64_t				sched_tick_max_delta;
+
+
+/*
+ *	sched_init_thread:
+ *
+ *	Perform periodic bookkeeping functions about ten
+ *	times per second.
+ */
+void
+sched_timeshare_maintenance_continue(void)
+{
+	uint64_t	sched_tick_ctime, late_time;
+
+	struct sched_update_scan_context scan_context = {
+		.earliest_bg_make_runnable_time = UINT64_MAX,
+		.earliest_normal_make_runnable_time = UINT64_MAX,
+		.earliest_rt_make_runnable_time = UINT64_MAX
+	};
+
+	sched_tick_ctime = mach_absolute_time();	
+
+	if (__improbable(sched_tick_last_abstime == 0)) {
+		sched_tick_last_abstime = sched_tick_ctime;
+		late_time = 0;
+		sched_tick_delta = 1;
+	} else {
+		late_time = sched_tick_ctime - sched_tick_last_abstime;
+		sched_tick_delta = late_time / sched_tick_interval;
+		/* Ensure a delta of 1, since the interval could be slightly
+		 * smaller than the sched_tick_interval due to dispatch
+		 * latencies.
+		 */
+		sched_tick_delta = MAX(sched_tick_delta, 1);
+
+		/* In the event interrupt latencies or platform
+		 * idle events that advanced the timebase resulted
+		 * in periods where no threads were dispatched,
+		 * cap the maximum "tick delta" at SCHED_TICK_MAX_DELTA
+		 * iterations.
+		 */
+		sched_tick_delta = MIN(sched_tick_delta, SCHED_TICK_MAX_DELTA);
+
+		sched_tick_last_abstime = sched_tick_ctime;
+		sched_tick_max_delta = MAX(sched_tick_delta, sched_tick_max_delta);
+	}
+
+	KERNEL_DEBUG_CONSTANT(MACHDBG_CODE(DBG_MACH_SCHED, MACH_SCHED_MAINTENANCE)|DBG_FUNC_START,
+	        sched_tick_delta, late_time, 0, 0, 0);
+
+	/* Add a number of pseudo-ticks corresponding to the elapsed interval
+	 * This could be greater than 1 if substantial intervals where
+	 * all processors are idle occur, which rarely occurs in practice.
+	 */
+
+	sched_tick += sched_tick_delta;
+
+	update_vm_info();
+
+	/*
+	 *  Compute various averages.
+	 */
+	compute_averages(sched_tick_delta);
+
+	/*
+	 *  Scan the run queues for threads which
+	 *  may need to be updated, and find the earliest runnable thread on the runqueue
+	 *  to report its latency.
+	 */
+	SCHED(thread_update_scan)(&scan_context);
+
+	SCHED(rt_runq_scan)(&scan_context);
+
+	uint64_t ctime = mach_absolute_time();
+
+	uint64_t bg_max_latency       = (ctime > scan_context.earliest_bg_make_runnable_time) ?
+	                                 ctime - scan_context.earliest_bg_make_runnable_time : 0;
+
+	uint64_t default_max_latency  = (ctime > scan_context.earliest_normal_make_runnable_time) ?
+	                                 ctime - scan_context.earliest_normal_make_runnable_time : 0;
+
+	uint64_t realtime_max_latency = (ctime > scan_context.earliest_rt_make_runnable_time) ?
+	                                 ctime - scan_context.earliest_rt_make_runnable_time : 0;
+
+	machine_max_runnable_latency(bg_max_latency, default_max_latency, realtime_max_latency);
+
+	/*
+	 * Check to see if the special sched VM group needs attention.
+	 */
+	sched_vm_group_maintenance();
+
+#if __arm__ || __arm64__
+	/* Check to see if the recommended cores failsafe is active */
+	sched_recommended_cores_maintenance();
+#endif /* __arm__ || __arm64__ */
+
+ 
+#if DEBUG || DEVELOPMENT
+#if __x86_64__
+#include <i386/misc_protos.h>
+	/* Check for long-duration interrupts */
+	mp_interrupt_watchdog();
+#endif /* __x86_64__ */
+#endif /* DEBUG || DEVELOPMENT */
+
+	KERNEL_DEBUG_CONSTANT(MACHDBG_CODE(DBG_MACH_SCHED, MACH_SCHED_MAINTENANCE) | DBG_FUNC_END,
+		sched_pri_shifts[TH_BUCKET_SHARE_FG], sched_pri_shifts[TH_BUCKET_SHARE_BG],
+		sched_pri_shifts[TH_BUCKET_SHARE_UT], sched_pri_shifts[TH_BUCKET_SHARE_DF], 0);
+
+	assert_wait((event_t)sched_timeshare_maintenance_continue, THREAD_UNINT);
+	thread_block((thread_continue_t)sched_timeshare_maintenance_continue);
+	/*NOTREACHED*/
+}
+
+static uint64_t sched_maintenance_wakeups;
+
+/*
+ * Determine if the set of routines formerly driven by a maintenance timer
+ * must be invoked, based on a deadline comparison. Signals the scheduler
+ * maintenance thread on deadline expiration. Must be invoked at an interval
+ * lower than the "sched_tick_interval", currently accomplished by
+ * invocation via the quantum expiration timer and at context switch time.
+ * Performance matters: this routine reuses a timestamp approximating the
+ * current absolute time received from the caller, and should perform
+ * no more than a comparison against the deadline in the common case.
+ */
+void
+sched_timeshare_consider_maintenance(uint64_t ctime) {
+
+	cpu_quiescent_counter_checkin(ctime);
+
+	uint64_t deadline = sched_maintenance_deadline;
+
+	if (__improbable(ctime >= deadline)) {
+		if (__improbable(current_thread() == sched_maintenance_thread))
+			return;
+		OSMemoryBarrier();
+
+		uint64_t ndeadline = ctime + sched_tick_interval;
+
+		if (__probable(__sync_bool_compare_and_swap(&sched_maintenance_deadline, deadline, ndeadline))) {
+			thread_wakeup((event_t)sched_timeshare_maintenance_continue);
+			sched_maintenance_wakeups++;
+		}
+	}
+
+	uint64_t load_compute_deadline = __c11_atomic_load(&sched_load_compute_deadline, memory_order_relaxed);
+
+	if (__improbable(load_compute_deadline && ctime >= load_compute_deadline)) {
+		uint64_t new_deadline = 0;
+		if (__c11_atomic_compare_exchange_strong(&sched_load_compute_deadline, &load_compute_deadline, new_deadline,
+		                                         memory_order_relaxed, memory_order_relaxed)) {
+			compute_sched_load();
+			new_deadline = ctime + sched_load_compute_interval_abs;
+			__c11_atomic_store(&sched_load_compute_deadline, new_deadline, memory_order_relaxed);
+		}
+	}
+
+#if __arm64__
+	uint64_t perf_deadline = __c11_atomic_load(&sched_perfcontrol_callback_deadline, memory_order_relaxed);
+
+	if (__improbable(perf_deadline && ctime >= perf_deadline)) {
+		/* CAS in 0, if success, make callback. Otherwise let the next context switch check again. */
+		if (__c11_atomic_compare_exchange_strong(&sched_perfcontrol_callback_deadline, &perf_deadline, 0,
+		                                         memory_order_relaxed, memory_order_relaxed)) {
+			machine_perfcontrol_deadline_passed(perf_deadline);
+		}
+	}
+#endif /* __arm64__ */
+
+}
+
+#endif /* CONFIG_SCHED_TIMESHARE_CORE */
+
+void
+sched_init_thread(void (*continuation)(void))
+{
+	thread_block(THREAD_CONTINUE_NULL);
 
-	thread_begin(self, self->last_processor);
+	thread_t thread = current_thread();
 
-	if (old_thread != THREAD_NULL)
-		thread_dispatch(old_thread);
+	thread_set_thread_name(thread, "sched_maintenance_thread");
 
-	funnel_refunnel_check(self, 4);
+	sched_maintenance_thread = thread;
 
-	if (old_thread != THREAD_NULL)
-		(void)spllo();
+	continuation();
 
-	call_continuation(continuation, parameter, self->wait_result);
 	/*NOTREACHED*/
 }
 
+#if defined(CONFIG_SCHED_TIMESHARE_CORE)
+
 /*
- *	Enqueue thread on run queue.  Thread must be locked,
- *	and not already be on a run queue.  Returns TRUE
- *	if a preemption is indicated based on the state
- *	of the run queue.
+ *	thread_update_scan / runq_scan:
+ *
+ *	Scan the run queues to account for timesharing threads 
+ *	which need to be updated.
+ *
+ *	Scanner runs in two passes.  Pass one squirrels likely
+ *	threads away in an array, pass two does the update.
+ *
+ *	This is necessary because the run queue is locked for
+ *	the candidate scan, but	the thread is locked for the update.
  *
- *	Run queue must be locked, see run_queue_remove()
- *	for more info.
+ *	Array should be sized to make forward progress, without
+ *	disabling preemption for long periods.
  */
-static boolean_t
-run_queue_enqueue(
-	register run_queue_t	rq,
-	register thread_t		thread,
-	integer_t				options)
+
+#define	THREAD_UPDATE_SIZE		128
+
+static thread_t thread_update_array[THREAD_UPDATE_SIZE];
+static uint32_t thread_update_count = 0;
+
+/* Returns TRUE if thread was added, FALSE if thread_update_array is full */
+boolean_t
+thread_update_add_thread(thread_t thread)
 {
-	register int			whichq = thread->sched_pri;
-	register queue_t		queue = &rq->queues[whichq];
-	boolean_t				result = FALSE;
-	
-	assert(whichq >= MINPRI && whichq <= MAXPRI);
+	if (thread_update_count == THREAD_UPDATE_SIZE)
+		return (FALSE);
 
-	assert(thread->runq == RUN_QUEUE_NULL);
-	if (queue_empty(queue)) {
-		enqueue_tail(queue, (queue_entry_t)thread);
+	thread_update_array[thread_update_count++] = thread;
+	thread_reference_internal(thread);
+	return (TRUE);
+}
 
-		setbit(MAXPRI - whichq, rq->bitmap);
-		if (whichq > rq->highq) {
-			rq->highq = whichq;
-			result = TRUE;
+void
+thread_update_process_threads(void)
+{
+	assert(thread_update_count <= THREAD_UPDATE_SIZE);
+
+	for (uint32_t i = 0 ; i < thread_update_count ; i++) {
+		thread_t thread = thread_update_array[i];
+		assert_thread_magic(thread);
+		thread_update_array[i] = THREAD_NULL;
+
+		spl_t s = splsched();
+		thread_lock(thread);
+		if (!(thread->state & (TH_WAIT)) && thread->sched_stamp != sched_tick) {
+			SCHED(update_priority)(thread);
 		}
-	}
-	else
-	if (options & SCHED_HEADQ)
-		enqueue_head(queue, (queue_entry_t)thread);
-	else
-		enqueue_tail(queue, (queue_entry_t)thread);
+		thread_unlock(thread);
+		splx(s);
 
-	thread->runq = rq;
-	if (thread->sched_mode & TH_MODE_PREEMPT)
-		rq->urgency++;
-	rq->count++;
+		thread_deallocate(thread);
+	}
 
-	return (result);
+	thread_update_count = 0;
 }
 
 /*
- *	Enqueue a thread for realtime execution, similar
- *	to above.  Handles preemption directly.
+ *	Scan a runq for candidate threads.
+ *
+ *	Returns TRUE if retry is needed.
  */
-static void
-realtime_schedule_insert(
-	register processor_set_t	pset,
-	register thread_t			thread)
+boolean_t
+runq_scan(
+          run_queue_t                   runq,
+          sched_update_scan_context_t   scan_context)
 {
-	register run_queue_t	rq = &pset->runq;
-	register int			whichq = thread->sched_pri;
-	register queue_t		queue = &rq->queues[whichq];
-	uint64_t				deadline = thread->realtime.deadline;
-	boolean_t				try_preempt = FALSE;
+	int count       = runq->count;
+	int queue_index;
 
-	assert(whichq >= BASEPRI_REALTIME && whichq <= MAXPRI);
+	assert(count >= 0);
 
-	assert(thread->runq == RUN_QUEUE_NULL);
-	if (queue_empty(queue)) {
-		enqueue_tail(queue, (queue_entry_t)thread);
+	if (count == 0)
+		return FALSE;
 
-		setbit(MAXPRI - whichq, rq->bitmap);
-		if (whichq > rq->highq)
-			rq->highq = whichq;
-		try_preempt = TRUE;
-	}
-	else {
-		register thread_t	entry = (thread_t)queue_first(queue);
+	for (queue_index = bitmap_first(runq->bitmap, NRQS);
+	     queue_index >= 0;
+	     queue_index = bitmap_next(runq->bitmap, queue_index)) {
 
-		while (TRUE) {
-			if (	queue_end(queue, (queue_entry_t)entry)	||
-						deadline < entry->realtime.deadline		) {
-				entry = (thread_t)queue_prev((queue_entry_t)entry);
-				break;
-			}
+		thread_t thread;
+		queue_t  queue = &runq->queues[queue_index];
 
-			entry = (thread_t)queue_next((queue_entry_t)entry);
-		}
+		qe_foreach_element(thread, queue, runq_links) {
+			assert(count > 0);
+			assert_thread_magic(thread);
 
-		if ((queue_entry_t)entry == queue)
-			try_preempt = TRUE;
+			if (thread->sched_stamp != sched_tick &&
+			    thread->sched_mode == TH_MODE_TIMESHARE) {
+				if (thread_update_add_thread(thread) == FALSE)
+					return TRUE;
+			}
 
-		insque((queue_entry_t)thread, (queue_entry_t)entry);
+			if (cpu_throttle_enabled && ((thread->sched_pri <= MAXPRI_THROTTLE) && (thread->base_pri <= MAXPRI_THROTTLE))) {
+				if (thread->last_made_runnable_time < scan_context->earliest_bg_make_runnable_time) {
+					scan_context->earliest_bg_make_runnable_time = thread->last_made_runnable_time;
+				}
+			} else {
+				if (thread->last_made_runnable_time < scan_context->earliest_normal_make_runnable_time) {
+					scan_context->earliest_normal_make_runnable_time = thread->last_made_runnable_time;
+				}
+			}
+			count--;
+		}
 	}
 
-	thread->runq = rq;
-	assert(thread->sched_mode & TH_MODE_PREEMPT);
-	rq->count++; rq->urgency++;
+	return FALSE;
+}
 
-	if (try_preempt) {
-		register processor_t	processor;
+#endif /* CONFIG_SCHED_TIMESHARE_CORE */
 
-		processor = current_processor();
-		if (	pset == processor->processor_set				&&
-				(thread->sched_pri > processor->current_pri	||
-					deadline < processor->deadline			)		) {
-			dispatch_counts.realtime_self++;
-			simple_unlock(&pset->sched_lock);
+boolean_t
+thread_eager_preemption(thread_t thread) 
+{
+	return ((thread->sched_flags & TH_SFLAG_EAGERPREEMPT) != 0);
+}
 
-			ast_on(AST_PREEMPT | AST_URGENT);
-			return;
-		}
+void
+thread_set_eager_preempt(thread_t thread) 
+{
+	spl_t x;
+	processor_t p;
+	ast_t ast = AST_NONE;
 
-		if (	pset->processor_count > 1			||
-				pset != processor->processor_set	) {
-			processor_t		myprocessor, lastprocessor;
-			queue_entry_t	next;
+	x = splsched();
+	p = current_processor();
 
-			myprocessor = processor;
-			processor = thread->last_processor;
-			if (	processor != myprocessor						&&
-					processor != PROCESSOR_NULL						&&
-					processor->processor_set == pset				&&
-					processor->state == PROCESSOR_RUNNING			&&
-					(thread->sched_pri > processor->current_pri	||
-						deadline < processor->deadline			)		) {
-				dispatch_counts.realtime_last++;
-				cause_ast_check(processor);
-				simple_unlock(&pset->sched_lock);
-				return;
-			}
+	thread_lock(thread);
+	thread->sched_flags |= TH_SFLAG_EAGERPREEMPT;
 
-			lastprocessor = processor;
-			queue = &pset->active_queue;
-			processor = (processor_t)queue_first(queue);
-			while (!queue_end(queue, (queue_entry_t)processor)) {
-				next = queue_next((queue_entry_t)processor);
-
-				if (	processor != myprocessor						&&
-						processor != lastprocessor						&&
-						(thread->sched_pri > processor->current_pri	||
-							deadline < processor->deadline			)		) {
-					if (!queue_end(queue, next)) {
-						remqueue(queue, (queue_entry_t)processor);
-						enqueue_tail(queue, (queue_entry_t)processor);
-					}
-					dispatch_counts.realtime_other++;
-					cause_ast_check(processor);
-					simple_unlock(&pset->sched_lock);
-					return;
-				}
+	if (thread == current_thread()) {
 
-				processor = (processor_t)next;
-			}
+		ast = csw_check(p, AST_NONE);
+		thread_unlock(thread);
+		if (ast != AST_NONE) {
+			(void) thread_block_reason(THREAD_CONTINUE_NULL, NULL, ast);
+		}
+	} else {
+		p = thread->last_processor;
+
+		if (p != PROCESSOR_NULL	&& p->state == PROCESSOR_RUNNING &&
+			p->active_thread == thread) {
+			cause_ast_check(p);
 		}
+		
+		thread_unlock(thread);
 	}
 
-	simple_unlock(&pset->sched_lock);
+	splx(x);
+}
+
+void
+thread_clear_eager_preempt(thread_t thread) 
+{
+	spl_t x;
+
+	x = splsched();
+	thread_lock(thread);
+
+	thread->sched_flags &= ~TH_SFLAG_EAGERPREEMPT;
+	
+	thread_unlock(thread);
+	splx(x);
 }
 
 /*
- *	thread_setrun:
- *
- *	Dispatch thread for execution, directly onto an idle
- *	processor if possible.  Else put on appropriate run
- *	queue. (local if bound, else processor set)
- *
- *	Thread must be locked.
+ * Scheduling statistics
  */
 void
-thread_setrun(
-	register thread_t			new_thread,
-	integer_t					options)
+sched_stats_handle_csw(processor_t processor, int reasons, int selfpri, int otherpri)
 {
-	register processor_t		processor;
-	register processor_set_t	pset;
-	register thread_t			thread;
-	ast_t						preempt = (options & SCHED_PREEMPT)?
-													AST_PREEMPT: AST_NONE;
-
-	assert(thread_runnable(new_thread));
+	struct processor_sched_statistics *stats;
+	boolean_t to_realtime = FALSE;
 	
-	/*
-	 *	Update priority if needed.
-	 */
-	if (new_thread->sched_stamp != sched_tick)
-		update_priority(new_thread);
+	stats = &processor->processor_data.sched_stats;
+	stats->csw_count++;
 
-	/*
-	 *	Check for urgent preemption.
-	 */
-	if (new_thread->sched_mode & TH_MODE_PREEMPT)
-		preempt = (AST_PREEMPT | AST_URGENT);
+	if (otherpri >= BASEPRI_REALTIME) {
+		stats->rt_sched_count++;
+		to_realtime = TRUE;
+	}
 
-	assert(new_thread->runq == RUN_QUEUE_NULL);
+	if ((reasons & AST_PREEMPT) != 0) {
+		stats->preempt_count++;
 
-	if ((processor = new_thread->bound_processor) == PROCESSOR_NULL) {
-	    /*
-	     *	First try to dispatch on
-		 *	the last processor.
-	     */
-	    pset = new_thread->processor_set;
-		processor = new_thread->last_processor;
-		if (	pset->processor_count > 1				&&
-				processor != PROCESSOR_NULL				&&
-				processor->state == PROCESSOR_IDLE		) {
-			processor_lock(processor);
-			simple_lock(&pset->sched_lock);
-			if (	processor->processor_set == pset		&&
-					processor->state == PROCESSOR_IDLE		) {
-				remqueue(&pset->idle_queue, (queue_entry_t)processor);
-				pset->idle_count--;
-				processor->next_thread = new_thread;
-				if (new_thread->sched_pri >= BASEPRI_RTQUEUES)
-					processor->deadline = new_thread->realtime.deadline;
-				else
-					processor->deadline = UINT64_MAX;
-				processor->state = PROCESSOR_DISPATCHING;
-				dispatch_counts.idle_pset_last++;
-				simple_unlock(&pset->sched_lock);
-				processor_unlock(processor);
-				if (processor != current_processor())
-					machine_signal_idle(processor);
-				return;
-			}
-			processor_unlock(processor);
-		}
-		else
-		simple_lock(&pset->sched_lock);
+		if (selfpri >= BASEPRI_REALTIME) {
+			stats->preempted_rt_count++;
+		} 
 
-		/*
-		 *	Next pick any idle processor
-		 *	in the processor set.
-		 */
-		if (pset->idle_count > 0) {
-			processor = (processor_t)dequeue_head(&pset->idle_queue);
-			pset->idle_count--;
-			processor->next_thread = new_thread;
-			if (new_thread->sched_pri >= BASEPRI_RTQUEUES)
-				processor->deadline = new_thread->realtime.deadline;
-			else
-				processor->deadline = UINT64_MAX;
-			processor->state = PROCESSOR_DISPATCHING;
-			dispatch_counts.idle_pset_any++;
-			simple_unlock(&pset->sched_lock);
-			if (processor != current_processor())	
-				machine_signal_idle(processor);
-			return;
+		if (to_realtime) {
+			stats->preempted_by_rt_count++;
 		}
 
-		if (new_thread->sched_pri >= BASEPRI_RTQUEUES)
-			realtime_schedule_insert(pset, new_thread);
-		else {
-			if (!run_queue_enqueue(&pset->runq, new_thread, options))
-				preempt = AST_NONE;
+	}
+}
 
-			/*
-			 *	Update the timesharing quanta.
-			 */
-			timeshare_quanta_update(pset);
-	
-			/*
-			 *	Preempt check.
-			 */
-			if (preempt != AST_NONE) {
-				/*
-				 * First try the current processor
-				 * if it is a member of the correct
-				 * processor set.
-				 */
-				processor = current_processor();
-				thread = processor->active_thread;
-				if (	pset == processor->processor_set	&&
-						csw_needed(thread, processor)		) {
-					dispatch_counts.pset_self++;
-					simple_unlock(&pset->sched_lock);
+void
+sched_stats_handle_runq_change(struct runq_stats *stats, int old_count) 
+{
+	uint64_t timestamp = mach_absolute_time();
 
-					ast_on(preempt);
-					return;
-				}
+	stats->count_sum += (timestamp - stats->last_change_timestamp) * old_count;
+	stats->last_change_timestamp = timestamp;
+}
 
-				/*
-				 * If that failed and we have other
-				 * processors available keep trying.
-				 */
-				if (	pset->processor_count > 1			||
-						pset != processor->processor_set	) {
-					queue_t			queue = &pset->active_queue;
-					processor_t		myprocessor, lastprocessor;
-					queue_entry_t	next;
+/*
+ *     For calls from assembly code
+ */
+#undef thread_wakeup
+void
+thread_wakeup(
+       event_t         x);
 
-					/*
-					 * Next try the last processor
-					 * dispatched on.
-					 */
-					myprocessor = processor;
-					processor = new_thread->last_processor;
-					if (	processor != myprocessor						&&
-							processor != PROCESSOR_NULL						&&
-							processor->processor_set == pset				&&
-							processor->state == PROCESSOR_RUNNING			&&
-							new_thread->sched_pri > processor->current_pri	) {
-						dispatch_counts.pset_last++;
-						cause_ast_check(processor);
-						simple_unlock(&pset->sched_lock);
-						return;
-					}
+void
+thread_wakeup(
+       event_t         x)
+{
+       thread_wakeup_with_result(x, THREAD_AWAKENED);
+}
 
-					/*
-					 * Lastly, pick any other
-					 * available processor.
-					 */
-					lastprocessor = processor;
-					processor = (processor_t)queue_first(queue);
-					while (!queue_end(queue, (queue_entry_t)processor)) {
-						next = queue_next((queue_entry_t)processor);
-
-						if (	processor != myprocessor			&&
-								processor != lastprocessor			&&
-								new_thread->sched_pri >
-											processor->current_pri		) {
-							if (!queue_end(queue, next)) {
-								remqueue(queue, (queue_entry_t)processor);
-								enqueue_tail(queue, (queue_entry_t)processor);
-							}
-							dispatch_counts.pset_other++;
-							cause_ast_check(processor);
-							simple_unlock(&pset->sched_lock);
-							return;
-						}
+boolean_t
+preemption_enabled(void)
+{
+	return (get_preemption_level() == 0 && ml_get_interrupts_enabled());
+}
 
-						processor = (processor_t)next;
-					}
-				}
-			}
+static void
+sched_timer_deadline_tracking_init(void) {
+	nanoseconds_to_absolutetime(TIMER_DEADLINE_TRACKING_BIN_1_DEFAULT, &timer_deadline_tracking_bin_1);
+	nanoseconds_to_absolutetime(TIMER_DEADLINE_TRACKING_BIN_2_DEFAULT, &timer_deadline_tracking_bin_2);
+}
 
-			simple_unlock(&pset->sched_lock);
-		}
-	}
-	else {
-	    /*
-	     *	Bound, can only run on bound processor.  Have to lock
-	     *  processor here because it may not be the current one.
-	     */
-		processor_lock(processor);
-		pset = processor->processor_set;
-		if (pset != PROCESSOR_SET_NULL) {
-			simple_lock(&pset->sched_lock);
-			if (processor->state == PROCESSOR_IDLE) {
-				remqueue(&pset->idle_queue, (queue_entry_t)processor);
-				pset->idle_count--;
-				processor->next_thread = new_thread;
-				processor->deadline = UINT64_MAX;
-				processor->state = PROCESSOR_DISPATCHING;
-				dispatch_counts.idle_bound++;
-				simple_unlock(&pset->sched_lock);
-				processor_unlock(processor);
-				if (processor != current_processor())	
-					machine_signal_idle(processor);
-				return;
-			}
-		}
-	  
-		if (!run_queue_enqueue(&processor->runq, new_thread, options))
-			preempt = AST_NONE;
+#if __arm__ || __arm64__
 
-		if (preempt != AST_NONE) {
-			if (processor == current_processor()) {
-				thread = processor->active_thread;
-				if (csw_needed(thread, processor)) {
-					dispatch_counts.bound_self++;
-					ast_on(preempt);
-				}
-			}
-			else
-			if (	processor->state == PROCESSOR_RUNNING			&&
-					new_thread->sched_pri > processor->current_pri	) {
-				dispatch_counts.bound_other++;
-				cause_ast_check(processor);
-			}
-		}
+uint32_t    perfcontrol_requested_recommended_cores = ALL_CORES_RECOMMENDED;
+uint32_t    perfcontrol_requested_recommended_core_count = MAX_CPUS;
+bool        perfcontrol_failsafe_active = false;
+bool        perfcontrol_sleep_override = false;
 
-		if (pset != PROCESSOR_SET_NULL)
-			simple_unlock(&pset->sched_lock);
+uint64_t    perfcontrol_failsafe_maintenance_runnable_time;
+uint64_t    perfcontrol_failsafe_activation_time;
+uint64_t    perfcontrol_failsafe_deactivation_time;
 
-		processor_unlock(processor);
-	}
-}
+/* data covering who likely caused it and how long they ran */
+#define FAILSAFE_NAME_LEN       33 /* (2*MAXCOMLEN)+1 from size of p_name */
+char        perfcontrol_failsafe_name[FAILSAFE_NAME_LEN];
+int         perfcontrol_failsafe_pid;
+uint64_t    perfcontrol_failsafe_tid;
+uint64_t    perfcontrol_failsafe_thread_timer_at_start;
+uint64_t    perfcontrol_failsafe_thread_timer_last_seen;
+uint32_t    perfcontrol_failsafe_recommended_at_trigger;
 
 /*
- *	Check for a possible preemption point in
- *	the (current) thread.
+ * Perf controller calls here to update the recommended core bitmask.
+ * If the failsafe is active, we don't immediately apply the new value.
+ * Instead, we store the new request and use it after the failsafe deactivates.
  *
- *	Called at splsched.
+ * If the failsafe is not active, immediately apply the update.
+ *
+ * No scheduler locks are held, no other locks are held that scheduler might depend on,
+ * interrupts are enabled
+ *
+ * currently prototype is in osfmk/arm/machine_routines.h
  */
-ast_t
-csw_check(
-	thread_t		thread,
-	processor_t		processor)
+void
+sched_perfcontrol_update_recommended_cores(uint32_t recommended_cores)
 {
-	int				current_pri = thread->sched_pri;
-	ast_t			result = AST_NONE;
-	run_queue_t		runq;
+	assert(preemption_enabled());
 
-	if (first_timeslice(processor)) {
-		runq = &processor->processor_set->runq;
-		if (runq->highq >= BASEPRI_RTQUEUES)
-			return (AST_PREEMPT | AST_URGENT);
+	spl_t s = splsched();
+	simple_lock(&sched_recommended_cores_lock);
 
-		if (runq->highq > current_pri) {
-			if (runq->urgency > 0)
-				return (AST_PREEMPT | AST_URGENT);
+	perfcontrol_requested_recommended_cores = recommended_cores;
+	perfcontrol_requested_recommended_core_count = __builtin_popcountll(recommended_cores);
 
-			result |= AST_PREEMPT;
-		}
+	if ((perfcontrol_failsafe_active == false) && (perfcontrol_sleep_override == false))
+		sched_update_recommended_cores(perfcontrol_requested_recommended_cores);
+	else
+		KERNEL_DEBUG_CONSTANT_IST(KDEBUG_TRACE,
+		        MACHDBG_CODE(DBG_MACH_SCHED,MACH_REC_CORES_FAILSAFE) | DBG_FUNC_NONE,
+		        perfcontrol_requested_recommended_cores,
+		        sched_maintenance_thread->last_made_runnable_time, 0, 0, 0);
 
-		runq = &processor->runq;
-		if (runq->highq > current_pri) {
-			if (runq->urgency > 0)
-				return (AST_PREEMPT | AST_URGENT);
+	simple_unlock(&sched_recommended_cores_lock);
+	splx(s);
+}
 
-			result |= AST_PREEMPT;
-		}
+void
+sched_override_recommended_cores_for_sleep(void)
+{
+	spl_t s = splsched();
+	simple_lock(&sched_recommended_cores_lock);
+
+	if (perfcontrol_sleep_override == false) {
+		perfcontrol_sleep_override = true;
+		sched_update_recommended_cores(ALL_CORES_RECOMMENDED);
 	}
-	else {
-		runq = &processor->processor_set->runq;
-		if (runq->highq >= current_pri) {
-			if (runq->urgency > 0)
-				return (AST_PREEMPT | AST_URGENT);
 
-			result |= AST_PREEMPT;
-		}
+	simple_unlock(&sched_recommended_cores_lock);
+	splx(s);
+}
 
-		runq = &processor->runq;
-		if (runq->highq >= current_pri) {
-			if (runq->urgency > 0)
-				return (AST_PREEMPT | AST_URGENT);
+void
+sched_restore_recommended_cores_after_sleep(void)
+{
+	spl_t s = splsched();
+	simple_lock(&sched_recommended_cores_lock);
 
-			result |= AST_PREEMPT;
-		}
+	if (perfcontrol_sleep_override == true) {
+		perfcontrol_sleep_override = false;
+		sched_update_recommended_cores(perfcontrol_requested_recommended_cores);
 	}
 
-	if (result != AST_NONE)
-		return (result);
-
-	if (thread->state & TH_SUSP)
-		result |= AST_PREEMPT;
-
-	return (result);
+	simple_unlock(&sched_recommended_cores_lock);
+	splx(s);
 }
 
 /*
- *	set_sched_pri:
- *
- *	Set the scheduled priority of the specified thread.
+ * Consider whether we need to activate the recommended cores failsafe
  *
- *	This may cause the thread to change queues.
- *
- *	Thread must be locked.
+ * Called from quantum timer interrupt context of a realtime thread
+ * No scheduler locks are held, interrupts are disabled
  */
 void
-set_sched_pri(
-	thread_t			thread,
-	int					priority)
+sched_consider_recommended_cores(uint64_t ctime, thread_t cur_thread)
 {
-	register struct run_queue	*rq = run_queue_remove(thread);
-
-	if (	!(thread->sched_mode & TH_MODE_TIMESHARE)				&&
-			(priority >= BASEPRI_PREEMPT						||
-			 (thread->task_priority < MINPRI_KERNEL			&&
-			  thread->task_priority >= BASEPRI_BACKGROUND	&&
-			  priority > thread->task_priority)					)	)
-		thread->sched_mode |= TH_MODE_PREEMPT;
-	else
-		thread->sched_mode &= ~TH_MODE_PREEMPT;
+	/*
+	 * Check if a realtime thread is starving the system
+	 * and bringing up non-recommended cores would help
+	 *
+	 * TODO: Is this the correct check for recommended == possible cores?
+	 * TODO: Validate the checks without the relevant lock are OK.
+	 */
 
-	thread->sched_pri = priority;
-	if (rq != RUN_QUEUE_NULL)
-		thread_setrun(thread, SCHED_PREEMPT | SCHED_TAILQ);
-	else
-	if (thread->state & TH_RUN) {
-		processor_t		processor = thread->last_processor;
+	if (__improbable(perfcontrol_failsafe_active == TRUE)) {
+		/* keep track of how long the responsible thread runs */
 
-		if (thread == current_thread()) {
-			ast_t		preempt = csw_check(thread, processor);
+		simple_lock(&sched_recommended_cores_lock);
 
-			if (preempt != AST_NONE)
-				ast_on(preempt);
-			processor->current_pri = priority;
+		if (perfcontrol_failsafe_active == TRUE &&
+		    cur_thread->thread_id == perfcontrol_failsafe_tid) {
+			perfcontrol_failsafe_thread_timer_last_seen = timer_grab(&cur_thread->user_timer) +
+			                                              timer_grab(&cur_thread->system_timer);
 		}
-		else
-		if (	processor != PROCESSOR_NULL						&&
-				processor->active_thread == thread	)
-			cause_ast_check(processor);
+
+		simple_unlock(&sched_recommended_cores_lock);
+
+		/* we're already trying to solve the problem, so bail */
+		return;
 	}
-}
 
-#if		0
+	/* The failsafe won't help if there are no more processors to enable */
+	if (__probable(perfcontrol_requested_recommended_core_count >= processor_count))
+		return;
 
-static void
-run_queue_check(
-	run_queue_t		rq,
-	thread_t		thread)
-{
-	queue_t			q;
-	queue_entry_t	qe;
+	uint64_t too_long_ago = ctime - perfcontrol_failsafe_starvation_threshold;
 
-	if (rq != thread->runq)
-		panic("run_queue_check: thread runq");
+	/* Use the maintenance thread as our canary in the coal mine */
+	thread_t m_thread = sched_maintenance_thread;
 
-	if (thread->sched_pri > MAXPRI || thread->sched_pri < MINPRI)
-		panic("run_queue_check: thread sched_pri");
+	/* If it doesn't look bad, nothing to see here */
+	if (__probable(m_thread->last_made_runnable_time >= too_long_ago))
+		return;
 
-	q = &rq->queues[thread->sched_pri];
-	qe = queue_first(q);
-	while (!queue_end(q, qe)) {
-		if (qe == (queue_entry_t)thread)
-			return;
+	/* It looks bad, take the lock to be sure */
+	thread_lock(m_thread);
 
-		qe = queue_next(qe);
+	if (m_thread->runq == PROCESSOR_NULL ||
+	   (m_thread->state & (TH_RUN|TH_WAIT)) != TH_RUN ||
+	    m_thread->last_made_runnable_time >= too_long_ago) {
+		/*
+		 * Maintenance thread is either on cpu or blocked, and
+		 * therefore wouldn't benefit from more cores
+		 */
+		thread_unlock(m_thread);
+		return;
 	}
 
-	panic("run_queue_check: end");
-}
-
-#endif	/* DEBUG */
+	uint64_t maintenance_runnable_time = m_thread->last_made_runnable_time;
 
-/*
- *	run_queue_remove:
- *
- *	Remove a thread from its current run queue and
- *	return the run queue if successful.
- *
- *	Thread must be locked.
- */
-run_queue_t
-run_queue_remove(
-	thread_t			thread)
-{
-	register run_queue_t	rq = thread->runq;
+	thread_unlock(m_thread);
 
 	/*
-	 *	If rq is RUN_QUEUE_NULL, the thread will stay out of the
-	 *	run queues because the caller locked the thread.  Otherwise
-	 *	the thread is on a run queue, but could be chosen for dispatch
-	 *	and removed.
+	 * There are cores disabled at perfcontrol's recommendation, but the
+	 * system is so overloaded that the maintenance thread can't run.
+	 * That likely means that perfcontrol can't run either, so it can't fix
+	 * the recommendation.  We have to kick in a failsafe to keep from starving.
+	 *
+	 * When the maintenance thread has been starved for too long,
+	 * ignore the recommendation from perfcontrol and light up all the cores.
+	 *
+	 * TODO: Consider weird states like boot, sleep, or debugger
 	 */
-	if (rq != RUN_QUEUE_NULL) {
-		processor_set_t		pset = thread->processor_set;
-		processor_t			processor = thread->bound_processor;
 
-		/*
-		 *	The run queues are locked by the pset scheduling
-		 *	lock, except when a processor is off-line the
-		 *	local run queue is locked by the processor lock.
-		 */
-		if (processor != PROCESSOR_NULL) {
-			processor_lock(processor);
-			pset = processor->processor_set;
-		}
+	simple_lock(&sched_recommended_cores_lock);
 
-		if (pset != PROCESSOR_SET_NULL)
-			simple_lock(&pset->sched_lock);
+	if (perfcontrol_failsafe_active == TRUE) {
+		simple_unlock(&sched_recommended_cores_lock);
+		return;
+	}
 
-		if (rq == thread->runq) {
-			/*
-			 *	Thread is on a run queue and we have a lock on
-			 *	that run queue.
-			 */
-			remqueue(&rq->queues[0], (queue_entry_t)thread);
-			rq->count--;
-			if (thread->sched_mode & TH_MODE_PREEMPT)
-				rq->urgency--;
-			assert(rq->urgency >= 0);
-
-			if (queue_empty(rq->queues + thread->sched_pri)) {
-				/* update run queue status */
-				if (thread->sched_pri != IDLEPRI)
-					clrbit(MAXPRI - thread->sched_pri, rq->bitmap);
-				rq->highq = MAXPRI - ffsbit(rq->bitmap);
-			}
+	KERNEL_DEBUG_CONSTANT_IST(KDEBUG_TRACE,
+	        MACHDBG_CODE(DBG_MACH_SCHED,MACH_REC_CORES_FAILSAFE) | DBG_FUNC_START,
+	        perfcontrol_requested_recommended_cores, maintenance_runnable_time, 0, 0, 0);
 
-			thread->runq = RUN_QUEUE_NULL;
-		}
-		else {
-			/*
-			 *	The thread left the run queue before we could
-			 * 	lock the run queue.
-			 */
-			assert(thread->runq == RUN_QUEUE_NULL);
-			rq = RUN_QUEUE_NULL;
-		}
+	perfcontrol_failsafe_active = TRUE;
+	perfcontrol_failsafe_activation_time = mach_absolute_time();
+	perfcontrol_failsafe_maintenance_runnable_time = maintenance_runnable_time;
+	perfcontrol_failsafe_recommended_at_trigger = perfcontrol_requested_recommended_cores;
+
+	/* Capture some data about who screwed up (assuming that the thread on core is at fault) */
+	task_t task = cur_thread->task;
+	perfcontrol_failsafe_pid = task_pid(task);
+	strlcpy(perfcontrol_failsafe_name, proc_name_address(task->bsd_info), sizeof(perfcontrol_failsafe_name));
+
+	perfcontrol_failsafe_tid = cur_thread->thread_id;
 
-		if (pset != PROCESSOR_SET_NULL)
-			simple_unlock(&pset->sched_lock);
+	/* Blame the thread for time it has run recently */
+	uint64_t recent_computation = (ctime - cur_thread->computation_epoch) + cur_thread->computation_metered;
 
-		if (processor != PROCESSOR_NULL)
-			processor_unlock(processor);
-	}
+	uint64_t last_seen = timer_grab(&cur_thread->user_timer) + timer_grab(&cur_thread->system_timer);
+
+	/* Compute the start time of the bad behavior in terms of the thread's on core time */
+	perfcontrol_failsafe_thread_timer_at_start  = last_seen - recent_computation;
+	perfcontrol_failsafe_thread_timer_last_seen = last_seen;
 
-	return (rq);
+	/* Ignore the previously recommended core configuration */
+	sched_update_recommended_cores(ALL_CORES_RECOMMENDED);
+
+	simple_unlock(&sched_recommended_cores_lock);
 }
 
 /*
- *	choose_thread:
- *
- *	Remove a thread to execute from the run queues
- *	and return it.
+ * Now that our bacon has been saved by the failsafe, consider whether to turn it off
  *
- *	Called with pset scheduling lock held.
+ * Runs in the context of the maintenance thread, no locks held
  */
-static thread_t
-choose_thread(
-	processor_set_t		pset,
-	processor_t			processor)
+static void
+sched_recommended_cores_maintenance(void)
 {
-	register run_queue_t	runq;
-	register thread_t		thread;
-	register queue_t		q;
+	/* Common case - no failsafe, nothing to be done here */
+	if (__probable(perfcontrol_failsafe_active == FALSE))
+		return;
 
-	runq = &processor->runq;
+	uint64_t ctime = mach_absolute_time();
 
-	if (runq->count > 0 && runq->highq >= pset->runq.highq) {
-		q = runq->queues + runq->highq;
+	boolean_t print_diagnostic = FALSE;
+	char p_name[FAILSAFE_NAME_LEN] = "";
 
-		thread = (thread_t)q->next;
-		((queue_entry_t)thread)->next->prev = q;
-		q->next = ((queue_entry_t)thread)->next;
-		thread->runq = RUN_QUEUE_NULL;
-		runq->count--;
-		if (thread->sched_mode & TH_MODE_PREEMPT)
-			runq->urgency--;
-		assert(runq->urgency >= 0);
-		if (queue_empty(q)) {
-			if (runq->highq != IDLEPRI)
-				clrbit(MAXPRI - runq->highq, runq->bitmap);
-			runq->highq = MAXPRI - ffsbit(runq->bitmap);
-		}
-
-		processor->deadline = UINT64_MAX;
+	spl_t s = splsched();
+	simple_lock(&sched_recommended_cores_lock);
 
-		return (thread);
-	}
+	/* Check again, under the lock, to avoid races */
+	if (perfcontrol_failsafe_active == FALSE)
+		goto out;
 
-	runq = &pset->runq;
+	/*
+	 * Ensure that the other cores get another few ticks to run some threads
+	 * If we don't have this hysteresis, the maintenance thread is the first
+	 * to run, and then it immediately kills the other cores
+	 */
+	if ((ctime - perfcontrol_failsafe_activation_time) < perfcontrol_failsafe_starvation_threshold)
+		goto out;
 
-	assert(runq->count > 0);
-	q = runq->queues + runq->highq;
+	/* Capture some diagnostic state under the lock so we can print it out later */
 
-	thread = (thread_t)q->next;
-	((queue_entry_t)thread)->next->prev = q;
-	q->next = ((queue_entry_t)thread)->next;
-	thread->runq = RUN_QUEUE_NULL;
-	runq->count--;
-	if (runq->highq >= BASEPRI_RTQUEUES)
-		processor->deadline = thread->realtime.deadline;
-	else
-		processor->deadline = UINT64_MAX;
-	if (thread->sched_mode & TH_MODE_PREEMPT)
-		runq->urgency--;
-	assert(runq->urgency >= 0);
-	if (queue_empty(q)) {
-		if (runq->highq != IDLEPRI)
-			clrbit(MAXPRI - runq->highq, runq->bitmap);
-		runq->highq = MAXPRI - ffsbit(runq->bitmap);
-	}
+	int      pid = perfcontrol_failsafe_pid;
+	uint64_t tid = perfcontrol_failsafe_tid;
 
-	timeshare_quanta_update(pset);
+	uint64_t thread_usage       = perfcontrol_failsafe_thread_timer_last_seen -
+	                              perfcontrol_failsafe_thread_timer_at_start;
+	uint32_t rec_cores_before   = perfcontrol_failsafe_recommended_at_trigger;
+	uint32_t rec_cores_after    = perfcontrol_requested_recommended_cores;
+	uint64_t failsafe_duration  = ctime - perfcontrol_failsafe_activation_time;
+	strlcpy(p_name, perfcontrol_failsafe_name, sizeof(p_name));
 
-	return (thread);
-}
+	print_diagnostic = TRUE;
 
-static processor_t
-delay_idle(
-	processor_t		processor,
-	thread_t		self)
-{
-	int				*gcount, *lcount;
-	uint64_t		abstime, spin, limit;
+	/* Deactivate the failsafe and reinstate the requested recommendation settings */
 
-	lcount = &processor->runq.count;
-	gcount = &processor->processor_set->runq.count;
+	perfcontrol_failsafe_deactivation_time = ctime;
+	perfcontrol_failsafe_active = FALSE;
 
-	abstime = mach_absolute_time();
-	limit = abstime + delay_idle_limit;
-	spin = abstime + delay_idle_spin;
+	KERNEL_DEBUG_CONSTANT_IST(KDEBUG_TRACE,
+	        MACHDBG_CODE(DBG_MACH_SCHED,MACH_REC_CORES_FAILSAFE) | DBG_FUNC_END,
+	        perfcontrol_requested_recommended_cores, failsafe_duration, 0, 0, 0);
 
-	timer_event((uint32_t)abstime, &processor->idle_thread->system_timer);
+	sched_update_recommended_cores(perfcontrol_requested_recommended_cores);
 
-	self->options |= TH_OPT_DELAYIDLE;
+out:
+	simple_unlock(&sched_recommended_cores_lock);
+	splx(s);
 
-	while (		*gcount == 0 && *lcount == 0	&&
-				(self->state & TH_WAIT)	!= 0	&&
-					abstime < limit				) {
-		if (abstime >= spin) {
-			(void)spllo();
+	if (print_diagnostic) {
+		uint64_t failsafe_duration_ms = 0, thread_usage_ms = 0;
 
-			(void)splsched();
-			processor = current_processor();
-			lcount = &processor->runq.count;
-			gcount = &processor->processor_set->runq.count;
+		absolutetime_to_nanoseconds(failsafe_duration, &failsafe_duration_ms);
+		failsafe_duration_ms = failsafe_duration_ms / NSEC_PER_MSEC;
 
-			abstime = mach_absolute_time();
-			spin = abstime + delay_idle_spin;
+		absolutetime_to_nanoseconds(thread_usage, &thread_usage_ms);
+		thread_usage_ms = thread_usage_ms / NSEC_PER_MSEC;
 
-			timer_event((uint32_t)abstime, &processor->idle_thread->system_timer);
-		}
-		else
-			abstime = mach_absolute_time();
+		printf("recommended core failsafe kicked in for %lld ms "
+		       "likely due to %s[%d] thread 0x%llx spending "
+		       "%lld ms on cpu at realtime priority - "
+		       "new recommendation: 0x%x -> 0x%x\n",
+		       failsafe_duration_ms, p_name, pid, tid, thread_usage_ms,
+		       rec_cores_before, rec_cores_after);
 	}
-
-	timer_event((uint32_t)abstime, &self->system_timer);
-
-	self->options &= ~TH_OPT_DELAYIDLE;
-
-	return (processor);
 }
 
 /*
- *	no_dispatch_count counts number of times processors go non-idle
- *	without being dispatched.  This should be very rare.
- */
-int	no_dispatch_count = 0;
-
-/*
- *	This is the idle processor thread, which just looks for other threads
- *	to execute.
+ * Apply a new recommended cores mask to the processors it affects
+ * Runs after considering failsafes and such
+ *
+ * Iterate over processors and update their ->is_recommended field.
+ * If a processor is running, we let it drain out at its next
+ * quantum expiration or blocking point. If a processor is idle, there
+ * may be more work for it to do, so IPI it.
+ *
+ * interrupts disabled, sched_recommended_cores_lock is held
  */
-void
-idle_thread(void)
+static void
+sched_update_recommended_cores(uint32_t recommended_cores)
 {
-	register processor_t		processor;
-	register thread_t			*threadp;
-	register int				*gcount;
-	register int				*lcount;
-	register thread_t			new_thread;
-	register int				state;
-	register processor_set_t 	pset;
-	ast_t						*myast = ast_pending();
-
-	processor = current_processor();
-
-	threadp = &processor->next_thread;
-	lcount = &processor->runq.count;
-	gcount = &processor->processor_set->runq.count;
+	processor_set_t pset, nset;
+	processor_t     processor;
+	uint64_t        needs_exit_idle_mask = 0x0;
 
+	processor = processor_list;
+	pset = processor->processor_set;
 
-	(void)splsched();			/* Turn interruptions off */
+	KDBG(MACHDBG_CODE(DBG_MACH_SCHED,MACH_SCHED_UPDATE_REC_CORES) | DBG_FUNC_START,
+	        recommended_cores, perfcontrol_failsafe_active, 0, 0);
 
-#ifdef __ppc__
-	pmsDown();					/* Step power down.  Note: interruptions must be disabled for this call */
-#endif
+	if (__builtin_popcount(recommended_cores) == 0) {
+		bit_set(recommended_cores, master_processor->cpu_id); /* add boot processor or we hang */
+	}
 
-	while (	(*threadp == THREAD_NULL)				&&
-				(*gcount == 0) && (*lcount == 0)	) {
+	/* First set recommended cores */
+	pset_lock(pset);
+	do {
 
-		/* check for ASTs while we wait */
-		if (*myast &~ (AST_SCHEDULING | AST_BSD)) {
-			/* no ASTs for us */
-			*myast &= AST_NONE;
-			(void)spllo();
+		nset = processor->processor_set;
+		if (nset != pset) {
+			pset_unlock(pset);
+			pset = nset;
+			pset_lock(pset);
 		}
-		else
-			machine_idle();
-
-		(void)splsched();
-	}
-
-	/*
-	 *	This is not a switch statement to avoid the
-	 *	bounds checking code in the common case.
-	 */
-	pset = processor->processor_set;
-	simple_lock(&pset->sched_lock);
 
-#ifdef __ppc__
-	pmsStep(0);					/* Step up out of idle power, may start timer for next step */
-#endif
+		if (bit_test(recommended_cores, processor->cpu_id)) {
+			processor->is_recommended = TRUE;
+			bit_set(pset->recommended_bitmask, processor->cpu_id);
 
-	state = processor->state;
-	if (state == PROCESSOR_DISPATCHING) {
-		/*
-		 *	Commmon case -- cpu dispatched.
-		 */
-		new_thread = *threadp;
-		*threadp = (volatile thread_t) THREAD_NULL;
-		processor->state = PROCESSOR_RUNNING;
-		enqueue_tail(&pset->active_queue, (queue_entry_t)processor);
-
-		if (	pset->runq.highq >= BASEPRI_RTQUEUES			&&
-				new_thread->sched_pri >= BASEPRI_RTQUEUES		) {
-			register run_queue_t	runq = &pset->runq;
-			register queue_t		q;
-
-			q = runq->queues + runq->highq;
-			if (((thread_t)q->next)->realtime.deadline <
-											processor->deadline) {
-				thread_t	thread = new_thread;
-
-				new_thread = (thread_t)q->next;
-				((queue_entry_t)new_thread)->next->prev = q;
-				q->next = ((queue_entry_t)new_thread)->next;
-				new_thread->runq = RUN_QUEUE_NULL;
-				processor->deadline = new_thread->realtime.deadline;
-				assert(new_thread->sched_mode & TH_MODE_PREEMPT);
-				runq->count--; runq->urgency--;
-				if (queue_empty(q)) {
-					if (runq->highq != IDLEPRI)
-						clrbit(MAXPRI - runq->highq, runq->bitmap);
-					runq->highq = MAXPRI - ffsbit(runq->bitmap);
+			if (processor->state == PROCESSOR_IDLE) {
+				if (processor != current_processor()) {
+					bit_set(needs_exit_idle_mask, processor->cpu_id);
 				}
-				dispatch_counts.missed_realtime++;
-				simple_unlock(&pset->sched_lock);
+			}
+		}
+	} while ((processor = processor->processor_list) != NULL);
+	pset_unlock(pset);
 
-				thread_lock(thread);
-				thread_setrun(thread, SCHED_HEADQ);
-				thread_unlock(thread);
+	/* Now shutdown not recommended cores */
+	processor = processor_list;
+	pset = processor->processor_set;
 
-				counter(c_idle_thread_handoff++);
-				thread_run(processor->idle_thread, (thread_continue_t)idle_thread, NULL, new_thread);
-				/*NOTREACHED*/
-			}
-			simple_unlock(&pset->sched_lock);
+	pset_lock(pset);
+	do {
 
-			counter(c_idle_thread_handoff++);
-			thread_run(processor->idle_thread, (thread_continue_t)idle_thread, NULL, new_thread);
-			/*NOTREACHED*/
+		nset = processor->processor_set;
+		if (nset != pset) {
+			pset_unlock(pset);
+			pset = nset;
+			pset_lock(pset);
 		}
 
-		if (	processor->runq.highq > new_thread->sched_pri		||
-				pset->runq.highq > new_thread->sched_pri				) {
-			thread_t	thread = new_thread;
+		if (!bit_test(recommended_cores, processor->cpu_id)) {
+			sched_ipi_type_t ipi_type = SCHED_IPI_NONE;
 
-			new_thread = choose_thread(pset, processor);
-			dispatch_counts.missed_other++;
-			simple_unlock(&pset->sched_lock);
+			processor->is_recommended = FALSE;
+			bit_clear(pset->recommended_bitmask, processor->cpu_id);
 
-			thread_lock(thread);
-			thread_setrun(thread, SCHED_HEADQ);
-			thread_unlock(thread);
-
-			counter(c_idle_thread_handoff++);
-			thread_run(processor->idle_thread, (thread_continue_t)idle_thread, NULL, new_thread);
-			/* NOTREACHED */
-		}
-		else {
-			simple_unlock(&pset->sched_lock);
+			if ((processor->state == PROCESSOR_RUNNING) || (processor->state == PROCESSOR_DISPATCHING)) {
+				ipi_type = SCHED_IPI_IMMEDIATE;
+			}
+			SCHED(processor_queue_shutdown)(processor);
+			/* pset unlocked */
 
-			counter(c_idle_thread_handoff++);
-			thread_run(processor->idle_thread, (thread_continue_t)idle_thread, NULL, new_thread);
-			/* NOTREACHED */
-		}
-	}
-	else
-	if (state == PROCESSOR_IDLE) {
-		/*
-		 *	Processor was not dispatched (Rare).
-		 *	Set it running again and force a
-		 *	reschedule.
-		 */
-		no_dispatch_count++;
-		pset->idle_count--;
-		remqueue(&pset->idle_queue, (queue_entry_t)processor);
-		processor->state = PROCESSOR_RUNNING;
-		enqueue_tail(&pset->active_queue, (queue_entry_t)processor);
-		simple_unlock(&pset->sched_lock);
+			SCHED(rt_queue_shutdown)(processor);
 
-		counter(c_idle_thread_block++);
-		thread_block((thread_continue_t)idle_thread);
-		/* NOTREACHED */
-	}
-	else
-	if (state == PROCESSOR_SHUTDOWN) {
-		/*
-		 *	Going off-line.  Force a
-		 *	reschedule.
-		 */
-		if ((new_thread = (thread_t)*threadp) != THREAD_NULL) {
-			*threadp = (volatile thread_t) THREAD_NULL;
-			processor->deadline = UINT64_MAX;
-			simple_unlock(&pset->sched_lock);
+			if (ipi_type != SCHED_IPI_NONE) {
+				if (processor == current_processor()) {
+					ast_on(AST_PREEMPT);
+				} else {
+					sched_ipi_perform(processor, ipi_type);
+				}
+			}
 
-			thread_lock(new_thread);
-			thread_setrun(new_thread, SCHED_HEADQ);
-			thread_unlock(new_thread);
+			pset_lock(pset);
 		}
-		else
-			simple_unlock(&pset->sched_lock);
+	} while ((processor = processor->processor_list) != NULL);
+	pset_unlock(pset);
 
-		counter(c_idle_thread_block++);
-		thread_block((thread_continue_t)idle_thread);
-		/* NOTREACHED */
+	/* Issue all pending IPIs now that the pset lock has been dropped */
+	for (int cpuid = lsb_first(needs_exit_idle_mask); cpuid >= 0; cpuid = lsb_next(needs_exit_idle_mask, cpuid)) {
+		processor = processor_array[cpuid];
+		machine_signal_idle(processor);
 	}
 
-	simple_unlock(&pset->sched_lock);
-
-	panic("idle_thread: state %d\n", processor->state);
-	/*NOTREACHED*/
+	KDBG(MACHDBG_CODE(DBG_MACH_SCHED,MACH_SCHED_UPDATE_REC_CORES) | DBG_FUNC_END,
+							  needs_exit_idle_mask, 0, 0, 0);
 }
+#endif /* __arm__ || __arm64__ */
 
-kern_return_t
-idle_thread_create(
-	processor_t		processor)
-{
-	kern_return_t	result;
-	thread_t		thread;
-	spl_t			s;
-
-	result = kernel_thread_create((thread_continue_t)idle_thread, NULL, MAXPRI_KERNEL, &thread);
-	if (result != KERN_SUCCESS)
-		return (result);
+void thread_set_options(uint32_t thopt) {
+ 	spl_t x;
+ 	thread_t t = current_thread();
+ 
+ 	x = splsched();
+ 	thread_lock(t);
+ 
+ 	t->options |= thopt;
+ 
+ 	thread_unlock(t);
+ 	splx(x);
+}
 
-	s = splsched();
-	thread_lock(thread);
-	thread->bound_processor = processor;
-	processor->idle_thread = thread;
-	thread->sched_pri = thread->priority = IDLEPRI;
-	thread->state = (TH_RUN | TH_IDLE);
-	thread_unlock(thread);
-	splx(s);
+void thread_set_pending_block_hint(thread_t thread, block_hint_t block_hint) {
+	thread->pending_block_hint = block_hint;
+}
 
-	thread_deallocate(thread);
+uint32_t qos_max_parallelism(int qos, uint64_t options)
+{
+    return SCHED(qos_max_parallelism)(qos, options);
+}
 
-	return (KERN_SUCCESS);
+uint32_t sched_qos_max_parallelism(__unused int qos, uint64_t options)
+{
+    host_basic_info_data_t hinfo;
+    mach_msg_type_number_t count = HOST_BASIC_INFO_COUNT;
+    /* Query the machine layer for core information */
+    __assert_only kern_return_t kret = host_info(host_self(), HOST_BASIC_INFO,
+            (host_info_t)&hinfo, &count);
+    assert(kret == KERN_SUCCESS);
+
+    /* We would not want multiple realtime threads running on the 
+     * same physical core; even for SMT capable machines.
+     */
+    if (options & QOS_PARALLELISM_REALTIME) {
+        return hinfo.physical_cpu;
+    }
+
+    if (options & QOS_PARALLELISM_COUNT_LOGICAL) {
+        return hinfo.logical_cpu;
+    } else {
+        return hinfo.physical_cpu;
+    }
 }
 
-static uint64_t		sched_tick_deadline;
+#if __arm64__
 
 /*
- * sched_startup:
+ * Set up or replace old timer with new timer
  *
- * Kicks off scheduler services.
- *
- * Called at splsched.
+ * Returns true if canceled old timer, false if it did not
  */
-void
-sched_startup(void)
+boolean_t
+sched_perfcontrol_update_callback_deadline(uint64_t new_deadline)
 {
-	kern_return_t	result;
-	thread_t		thread;
-
-	result = kernel_thread_start_priority((thread_continue_t)sched_tick_thread, NULL, MAXPRI_KERNEL, &thread);
-	if (result != KERN_SUCCESS)
-		panic("sched_startup");
-
-	thread_deallocate(thread);
-
 	/*
-	 * Yield to the sched_tick_thread while it times
-	 * a series of context switches back.  It stores
-	 * the baseline value in sched_cswtime.
-	 *
-	 * The current thread is the only other thread
-	 * active at this point.
+	 * Exchange deadline for new deadline, if old deadline was nonzero,
+	 * then I cancelled the callback, otherwise I didn't
 	 */
-	while (sched_cswtime == 0)
-		thread_block(THREAD_CONTINUE_NULL);
 
-	thread_daemon_init();
+	uint64_t old_deadline = __c11_atomic_load(&sched_perfcontrol_callback_deadline,
+	                                          memory_order_relaxed);
 
-	thread_call_initialize();
-}
 
-/*
- *	sched_tick_thread:
- *
- *	Perform periodic bookkeeping functions about ten
- *	times per second.
- */
-static void
-sched_tick_continue(void)
-{
-	uint64_t			abstime = mach_absolute_time();
+	while (!__c11_atomic_compare_exchange_weak(&sched_perfcontrol_callback_deadline,
+	                                           &old_deadline, new_deadline,
+	                                           memory_order_relaxed, memory_order_relaxed));
 
-	sched_tick++;
 
-	/*
-	 *  Compute various averages.
-	 */
-	compute_averages();
+	/* now old_deadline contains previous value, which might not be the same if it raced */
 
-	/*
-	 *  Scan the run queues for threads which
-	 *  may need to be updated.
-	 */
-	thread_update_scan();
+	return (old_deadline != 0) ? TRUE : FALSE;
+}
 
-	clock_deadline_for_periodic_event(sched_tick_interval, abstime,
-														&sched_tick_deadline);
+#endif /* __arm64__ */
 
-	assert_wait_deadline((event_t)sched_tick_thread, THREAD_UNINT, sched_tick_deadline);
-	thread_block((thread_continue_t)sched_tick_continue);
-	/*NOTREACHED*/
+void
+sched_update_pset_load_average(processor_set_t pset)
+{
+	int load = ((bit_count(pset->cpu_state_map[PROCESSOR_RUNNING]) + pset->pset_runq.count + rt_runq_count(pset)) << PSET_LOAD_NUMERATOR_SHIFT);
+	int new_load_average = (pset->load_average + load) >> 1;
+
+	pset->load_average = new_load_average;
+
+#if (DEVELOPMENT || DEBUG)
+#endif
 }
 
-/*
- * Time a series of context switches to determine
- * a baseline.  Toss the high and low and return
- * the one-way value.
- */
-static uint32_t
-time_cswitch(void)
+/* pset is locked */
+static processor_t
+choose_processor_for_realtime_thread(processor_set_t pset)
 {
-	uint32_t	new, hi, low, accum;
-	uint64_t	abstime;
-	int			i, tries = 7;
+	uint64_t cpu_map = (pset->cpu_bitmask & pset->recommended_bitmask & ~pset->pending_AST_cpu_mask);
 
-	accum = hi = low = 0;
-	for (i = 0; i < tries; ++i) {
-		abstime = mach_absolute_time();
-		thread_block(THREAD_CONTINUE_NULL);
+	for (int cpuid = lsb_first(cpu_map); cpuid >= 0; cpuid = lsb_next(cpu_map, cpuid)) {
+		processor_t processor = processor_array[cpuid];
 
-		new = mach_absolute_time() - abstime;
+		if (processor->processor_primary != processor) {
+			continue;
+		}
 
-		if (i == 0)
-			accum = hi = low = new;
-		else {
-			if (new < low)
-				low = new;
-			else
-			if (new > hi)
-				hi = new;
-			accum += new;
+		if (processor->state == PROCESSOR_IDLE) {
+			return processor;
 		}
-	}
 
-	return ((accum - hi - low) / (2 * (tries - 2)));
-}
+		if ((processor->state != PROCESSOR_RUNNING) && (processor->state != PROCESSOR_DISPATCHING)) {
+			continue;
+		}
 
-void
-sched_tick_thread(void)
-{
-	sched_cswtime = time_cswitch();
+		if (processor->current_pri >= BASEPRI_RTQUEUES) {
+			continue;
+		}
 
-	sched_tick_deadline = mach_absolute_time();
+		return processor;
 
-	sched_tick_continue();
-	/*NOTREACHED*/
-}
+	}
 
-/*
- *	thread_update_scan / runq_scan:
- *
- *	Scan the run queues to account for timesharing threads 
- *	which need to be updated.
- *
- *	Scanner runs in two passes.  Pass one squirrels likely
- *	threads away in an array, pass two does the update.
- *
- *	This is necessary because the run queue is locked for
- *	the candidate scan, but	the thread is locked for the update.
- *
- *	Array should be sized to make forward progress, without
- *	disabling preemption for long periods.
- */
+	if (!sched_allow_rt_smt) {
+		return PROCESSOR_NULL;
+	}
 
-#define	THREAD_UPDATE_SIZE		128
+	/* Consider secondary processors */
+	for (int cpuid = lsb_first(cpu_map); cpuid >= 0; cpuid = lsb_next(cpu_map, cpuid)) {
+		processor_t processor = processor_array[cpuid];
 
-static thread_t		thread_update_array[THREAD_UPDATE_SIZE];
-static int			thread_update_count = 0;
+		if (processor->processor_primary == processor) {
+			continue;
+		}
 
-/*
- *	Scan a runq for candidate threads.
- *
- *	Returns TRUE if retry is needed.
- */
-static boolean_t
-runq_scan(
-	run_queue_t				runq)
-{
-	register int			count;
-	register queue_t		q;
-	register thread_t		thread;
-
-	if ((count = runq->count) > 0) {
-	    q = runq->queues + runq->highq;
-		while (count > 0) {
-			queue_iterate(q, thread, thread_t, links) {
-				if (		thread->sched_stamp != sched_tick		&&
-						(thread->sched_mode & TH_MODE_TIMESHARE)	) {
-					if (thread_update_count == THREAD_UPDATE_SIZE)
-						return (TRUE);
-
-					thread_update_array[thread_update_count++] = thread;
-					thread_reference_internal(thread);
-				}
+		if (processor->state == PROCESSOR_IDLE) {
+			return processor;
+		}
 
-				count--;
-			}
+		if ((processor->state != PROCESSOR_RUNNING) && (processor->state != PROCESSOR_DISPATCHING)) {
+			continue;
+		}
 
-			q--;
+		if (processor->current_pri >= BASEPRI_RTQUEUES) {
+			continue;
 		}
+
+		return processor;
+
 	}
 
-	return (FALSE);
+	return PROCESSOR_NULL;
 }
 
-static void
-thread_update_scan(void)
+/* pset is locked */
+static bool
+all_available_primaries_are_running_realtime_threads(processor_set_t pset)
 {
-	register boolean_t			restart_needed;
-	register processor_set_t	pset = &default_pset;
-	register processor_t		processor;
-	register thread_t			thread;
-	spl_t						s;
-
-	do {
-		s = splsched();
-		simple_lock(&pset->sched_lock);
-	    restart_needed = runq_scan(&pset->runq);
-		simple_unlock(&pset->sched_lock);
-
-		if (!restart_needed) {
-			simple_lock(&pset->sched_lock);
-			processor = (processor_t)queue_first(&pset->processors);
-			while (!queue_end(&pset->processors, (queue_entry_t)processor)) {
-				if ((restart_needed = runq_scan(&processor->runq)) != 0)
-					break;
-
-				thread = processor->idle_thread;
-				if (thread->sched_stamp != sched_tick) {
-					if (thread_update_count == THREAD_UPDATE_SIZE) {
-						restart_needed = TRUE;
-						break;
-					}
+	uint64_t cpu_map = (pset->cpu_bitmask & pset->recommended_bitmask);
 
-					thread_update_array[thread_update_count++] = thread;
-					thread_reference_internal(thread);
-				}
+	for (int cpuid = lsb_first(cpu_map); cpuid >= 0; cpuid = lsb_next(cpu_map, cpuid)) {
+		processor_t processor = processor_array[cpuid];
 
-				processor = (processor_t)queue_next(&processor->processors);
-			}
-			simple_unlock(&pset->sched_lock);
+		if (processor->processor_primary != processor) {
+			continue;
 		}
-		splx(s);
-
-	    /*
-	     *	Ok, we now have a collection of candidates -- fix them.
-	     */
-	    while (thread_update_count > 0) {
-			thread = thread_update_array[--thread_update_count];
-			thread_update_array[thread_update_count] = THREAD_NULL;
 
-			s = splsched();
-			thread_lock(thread);
-			if (	!(thread->state & (TH_WAIT|TH_SUSP))	&&
-						thread->sched_stamp != sched_tick	)
-				update_priority(thread);
-			thread_unlock(thread);
-			splx(s);
+		if (processor->state == PROCESSOR_IDLE) {
+			return false;
+		}
 
-			thread_deallocate(thread);
-	    }
-	} while (restart_needed);
-}
-		
-/*
- *	Just in case someone doesn't use the macro
- */
-#undef	thread_wakeup
-void
-thread_wakeup(
-	event_t		x);
+		if (processor->state == PROCESSOR_DISPATCHING) {
+			return false;
+		}
 
-void
-thread_wakeup(
-	event_t		x)
-{
-	thread_wakeup_with_result(x, THREAD_AWAKENED);
-}
+		if (processor->state != PROCESSOR_RUNNING) {
+			/*
+			 * All other processor states are considered unavailable to run
+			 * realtime threads.  In particular, we prefer an available secondary
+			 * processor over the risk of leaving a realtime thread on the run queue
+			 * while waiting for a processor in PROCESSOR_START state,
+			 * which should anyway be a rare case.
+			 */
+			continue;
+		}
 
-boolean_t
-preemption_enabled(void)
-{
-	return (get_preemption_level() == 0 && ml_get_interrupts_enabled());
-}
+		if (processor->current_pri < BASEPRI_RTQUEUES) {
+			return false;
+		}
+	}
 
-#if	DEBUG
-static boolean_t
-thread_runnable(
-	thread_t	thread)
-{
-	return ((thread->state & (TH_RUN|TH_WAIT)) == TH_RUN);
+	return true;
 }
-#endif	/* DEBUG */
-
-#if	MACH_KDB
-#include <ddb/db_output.h>
-#define	printf		kdbprintf
-void			db_sched(void);
 
-void
-db_sched(void)
-{
-	iprintf("Scheduling Statistics:\n");
-	db_indent += 2;
-	iprintf("Thread invocations:  csw %d same %d\n",
-		c_thread_invoke_csw, c_thread_invoke_same);
-#if	MACH_COUNTERS
-	iprintf("Thread block:  calls %d\n",
-		c_thread_block_calls);
-	iprintf("Idle thread:\n\thandoff %d block %d no_dispatch %d\n",
-		c_idle_thread_handoff,
-		c_idle_thread_block, no_dispatch_count);
-	iprintf("Sched thread blocks:  %d\n", c_sched_thread_block);
-#endif	/* MACH_COUNTERS */
-	db_indent -= 2;
-}
-
-#include <ddb/db_output.h>
-void		db_show_thread_log(void);
 
-void
-db_show_thread_log(void)
-{
-}
-#endif	/* MACH_KDB */