[apple/xnu.git] / bsd / kern / kern_overrides.c

/*
 * Copyright (c) 2013 Apple Computer, Inc. All rights reserved.
 *
 * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
 *
 * This file contains Original Code and/or Modifications of Original Code
 * as defined in and that are subject to the Apple Public Source License
 * Version 2.0 (the 'License'). You may not use this file except in
 * compliance with the License. The rights granted to you under the License
 * may not be used to create, or enable the creation or redistribution of,
 * unlawful or unlicensed copies of an Apple operating system, or to
 * circumvent, violate, or enable the circumvention or violation of, any
 * terms of an Apple operating system software license agreement.
 *
 * Please obtain a copy of the License at
 * http://www.opensource.apple.com/apsl/ and read it before using this file.
 *
 * The Original Code and all software distributed under the License are
 * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
 * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
 * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
 * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
 * Please see the License for the specific language governing rights and
 * limitations under the License.
 *
 * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
 */

#include <sys/param.h>
#include <sys/systm.h>
#include <sys/kernel.h>
#include <sys/malloc.h>
#include <sys/proc_internal.h>
#include <sys/proc.h>
#include <sys/kauth.h>
#include <sys/unistd.h>
#include <sys/priv.h>

#include <mach/mach_types.h>
#include <mach/vm_param.h>
#include <kern/task.h>
#include <kern/locks.h>
#include <kern/assert.h>
#include <kern/sched_prim.h>

#include <sys/kern_overrides.h>
#include <sys/bsdtask_info.h>
#include <sys/kdebug.h>
#include <sys/sysproto.h>
#include <sys/msgbuf.h>
#include <sys/kern_memorystatus.h>

/* Mutex for global system override state */
static lck_mtx_t        sys_override_lock;
static lck_grp_t        *sys_override_mtx_grp;
static lck_attr_t       *sys_override_mtx_attr;
static lck_grp_attr_t   *sys_override_mtx_grp_attr;

/*
 * Assertion counts for system properties (add new ones for each new mechanism)
 *
 * The assertion count management for system overrides is as follows:
 *
 * - All assertion counts are protected by the sys_override_lock.
 *
 * - Each caller of system_override() increments the assertion count for the
 *   mechanism it specified in the flags. The caller then blocks for the
 *   timeout specified in the system call.
 *
 * - At the end of the timeout, the caller thread wakes up and decrements the
 *   assertion count for the mechanism it originally took an assertion on.
 *
 * - If another caller calls the system_override() to disable the override
 *   for a mechanism, it simply disables the mechanism without changing any
 *   assertion counts. That way, the assertion counts are properly balanced.
 *
 * One thing to note is that a SYS_OVERRIDE_DISABLE disables the overrides
 * for a mechanism irrespective of how many clients requested that override.
 * That makes the implementation simpler and avoids keeping a lot of process
 * specific state in the kernel.
 *
 */
static int64_t          io_throttle_assert_cnt;
static int64_t          cpu_throttle_assert_cnt;
static int64_t          fast_jetsam_assert_cnt;

/* Wait Channel for system override */
static uint64_t         sys_override_wait;

/* Global variable to indicate if system_override is enabled */
int                     sys_override_enabled;

/* Helper routines */
static void system_override_begin(uint64_t flags);
static void system_override_end(uint64_t flags);
static void system_override_abort(uint64_t flags);
static void system_override_callouts(uint64_t flags, boolean_t enable_override);
static __attribute__((noinline)) void PROCESS_OVERRIDING_SYSTEM_DEFAULTS(uint64_t timeout);

void
init_system_override()
{
	sys_override_mtx_grp_attr = lck_grp_attr_alloc_init();
	sys_override_mtx_grp = lck_grp_alloc_init("system_override", sys_override_mtx_grp_attr);
	sys_override_mtx_attr = lck_attr_alloc_init();
	lck_mtx_init(&sys_override_lock, sys_override_mtx_grp, sys_override_mtx_attr);
	io_throttle_assert_cnt = cpu_throttle_assert_cnt = fast_jetsam_assert_cnt = 0;
	sys_override_enabled = 1;
}

/* system call implementation */
int
system_override(__unused struct proc *p, struct system_override_args * uap, __unused int32_t *retval)
{
	uint64_t timeout = uap->timeout;
	uint64_t flags = uap->flags;
	int error = 0;

	/* Check credentials for caller. Only entitled processes are allowed to make this call. */
	if ((error = priv_check_cred(kauth_cred_get(), PRIV_SYSTEM_OVERRIDE, 0))) {
		goto out;
	}

	/* Check to see if sane flags are specified. */
	if ((flags & ~SYS_OVERRIDE_FLAGS_MASK) != 0) {
		error = EINVAL;
		goto out;
	}

	/* Make sure that the system override syscall has been initialized */
	if (!sys_override_enabled) {
		error = EINVAL;
		goto out;
	}

	lck_mtx_lock(&sys_override_lock);

	if (flags & SYS_OVERRIDE_DISABLE) {
		flags &= ~SYS_OVERRIDE_DISABLE;
		system_override_abort(flags);
	} else {
		system_override_begin(flags);
		PROCESS_OVERRIDING_SYSTEM_DEFAULTS(timeout);
		system_override_end(flags);
	}

	lck_mtx_unlock(&sys_override_lock);

out:
	return error;
}

/*
 * Helper routines for enabling/disabling system overrides for various mechanisms.
 * These routines should be called with the sys_override_lock held. Each subsystem
 * which is hooked into the override service provides two routines:
 *
 * - void sys_override_foo_init(void);
 * Routine to initialize the subsystem or the data needed for the override to work.
 * This routine is optional and if a subsystem needs it, it should be invoked from
 * init_system_override().
 *
 * - void sys_override_foo(boolean_t enable_override);
 * Routine to enable/disable the override mechanism for that subsystem. A value of
 * true indicates that the mechanism should be overridden and the special behavior
 * should begin. A false value indicates that the subsystem should return to default
 * behavior. This routine is mandatory and should be invoked as part of the helper
 * routines if the flags passed in the syscall match the subsystem. Also, this
 * routine should preferably be idempotent.
 */

static void
system_override_callouts(uint64_t flags, boolean_t enable_override)
{
	switch (flags) {
	case SYS_OVERRIDE_IO_THROTTLE:
		if (enable_override) {
			KERNEL_DEBUG_CONSTANT(FSDBG_CODE(DBG_THROTTLE, IO_THROTTLE_DISABLE) | DBG_FUNC_START,
			    current_proc()->p_pid, 0, 0, 0, 0);
		} else {
			KERNEL_DEBUG_CONSTANT(FSDBG_CODE(DBG_THROTTLE, IO_THROTTLE_DISABLE) | DBG_FUNC_END,
			    current_proc()->p_pid, 0, 0, 0, 0);
		}
		sys_override_io_throttle(enable_override);
		break;

	case SYS_OVERRIDE_CPU_THROTTLE:
		if (enable_override) {
			KERNEL_DEBUG_CONSTANT(MACHDBG_CODE(DBG_MACH_SCHED, MACH_CPU_THROTTLE_DISABLE) | DBG_FUNC_START,
			    current_proc()->p_pid, 0, 0, 0, 0);
		} else {
			KERNEL_DEBUG_CONSTANT(MACHDBG_CODE(DBG_MACH_SCHED, MACH_CPU_THROTTLE_DISABLE) | DBG_FUNC_END,
			    current_proc()->p_pid, 0, 0, 0, 0);
		}
		sys_override_cpu_throttle(enable_override);
		break;

	case SYS_OVERRIDE_FAST_JETSAM:
		if (enable_override) {
			KERNEL_DEBUG_CONSTANT(BSDDBG_CODE(DBG_BSD_MEMSTAT, BSD_MEMSTAT_FAST_JETSAM) | DBG_FUNC_START,
			    current_proc()->p_pid, 0, 0, 0, 0);
		} else {
			KERNEL_DEBUG_CONSTANT(BSDDBG_CODE(DBG_BSD_MEMSTAT, BSD_MEMSTAT_FAST_JETSAM) | DBG_FUNC_END,
			    current_proc()->p_pid, 0, 0, 0, 0);
		}
#if CONFIG_JETSAM
		memorystatus_fast_jetsam_override(enable_override);
#endif /* CONFIG_JETSAM */
		break;

	default:
		panic("Unknown option to system_override_callouts(): %llu\n", flags);
	}
}

/*
 * system_override_begin(uint64_t flags)
 *
 * Routine to start a system override if the assertion count
 * transitions from 0->1 for a specified mechanism.
 */
static void
system_override_begin(uint64_t flags)
{
	lck_mtx_assert(&sys_override_lock, LCK_MTX_ASSERT_OWNED);

	if (flags & SYS_OVERRIDE_IO_THROTTLE) {
		if (io_throttle_assert_cnt == 0) {
			system_override_callouts(SYS_OVERRIDE_IO_THROTTLE, true);
		}
		io_throttle_assert_cnt++;
	}

	if (flags & SYS_OVERRIDE_CPU_THROTTLE) {
		if (cpu_throttle_assert_cnt == 0) {
			system_override_callouts(SYS_OVERRIDE_CPU_THROTTLE, true);
		}
		cpu_throttle_assert_cnt++;
	}

	if (flags & SYS_OVERRIDE_FAST_JETSAM) {
		if (fast_jetsam_assert_cnt == 0) {
			system_override_callouts(SYS_OVERRIDE_FAST_JETSAM, true);
		}
		fast_jetsam_assert_cnt++;
	}
}

/*
 * system_override_end(uint64_t flags)
 *
 * Routine to end a system override if the assertion count
 * transitions from 1->0 for a specified mechanism.
 */
static void
system_override_end(uint64_t flags)
{
	lck_mtx_assert(&sys_override_lock, LCK_MTX_ASSERT_OWNED);

	if (flags & SYS_OVERRIDE_IO_THROTTLE) {
		assert(io_throttle_assert_cnt > 0);
		io_throttle_assert_cnt--;
		if (io_throttle_assert_cnt == 0) {
			system_override_callouts(SYS_OVERRIDE_IO_THROTTLE, false);
		}
	}

	if (flags & SYS_OVERRIDE_CPU_THROTTLE) {
		assert(cpu_throttle_assert_cnt > 0);
		cpu_throttle_assert_cnt--;
		if (cpu_throttle_assert_cnt == 0) {
			system_override_callouts(SYS_OVERRIDE_CPU_THROTTLE, false);
		}
	}

	if (flags & SYS_OVERRIDE_FAST_JETSAM) {
		assert(fast_jetsam_assert_cnt > 0);
		fast_jetsam_assert_cnt--;
		if (fast_jetsam_assert_cnt == 0) {
			system_override_callouts(SYS_OVERRIDE_FAST_JETSAM, false);
		}
	}
}

/*
 * system_override_abort(uint64_t flags)
 *
 * Routine to abort a system override (if one was active)
 * irrespective of the assertion counts and number of blocked
 * requestors.
 */
static void
system_override_abort(uint64_t flags)
{
	lck_mtx_assert(&sys_override_lock, LCK_MTX_ASSERT_OWNED);

	if ((flags & SYS_OVERRIDE_IO_THROTTLE) && (io_throttle_assert_cnt > 0)) {
		system_override_callouts(SYS_OVERRIDE_IO_THROTTLE, false);
	}

	if ((flags & SYS_OVERRIDE_CPU_THROTTLE) && (cpu_throttle_assert_cnt > 0)) {
		system_override_callouts(SYS_OVERRIDE_CPU_THROTTLE, false);
	}

	if ((flags & SYS_OVERRIDE_FAST_JETSAM) && (fast_jetsam_assert_cnt > 0)) {
		system_override_callouts(SYS_OVERRIDE_FAST_JETSAM, false);
	}
}

static __attribute__((noinline)) void
PROCESS_OVERRIDING_SYSTEM_DEFAULTS(uint64_t timeout)
{
	struct timespec ts;
	ts.tv_sec = timeout / NSEC_PER_SEC;
	ts.tv_nsec = timeout - ((long)ts.tv_sec * NSEC_PER_SEC);
	msleep((caddr_t)&sys_override_wait, &sys_override_lock, PRIBIO | PCATCH, "system_override", &ts);
}
Commit	Line	Data
39236c6e A	1	/*
	2	* Copyright (c) 2013 Apple Computer, Inc. All rights reserved.
	3	*
	4	* @APPLE_OSREFERENCE_LICENSE_HEADER_START@
0a7de745	5	*
39236c6e A	6	* This file contains Original Code and/or Modifications of Original Code
	7	* as defined in and that are subject to the Apple Public Source License
	8	* Version 2.0 (the 'License'). You may not use this file except in
	9	* compliance with the License. The rights granted to you under the License
	10	* may not be used to create, or enable the creation or redistribution of,
	11	* unlawful or unlicensed copies of an Apple operating system, or to
	12	* circumvent, violate, or enable the circumvention or violation of, any
	13	* terms of an Apple operating system software license agreement.
0a7de745	14	*
39236c6e A	15	* Please obtain a copy of the License at
39236c6e A	16	* http://www.opensource.apple.com/apsl/ and read it before using this file.
0a7de745	17	*
39236c6e A	18	* The Original Code and all software distributed under the License are
	19	* distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
	20	* EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
	21	* INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
	22	* FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
	23	* Please see the License for the specific language governing rights and
	24	* limitations under the License.
0a7de745	25	*
39236c6e A	26	* @APPLE_OSREFERENCE_LICENSE_HEADER_END@
	27	*/
	28
39236c6e A	29	#include <sys/param.h>
	30	#include <sys/systm.h>
	31	#include <sys/kernel.h>
	32	#include <sys/malloc.h>
	33	#include <sys/proc_internal.h>
	34	#include <sys/proc.h>
	35	#include <sys/kauth.h>
	36	#include <sys/unistd.h>
	37	#include <sys/priv.h>
39236c6e A	38
	39	#include <mach/mach_types.h>
	40	#include <mach/vm_param.h>
	41	#include <kern/task.h>
fe8ab488	42	#include <kern/locks.h>
39236c6e A	43	#include <kern/assert.h>
	44	#include <kern/sched_prim.h>
	45
	46	#include <sys/kern_overrides.h>
	47	#include <sys/bsdtask_info.h>
	48	#include <sys/kdebug.h>
	49	#include <sys/sysproto.h>
	50	#include <sys/msgbuf.h>
d9a64523	51	#include <sys/kern_memorystatus.h>
39236c6e A	52
39236c6e A	53	/* Mutex for global system override state */
0a7de745	54	static lck_mtx_t sys_override_lock;
39236c6e A	55	static lck_grp_t *sys_override_mtx_grp;
	56	static lck_attr_t *sys_override_mtx_attr;
	57	static lck_grp_attr_t *sys_override_mtx_grp_attr;
	58
0a7de745	59	/*
d9a64523 A	60	* Assertion counts for system properties (add new ones for each new mechanism)
	61	*
	62	* The assertion count management for system overrides is as follows:
	63	*
	64	* - All assertion counts are protected by the sys_override_lock.
	65	*
0a7de745 A	66	* - Each caller of system_override() increments the assertion count for the
	67	* mechanism it specified in the flags. The caller then blocks for the
	68	* timeout specified in the system call.
d9a64523	69	*
0a7de745	70	* - At the end of the timeout, the caller thread wakes up and decrements the
d9a64523 A	71	* assertion count for the mechanism it originally took an assertion on.
d9a64523 A	72	*
0a7de745 A	73	* - If another caller calls the system_override() to disable the override
	74	* for a mechanism, it simply disables the mechanism without changing any
	75	* assertion counts. That way, the assertion counts are properly balanced.
d9a64523	76	*
0a7de745	77	* One thing to note is that a SYS_OVERRIDE_DISABLE disables the overrides
d9a64523	78	* for a mechanism irrespective of how many clients requested that override.
0a7de745	79	* That makes the implementation simpler and avoids keeping a lot of process
d9a64523 A	80	* specific state in the kernel.
	81	*
	82	*/
0a7de745 A	83	static int64_t io_throttle_assert_cnt;
	84	static int64_t cpu_throttle_assert_cnt;
	85	static int64_t fast_jetsam_assert_cnt;
39236c6e A	86
39236c6e A	87	/* Wait Channel for system override */
0a7de745	88	static uint64_t sys_override_wait;
39236c6e	89
fe8ab488	90	/* Global variable to indicate if system_override is enabled */
0a7de745	91	int sys_override_enabled;
fe8ab488	92
d9a64523 A	93	/* Helper routines */
	94	static void system_override_begin(uint64_t flags);
	95	static void system_override_end(uint64_t flags);
	96	static void system_override_abort(uint64_t flags);
	97	static void system_override_callouts(uint64_t flags, boolean_t enable_override);
39236c6e A	98	static __attribute__((noinline)) void PROCESS_OVERRIDING_SYSTEM_DEFAULTS(uint64_t timeout);
39236c6e A	99
39236c6e A	100	void
	101	init_system_override()
	102	{
	103	sys_override_mtx_grp_attr = lck_grp_attr_alloc_init();
	104	sys_override_mtx_grp = lck_grp_alloc_init("system_override", sys_override_mtx_grp_attr);
	105	sys_override_mtx_attr = lck_attr_alloc_init();
	106	lck_mtx_init(&sys_override_lock, sys_override_mtx_grp, sys_override_mtx_attr);
d9a64523	107	io_throttle_assert_cnt = cpu_throttle_assert_cnt = fast_jetsam_assert_cnt = 0;
fe8ab488	108	sys_override_enabled = 1;
39236c6e A	109	}
	110
	111	/* system call implementation */
	112	int
	113	system_override(__unused struct proc p, struct system_override_args uap, __unused int32_t *retval)
	114	{
	115	uint64_t timeout = uap->timeout;
	116	uint64_t flags = uap->flags;
	117	int error = 0;
	118
	119	/* Check credentials for caller. Only entitled processes are allowed to make this call. */
	120	if ((error = priv_check_cred(kauth_cred_get(), PRIV_SYSTEM_OVERRIDE, 0))) {
	121	goto out;
0a7de745	122	}
39236c6e	123
d9a64523	124	/* Check to see if sane flags are specified. */
fe8ab488	125	if ((flags & ~SYS_OVERRIDE_FLAGS_MASK) != 0) {
39236c6e A	126	error = EINVAL;
	127	goto out;
	128	}
	129
d9a64523 A	130	/* Make sure that the system override syscall has been initialized */
	131	if (!sys_override_enabled) {
	132	error = EINVAL;
fe8ab488 A	133	goto out;
	134	}
	135
39236c6e A	136	lck_mtx_lock(&sys_override_lock);
39236c6e A	137
d9a64523 A	138	if (flags & SYS_OVERRIDE_DISABLE) {
	139	flags &= ~SYS_OVERRIDE_DISABLE;
	140	system_override_abort(flags);
	141	} else {
	142	system_override_begin(flags);
	143	PROCESS_OVERRIDING_SYSTEM_DEFAULTS(timeout);
	144	system_override_end(flags);
	145	}
39236c6e A	146
	147	lck_mtx_unlock(&sys_override_lock);
	148
	149	out:
	150	return error;
	151	}
	152
	153	/*
d9a64523	154	* Helper routines for enabling/disabling system overrides for various mechanisms.
0a7de745	155	* These routines should be called with the sys_override_lock held. Each subsystem
d9a64523	156	* which is hooked into the override service provides two routines:
0a7de745	157	*
d9a64523 A	158	* - void sys_override_foo_init(void);
d9a64523 A	159	* Routine to initialize the subsystem or the data needed for the override to work.
0a7de745	160	* This routine is optional and if a subsystem needs it, it should be invoked from
d9a64523	161	* init_system_override().
0a7de745	162	*
d9a64523	163	* - void sys_override_foo(boolean_t enable_override);
0a7de745 A	164	* Routine to enable/disable the override mechanism for that subsystem. A value of
	165	* true indicates that the mechanism should be overridden and the special behavior
	166	* should begin. A false value indicates that the subsystem should return to default
	167	* behavior. This routine is mandatory and should be invoked as part of the helper
	168	* routines if the flags passed in the syscall match the subsystem. Also, this
d9a64523 A	169	* routine should preferably be idempotent.
	170	*/
	171
	172	static void
	173	system_override_callouts(uint64_t flags, boolean_t enable_override)
	174	{
	175	switch (flags) {
0a7de745 A	176	case SYS_OVERRIDE_IO_THROTTLE:
	177	if (enable_override) {
	178	KERNEL_DEBUG_CONSTANT(FSDBG_CODE(DBG_THROTTLE, IO_THROTTLE_DISABLE) \| DBG_FUNC_START,
	179	current_proc()->p_pid, 0, 0, 0, 0);
	180	} else {
	181	KERNEL_DEBUG_CONSTANT(FSDBG_CODE(DBG_THROTTLE, IO_THROTTLE_DISABLE) \| DBG_FUNC_END,
	182	current_proc()->p_pid, 0, 0, 0, 0);
	183	}
	184	sys_override_io_throttle(enable_override);
	185	break;
	186
	187	case SYS_OVERRIDE_CPU_THROTTLE:
	188	if (enable_override) {
	189	KERNEL_DEBUG_CONSTANT(MACHDBG_CODE(DBG_MACH_SCHED, MACH_CPU_THROTTLE_DISABLE) \| DBG_FUNC_START,
	190	current_proc()->p_pid, 0, 0, 0, 0);
	191	} else {
	192	KERNEL_DEBUG_CONSTANT(MACHDBG_CODE(DBG_MACH_SCHED, MACH_CPU_THROTTLE_DISABLE) \| DBG_FUNC_END,
	193	current_proc()->p_pid, 0, 0, 0, 0);
	194	}
	195	sys_override_cpu_throttle(enable_override);
	196	break;
	197
	198	case SYS_OVERRIDE_FAST_JETSAM:
	199	if (enable_override) {
	200	KERNEL_DEBUG_CONSTANT(BSDDBG_CODE(DBG_BSD_MEMSTAT, BSD_MEMSTAT_FAST_JETSAM) \| DBG_FUNC_START,
	201	current_proc()->p_pid, 0, 0, 0, 0);
	202	} else {
	203	KERNEL_DEBUG_CONSTANT(BSDDBG_CODE(DBG_BSD_MEMSTAT, BSD_MEMSTAT_FAST_JETSAM) \| DBG_FUNC_END,
	204	current_proc()->p_pid, 0, 0, 0, 0);
	205	}
d9a64523	206	#if CONFIG_JETSAM
0a7de745	207	memorystatus_fast_jetsam_override(enable_override);
d9a64523	208	#endif /* CONFIG_JETSAM */
0a7de745	209	break;
d9a64523	210
0a7de745 A	211	default:
0a7de745 A	212	panic("Unknown option to system_override_callouts(): %llu\n", flags);
d9a64523 A	213	}
	214	}
	215
	216	/*
	217	* system_override_begin(uint64_t flags)
	218	*
0a7de745	219	* Routine to start a system override if the assertion count
d9a64523	220	* transitions from 0->1 for a specified mechanism.
39236c6e A	221	*/
39236c6e A	222	static void
d9a64523	223	system_override_begin(uint64_t flags)
39236c6e	224	{
d9a64523	225	lck_mtx_assert(&sys_override_lock, LCK_MTX_ASSERT_OWNED);
0a7de745	226
39236c6e	227	if (flags & SYS_OVERRIDE_IO_THROTTLE) {
d9a64523 A	228	if (io_throttle_assert_cnt == 0) {
d9a64523 A	229	system_override_callouts(SYS_OVERRIDE_IO_THROTTLE, true);
39236c6e A	230	}
	231	io_throttle_assert_cnt++;
	232	}
0a7de745	233
39236c6e	234	if (flags & SYS_OVERRIDE_CPU_THROTTLE) {
d9a64523 A	235	if (cpu_throttle_assert_cnt == 0) {
d9a64523 A	236	system_override_callouts(SYS_OVERRIDE_CPU_THROTTLE, true);
39236c6e A	237	}
	238	cpu_throttle_assert_cnt++;
	239	}
0a7de745	240
d9a64523 A	241	if (flags & SYS_OVERRIDE_FAST_JETSAM) {
	242	if (fast_jetsam_assert_cnt == 0) {
	243	system_override_callouts(SYS_OVERRIDE_FAST_JETSAM, true);
	244	}
	245	fast_jetsam_assert_cnt++;
	246	}
39236c6e A	247	}
	248
	249	/*
d9a64523 A	250	* system_override_end(uint64_t flags)
d9a64523 A	251	*
0a7de745	252	* Routine to end a system override if the assertion count
d9a64523	253	* transitions from 1->0 for a specified mechanism.
39236c6e A	254	*/
39236c6e A	255	static void
d9a64523	256	system_override_end(uint64_t flags)
39236c6e	257	{
d9a64523	258	lck_mtx_assert(&sys_override_lock, LCK_MTX_ASSERT_OWNED);
0a7de745	259
39236c6e A	260	if (flags & SYS_OVERRIDE_IO_THROTTLE) {
	261	assert(io_throttle_assert_cnt > 0);
	262	io_throttle_assert_cnt--;
d9a64523 A	263	if (io_throttle_assert_cnt == 0) {
d9a64523 A	264	system_override_callouts(SYS_OVERRIDE_IO_THROTTLE, false);
39236c6e A	265	}
	266	}
	267
	268	if (flags & SYS_OVERRIDE_CPU_THROTTLE) {
	269	assert(cpu_throttle_assert_cnt > 0);
	270	cpu_throttle_assert_cnt--;
d9a64523 A	271	if (cpu_throttle_assert_cnt == 0) {
d9a64523 A	272	system_override_callouts(SYS_OVERRIDE_CPU_THROTTLE, false);
39236c6e A	273	}
39236c6e A	274	}
d9a64523 A	275
	276	if (flags & SYS_OVERRIDE_FAST_JETSAM) {
	277	assert(fast_jetsam_assert_cnt > 0);
	278	fast_jetsam_assert_cnt--;
	279	if (fast_jetsam_assert_cnt == 0) {
	280	system_override_callouts(SYS_OVERRIDE_FAST_JETSAM, false);
	281	}
	282	}
d9a64523 A	283	}
	284
	285	/*
	286	* system_override_abort(uint64_t flags)
	287	*
0a7de745 A	288	* Routine to abort a system override (if one was active)
0a7de745 A	289	* irrespective of the assertion counts and number of blocked
d9a64523 A	290	* requestors.
	291	*/
	292	static void
	293	system_override_abort(uint64_t flags)
	294	{
d9a64523	295	lck_mtx_assert(&sys_override_lock, LCK_MTX_ASSERT_OWNED);
0a7de745	296
d9a64523 A	297	if ((flags & SYS_OVERRIDE_IO_THROTTLE) && (io_throttle_assert_cnt > 0)) {
	298	system_override_callouts(SYS_OVERRIDE_IO_THROTTLE, false);
	299	}
	300
0a7de745	301	if ((flags & SYS_OVERRIDE_CPU_THROTTLE) && (cpu_throttle_assert_cnt > 0)) {
d9a64523 A	302	system_override_callouts(SYS_OVERRIDE_CPU_THROTTLE, false);
	303	}
	304
0a7de745	305	if ((flags & SYS_OVERRIDE_FAST_JETSAM) && (fast_jetsam_assert_cnt > 0)) {
d9a64523 A	306	system_override_callouts(SYS_OVERRIDE_FAST_JETSAM, false);
d9a64523 A	307	}
39236c6e A	308	}
	309
	310	static __attribute__((noinline)) void
	311	PROCESS_OVERRIDING_SYSTEM_DEFAULTS(uint64_t timeout)
	312	{
	313	struct timespec ts;
	314	ts.tv_sec = timeout / NSEC_PER_SEC;
	315	ts.tv_nsec = timeout - ((long)ts.tv_sec * NSEC_PER_SEC);
	316	msleep((caddr_t)&sys_override_wait, &sys_override_lock, PRIBIO \| PCATCH, "system_override", &ts);
	317	}