[apple/xnu.git] / tests / turnstiles_test.c

/*
 * turnstiles_test: Tests turnstile kernel primitive.
 */

#ifdef T_NAMESPACE
#undef T_NAMESPACE
#endif

#include <darwintest.h>
#include <darwintest_multiprocess.h>

#include <pthread.h>
#include <launch.h>
#include <servers/bootstrap.h>
#include <stdlib.h>
#include <sys/event.h>
#include <unistd.h>
#include <crt_externs.h>
#include <sys/sysctl.h>
#include <sys/types.h>

#define SYSCTL_TURNSTILE_TEST_USER_DEFAULT            1
#define SYSCTL_TURNSTILE_TEST_USER_HASHTABLE          2
#define SYSCTL_TURNSTILE_TEST_KERNEL_DEFAULT          3
#define SYSCTL_TURNSTILE_TEST_KERNEL_HASHTABLE        4

T_GLOBAL_META(T_META_NAMESPACE("xnu.turnstiles_test"));

static void
thread_create_at_qos(qos_class_t qos, void * (*function)(void *), int type)
{
	qos_class_t qos_thread;
	pthread_t thread;
	pthread_attr_t attr;
	int ret;

	ret = setpriority(PRIO_DARWIN_ROLE, 0, PRIO_DARWIN_ROLE_UI_FOCAL);
	if (ret != 0) {
		T_LOG("set priority failed\n");
	}

	pthread_attr_init(&attr);
	pthread_attr_set_qos_class_np(&attr, qos, 0);
	pthread_create(&thread, &attr, function, (void *)type);

	T_LOG("pthread created\n");
	pthread_get_qos_class_np(thread, &qos_thread, NULL);
	T_EXPECT_EQ(qos_thread, (qos_class_t)qos, NULL);
}

static int
get_sched_pri(thread_t thread_port)
{
	kern_return_t kr;

	thread_extended_info_data_t extended_info;
	mach_msg_type_number_t count = THREAD_EXTENDED_INFO_COUNT;
	kr = thread_info(thread_port, THREAD_EXTENDED_INFO,
	    (thread_info_t)&extended_info, &count);

	T_QUIET; T_ASSERT_MACH_SUCCESS(kr, "thread_info");
	return extended_info.pth_curpri;
}

static int
get_base_pri(thread_t thread_port)
{
	kern_return_t kr;

	thread_extended_info_data_t extended_info;
	mach_msg_type_number_t count = THREAD_EXTENDED_INFO_COUNT;
	kr = thread_info(thread_port, THREAD_EXTENDED_INFO,
	    (thread_info_t)&extended_info, &count);

	T_QUIET; T_ASSERT_MACH_SUCCESS(kr, "thread_info");
	return extended_info.pth_priority;
}

static void
turnstile_prim_lock(int type)
{
	int ret;
	uint64_t tid;
	int in_val = type;
	pthread_threadid_np(NULL, &tid);
	T_LOG("sysctlbyname lock type %d called from thread %llu \n", type, tid);
	ret = sysctlbyname("kern.turnstiles_test_lock", NULL, 0, &in_val, sizeof(in_val));
	T_LOG("sysctlbyname lock returned from thread %llu with value %d \n", tid, ret);
}

static void
turnstile_prim_unlock(int type)
{
	int ret;
	uint64_t tid;
	int in_val = type;
	pthread_threadid_np(NULL, &tid);
	T_LOG("sysctlbyname unlock type %d called from thread %llu \n", type, tid);
	ret = sysctlbyname("kern.turnstiles_test_unlock", NULL, 0, &in_val, sizeof(in_val));
	T_LOG("sysctlbyname unlock returned from thread %llu with value %d \n", tid, ret);
}

struct thread_data {
	int pri_to_set;
	int lock1;
	int lock2;
	unsigned int sleep;
	int sched_pri_to_check;
	int base_pri_to_check;
};

static void *
chain_locking(void* args)
{
	struct thread_data* data = (struct thread_data*) args;
	int policy, pri;
	int ret;
	struct sched_param param;

	/* Change our priority to pri_to_set */
	ret = pthread_getschedparam(pthread_self(), &policy, &param);
	T_QUIET; T_ASSERT_MACH_SUCCESS(ret, "pthread_getschedparam");

	param.sched_priority = data->pri_to_set;

	/* this sets both sched and base pri */
	ret = pthread_setschedparam(pthread_self(), policy, &param);
	T_QUIET; T_ASSERT_MACH_SUCCESS(ret, "pthread_setschedparam");

	pri = get_sched_pri(mach_thread_self());

	T_ASSERT_EQ(pri, data->pri_to_set, "Priority before holding locks");

	/* take lock1 */
	if (data->lock1) {
		turnstile_prim_lock(data->lock1);
	}

	/* take lock2 */
	if (data->lock2) {
		turnstile_prim_lock(data->lock2);
	}

	if (data->sleep) {
		sleep(data->sleep);
	}

	if (data->sched_pri_to_check) {
		pri = get_sched_pri(mach_thread_self());
		T_ASSERT_EQ(pri, data->sched_pri_to_check, "Sched priority while holding locks");
	}

	if (data->base_pri_to_check) {
		pri = get_base_pri(mach_thread_self());
		T_ASSERT_EQ(pri, data->base_pri_to_check, "Base priority while holding locks");
	}

	if (data->lock2) {
		turnstile_prim_unlock(data->lock2);
	}

	if (data->lock1) {
		turnstile_prim_unlock(data->lock1);
	}

	pri = get_sched_pri(mach_thread_self());
	T_ASSERT_EQ(pri, data->pri_to_set, "Priority after releasing locks");

	return NULL;
}

static void *
take_lock_check_priority(void * arg)
{
	int old_pri = get_base_pri(mach_thread_self());
	int unboosted_pri;
	int boosted_pri;
	int after_unlock_pri;
	uint64_t tid;
	int type = (int)arg;

	pthread_threadid_np(NULL, &tid);

	T_ASSERT_EQ(old_pri, 37, "thread(%llu) priority before acquiring the lock is %d\n", tid, old_pri);

	/* Take the test lock */
	turnstile_prim_lock(type);

	unboosted_pri = get_base_pri(mach_thread_self());
	T_ASSERT_EQ(unboosted_pri, 37, "thread(%llu) priority after acquiring the lock (uncontended) is %d\n", tid, unboosted_pri);

	sleep(8);

	/* Check for elevated priority */
	boosted_pri =  get_base_pri(mach_thread_self());
	T_ASSERT_EQ(boosted_pri, 47, "thread(%llu) priority after contention by 47 thread is %d\n", tid, boosted_pri);

	/* Drop the lock */
	turnstile_prim_unlock(type);

	/* Check for regular priority */
	after_unlock_pri =  get_base_pri(mach_thread_self());
	T_ASSERT_EQ(after_unlock_pri, 37, "thread(%llu) priority after dropping lock is %d\n", tid, after_unlock_pri);

	return NULL;
}

static void *
try_to_take_lock_and_unlock(void *arg)
{
	uint64_t tid;
	int type = (int)arg;

	pthread_threadid_np(NULL, &tid);
	sleep(4);

	int old_pri = get_base_pri(mach_thread_self());
	T_ASSERT_EQ(old_pri, 47, "thread(%llu) priority before acquiring the lock is %d\n", tid, old_pri);

	/* Try taking the test lock */
	turnstile_prim_lock(type);
	sleep(2);
	turnstile_prim_unlock(type);
	return NULL;
}

static void *
take_lock_and_exit(void * arg)
{
	int old_pri = get_base_pri(mach_thread_self());
	int unboosted_pri;
	int boosted_pri;
	uint64_t tid;
	int type = (int)arg;

	pthread_threadid_np(NULL, &tid);

	T_ASSERT_EQ(old_pri, 37, "thread(%llu) priority before acquiring the lock is %d\n", tid, old_pri);

	/* Take the test lock */
	turnstile_prim_lock(type);

	unboosted_pri =  get_base_pri(mach_thread_self());
	T_ASSERT_EQ(unboosted_pri, 37, "thread(%llu) priority after acquiring the lock (uncontended) is %d\n", tid, unboosted_pri);

	sleep(8);

	/* Check for elevated priority */
	boosted_pri =  get_base_pri(mach_thread_self());
	T_ASSERT_EQ(boosted_pri, 47, "thread(%llu) priority after contention by 47 thread is %d\n", tid, boosted_pri);

	/* return without unlocking the lock */
	return NULL;
}

static void *
unlock_an_owner_exited_lock(void *arg)
{
	uint64_t tid;
	int type = (int)arg;

	pthread_threadid_np(NULL, &tid);
	sleep(12);

	int old_pri = get_base_pri(mach_thread_self());
	T_ASSERT_EQ(old_pri, 47, "thread(%llu) priority before acquiring the lock is %d\n", tid, old_pri);

	/* Unlock the test lock causing the turnstile code to call thread_deallocate_safe */
	turnstile_prim_unlock(type);
	return NULL;
}

/*
 * Test 1: test if lock contended by a UI thread boosts the owner to UI qos.
 */
static void
test1(int type)
{
	T_LOG("Test 1: test if lock contended by a UI thread boosts the owner to UI qos");

	/* Create a thread at IN and take lock */
	thread_create_at_qos(QOS_CLASS_USER_INITIATED, &take_lock_check_priority, type);

	/* Create a thread at UI and try to take lock */
	thread_create_at_qos(QOS_CLASS_USER_INTERACTIVE, &try_to_take_lock_and_unlock, type);

	sleep(12);
	return;
}

/*
 * Test 2: test if lock contended by a 2 UI thread boosts the owner to UI qos.
 */
static void
test2(int type)
{
	T_LOG("Test 2: test if lock contended by a 2 UI thread boosts the owner to UI qos");

	/* Create a thread at IN and take lock */
	thread_create_at_qos(QOS_CLASS_USER_INITIATED, &take_lock_check_priority, type);

	/* Create a thread at UI and try to take lock */
	thread_create_at_qos(QOS_CLASS_USER_INTERACTIVE, &try_to_take_lock_and_unlock, type);

	/* Create a thread at UI and try to take lock */
	thread_create_at_qos(QOS_CLASS_USER_INTERACTIVE, &try_to_take_lock_and_unlock, type);

	sleep(16);
	return;
}

/*
 * Test 3: test if lock owner thread exiting without unlocking allows turnstile to work correctly.
 */
static void
test3(int type)
{
	T_LOG("Test 3: test if lock owner thread exiting without unlocking allows turnstile to work correctly");

	/* Create a thread at IN and take lock */
	thread_create_at_qos(QOS_CLASS_USER_INITIATED, &take_lock_and_exit, type);

	/* Create a thread at UI and try to take lock */
	thread_create_at_qos(QOS_CLASS_USER_INTERACTIVE, &try_to_take_lock_and_unlock, type);

	/* Create a thread at UI and try to take lock */
	thread_create_at_qos(QOS_CLASS_USER_INTERACTIVE, &unlock_an_owner_exited_lock, type);

	sleep(16);
	return;
}

/*
 * Test 4: test if a chain of user-space turnstile primitives followed by kernel primitives works correctly.
 */
static void
test4(void)
{
	pthread_t threads[5] = {};
	struct thread_data data[5] = {};

	T_LOG("Test 4: test if a chain of user-space turnstile primitives followed by kernel primitives works correctly");

	/*
	 * Chain: t4->ud->t3->uh->t2->kh->t1->kd->t0
	 * ud and uh (user space turnstiles) will push base pri and sched pri
	 * kd and kh (kernel space turnstiles) will push sched pri
	 * sched pri should be propagated up to the end
	 * kh is the breaking point of the chain for sched pri
	 */


	/* Create a thread at priority 4 and take SYSCTL_TURNSTILE_TEST_KERNEL_DEFAULT lock */
	data[0].pri_to_set = 4;
	data[0].lock1 = SYSCTL_TURNSTILE_TEST_KERNEL_DEFAULT; /* this should be not locked */
	data[0].lock2 = NULL;
	data[0].sleep = 10; /* long sleep, nothing is blocking this thread */
	data[0].sched_pri_to_check = 60;
	data[0].base_pri_to_check = 4;
	pthread_create(&threads[0], NULL, chain_locking, (void *)&data[0]);
	sleep(2); /* give the thread time to acquire the lock */

	/* Create a thread at priority 31 and take SYSCTL_TURNSTILE_TEST_KERNEL_HASHTABLE lock followed by SYSCTL_TURNSTILE_TEST_KERNEL_DEFAULT */
	data[1].pri_to_set = 31;
	data[1].lock1 = SYSCTL_TURNSTILE_TEST_KERNEL_HASHTABLE; /* this should be not locked */
	data[1].lock2 = SYSCTL_TURNSTILE_TEST_KERNEL_DEFAULT; /* this should be locked */
	data[1].sleep = 0; /* no need to sleep, everything should be pushing by the time it acquires the lock */
	data[1].sched_pri_to_check = 60;
	data[1].base_pri_to_check = 31;
	pthread_create(&threads[1], NULL, chain_locking, (void *)&data[1]);
	sleep(2); /* give the thread time to acquire the lock */

	/* Create a thread at priority 40 and take SYSCTL_TURNSTILE_TEST_USER_HASHTABLE lock followed by SYSCTL_TURNSTILE_TEST_KERNEL_HASHTABLE */
	data[2].pri_to_set = 40;
	data[2].lock1 = SYSCTL_TURNSTILE_TEST_USER_HASHTABLE; /* this should be not locked */
	data[2].lock2 = SYSCTL_TURNSTILE_TEST_KERNEL_HASHTABLE; /* this should be locked */
	data[2].sleep = 0; /* no need to sleep, everything should be pushing by the time it acquires the lock */
	data[2].sched_pri_to_check = 60;
	data[2].base_pri_to_check = 60;
	pthread_create(&threads[2], NULL, chain_locking, (void *)&data[2]);
	sleep(2); /* give the thread time to acquire the lock */

	/* Create a thread at priority 47 and take SYSCTL_TURNSTILE_TEST_USER_DEFAULT lock followed by SYSCTL_TURNSTILE_TEST_USER_HASHTABLE */
	data[3].pri_to_set = 47;
	data[3].lock1 = SYSCTL_TURNSTILE_TEST_USER_DEFAULT; /* this should be not locked */
	data[3].lock2 = SYSCTL_TURNSTILE_TEST_USER_HASHTABLE; /* this should be locked */
	data[3].sleep = 0; /* no need to sleep, everything should be pushing by the time it acquires the lock */
	data[3].sched_pri_to_check = 60;
	data[3].base_pri_to_check = 60;
	pthread_create(&threads[3], NULL, chain_locking, (void *)&data[3]);
	sleep(2); /* give the thread time to acquire the lock */

	/* Create a thread at priority 60 and take SYSCTL_TURNSTILE_TEST_USER_DEFAULT */
	data[4].pri_to_set = 60;
	data[4].lock1 = SYSCTL_TURNSTILE_TEST_USER_DEFAULT; /* this should be locked */
	data[4].lock2 = NULL;
	data[4].sleep = 0; /* no need to sleep, nothing should be pushing by the time it acquires the lock */
	data[4].sched_pri_to_check = 60; /* this is its own priority */
	data[4].base_pri_to_check = 60;
	pthread_create(&threads[4], NULL, chain_locking, (void *)&data[4]);

	sleep(16);
	return;
}

/*
 * Test 5: test if a chain of user-space turnstile primitives interleaved by kernel primitives works correctly.
 */
static void
test5(void)
{
	pthread_t threads[5] = {};
	struct thread_data data[5] = {};

	T_LOG("Test 5: test if a chain of user-space turnstile primitives interleaved by kernel primitives works correctly");

	/*
	 * Chain: t4->ud->t3->kh->t2->uh->t1->kd->t0
	 * ud and uh (user space turnstiles) will push base pri and sched pri
	 * kd and kh (kernel space turnstiles) will push sched pri
	 * uh is the breaking point of the chain for sched pri
	 */

	/* Create a thread at priority 4 and take SYSCTL_TURNSTILE_TEST_KERNEL_DEFAULT lock */
	data[0].pri_to_set = 4;
	data[0].lock1 = SYSCTL_TURNSTILE_TEST_KERNEL_DEFAULT; /* this should be not locked */
	data[0].lock2 = NULL;
	data[0].sleep = 10; /* long sleep, nothing is blocking this thread */
	data[0].sched_pri_to_check = 41;
	data[0].base_pri_to_check = 4;
	pthread_create(&threads[0], NULL, chain_locking, (void *)&data[0]);
	sleep(2); /* give the thread time to acquire the lock */

	/* Create a thread at priority 31 and take SYSCTL_TURNSTILE_TEST_USER_HASHTABLE lock followed by SYSCTL_TURNSTILE_TEST_KERNEL_DEFAULT */
	data[1].pri_to_set = 31;
	data[1].lock1 = SYSCTL_TURNSTILE_TEST_USER_HASHTABLE; /* this should be not locked */
	data[1].lock2 = SYSCTL_TURNSTILE_TEST_KERNEL_DEFAULT; /* this should be locked */
	data[1].sleep = 0; /* no need to sleep, everything should be pushing by the time it acquires the lock */
	data[1].sched_pri_to_check = 41;
	data[1].base_pri_to_check = 41;
	pthread_create(&threads[1], NULL, chain_locking, (void *)&data[1]);
	sleep(2); /* give the thread time to acquire the lock */

	/* Create a thread at priority 41 and take SYSCTL_TURNSTILE_TEST_KERNEL_HASHTABLE lock followed by SYSCTL_TURNSTILE_TEST_USER_HASHTABLE */
	data[2].pri_to_set = 41;
	data[2].lock1 = SYSCTL_TURNSTILE_TEST_KERNEL_HASHTABLE; /* this should be not locked */
	data[2].lock2 = SYSCTL_TURNSTILE_TEST_USER_HASHTABLE; /* this should be locked */
	data[2].sleep = 0; /* no need to sleep, everything should be pushing by the time it acquires the lock */
	data[2].sched_pri_to_check = 60;
	data[2].base_pri_to_check = 41;
	pthread_create(&threads[2], NULL, chain_locking, (void *)&data[2]);
	sleep(2); /* give the thread time to acquire the lock */

	/* Create a thread at priority 47 and take SYSCTL_TURNSTILE_TEST_USER_DEFAULT lock followed by SYSCTL_TURNSTILE_TEST_KERNEL_HASHTABLE */
	data[3].pri_to_set = 47;
	data[3].lock1 = SYSCTL_TURNSTILE_TEST_USER_DEFAULT; /* this should be not locked */
	data[3].lock2 = SYSCTL_TURNSTILE_TEST_KERNEL_HASHTABLE; /* this should be locked */
	data[3].sleep = 0; /* no need to sleep, everything should be pushing by the time it acquires the lock */
	data[3].sched_pri_to_check = 60;
	data[3].base_pri_to_check = 60;
	pthread_create(&threads[3], NULL, chain_locking, (void *)&data[3]);
	sleep(2); /* give the thread time to acquire the lock */

	/* Create a thread at priority 60 and take SYSCTL_TURNSTILE_TEST_USER_DEFAULT */
	data[4].pri_to_set = 60;
	data[4].lock1 = SYSCTL_TURNSTILE_TEST_USER_DEFAULT; /* this should be locked */
	data[4].lock2 = NULL;
	data[4].sleep = 0; /* no need to sleep, nothing should be pushing by the time it acquires the lock */
	data[4].sched_pri_to_check = 60; /* this is its own priority */
	data[4].base_pri_to_check = 60;
	pthread_create(&threads[4], NULL, chain_locking, (void *)&data[4]);

	sleep(16);
	return;
}

T_DECL(turnstile_test, "Turnstile test", T_META_ASROOT(YES))
{
	test1(SYSCTL_TURNSTILE_TEST_USER_DEFAULT);
	test2(SYSCTL_TURNSTILE_TEST_USER_DEFAULT);
	test3(SYSCTL_TURNSTILE_TEST_USER_DEFAULT);

	test1(SYSCTL_TURNSTILE_TEST_USER_HASHTABLE);
	test2(SYSCTL_TURNSTILE_TEST_USER_HASHTABLE);
	test3(SYSCTL_TURNSTILE_TEST_USER_HASHTABLE);

	/*
	 * rdar://problem/46302128
	 * These tests are using a sysctl to lock a dummy kernel resource that uses turnstile.
	 * However a thread holding a kernel push from turnstile should never return in
	 * userspace, and rdar://problem/24194397 adds an assert for it.
	 */
	//test4();
	//test5();
}
Commit	Line	Data
d9a64523 A	1	/*
	2	* turnstiles_test: Tests turnstile kernel primitive.
	3	*/
	4
	5	#ifdef T_NAMESPACE
	6	#undef T_NAMESPACE
	7	#endif
	8
	9	#include <darwintest.h>
	10	#include <darwintest_multiprocess.h>
	11
	12	#include <pthread.h>
	13	#include <launch.h>
	14	#include <servers/bootstrap.h>
	15	#include <stdlib.h>
	16	#include <sys/event.h>
	17	#include <unistd.h>
	18	#include <crt_externs.h>
	19	#include <sys/sysctl.h>
	20	#include <sys/types.h>
	21
cb323159 A	22	#define SYSCTL_TURNSTILE_TEST_USER_DEFAULT 1
	23	#define SYSCTL_TURNSTILE_TEST_USER_HASHTABLE 2
	24	#define SYSCTL_TURNSTILE_TEST_KERNEL_DEFAULT 3
	25	#define SYSCTL_TURNSTILE_TEST_KERNEL_HASHTABLE 4
d9a64523 A	26
	27	T_GLOBAL_META(T_META_NAMESPACE("xnu.turnstiles_test"));
	28
	29	static void
	30	thread_create_at_qos(qos_class_t qos, void * (function)(void ), int type)
	31	{
	32	qos_class_t qos_thread;
	33	pthread_t thread;
0a7de745	34	pthread_attr_t attr;
d9a64523 A	35	int ret;
	36
	37	ret = setpriority(PRIO_DARWIN_ROLE, 0, PRIO_DARWIN_ROLE_UI_FOCAL);
	38	if (ret != 0) {
	39	T_LOG("set priority failed\n");
	40	}
	41
0a7de745 A	42	pthread_attr_init(&attr);
	43	pthread_attr_set_qos_class_np(&attr, qos, 0);
	44	pthread_create(&thread, &attr, function, (void *)type);
d9a64523 A	45
	46	T_LOG("pthread created\n");
	47	pthread_get_qos_class_np(thread, &qos_thread, NULL);
0a7de745	48	T_EXPECT_EQ(qos_thread, (qos_class_t)qos, NULL);
d9a64523 A	49	}
	50
	51	static int
cb323159	52	get_sched_pri(thread_t thread_port)
0a7de745	53	{
d9a64523 A	54	kern_return_t kr;
	55
	56	thread_extended_info_data_t extended_info;
	57	mach_msg_type_number_t count = THREAD_EXTENDED_INFO_COUNT;
	58	kr = thread_info(thread_port, THREAD_EXTENDED_INFO,
0a7de745	59	(thread_info_t)&extended_info, &count);
d9a64523 A	60
	61	T_QUIET; T_ASSERT_MACH_SUCCESS(kr, "thread_info");
	62	return extended_info.pth_curpri;
	63	}
	64
cb323159 A	65	static int
	66	get_base_pri(thread_t thread_port)
	67	{
	68	kern_return_t kr;
	69
	70	thread_extended_info_data_t extended_info;
	71	mach_msg_type_number_t count = THREAD_EXTENDED_INFO_COUNT;
	72	kr = thread_info(thread_port, THREAD_EXTENDED_INFO,
	73	(thread_info_t)&extended_info, &count);
	74
	75	T_QUIET; T_ASSERT_MACH_SUCCESS(kr, "thread_info");
	76	return extended_info.pth_priority;
	77	}
	78
d9a64523 A	79	static void
	80	turnstile_prim_lock(int type)
	81	{
	82	int ret;
	83	uint64_t tid;
	84	int in_val = type;
	85	pthread_threadid_np(NULL, &tid);
cb323159	86	T_LOG("sysctlbyname lock type %d called from thread %llu \n", type, tid);
d9a64523 A	87	ret = sysctlbyname("kern.turnstiles_test_lock", NULL, 0, &in_val, sizeof(in_val));
	88	T_LOG("sysctlbyname lock returned from thread %llu with value %d \n", tid, ret);
	89	}
	90
	91	static void
	92	turnstile_prim_unlock(int type)
	93	{
	94	int ret;
	95	uint64_t tid;
	96	int in_val = type;
	97	pthread_threadid_np(NULL, &tid);
cb323159	98	T_LOG("sysctlbyname unlock type %d called from thread %llu \n", type, tid);
d9a64523 A	99	ret = sysctlbyname("kern.turnstiles_test_unlock", NULL, 0, &in_val, sizeof(in_val));
	100	T_LOG("sysctlbyname unlock returned from thread %llu with value %d \n", tid, ret);
	101	}
	102
cb323159 A	103	struct thread_data {
	104	int pri_to_set;
	105	int lock1;
	106	int lock2;
	107	unsigned int sleep;
	108	int sched_pri_to_check;
	109	int base_pri_to_check;
	110	};
	111
	112	static void *
	113	chain_locking(void* args)
	114	{
	115	struct thread_data* data = (struct thread_data*) args;
	116	int policy, pri;
	117	int ret;
	118	struct sched_param param;
	119
	120	/* Change our priority to pri_to_set */
	121	ret = pthread_getschedparam(pthread_self(), &policy, &param);
	122	T_QUIET; T_ASSERT_MACH_SUCCESS(ret, "pthread_getschedparam");
	123
	124	param.sched_priority = data->pri_to_set;
	125
	126	/* this sets both sched and base pri */
	127	ret = pthread_setschedparam(pthread_self(), policy, &param);
	128	T_QUIET; T_ASSERT_MACH_SUCCESS(ret, "pthread_setschedparam");
	129
	130	pri = get_sched_pri(mach_thread_self());
	131
	132	T_ASSERT_EQ(pri, data->pri_to_set, "Priority before holding locks");
	133
	134	/* take lock1 */
	135	if (data->lock1) {
	136	turnstile_prim_lock(data->lock1);
	137	}
	138
	139	/* take lock2 */
	140	if (data->lock2) {
	141	turnstile_prim_lock(data->lock2);
	142	}
	143
	144	if (data->sleep) {
	145	sleep(data->sleep);
	146	}
	147
	148	if (data->sched_pri_to_check) {
	149	pri = get_sched_pri(mach_thread_self());
	150	T_ASSERT_EQ(pri, data->sched_pri_to_check, "Sched priority while holding locks");
	151	}
	152
	153	if (data->base_pri_to_check) {
	154	pri = get_base_pri(mach_thread_self());
	155	T_ASSERT_EQ(pri, data->base_pri_to_check, "Base priority while holding locks");
	156	}
	157
	158	if (data->lock2) {
	159	turnstile_prim_unlock(data->lock2);
	160	}
	161
	162	if (data->lock1) {
	163	turnstile_prim_unlock(data->lock1);
	164	}
	165
	166	pri = get_sched_pri(mach_thread_self());
167	T_ASSERT_EQ(pri, data->pri_to_set, "Priority after releasing locks");
168
169	return NULL;
170	}
171
d9a64523 A	172	static void *
	173	take_lock_check_priority(void * arg)
	174	{
cb323159	175	int old_pri = get_base_pri(mach_thread_self());
d9a64523 A	176	int unboosted_pri;
	177	int boosted_pri;
	178	int after_unlock_pri;
	179	uint64_t tid;
	180	int type = (int)arg;
	181
	182	pthread_threadid_np(NULL, &tid);
	183
	184	T_ASSERT_EQ(old_pri, 37, "thread(%llu) priority before acquiring the lock is %d\n", tid, old_pri);
	185
	186	/* Take the test lock */
	187	turnstile_prim_lock(type);
	188
cb323159	189	unboosted_pri = get_base_pri(mach_thread_self());
d9a64523 A	190	T_ASSERT_EQ(unboosted_pri, 37, "thread(%llu) priority after acquiring the lock (uncontended) is %d\n", tid, unboosted_pri);
	191
	192	sleep(8);
	193
	194	/* Check for elevated priority */
cb323159	195	boosted_pri = get_base_pri(mach_thread_self());
d9a64523 A	196	T_ASSERT_EQ(boosted_pri, 47, "thread(%llu) priority after contention by 47 thread is %d\n", tid, boosted_pri);
	197
	198	/* Drop the lock */
	199	turnstile_prim_unlock(type);
	200
	201	/* Check for regular priority */
cb323159	202	after_unlock_pri = get_base_pri(mach_thread_self());
d9a64523 A	203	T_ASSERT_EQ(after_unlock_pri, 37, "thread(%llu) priority after dropping lock is %d\n", tid, after_unlock_pri);
	204
	205	return NULL;
	206	}
	207
	208	static void *
	209	try_to_take_lock_and_unlock(void *arg)
	210	{
	211	uint64_t tid;
	212	int type = (int)arg;
	213
	214	pthread_threadid_np(NULL, &tid);
	215	sleep(4);
	216
cb323159	217	int old_pri = get_base_pri(mach_thread_self());
d9a64523 A	218	T_ASSERT_EQ(old_pri, 47, "thread(%llu) priority before acquiring the lock is %d\n", tid, old_pri);
	219
	220	/* Try taking the test lock */
	221	turnstile_prim_lock(type);
0a7de745	222	sleep(2);
d9a64523 A	223	turnstile_prim_unlock(type);
	224	return NULL;
	225	}
	226
	227	static void *
	228	take_lock_and_exit(void * arg)
	229	{
cb323159	230	int old_pri = get_base_pri(mach_thread_self());
d9a64523 A	231	int unboosted_pri;
	232	int boosted_pri;
	233	uint64_t tid;
	234	int type = (int)arg;
	235
	236	pthread_threadid_np(NULL, &tid);
	237
	238	T_ASSERT_EQ(old_pri, 37, "thread(%llu) priority before acquiring the lock is %d\n", tid, old_pri);
	239
	240	/* Take the test lock */
	241	turnstile_prim_lock(type);
	242
cb323159	243	unboosted_pri = get_base_pri(mach_thread_self());
d9a64523 A	244	T_ASSERT_EQ(unboosted_pri, 37, "thread(%llu) priority after acquiring the lock (uncontended) is %d\n", tid, unboosted_pri);
	245
	246	sleep(8);
	247
	248	/* Check for elevated priority */
cb323159	249	boosted_pri = get_base_pri(mach_thread_self());
d9a64523 A	250	T_ASSERT_EQ(boosted_pri, 47, "thread(%llu) priority after contention by 47 thread is %d\n", tid, boosted_pri);
	251
	252	/* return without unlocking the lock */
	253	return NULL;
	254	}
	255
	256	static void *
	257	unlock_an_owner_exited_lock(void *arg)
	258	{
	259	uint64_t tid;
	260	int type = (int)arg;
	261
	262	pthread_threadid_np(NULL, &tid);
	263	sleep(12);
	264
cb323159	265	int old_pri = get_base_pri(mach_thread_self());
d9a64523 A	266	T_ASSERT_EQ(old_pri, 47, "thread(%llu) priority before acquiring the lock is %d\n", tid, old_pri);
	267
	268	/* Unlock the test lock causing the turnstile code to call thread_deallocate_safe */
	269	turnstile_prim_unlock(type);
	270	return NULL;
	271	}
	272
	273	/*
	274	* Test 1: test if lock contended by a UI thread boosts the owner to UI qos.
	275	*/
	276	static void
	277	test1(int type)
	278	{
	279	T_LOG("Test 1: test if lock contended by a UI thread boosts the owner to UI qos");
	280
	281	/* Create a thread at IN and take lock */
	282	thread_create_at_qos(QOS_CLASS_USER_INITIATED, &take_lock_check_priority, type);
	283
	284	/* Create a thread at UI and try to take lock */
	285	thread_create_at_qos(QOS_CLASS_USER_INTERACTIVE, &try_to_take_lock_and_unlock, type);
	286
	287	sleep(12);
	288	return;
	289	}
	290
	291	/*
	292	* Test 2: test if lock contended by a 2 UI thread boosts the owner to UI qos.
	293	*/
	294	static void
	295	test2(int type)
	296	{
	297	T_LOG("Test 2: test if lock contended by a 2 UI thread boosts the owner to UI qos");
	298
	299	/* Create a thread at IN and take lock */
	300	thread_create_at_qos(QOS_CLASS_USER_INITIATED, &take_lock_check_priority, type);
	301
	302	/* Create a thread at UI and try to take lock */
	303	thread_create_at_qos(QOS_CLASS_USER_INTERACTIVE, &try_to_take_lock_and_unlock, type);
	304
	305	/* Create a thread at UI and try to take lock */
	306	thread_create_at_qos(QOS_CLASS_USER_INTERACTIVE, &try_to_take_lock_and_unlock, type);
	307
	308	sleep(16);
	309	return;
	310	}
	311
	312	/*
	313	* Test 3: test if lock owner thread exiting without unlocking allows turnstile to work correctly.
	314	*/
	315	static void
	316	test3(int type)
	317	{
	318	T_LOG("Test 3: test if lock owner thread exiting without unlocking allows turnstile to work correctly");
	319
	320	/* Create a thread at IN and take lock */
	321	thread_create_at_qos(QOS_CLASS_USER_INITIATED, &take_lock_and_exit, type);
	322
	323	/* Create a thread at UI and try to take lock */
	324	thread_create_at_qos(QOS_CLASS_USER_INTERACTIVE, &try_to_take_lock_and_unlock, type);
	325
	326	/* Create a thread at UI and try to take lock */
	327	thread_create_at_qos(QOS_CLASS_USER_INTERACTIVE, &unlock_an_owner_exited_lock, type);
	328
	329	sleep(16);
330	return;
331	}
332
cb323159 A	333	/*
	334	* Test 4: test if a chain of user-space turnstile primitives followed by kernel primitives works correctly.
	335	*/
	336	static void
	337	test4(void)
d9a64523	338	{
cb323159 A	339	pthread_t threads[5] = {};
	340	struct thread_data data[5] = {};
	341
	342	T_LOG("Test 4: test if a chain of user-space turnstile primitives followed by kernel primitives works correctly");
	343
	344	/*
	345	* Chain: t4->ud->t3->uh->t2->kh->t1->kd->t0
	346	* ud and uh (user space turnstiles) will push base pri and sched pri
	347	* kd and kh (kernel space turnstiles) will push sched pri
	348	* sched pri should be propagated up to the end
	349	* kh is the breaking point of the chain for sched pri
	350	*/
	351
	352
	353	/* Create a thread at priority 4 and take SYSCTL_TURNSTILE_TEST_KERNEL_DEFAULT lock */
	354	data[0].pri_to_set = 4;
	355	data[0].lock1 = SYSCTL_TURNSTILE_TEST_KERNEL_DEFAULT; /* this should be not locked */
	356	data[0].lock2 = NULL;
	357	data[0].sleep = 10; /* long sleep, nothing is blocking this thread */
	358	data[0].sched_pri_to_check = 60;
	359	data[0].base_pri_to_check = 4;
	360	pthread_create(&threads[0], NULL, chain_locking, (void *)&data[0]);
	361	sleep(2); /* give the thread time to acquire the lock */
	362
	363	/* Create a thread at priority 31 and take SYSCTL_TURNSTILE_TEST_KERNEL_HASHTABLE lock followed by SYSCTL_TURNSTILE_TEST_KERNEL_DEFAULT */
	364	data[1].pri_to_set = 31;
	365	data[1].lock1 = SYSCTL_TURNSTILE_TEST_KERNEL_HASHTABLE; /* this should be not locked */
	366	data[1].lock2 = SYSCTL_TURNSTILE_TEST_KERNEL_DEFAULT; /* this should be locked */
	367	data[1].sleep = 0; /* no need to sleep, everything should be pushing by the time it acquires the lock */
	368	data[1].sched_pri_to_check = 60;
	369	data[1].base_pri_to_check = 31;
	370	pthread_create(&threads[1], NULL, chain_locking, (void *)&data[1]);
	371	sleep(2); /* give the thread time to acquire the lock */
	372
	373	/* Create a thread at priority 40 and take SYSCTL_TURNSTILE_TEST_USER_HASHTABLE lock followed by SYSCTL_TURNSTILE_TEST_KERNEL_HASHTABLE */
	374	data[2].pri_to_set = 40;
	375	data[2].lock1 = SYSCTL_TURNSTILE_TEST_USER_HASHTABLE; /* this should be not locked */
	376	data[2].lock2 = SYSCTL_TURNSTILE_TEST_KERNEL_HASHTABLE; /* this should be locked */
	377	data[2].sleep = 0; /* no need to sleep, everything should be pushing by the time it acquires the lock */
	378	data[2].sched_pri_to_check = 60;
	379	data[2].base_pri_to_check = 60;
	380	pthread_create(&threads[2], NULL, chain_locking, (void *)&data[2]);
	381	sleep(2); /* give the thread time to acquire the lock */
	382
	383	/* Create a thread at priority 47 and take SYSCTL_TURNSTILE_TEST_USER_DEFAULT lock followed by SYSCTL_TURNSTILE_TEST_USER_HASHTABLE */
	384	data[3].pri_to_set = 47;
	385	data[3].lock1 = SYSCTL_TURNSTILE_TEST_USER_DEFAULT; /* this should be not locked */
	386	data[3].lock2 = SYSCTL_TURNSTILE_TEST_USER_HASHTABLE; /* this should be locked */
	387	data[3].sleep = 0; /* no need to sleep, everything should be pushing by the time it acquires the lock */
	388	data[3].sched_pri_to_check = 60;
	389	data[3].base_pri_to_check = 60;
	390	pthread_create(&threads[3], NULL, chain_locking, (void *)&data[3]);
	391	sleep(2); /* give the thread time to acquire the lock */
	392
	393	/* Create a thread at priority 60 and take SYSCTL_TURNSTILE_TEST_USER_DEFAULT */
	394	data[4].pri_to_set = 60;
	395	data[4].lock1 = SYSCTL_TURNSTILE_TEST_USER_DEFAULT; /* this should be locked */
	396	data[4].lock2 = NULL;
	397	data[4].sleep = 0; /* no need to sleep, nothing should be pushing by the time it acquires the lock */
	398	data[4].sched_pri_to_check = 60; /* this is its own priority */
	399	data[4].base_pri_to_check = 60;
	400	pthread_create(&threads[4], NULL, chain_locking, (void *)&data[4]);
d9a64523	401
cb323159 A	402	sleep(16);
	403	return;
	404	}
	405
	406	/*
	407	* Test 5: test if a chain of user-space turnstile primitives interleaved by kernel primitives works correctly.
	408	*/
	409	static void
	410	test5(void)
	411	{
	412	pthread_t threads[5] = {};
	413	struct thread_data data[5] = {};
	414
	415	T_LOG("Test 5: test if a chain of user-space turnstile primitives interleaved by kernel primitives works correctly");
	416
	417	/*
	418	* Chain: t4->ud->t3->kh->t2->uh->t1->kd->t0
	419	* ud and uh (user space turnstiles) will push base pri and sched pri
	420	* kd and kh (kernel space turnstiles) will push sched pri
	421	* uh is the breaking point of the chain for sched pri
	422	*/
	423
	424	/* Create a thread at priority 4 and take SYSCTL_TURNSTILE_TEST_KERNEL_DEFAULT lock */
	425	data[0].pri_to_set = 4;
	426	data[0].lock1 = SYSCTL_TURNSTILE_TEST_KERNEL_DEFAULT; /* this should be not locked */
	427	data[0].lock2 = NULL;
	428	data[0].sleep = 10; /* long sleep, nothing is blocking this thread */
	429	data[0].sched_pri_to_check = 41;
	430	data[0].base_pri_to_check = 4;
	431	pthread_create(&threads[0], NULL, chain_locking, (void *)&data[0]);
	432	sleep(2); /* give the thread time to acquire the lock */
	433
	434	/* Create a thread at priority 31 and take SYSCTL_TURNSTILE_TEST_USER_HASHTABLE lock followed by SYSCTL_TURNSTILE_TEST_KERNEL_DEFAULT */
	435	data[1].pri_to_set = 31;
	436	data[1].lock1 = SYSCTL_TURNSTILE_TEST_USER_HASHTABLE; /* this should be not locked */
	437	data[1].lock2 = SYSCTL_TURNSTILE_TEST_KERNEL_DEFAULT; /* this should be locked */
	438	data[1].sleep = 0; /* no need to sleep, everything should be pushing by the time it acquires the lock */
	439	data[1].sched_pri_to_check = 41;
	440	data[1].base_pri_to_check = 41;
	441	pthread_create(&threads[1], NULL, chain_locking, (void *)&data[1]);
	442	sleep(2); /* give the thread time to acquire the lock */
	443
	444	/* Create a thread at priority 41 and take SYSCTL_TURNSTILE_TEST_KERNEL_HASHTABLE lock followed by SYSCTL_TURNSTILE_TEST_USER_HASHTABLE */
	445	data[2].pri_to_set = 41;
	446	data[2].lock1 = SYSCTL_TURNSTILE_TEST_KERNEL_HASHTABLE; /* this should be not locked */
	447	data[2].lock2 = SYSCTL_TURNSTILE_TEST_USER_HASHTABLE; /* this should be locked */
	448	data[2].sleep = 0; /* no need to sleep, everything should be pushing by the time it acquires the lock */
	449	data[2].sched_pri_to_check = 60;
	450	data[2].base_pri_to_check = 41;
	451	pthread_create(&threads[2], NULL, chain_locking, (void *)&data[2]);
	452	sleep(2); /* give the thread time to acquire the lock */
	453
	454	/* Create a thread at priority 47 and take SYSCTL_TURNSTILE_TEST_USER_DEFAULT lock followed by SYSCTL_TURNSTILE_TEST_KERNEL_HASHTABLE */
	455	data[3].pri_to_set = 47;
	456	data[3].lock1 = SYSCTL_TURNSTILE_TEST_USER_DEFAULT; /* this should be not locked */
	457	data[3].lock2 = SYSCTL_TURNSTILE_TEST_KERNEL_HASHTABLE; /* this should be locked */
	458	data[3].sleep = 0; /* no need to sleep, everything should be pushing by the time it acquires the lock */
	459	data[3].sched_pri_to_check = 60;
	460	data[3].base_pri_to_check = 60;
	461	pthread_create(&threads[3], NULL, chain_locking, (void *)&data[3]);
	462	sleep(2); /* give the thread time to acquire the lock */
	463
	464	/* Create a thread at priority 60 and take SYSCTL_TURNSTILE_TEST_USER_DEFAULT */
	465	data[4].pri_to_set = 60;
466	data[4].lock1 = SYSCTL_TURNSTILE_TEST_USER_DEFAULT; /* this should be locked */
467	data[4].lock2 = NULL;
468	data[4].sleep = 0; /* no need to sleep, nothing should be pushing by the time it acquires the lock */
469	data[4].sched_pri_to_check = 60; /* this is its own priority */
470	data[4].base_pri_to_check = 60;
471	pthread_create(&threads[4], NULL, chain_locking, (void *)&data[4]);
472
473	sleep(16);
474	return;
475	}
476
477	T_DECL(turnstile_test, "Turnstile test", T_META_ASROOT(YES))
478	{
479	test1(SYSCTL_TURNSTILE_TEST_USER_DEFAULT);
480	test2(SYSCTL_TURNSTILE_TEST_USER_DEFAULT);
481	test3(SYSCTL_TURNSTILE_TEST_USER_DEFAULT);
482
483	test1(SYSCTL_TURNSTILE_TEST_USER_HASHTABLE);
484	test2(SYSCTL_TURNSTILE_TEST_USER_HASHTABLE);
485	test3(SYSCTL_TURNSTILE_TEST_USER_HASHTABLE);
486
487	/*
488	* rdar://problem/46302128
489	* These tests are using a sysctl to lock a dummy kernel resource that uses turnstile.
490	* However a thread holding a kernel push from turnstile should never return in
491	* userspace, and rdar://problem/24194397 adds an assert for it.
492	*/
493	//test4();
494	//test5();
d9a64523	495	}