/*
- * Copyright (c) 2015 Apple Inc. All rights reserved.
+ * Copyright (c) 2019 Apple Inc. All rights reserved.
*
* @APPLE_OSREFERENCE_LICENSE_HEADER_START@
*
#include <vm/vm_page.h>
#include <vm/vm_object.h>
#include <kern/priority_queue.h>
+#include <string.h>
#if !(DEVELOPMENT || DEBUG)
#error "Testing is not enabled on RELEASE configurations"
kern_return_t RandomULong_test(void);
kern_return_t kcdata_api_test(void);
kern_return_t priority_queue_test(void);
+kern_return_t ts_kernel_primitive_test(void);
+kern_return_t ts_kernel_sleep_inheritor_test(void);
+kern_return_t ts_kernel_gate_test(void);
+kern_return_t ts_kernel_turnstile_chain_test(void);
+kern_return_t ts_kernel_timingsafe_bcmp_test(void);
+
+extern kern_return_t kprintf_hhx_test(void);
#if defined(__arm__) || defined(__arm64__)
kern_return_t pmap_coredump_test(void);
#if __ARM_PAN_AVAILABLE__
extern kern_return_t arm64_pan_test(void);
#endif
+#if defined(HAS_APPLE_PAC)
+extern kern_return_t arm64_ropjop_test(void);
+#endif /* defined(HAS_APPLE_PAC) */
#endif /* __arm64__ */
extern kern_return_t test_thread_call(void);
-struct xnupost_panic_widget xt_panic_widgets = {NULL, NULL, NULL, NULL};
+struct xnupost_panic_widget xt_panic_widgets = {.xtp_context_p = NULL,
+ .xtp_outval_p = NULL,
+ .xtp_func_name = NULL,
+ .xtp_func = NULL};
struct xnupost_test kernel_post_tests[] = {XNUPOST_TEST_CONFIG_BASIC(zalloc_test),
XNUPOST_TEST_CONFIG_BASIC(RandomULong_test),
#if __ARM_PAN_AVAILABLE__
XNUPOST_TEST_CONFIG_BASIC(arm64_pan_test),
#endif
+#if defined(HAS_APPLE_PAC)
+ XNUPOST_TEST_CONFIG_BASIC(arm64_ropjop_test),
+#endif /* defined(HAS_APPLE_PAC) */
#endif /* __arm64__ */
XNUPOST_TEST_CONFIG_BASIC(kcdata_api_test),
XNUPOST_TEST_CONFIG_BASIC(console_serial_test),
XNUPOST_TEST_CONFIG_BASIC(bitmap_post_test),
//XNUPOST_TEST_CONFIG_TEST_PANIC(kcdata_api_assert_tests)
XNUPOST_TEST_CONFIG_BASIC(test_thread_call),
- XNUPOST_TEST_CONFIG_BASIC(priority_queue_test), };
+ XNUPOST_TEST_CONFIG_BASIC(priority_queue_test),
+ XNUPOST_TEST_CONFIG_BASIC(ts_kernel_primitive_test),
+ XNUPOST_TEST_CONFIG_BASIC(ts_kernel_sleep_inheritor_test),
+ XNUPOST_TEST_CONFIG_BASIC(ts_kernel_gate_test),
+ XNUPOST_TEST_CONFIG_BASIC(ts_kernel_turnstile_chain_test),
+ XNUPOST_TEST_CONFIG_BASIC(ts_kernel_timingsafe_bcmp_test),
+ XNUPOST_TEST_CONFIG_BASIC(kprintf_hhx_test), };
uint32_t kernel_post_tests_count = sizeof(kernel_post_tests) / sizeof(xnupost_test_data_t);
} __attribute__((packed));
struct kcdata_subtype_descriptor test_disk_io_stats_def[] = {
- {KCS_SUBTYPE_FLAGS_NONE, KC_ST_UINT64, 0 * sizeof(uint64_t), sizeof(uint64_t), "disk_reads_count"},
- {KCS_SUBTYPE_FLAGS_NONE, KC_ST_UINT64, 1 * sizeof(uint64_t), sizeof(uint64_t), "disk_reads_size"},
- {KCS_SUBTYPE_FLAGS_ARRAY, KC_ST_UINT64, 2 * sizeof(uint64_t), KCS_SUBTYPE_PACK_SIZE(4, sizeof(uint64_t)), "io_priority_count"},
- {KCS_SUBTYPE_FLAGS_ARRAY, KC_ST_UINT64, (2 + 4) * sizeof(uint64_t), sizeof(uint64_t), "io_priority_size"},
+ {
+ .kcs_flags = KCS_SUBTYPE_FLAGS_NONE,
+ .kcs_elem_type = KC_ST_UINT64,
+ .kcs_elem_offset = 0 * sizeof(uint64_t),
+ .kcs_elem_size = sizeof(uint64_t),
+ .kcs_name = "disk_reads_count"
+ },
+ {
+ .kcs_flags = KCS_SUBTYPE_FLAGS_NONE,
+ .kcs_elem_type = KC_ST_UINT64,
+ .kcs_elem_offset = 1 * sizeof(uint64_t),
+ .kcs_elem_size = sizeof(uint64_t),
+ .kcs_name = "disk_reads_size"
+ },
+ {
+ .kcs_flags = KCS_SUBTYPE_FLAGS_ARRAY,
+ .kcs_elem_type = KC_ST_UINT64,
+ .kcs_elem_offset = 2 * sizeof(uint64_t),
+ .kcs_elem_size = KCS_SUBTYPE_PACK_SIZE(4, sizeof(uint64_t)),
+ .kcs_name = "io_priority_count"
+ },
+ {
+ .kcs_flags = KCS_SUBTYPE_FLAGS_ARRAY,
+ .kcs_elem_type = KC_ST_UINT64,
+ .kcs_elem_offset = (2 + 4) * sizeof(uint64_t),
+ .kcs_elem_size = sizeof(uint64_t),
+ .kcs_name = "io_priority_size"
+ },
};
kern_return_t
return KERN_SUCCESS;
}
#endif
+
+struct ts_kern_prim_test_args {
+ int *end_barrier;
+ int *notify_b;
+ int *wait_event_b;
+ int before_num;
+ int *notify_a;
+ int *wait_event_a;
+ int after_num;
+ int priority_to_check;
+};
+
+static void
+wait_threads(
+ int* var,
+ int num)
+{
+ if (var != NULL) {
+ while (os_atomic_load(var, acquire) != num) {
+ assert_wait((event_t) var, THREAD_UNINT);
+ if (os_atomic_load(var, acquire) != num) {
+ (void) thread_block(THREAD_CONTINUE_NULL);
+ } else {
+ clear_wait(current_thread(), THREAD_AWAKENED);
+ }
+ }
+ }
+}
+
+static void
+wake_threads(
+ int* var)
+{
+ if (var) {
+ os_atomic_inc(var, relaxed);
+ thread_wakeup((event_t) var);
+ }
+}
+
+extern void IOSleep(int);
+
+static void
+thread_lock_unlock_kernel_primitive(
+ void *args,
+ __unused wait_result_t wr)
+{
+ thread_t thread = current_thread();
+ struct ts_kern_prim_test_args *info = (struct ts_kern_prim_test_args*) args;
+ int pri;
+
+ thread_lock(thread);
+ pri = thread->sched_pri;
+ thread_unlock(thread);
+
+ wait_threads(info->wait_event_b, info->before_num);
+ wake_threads(info->notify_b);
+
+ tstile_test_prim_lock(SYSCTL_TURNSTILE_TEST_KERNEL_DEFAULT);
+
+ wake_threads(info->notify_a);
+ wait_threads(info->wait_event_a, info->after_num);
+
+ IOSleep(100);
+
+ if (info->priority_to_check) {
+ thread_lock(thread);
+ pri = thread->sched_pri;
+ thread_unlock(thread);
+ T_ASSERT(pri == info->priority_to_check, "Priority thread: current sched %d sched wanted %d", pri, info->priority_to_check);
+ }
+
+ tstile_test_prim_unlock(SYSCTL_TURNSTILE_TEST_KERNEL_DEFAULT);
+
+ wake_threads(info->end_barrier);
+ thread_terminate_self();
+}
+
+kern_return_t
+ts_kernel_primitive_test(void)
+{
+ thread_t owner, thread1, thread2;
+ struct ts_kern_prim_test_args targs[2] = {};
+ kern_return_t result;
+ int end_barrier = 0;
+ int owner_locked = 0;
+ int waiters_ready = 0;
+
+ T_LOG("Testing turnstile kernel primitive");
+
+ targs[0].notify_b = NULL;
+ targs[0].wait_event_b = NULL;
+ targs[0].before_num = 0;
+ targs[0].notify_a = &owner_locked;
+ targs[0].wait_event_a = &waiters_ready;
+ targs[0].after_num = 2;
+ targs[0].priority_to_check = 90;
+ targs[0].end_barrier = &end_barrier;
+
+ // Start owner with priority 80
+ result = kernel_thread_start_priority((thread_continue_t)thread_lock_unlock_kernel_primitive, &targs[0], 80, &owner);
+ T_ASSERT(result == KERN_SUCCESS, "Starting owner");
+
+ targs[1].notify_b = &waiters_ready;
+ targs[1].wait_event_b = &owner_locked;
+ targs[1].before_num = 1;
+ targs[1].notify_a = NULL;
+ targs[1].wait_event_a = NULL;
+ targs[1].after_num = 0;
+ targs[1].priority_to_check = 0;
+ targs[1].end_barrier = &end_barrier;
+
+ // Start waiters with priority 85 and 90
+ result = kernel_thread_start_priority((thread_continue_t)thread_lock_unlock_kernel_primitive, &targs[1], 85, &thread1);
+ T_ASSERT(result == KERN_SUCCESS, "Starting thread1");
+
+ result = kernel_thread_start_priority((thread_continue_t)thread_lock_unlock_kernel_primitive, &targs[1], 90, &thread2);
+ T_ASSERT(result == KERN_SUCCESS, "Starting thread2");
+
+ wait_threads(&end_barrier, 3);
+
+ return KERN_SUCCESS;
+}
+
+#define MTX_LOCK 0
+#define RW_LOCK 1
+
+#define NUM_THREADS 4
+
+struct synch_test_common {
+ unsigned int nthreads;
+ thread_t *threads;
+ int max_pri;
+ int test_done;
+};
+
+static kern_return_t
+init_synch_test_common(struct synch_test_common *info, unsigned int nthreads)
+{
+ info->nthreads = nthreads;
+ info->threads = kalloc(sizeof(thread_t) * nthreads);
+ if (!info->threads) {
+ return ENOMEM;
+ }
+
+ return KERN_SUCCESS;
+}
+
+static void
+destroy_synch_test_common(struct synch_test_common *info)
+{
+ kfree(info->threads, sizeof(thread_t) * info->nthreads);
+}
+
+static void
+start_threads(thread_continue_t func, struct synch_test_common *info, bool sleep_after_first)
+{
+ thread_t thread;
+ kern_return_t result;
+ uint i;
+ int priority = 75;
+
+ info->test_done = 0;
+
+ for (i = 0; i < info->nthreads; i++) {
+ info->threads[i] = NULL;
+ }
+
+ info->max_pri = priority + (info->nthreads - 1) * 5;
+ if (info->max_pri > 95) {
+ info->max_pri = 95;
+ }
+
+ for (i = 0; i < info->nthreads; i++) {
+ result = kernel_thread_start_priority((thread_continue_t)func, info, priority, &thread);
+ os_atomic_store(&info->threads[i], thread, release);
+ T_ASSERT(result == KERN_SUCCESS, "Starting thread %d, priority %d, %p", i, priority, thread);
+
+ priority += 5;
+
+ if (i == 0 && sleep_after_first) {
+ IOSleep(100);
+ }
+ }
+}
+
+static unsigned int
+get_max_pri(struct synch_test_common * info)
+{
+ return info->max_pri;
+}
+
+static void
+wait_all_thread(struct synch_test_common * info)
+{
+ wait_threads(&info->test_done, info->nthreads);
+}
+
+static void
+notify_waiter(struct synch_test_common * info)
+{
+ wake_threads(&info->test_done);
+}
+
+static void
+wait_for_waiters(struct synch_test_common *info)
+{
+ uint i, j;
+ thread_t thread;
+
+ for (i = 0; i < info->nthreads; i++) {
+ j = 0;
+ while (os_atomic_load(&info->threads[i], acquire) == NULL) {
+ if (j % 100 == 0) {
+ IOSleep(10);
+ }
+ j++;
+ }
+
+ if (info->threads[i] != current_thread()) {
+ j = 0;
+ do {
+ thread = os_atomic_load(&info->threads[i], relaxed);
+ if (thread == (thread_t) 1) {
+ break;
+ }
+
+ if (!(thread->state & TH_RUN)) {
+ break;
+ }
+
+ if (j % 100 == 0) {
+ IOSleep(100);
+ }
+ j++;
+
+ if (thread->started == FALSE) {
+ continue;
+ }
+ } while (thread->state & TH_RUN);
+ }
+ }
+}
+
+static void
+exclude_current_waiter(struct synch_test_common *info)
+{
+ uint i, j;
+
+ for (i = 0; i < info->nthreads; i++) {
+ j = 0;
+ while (os_atomic_load(&info->threads[i], acquire) == NULL) {
+ if (j % 100 == 0) {
+ IOSleep(10);
+ }
+ j++;
+ }
+
+ if (os_atomic_load(&info->threads[i], acquire) == current_thread()) {
+ os_atomic_store(&info->threads[i], (thread_t)1, release);
+ return;
+ }
+ }
+}
+
+struct info_sleep_inheritor_test {
+ struct synch_test_common head;
+ lck_mtx_t mtx_lock;
+ lck_rw_t rw_lock;
+ decl_lck_mtx_gate_data(, gate);
+ boolean_t gate_closed;
+ int prim_type;
+ boolean_t work_to_do;
+ unsigned int max_pri;
+ unsigned int steal_pri;
+ int synch_value;
+ int synch;
+ int value;
+ int handoff_failure;
+ thread_t thread_inheritor;
+};
+
+static void
+primitive_lock(struct info_sleep_inheritor_test *info)
+{
+ switch (info->prim_type) {
+ case MTX_LOCK:
+ lck_mtx_lock(&info->mtx_lock);
+ break;
+ case RW_LOCK:
+ lck_rw_lock(&info->rw_lock, LCK_RW_TYPE_EXCLUSIVE);
+ break;
+ default:
+ panic("invalid type %d", info->prim_type);
+ }
+}
+
+static void
+primitive_unlock(struct info_sleep_inheritor_test *info)
+{
+ switch (info->prim_type) {
+ case MTX_LOCK:
+ lck_mtx_unlock(&info->mtx_lock);
+ break;
+ case RW_LOCK:
+ lck_rw_unlock(&info->rw_lock, LCK_RW_TYPE_EXCLUSIVE);
+ break;
+ default:
+ panic("invalid type %d", info->prim_type);
+ }
+}
+
+static wait_result_t
+primitive_sleep_with_inheritor(struct info_sleep_inheritor_test *info)
+{
+ wait_result_t ret = KERN_SUCCESS;
+ switch (info->prim_type) {
+ case MTX_LOCK:
+ ret = lck_mtx_sleep_with_inheritor(&info->mtx_lock, LCK_SLEEP_DEFAULT, (event_t) &info->thread_inheritor, info->thread_inheritor, THREAD_UNINT | THREAD_WAIT_NOREPORT_USER, TIMEOUT_WAIT_FOREVER);
+ break;
+ case RW_LOCK:
+ ret = lck_rw_sleep_with_inheritor(&info->rw_lock, LCK_SLEEP_DEFAULT, (event_t) &info->thread_inheritor, info->thread_inheritor, THREAD_UNINT | THREAD_WAIT_NOREPORT_USER, TIMEOUT_WAIT_FOREVER);
+ break;
+ default:
+ panic("invalid type %d", info->prim_type);
+ }
+
+ return ret;
+}
+
+static void
+primitive_wakeup_one_with_inheritor(struct info_sleep_inheritor_test *info)
+{
+ switch (info->prim_type) {
+ case MTX_LOCK:
+ case RW_LOCK:
+ wakeup_one_with_inheritor((event_t) &info->thread_inheritor, THREAD_AWAKENED, LCK_WAKE_DEFAULT, &info->thread_inheritor);
+ break;
+ default:
+ panic("invalid type %d", info->prim_type);
+ }
+}
+
+static void
+primitive_wakeup_all_with_inheritor(struct info_sleep_inheritor_test *info)
+{
+ switch (info->prim_type) {
+ case MTX_LOCK:
+ case RW_LOCK:
+ wakeup_all_with_inheritor((event_t) &info->thread_inheritor, THREAD_AWAKENED);
+ break;
+ default:
+ panic("invalid type %d", info->prim_type);
+ }
+ return;
+}
+
+static void
+primitive_change_sleep_inheritor(struct info_sleep_inheritor_test *info)
+{
+ switch (info->prim_type) {
+ case MTX_LOCK:
+ case RW_LOCK:
+ change_sleep_inheritor((event_t) &info->thread_inheritor, info->thread_inheritor);
+ break;
+ default:
+ panic("invalid type %d", info->prim_type);
+ }
+ return;
+}
+
+static kern_return_t
+primitive_gate_try_close(struct info_sleep_inheritor_test *info)
+{
+ kern_return_t ret = KERN_SUCCESS;
+ switch (info->prim_type) {
+ case MTX_LOCK:
+ ret = lck_mtx_gate_try_close(&info->mtx_lock, &info->gate);
+ break;
+ case RW_LOCK:
+ ret = lck_rw_gate_try_close(&info->rw_lock, &info->gate);
+ break;
+ default:
+ panic("invalid type %d", info->prim_type);
+ }
+ return ret;
+}
+
+static gate_wait_result_t
+primitive_gate_wait(struct info_sleep_inheritor_test *info)
+{
+ gate_wait_result_t ret = GATE_OPENED;
+ switch (info->prim_type) {
+ case MTX_LOCK:
+ ret = lck_mtx_gate_wait(&info->mtx_lock, &info->gate, LCK_SLEEP_DEFAULT, THREAD_UNINT | THREAD_WAIT_NOREPORT_USER, TIMEOUT_WAIT_FOREVER);
+ break;
+ case RW_LOCK:
+ ret = lck_rw_gate_wait(&info->rw_lock, &info->gate, LCK_SLEEP_DEFAULT, THREAD_UNINT | THREAD_WAIT_NOREPORT_USER, TIMEOUT_WAIT_FOREVER);
+ break;
+ default:
+ panic("invalid type %d", info->prim_type);
+ }
+ return ret;
+}
+
+static void
+primitive_gate_open(struct info_sleep_inheritor_test *info)
+{
+ switch (info->prim_type) {
+ case MTX_LOCK:
+ lck_mtx_gate_open(&info->mtx_lock, &info->gate);
+ break;
+ case RW_LOCK:
+ lck_rw_gate_open(&info->rw_lock, &info->gate);
+ break;
+ default:
+ panic("invalid type %d", info->prim_type);
+ }
+}
+
+static void
+primitive_gate_close(struct info_sleep_inheritor_test *info)
+{
+ switch (info->prim_type) {
+ case MTX_LOCK:
+ lck_mtx_gate_close(&info->mtx_lock, &info->gate);
+ break;
+ case RW_LOCK:
+ lck_rw_gate_close(&info->rw_lock, &info->gate);
+ break;
+ default:
+ panic("invalid type %d", info->prim_type);
+ }
+}
+
+static void
+primitive_gate_steal(struct info_sleep_inheritor_test *info)
+{
+ switch (info->prim_type) {
+ case MTX_LOCK:
+ lck_mtx_gate_steal(&info->mtx_lock, &info->gate);
+ break;
+ case RW_LOCK:
+ lck_rw_gate_steal(&info->rw_lock, &info->gate);
+ break;
+ default:
+ panic("invalid type %d", info->prim_type);
+ }
+}
+
+static kern_return_t
+primitive_gate_handoff(struct info_sleep_inheritor_test *info, int flags)
+{
+ kern_return_t ret = KERN_SUCCESS;
+ switch (info->prim_type) {
+ case MTX_LOCK:
+ ret = lck_mtx_gate_handoff(&info->mtx_lock, &info->gate, flags);
+ break;
+ case RW_LOCK:
+ ret = lck_rw_gate_handoff(&info->rw_lock, &info->gate, flags);
+ break;
+ default:
+ panic("invalid type %d", info->prim_type);
+ }
+ return ret;
+}
+
+static void
+primitive_gate_assert(struct info_sleep_inheritor_test *info, int type)
+{
+ switch (info->prim_type) {
+ case MTX_LOCK:
+ lck_mtx_gate_assert(&info->mtx_lock, &info->gate, type);
+ break;
+ case RW_LOCK:
+ lck_rw_gate_assert(&info->rw_lock, &info->gate, type);
+ break;
+ default:
+ panic("invalid type %d", info->prim_type);
+ }
+}
+
+static void
+primitive_gate_init(struct info_sleep_inheritor_test *info)
+{
+ switch (info->prim_type) {
+ case MTX_LOCK:
+ lck_mtx_gate_init(&info->mtx_lock, &info->gate);
+ break;
+ case RW_LOCK:
+ lck_rw_gate_init(&info->rw_lock, &info->gate);
+ break;
+ default:
+ panic("invalid type %d", info->prim_type);
+ }
+}
+
+static void
+primitive_gate_destroy(struct info_sleep_inheritor_test *info)
+{
+ switch (info->prim_type) {
+ case MTX_LOCK:
+ lck_mtx_gate_destroy(&info->mtx_lock, &info->gate);
+ break;
+ case RW_LOCK:
+ lck_rw_gate_destroy(&info->rw_lock, &info->gate);
+ break;
+ default:
+ panic("invalid type %d", info->prim_type);
+ }
+}
+
+static void
+thread_inheritor_like_mutex(
+ void *args,
+ __unused wait_result_t wr)
+{
+ wait_result_t wait;
+
+ struct info_sleep_inheritor_test *info = (struct info_sleep_inheritor_test*) args;
+ uint my_pri = current_thread()->sched_pri;
+
+ T_LOG("Started thread pri %d %p", my_pri, current_thread());
+
+ /*
+ * spin here to start concurrently
+ */
+ wake_threads(&info->synch);
+ wait_threads(&info->synch, info->synch_value);
+
+ primitive_lock(info);
+
+ if (info->thread_inheritor == NULL) {
+ info->thread_inheritor = current_thread();
+ } else {
+ wait = primitive_sleep_with_inheritor(info);
+ T_ASSERT(wait == THREAD_AWAKENED || wait == THREAD_NOT_WAITING, "sleep_with_inheritor return");
+ }
+ primitive_unlock(info);
+
+ IOSleep(100);
+ info->value++;
+
+ primitive_lock(info);
+
+ T_ASSERT(info->thread_inheritor == current_thread(), "thread_inheritor is %p", info->thread_inheritor);
+ primitive_wakeup_one_with_inheritor(info);
+ T_LOG("woken up %p", info->thread_inheritor);
+
+ if (info->thread_inheritor == NULL) {
+ T_ASSERT(info->handoff_failure == 0, "handoff failures");
+ info->handoff_failure++;
+ } else {
+ T_ASSERT(info->thread_inheritor != current_thread(), "thread_inheritor is %p", info->thread_inheritor);
+ thread_deallocate(info->thread_inheritor);
+ }
+
+ primitive_unlock(info);
+
+ assert(current_thread()->kern_promotion_schedpri == 0);
+ notify_waiter((struct synch_test_common *)info);
+
+ thread_terminate_self();
+}
+
+static void
+thread_just_inheritor_do_work(
+ void *args,
+ __unused wait_result_t wr)
+{
+ struct info_sleep_inheritor_test *info = (struct info_sleep_inheritor_test*) args;
+ uint my_pri = current_thread()->sched_pri;
+ uint max_pri;
+
+ T_LOG("Started thread pri %d %p", my_pri, current_thread());
+ primitive_lock(info);
+
+ if (info->thread_inheritor == NULL) {
+ info->thread_inheritor = current_thread();
+ primitive_unlock(info);
+ T_LOG("Thread pri %d first to run %p", my_pri, current_thread());
+
+ wait_threads(&info->synch, info->synch_value - 1);
+
+ wait_for_waiters((struct synch_test_common *)info);
+
+ max_pri = get_max_pri((struct synch_test_common *) info);
+ T_ASSERT((uint) current_thread()->sched_pri == max_pri, "sleep_inheritor inheritor priority current is %d, should be %d", current_thread()->sched_pri, max_pri);
+
+ os_atomic_store(&info->synch, 0, relaxed);
+ primitive_lock(info);
+ primitive_wakeup_all_with_inheritor(info);
+ } else {
+ wake_threads(&info->synch);
+ primitive_sleep_with_inheritor(info);
+ }
+
+ primitive_unlock(info);
+
+ assert(current_thread()->kern_promotion_schedpri == 0);
+ notify_waiter((struct synch_test_common *)info);
+
+ thread_terminate_self();
+}
+
+static void
+thread_steal_work(
+ void *args,
+ __unused wait_result_t wr)
+{
+ struct info_sleep_inheritor_test *info = (struct info_sleep_inheritor_test*) args;
+ uint my_pri = current_thread()->sched_pri;
+
+ T_LOG("Started thread pri %d %p", my_pri, current_thread());
+ primitive_lock(info);
+
+ if (info->thread_inheritor == NULL) {
+ info->thread_inheritor = current_thread();
+ exclude_current_waiter((struct synch_test_common *)info);
+
+ T_LOG("Thread pri %d first to run %p", my_pri, current_thread());
+ primitive_unlock(info);
+
+ wait_threads(&info->synch, info->synch_value - 2);
+
+ wait_for_waiters((struct synch_test_common *)info);
+ T_LOG("Thread pri %d first to run %p", my_pri, current_thread());
+ primitive_lock(info);
+ if (info->thread_inheritor == current_thread()) {
+ primitive_wakeup_all_with_inheritor(info);
+ }
+ } else {
+ if (info->steal_pri == 0) {
+ info->steal_pri = my_pri;
+ info->thread_inheritor = current_thread();
+ primitive_change_sleep_inheritor(info);
+ exclude_current_waiter((struct synch_test_common *)info);
+
+ primitive_unlock(info);
+
+ wait_threads(&info->synch, info->synch_value - 2);
+
+ T_LOG("Thread pri %d stole push %p", my_pri, current_thread());
+ wait_for_waiters((struct synch_test_common *)info);
+
+ T_ASSERT((uint) current_thread()->sched_pri == info->steal_pri, "sleep_inheritor inheritor priority current is %d, should be %d", current_thread()->sched_pri, info->steal_pri);
+
+ primitive_lock(info);
+ primitive_wakeup_all_with_inheritor(info);
+ } else {
+ if (my_pri > info->steal_pri) {
+ info->steal_pri = my_pri;
+ }
+ wake_threads(&info->synch);
+ primitive_sleep_with_inheritor(info);
+ exclude_current_waiter((struct synch_test_common *)info);
+ }
+ }
+ primitive_unlock(info);
+
+ assert(current_thread()->kern_promotion_schedpri == 0);
+ notify_waiter((struct synch_test_common *)info);
+
+ thread_terminate_self();
+}
+
+static void
+thread_no_inheritor_work(
+ void *args,
+ __unused wait_result_t wr)
+{
+ struct info_sleep_inheritor_test *info = (struct info_sleep_inheritor_test*) args;
+ uint my_pri = current_thread()->sched_pri;
+
+ T_LOG("Started thread pri %d %p", my_pri, current_thread());
+ primitive_lock(info);
+
+ info->value--;
+ if (info->value == 0) {
+ primitive_wakeup_all_with_inheritor(info);
+ } else {
+ info->thread_inheritor = NULL;
+ primitive_sleep_with_inheritor(info);
+ }
+
+ primitive_unlock(info);
+
+ assert(current_thread()->kern_promotion_schedpri == 0);
+ notify_waiter((struct synch_test_common *)info);
+
+ thread_terminate_self();
+}
+
+static void
+thread_mtx_work(
+ void *args,
+ __unused wait_result_t wr)
+{
+ struct info_sleep_inheritor_test *info = (struct info_sleep_inheritor_test*) args;
+ uint my_pri = current_thread()->sched_pri;
+ int i;
+ u_int8_t rand;
+ unsigned int mod_rand;
+ uint max_pri;
+
+ T_LOG("Started thread pri %d %p", my_pri, current_thread());
+
+ for (i = 0; i < 10; i++) {
+ lck_mtx_lock(&info->mtx_lock);
+ if (info->thread_inheritor == NULL) {
+ info->thread_inheritor = current_thread();
+ lck_mtx_unlock(&info->mtx_lock);
+
+ T_LOG("Thread pri %d first to run %p", my_pri, current_thread());
+
+ wait_threads(&info->synch, info->synch_value - 1);
+ wait_for_waiters((struct synch_test_common *)info);
+ max_pri = get_max_pri((struct synch_test_common *) info);
+ T_ASSERT((uint) current_thread()->sched_pri == max_pri, "sleep_inheritor inheritor priority current is %d, should be %d", current_thread()->sched_pri, max_pri);
+
+ os_atomic_store(&info->synch, 0, relaxed);
+
+ lck_mtx_lock(&info->mtx_lock);
+ info->thread_inheritor = NULL;
+ wakeup_all_with_inheritor((event_t) &info->thread_inheritor, THREAD_AWAKENED);
+ lck_mtx_unlock(&info->mtx_lock);
+ continue;
+ }
+
+ read_random(&rand, sizeof(rand));
+ mod_rand = rand % 2;
+
+ wake_threads(&info->synch);
+ switch (mod_rand) {
+ case 0:
+ lck_mtx_sleep_with_inheritor(&info->mtx_lock, LCK_SLEEP_DEFAULT, (event_t) &info->thread_inheritor, info->thread_inheritor, THREAD_UNINT | THREAD_WAIT_NOREPORT_USER, TIMEOUT_WAIT_FOREVER);
+ lck_mtx_unlock(&info->mtx_lock);
+ break;
+ case 1:
+ lck_mtx_sleep_with_inheritor(&info->mtx_lock, LCK_SLEEP_UNLOCK, (event_t) &info->thread_inheritor, info->thread_inheritor, THREAD_UNINT | THREAD_WAIT_NOREPORT_USER, TIMEOUT_WAIT_FOREVER);
+ break;
+ default:
+ panic("rand()mod4 returned %u (random %u)", mod_rand, rand);
+ }
+ }
+
+ /*
+ * spin here to stop using the lock as mutex
+ */
+ wake_threads(&info->synch);
+ wait_threads(&info->synch, info->synch_value);
+
+ for (i = 0; i < 10; i++) {
+ /* read_random might sleep so read it before acquiring the mtx as spin */
+ read_random(&rand, sizeof(rand));
+
+ lck_mtx_lock_spin(&info->mtx_lock);
+ if (info->thread_inheritor == NULL) {
+ info->thread_inheritor = current_thread();
+ lck_mtx_unlock(&info->mtx_lock);
+
+ T_LOG("Thread pri %d first to run %p", my_pri, current_thread());
+ wait_for_waiters((struct synch_test_common *)info);
+ max_pri = get_max_pri((struct synch_test_common *) info);
+ T_ASSERT((uint) current_thread()->sched_pri == max_pri, "sleep_inheritor inheritor priority current is %d, should be %d", current_thread()->sched_pri, max_pri);
+
+ lck_mtx_lock_spin(&info->mtx_lock);
+ info->thread_inheritor = NULL;
+ wakeup_all_with_inheritor((event_t) &info->thread_inheritor, THREAD_AWAKENED);
+ lck_mtx_unlock(&info->mtx_lock);
+ continue;
+ }
+
+ mod_rand = rand % 2;
+ switch (mod_rand) {
+ case 0:
+ lck_mtx_sleep_with_inheritor(&info->mtx_lock, LCK_SLEEP_SPIN, (event_t) &info->thread_inheritor, info->thread_inheritor, THREAD_UNINT | THREAD_WAIT_NOREPORT_USER, TIMEOUT_WAIT_FOREVER);
+ lck_mtx_unlock(&info->mtx_lock);
+ break;
+ case 1:
+ lck_mtx_sleep_with_inheritor(&info->mtx_lock, LCK_SLEEP_SPIN_ALWAYS, (event_t) &info->thread_inheritor, info->thread_inheritor, THREAD_UNINT | THREAD_WAIT_NOREPORT_USER, TIMEOUT_WAIT_FOREVER);
+ lck_mtx_unlock(&info->mtx_lock);
+ break;
+ default:
+ panic("rand()mod4 returned %u (random %u)", mod_rand, rand);
+ }
+ }
+ assert(current_thread()->kern_promotion_schedpri == 0);
+ notify_waiter((struct synch_test_common *)info);
+
+ thread_terminate_self();
+}
+
+static void
+thread_rw_work(
+ void *args,
+ __unused wait_result_t wr)
+{
+ struct info_sleep_inheritor_test *info = (struct info_sleep_inheritor_test*) args;
+ uint my_pri = current_thread()->sched_pri;
+ int i;
+ lck_rw_type_t type;
+ u_int8_t rand;
+ unsigned int mod_rand;
+ uint max_pri;
+
+ T_LOG("Started thread pri %d %p", my_pri, current_thread());
+
+ for (i = 0; i < 10; i++) {
+try_again:
+ type = LCK_RW_TYPE_SHARED;
+ lck_rw_lock(&info->rw_lock, type);
+ if (info->thread_inheritor == NULL) {
+ type = LCK_RW_TYPE_EXCLUSIVE;
+
+ if (lck_rw_lock_shared_to_exclusive(&info->rw_lock)) {
+ if (info->thread_inheritor == NULL) {
+ info->thread_inheritor = current_thread();
+ lck_rw_unlock(&info->rw_lock, type);
+ wait_threads(&info->synch, info->synch_value - 1);
+
+ T_LOG("Thread pri %d first to run %p", my_pri, current_thread());
+ wait_for_waiters((struct synch_test_common *)info);
+ max_pri = get_max_pri((struct synch_test_common *) info);
+ T_ASSERT((uint) current_thread()->sched_pri == max_pri, "sleep_inheritor inheritor priority current is %d, should be %d", current_thread()->sched_pri, max_pri);
+
+ os_atomic_store(&info->synch, 0, relaxed);
+
+ lck_rw_lock(&info->rw_lock, type);
+ info->thread_inheritor = NULL;
+ wakeup_all_with_inheritor((event_t) &info->thread_inheritor, THREAD_AWAKENED);
+ lck_rw_unlock(&info->rw_lock, type);
+ continue;
+ }
+ } else {
+ goto try_again;
+ }
+ }
+
+ read_random(&rand, sizeof(rand));
+ mod_rand = rand % 4;
+
+ wake_threads(&info->synch);
+ switch (mod_rand) {
+ case 0:
+ lck_rw_sleep_with_inheritor(&info->rw_lock, LCK_SLEEP_DEFAULT, (event_t) &info->thread_inheritor, info->thread_inheritor, THREAD_UNINT | THREAD_WAIT_NOREPORT_USER, TIMEOUT_WAIT_FOREVER);
+ lck_rw_unlock(&info->rw_lock, type);
+ break;
+ case 1:
+ lck_rw_sleep_with_inheritor(&info->rw_lock, LCK_SLEEP_UNLOCK, (event_t) &info->thread_inheritor, info->thread_inheritor, THREAD_UNINT | THREAD_WAIT_NOREPORT_USER, TIMEOUT_WAIT_FOREVER);
+ break;
+ case 2:
+ lck_rw_sleep_with_inheritor(&info->rw_lock, LCK_SLEEP_SHARED, (event_t) &info->thread_inheritor, info->thread_inheritor, THREAD_UNINT | THREAD_WAIT_NOREPORT_USER, TIMEOUT_WAIT_FOREVER);
+ lck_rw_unlock(&info->rw_lock, LCK_RW_TYPE_SHARED);
+ break;
+ case 3:
+ lck_rw_sleep_with_inheritor(&info->rw_lock, LCK_SLEEP_EXCLUSIVE, (event_t) &info->thread_inheritor, info->thread_inheritor, THREAD_UNINT | THREAD_WAIT_NOREPORT_USER, TIMEOUT_WAIT_FOREVER);
+ lck_rw_unlock(&info->rw_lock, LCK_RW_TYPE_EXCLUSIVE);
+ break;
+ default:
+ panic("rand()mod4 returned %u (random %u)", mod_rand, rand);
+ }
+ }
+
+ assert(current_thread()->kern_promotion_schedpri == 0);
+ notify_waiter((struct synch_test_common *)info);
+
+ thread_terminate_self();
+}
+
+static void
+test_sleep_with_wake_all(struct info_sleep_inheritor_test *info, int prim_type)
+{
+ info->prim_type = prim_type;
+ info->synch = 0;
+ info->synch_value = info->head.nthreads;
+
+ info->thread_inheritor = NULL;
+
+ start_threads((thread_continue_t)thread_just_inheritor_do_work, (struct synch_test_common *)info, TRUE);
+ wait_all_thread((struct synch_test_common *)info);
+}
+
+static void
+test_sleep_with_wake_one(struct info_sleep_inheritor_test *info, int prim_type)
+{
+ info->prim_type = prim_type;
+
+ info->synch = 0;
+ info->synch_value = info->head.nthreads;
+ info->value = 0;
+ info->handoff_failure = 0;
+ info->thread_inheritor = NULL;
+
+ start_threads((thread_continue_t)thread_inheritor_like_mutex, (struct synch_test_common *)info, FALSE);
+ wait_all_thread((struct synch_test_common *)info);
+
+ T_ASSERT(info->value == (int)info->head.nthreads, "value protected by sleep");
+ T_ASSERT(info->handoff_failure == 1, "handoff failures");
+}
+
+static void
+test_change_sleep_inheritor(struct info_sleep_inheritor_test *info, int prim_type)
+{
+ info->prim_type = prim_type;
+
+ info->thread_inheritor = NULL;
+ info->steal_pri = 0;
+ info->synch = 0;
+ info->synch_value = info->head.nthreads;
+
+ start_threads((thread_continue_t)thread_steal_work, (struct synch_test_common *)info, FALSE);
+ wait_all_thread((struct synch_test_common *)info);
+}
+
+static void
+test_no_inheritor(struct info_sleep_inheritor_test *info, int prim_type)
+{
+ info->prim_type = prim_type;
+ info->synch = 0;
+ info->synch_value = info->head.nthreads;
+
+ info->thread_inheritor = NULL;
+ info->value = info->head.nthreads;
+
+ start_threads((thread_continue_t)thread_no_inheritor_work, (struct synch_test_common *)info, FALSE);
+ wait_all_thread((struct synch_test_common *)info);
+}
+
+static void
+test_rw_lock(struct info_sleep_inheritor_test *info)
+{
+ info->thread_inheritor = NULL;
+ info->value = info->head.nthreads;
+ info->synch = 0;
+ info->synch_value = info->head.nthreads;
+
+ start_threads((thread_continue_t)thread_rw_work, (struct synch_test_common *)info, FALSE);
+ wait_all_thread((struct synch_test_common *)info);
+}
+
+static void
+test_mtx_lock(struct info_sleep_inheritor_test *info)
+{
+ info->thread_inheritor = NULL;
+ info->value = info->head.nthreads;
+ info->synch = 0;
+ info->synch_value = info->head.nthreads;
+
+ start_threads((thread_continue_t)thread_mtx_work, (struct synch_test_common *)info, FALSE);
+ wait_all_thread((struct synch_test_common *)info);
+}
+
+kern_return_t
+ts_kernel_sleep_inheritor_test(void)
+{
+ struct info_sleep_inheritor_test info = {};
+
+ init_synch_test_common((struct synch_test_common *)&info, NUM_THREADS);
+
+ lck_attr_t* lck_attr = lck_attr_alloc_init();
+ lck_grp_attr_t* lck_grp_attr = lck_grp_attr_alloc_init();
+ lck_grp_t* lck_grp = lck_grp_alloc_init("test sleep_inheritor", lck_grp_attr);
+
+ lck_mtx_init(&info.mtx_lock, lck_grp, lck_attr);
+ lck_rw_init(&info.rw_lock, lck_grp, lck_attr);
+
+ /*
+ * Testing lck_mtx_sleep_with_inheritor and wakeup_all_with_inheritor
+ */
+ T_LOG("Testing mtx sleep with inheritor and wake_all_with_inheritor");
+ test_sleep_with_wake_all(&info, MTX_LOCK);
+
+ /*
+ * Testing rw_mtx_sleep_with_inheritor and wakeup_all_with_inheritor
+ */
+ T_LOG("Testing rw sleep with inheritor and wake_all_with_inheritor");
+ test_sleep_with_wake_all(&info, RW_LOCK);
+
+ /*
+ * Testing lck_mtx_sleep_with_inheritor and wakeup_one_with_inheritor
+ */
+ T_LOG("Testing mtx sleep with inheritor and wake_one_with_inheritor");
+ test_sleep_with_wake_one(&info, MTX_LOCK);
+
+ /*
+ * Testing lck_rw_sleep_with_inheritor and wakeup_one_with_inheritor
+ */
+ T_LOG("Testing rw sleep with inheritor and wake_one_with_inheritor");
+ test_sleep_with_wake_one(&info, RW_LOCK);
+
+ /*
+ * Testing lck_mtx_sleep_with_inheritor and wakeup_all_with_inheritor
+ * and change_sleep_inheritor
+ */
+ T_LOG("Testing change_sleep_inheritor with mxt sleep");
+ test_change_sleep_inheritor(&info, MTX_LOCK);
+
+ /*
+ * Testing lck_mtx_sleep_with_inheritor and wakeup_all_with_inheritor
+ * and change_sleep_inheritor
+ */
+ T_LOG("Testing change_sleep_inheritor with rw sleep");
+ test_change_sleep_inheritor(&info, RW_LOCK);
+
+ /*
+ * Testing lck_mtx_sleep_with_inheritor and wakeup_all_with_inheritor
+ * with inheritor NULL
+ */
+ T_LOG("Testing inheritor NULL");
+ test_no_inheritor(&info, MTX_LOCK);
+
+ /*
+ * Testing lck_mtx_sleep_with_inheritor and wakeup_all_with_inheritor
+ * with inheritor NULL
+ */
+ T_LOG("Testing inheritor NULL");
+ test_no_inheritor(&info, RW_LOCK);
+
+ /*
+ * Testing mtx locking combinations
+ */
+ T_LOG("Testing mtx locking combinations");
+ test_mtx_lock(&info);
+
+ /*
+ * Testing rw locking combinations
+ */
+ T_LOG("Testing rw locking combinations");
+ test_rw_lock(&info);
+
+ destroy_synch_test_common((struct synch_test_common *)&info);
+
+ lck_attr_free(lck_attr);
+ lck_grp_attr_free(lck_grp_attr);
+ lck_rw_destroy(&info.rw_lock, lck_grp);
+ lck_mtx_destroy(&info.mtx_lock, lck_grp);
+ lck_grp_free(lck_grp);
+
+ return KERN_SUCCESS;
+}
+
+static void
+thread_gate_aggressive(
+ void *args,
+ __unused wait_result_t wr)
+{
+ struct info_sleep_inheritor_test *info = (struct info_sleep_inheritor_test*) args;
+ uint my_pri = current_thread()->sched_pri;
+
+ T_LOG("Started thread pri %d %p", my_pri, current_thread());
+
+ primitive_lock(info);
+ if (info->thread_inheritor == NULL) {
+ info->thread_inheritor = current_thread();
+ primitive_gate_assert(info, GATE_ASSERT_OPEN);
+ primitive_gate_close(info);
+ exclude_current_waiter((struct synch_test_common *)info);
+
+ primitive_unlock(info);
+
+ wait_threads(&info->synch, info->synch_value - 2);
+ wait_for_waiters((struct synch_test_common *)info);
+ T_LOG("Thread pri %d first to run %p", my_pri, current_thread());
+
+ primitive_lock(info);
+ if (info->thread_inheritor == current_thread()) {
+ primitive_gate_open(info);
+ }
+ } else {
+ if (info->steal_pri == 0) {
+ info->steal_pri = my_pri;
+ info->thread_inheritor = current_thread();
+ primitive_gate_steal(info);
+ exclude_current_waiter((struct synch_test_common *)info);
+
+ primitive_unlock(info);
+ wait_threads(&info->synch, info->synch_value - 2);
+
+ T_LOG("Thread pri %d stole push %p", my_pri, current_thread());
+ wait_for_waiters((struct synch_test_common *)info);
+ T_ASSERT((uint) current_thread()->sched_pri == info->steal_pri, "gate keeper priority current is %d, should be %d", current_thread()->sched_pri, info->steal_pri);
+
+ primitive_lock(info);
+ primitive_gate_open(info);
+ } else {
+ if (my_pri > info->steal_pri) {
+ info->steal_pri = my_pri;
+ }
+ wake_threads(&info->synch);
+ primitive_gate_wait(info);
+ exclude_current_waiter((struct synch_test_common *)info);
+ }
+ }
+ primitive_unlock(info);
+
+ assert(current_thread()->kern_promotion_schedpri == 0);
+ notify_waiter((struct synch_test_common *)info);
+
+ thread_terminate_self();
+}
+
+static void
+thread_gate_like_mutex(
+ void *args,
+ __unused wait_result_t wr)
+{
+ gate_wait_result_t wait;
+ kern_return_t ret;
+ uint my_pri = current_thread()->sched_pri;
+
+ struct info_sleep_inheritor_test *info = (struct info_sleep_inheritor_test*) args;
+
+ T_LOG("Started thread pri %d %p", my_pri, current_thread());
+
+ /*
+ * spin here to start concurrently
+ */
+ wake_threads(&info->synch);
+ wait_threads(&info->synch, info->synch_value);
+
+ primitive_lock(info);
+
+ if (primitive_gate_try_close(info) != KERN_SUCCESS) {
+ wait = primitive_gate_wait(info);
+ T_ASSERT(wait == GATE_HANDOFF, "gate_wait return");
+ }
+
+ primitive_gate_assert(info, GATE_ASSERT_HELD);
+
+ primitive_unlock(info);
+
+ IOSleep(100);
+ info->value++;
+
+ primitive_lock(info);
+
+ ret = primitive_gate_handoff(info, GATE_HANDOFF_DEFAULT);
+ if (ret == KERN_NOT_WAITING) {
+ T_ASSERT(info->handoff_failure == 0, "handoff failures");
+ primitive_gate_handoff(info, GATE_HANDOFF_OPEN_IF_NO_WAITERS);
+ info->handoff_failure++;
+ }
+
+ primitive_unlock(info);
+ notify_waiter((struct synch_test_common *)info);
+
+ thread_terminate_self();
+}
+
+static void
+thread_just_one_do_work(
+ void *args,
+ __unused wait_result_t wr)
+{
+ struct info_sleep_inheritor_test *info = (struct info_sleep_inheritor_test*) args;
+ uint my_pri = current_thread()->sched_pri;
+ uint max_pri;
+
+ T_LOG("Started thread pri %d %p", my_pri, current_thread());
+
+ primitive_lock(info);
+check_again:
+ if (info->work_to_do) {
+ if (primitive_gate_try_close(info) == KERN_SUCCESS) {
+ primitive_gate_assert(info, GATE_ASSERT_HELD);
+ primitive_unlock(info);
+
+ T_LOG("Thread pri %d acquired the gate %p", my_pri, current_thread());
+ wait_threads(&info->synch, info->synch_value - 1);
+ wait_for_waiters((struct synch_test_common *)info);
+ max_pri = get_max_pri((struct synch_test_common *) info);
+ T_ASSERT((uint) current_thread()->sched_pri == max_pri, "gate owner priority current is %d, should be %d", current_thread()->sched_pri, max_pri);
+ os_atomic_store(&info->synch, 0, relaxed);
+
+ primitive_lock(info);
+ info->work_to_do = FALSE;
+ primitive_gate_open(info);
+ } else {
+ primitive_gate_assert(info, GATE_ASSERT_CLOSED);
+ wake_threads(&info->synch);
+ primitive_gate_wait(info);
+ goto check_again;
+ }
+ }
+ primitive_unlock(info);
+
+ assert(current_thread()->kern_promotion_schedpri == 0);
+ notify_waiter((struct synch_test_common *)info);
+ thread_terminate_self();
+}
+
+static void
+test_gate_push(struct info_sleep_inheritor_test *info, int prim_type)
+{
+ info->prim_type = prim_type;
+
+ primitive_gate_init(info);
+ info->work_to_do = TRUE;
+ info->synch = 0;
+ info->synch_value = NUM_THREADS;
+
+ start_threads((thread_continue_t)thread_just_one_do_work, (struct synch_test_common *) info, TRUE);
+ wait_all_thread((struct synch_test_common *)info);
+
+ primitive_gate_destroy(info);
+}
+
+static void
+test_gate_handoff(struct info_sleep_inheritor_test *info, int prim_type)
+{
+ info->prim_type = prim_type;
+
+ primitive_gate_init(info);
+
+ info->synch = 0;
+ info->synch_value = NUM_THREADS;
+ info->value = 0;
+ info->handoff_failure = 0;
+
+ start_threads((thread_continue_t)thread_gate_like_mutex, (struct synch_test_common *)info, false);
+ wait_all_thread((struct synch_test_common *)info);
+
+ T_ASSERT(info->value == NUM_THREADS, "value protected by gate");
+ T_ASSERT(info->handoff_failure == 1, "handoff failures");
+
+ primitive_gate_destroy(info);
+}
+
+static void
+test_gate_steal(struct info_sleep_inheritor_test *info, int prim_type)
+{
+ info->prim_type = prim_type;
+
+ primitive_gate_init(info);
+
+ info->synch = 0;
+ info->synch_value = NUM_THREADS;
+ info->thread_inheritor = NULL;
+ info->steal_pri = 0;
+
+ start_threads((thread_continue_t)thread_gate_aggressive, (struct synch_test_common *)info, FALSE);
+ wait_all_thread((struct synch_test_common *)info);
+
+ primitive_gate_destroy(info);
+}
+
+kern_return_t
+ts_kernel_gate_test(void)
+{
+ struct info_sleep_inheritor_test info = {};
+
+ T_LOG("Testing gate primitive");
+
+ init_synch_test_common((struct synch_test_common *)&info, NUM_THREADS);
+
+ lck_attr_t* lck_attr = lck_attr_alloc_init();
+ lck_grp_attr_t* lck_grp_attr = lck_grp_attr_alloc_init();
+ lck_grp_t* lck_grp = lck_grp_alloc_init("test gate", lck_grp_attr);
+
+ lck_mtx_init(&info.mtx_lock, lck_grp, lck_attr);
+ lck_rw_init(&info.rw_lock, lck_grp, lck_attr);
+
+ /*
+ * Testing the priority inherited by the keeper
+ * lck_mtx_gate_try_close, lck_mtx_gate_open, lck_mtx_gate_wait
+ */
+ T_LOG("Testing gate push, lck");
+ test_gate_push(&info, MTX_LOCK);
+
+ T_LOG("Testing gate push, rw");
+ test_gate_push(&info, RW_LOCK);
+
+ /*
+ * Testing the handoff
+ * lck_mtx_gate_wait, lck_mtx_gate_handoff
+ */
+ T_LOG("Testing gate handoff, lck");
+ test_gate_handoff(&info, MTX_LOCK);
+
+ T_LOG("Testing gate handoff, rw");
+ test_gate_handoff(&info, RW_LOCK);
+
+ /*
+ * Testing the steal
+ * lck_mtx_gate_close, lck_mtx_gate_wait, lck_mtx_gate_steal, lck_mtx_gate_handoff
+ */
+ T_LOG("Testing gate steal, lck");
+ test_gate_steal(&info, MTX_LOCK);
+
+ T_LOG("Testing gate steal, rw");
+ test_gate_steal(&info, RW_LOCK);
+
+ destroy_synch_test_common((struct synch_test_common *)&info);
+
+ lck_attr_free(lck_attr);
+ lck_grp_attr_free(lck_grp_attr);
+ lck_mtx_destroy(&info.mtx_lock, lck_grp);
+ lck_grp_free(lck_grp);
+
+ return KERN_SUCCESS;
+}
+
+#define NUM_THREAD_CHAIN 6
+
+struct turnstile_chain_test {
+ struct synch_test_common head;
+ lck_mtx_t mtx_lock;
+ int synch_value;
+ int synch;
+ int synch2;
+ gate_t gates[NUM_THREAD_CHAIN];
+};
+
+static void
+thread_sleep_gate_chain_work(
+ void *args,
+ __unused wait_result_t wr)
+{
+ struct turnstile_chain_test *info = (struct turnstile_chain_test*) args;
+ thread_t self = current_thread();
+ uint my_pri = self->sched_pri;
+ uint max_pri;
+ uint i;
+ thread_t inheritor = NULL, woken_up;
+ event_t wait_event, wake_event;
+ kern_return_t ret;
+
+ T_LOG("Started thread pri %d %p", my_pri, self);
+
+ /*
+ * Need to use the threads ids, wait for all of them to be populated
+ */
+
+ while (os_atomic_load(&info->head.threads[info->head.nthreads - 1], acquire) == NULL) {
+ IOSleep(10);
+ }
+
+ max_pri = get_max_pri((struct synch_test_common *) info);
+
+ for (i = 0; i < info->head.nthreads; i = i + 2) {
+ // even threads will close a gate
+ if (info->head.threads[i] == self) {
+ lck_mtx_lock(&info->mtx_lock);
+ lck_mtx_gate_close(&info->mtx_lock, &info->gates[i]);
+ lck_mtx_unlock(&info->mtx_lock);
+ break;
+ }
+ }
+
+ wake_threads(&info->synch2);
+ wait_threads(&info->synch2, info->synch_value);
+
+ if (self == os_atomic_load(&info->head.threads[0], acquire)) {
+ wait_threads(&info->synch, info->synch_value - 1);
+ wait_for_waiters((struct synch_test_common *)info);
+
+ T_ASSERT((uint) self->sched_pri == max_pri, "sleep_inheritor inheritor priority current is %d, should be %d", self->sched_pri, max_pri);
+
+ lck_mtx_lock(&info->mtx_lock);
+ lck_mtx_gate_open(&info->mtx_lock, &info->gates[0]);
+ lck_mtx_unlock(&info->mtx_lock);
+ } else {
+ wait_event = NULL;
+ wake_event = NULL;
+ for (i = 0; i < info->head.nthreads; i++) {
+ if (info->head.threads[i] == self) {
+ inheritor = info->head.threads[i - 1];
+ wait_event = (event_t) &info->head.threads[i - 1];
+ wake_event = (event_t) &info->head.threads[i];
+ break;
+ }
+ }
+ assert(wait_event != NULL);
+
+ lck_mtx_lock(&info->mtx_lock);
+ wake_threads(&info->synch);
+
+ if (i % 2 != 0) {
+ lck_mtx_gate_wait(&info->mtx_lock, &info->gates[i - 1], LCK_SLEEP_UNLOCK, THREAD_UNINT | THREAD_WAIT_NOREPORT_USER, TIMEOUT_WAIT_FOREVER);
+ T_ASSERT((uint) self->sched_pri == max_pri, "sleep_inheritor inheritor priority current is %d, should be %d", self->sched_pri, max_pri);
+
+ ret = wakeup_one_with_inheritor(wake_event, THREAD_AWAKENED, LCK_WAKE_DO_NOT_TRANSFER_PUSH, &woken_up);
+ if (ret == KERN_SUCCESS) {
+ T_ASSERT(i != (info->head.nthreads - 1), "thread id");
+ T_ASSERT(woken_up == info->head.threads[i + 1], "wakeup_one_with_inheritor woke next");
+ } else {
+ T_ASSERT(i == (info->head.nthreads - 1), "thread id");
+ }
+
+ // i am still the inheritor, wake all to drop inheritership
+ ret = wakeup_all_with_inheritor(wake_event, LCK_WAKE_DEFAULT);
+ T_ASSERT(ret == KERN_NOT_WAITING, "waiters on event");
+ } else {
+ // I previously closed a gate
+ lck_mtx_sleep_with_inheritor(&info->mtx_lock, LCK_SLEEP_UNLOCK, wait_event, inheritor, THREAD_UNINT | THREAD_WAIT_NOREPORT_USER, TIMEOUT_WAIT_FOREVER);
+ T_ASSERT((uint) self->sched_pri == max_pri, "sleep_inheritor inheritor priority current is %d, should be %d", self->sched_pri, max_pri);
+
+ lck_mtx_lock(&info->mtx_lock);
+ lck_mtx_gate_open(&info->mtx_lock, &info->gates[i]);
+ lck_mtx_unlock(&info->mtx_lock);
+ }
+ }
+
+ assert(current_thread()->kern_promotion_schedpri == 0);
+ notify_waiter((struct synch_test_common *)info);
+
+ thread_terminate_self();
+}
+
+static void
+thread_gate_chain_work(
+ void *args,
+ __unused wait_result_t wr)
+{
+ struct turnstile_chain_test *info = (struct turnstile_chain_test*) args;
+ thread_t self = current_thread();
+ uint my_pri = self->sched_pri;
+ uint max_pri;
+ uint i;
+ T_LOG("Started thread pri %d %p", my_pri, self);
+
+
+ /*
+ * Need to use the threads ids, wait for all of them to be populated
+ */
+ while (os_atomic_load(&info->head.threads[info->head.nthreads - 1], acquire) == NULL) {
+ IOSleep(10);
+ }
+
+ max_pri = get_max_pri((struct synch_test_common *) info);
+
+ for (i = 0; i < info->head.nthreads; i++) {
+ if (info->head.threads[i] == self) {
+ lck_mtx_lock(&info->mtx_lock);
+ lck_mtx_gate_close(&info->mtx_lock, &info->gates[i]);
+ lck_mtx_unlock(&info->mtx_lock);
+ break;
+ }
+ }
+ assert(i != info->head.nthreads);
+
+ wake_threads(&info->synch2);
+ wait_threads(&info->synch2, info->synch_value);
+
+ if (self == os_atomic_load(&info->head.threads[0], acquire)) {
+ wait_threads(&info->synch, info->synch_value - 1);
+
+ wait_for_waiters((struct synch_test_common *)info);
+
+ T_ASSERT((uint) self->sched_pri == max_pri, "sleep_inheritor inheritor priority current is %d, should be %d", self->sched_pri, max_pri);
+
+ lck_mtx_lock(&info->mtx_lock);
+ lck_mtx_gate_open(&info->mtx_lock, &info->gates[0]);
+ lck_mtx_unlock(&info->mtx_lock);
+ } else {
+ lck_mtx_lock(&info->mtx_lock);
+ wake_threads(&info->synch);
+ lck_mtx_gate_wait(&info->mtx_lock, &info->gates[i - 1], LCK_SLEEP_UNLOCK, THREAD_UNINT | THREAD_WAIT_NOREPORT_USER, TIMEOUT_WAIT_FOREVER);
+
+ T_ASSERT((uint) self->sched_pri == max_pri, "sleep_inheritor inheritor priority current is %d, should be %d", self->sched_pri, max_pri);
+
+ lck_mtx_lock(&info->mtx_lock);
+ lck_mtx_gate_open(&info->mtx_lock, &info->gates[i]);
+ lck_mtx_unlock(&info->mtx_lock);
+ }
+
+ assert(current_thread()->kern_promotion_schedpri == 0);
+ notify_waiter((struct synch_test_common *)info);
+
+ thread_terminate_self();
+}
+
+static void
+thread_sleep_chain_work(
+ void *args,
+ __unused wait_result_t wr)
+{
+ struct turnstile_chain_test *info = (struct turnstile_chain_test*) args;
+ thread_t self = current_thread();
+ uint my_pri = self->sched_pri;
+ uint max_pri;
+ event_t wait_event, wake_event;
+ uint i;
+ thread_t inheritor = NULL, woken_up = NULL;
+ kern_return_t ret;
+
+ T_LOG("Started thread pri %d %p", my_pri, self);
+
+ /*
+ * Need to use the threads ids, wait for all of them to be populated
+ */
+ while (os_atomic_load(&info->head.threads[info->head.nthreads - 1], acquire) == NULL) {
+ IOSleep(10);
+ }
+
+ max_pri = get_max_pri((struct synch_test_common *) info);
+
+ if (self == os_atomic_load(&info->head.threads[0], acquire)) {
+ wait_threads(&info->synch, info->synch_value - 1);
+
+ wait_for_waiters((struct synch_test_common *)info);
+
+ T_ASSERT((uint) self->sched_pri == max_pri, "sleep_inheritor inheritor priority current is %d, should be %d", self->sched_pri, max_pri);
+
+ ret = wakeup_one_with_inheritor((event_t) &info->head.threads[0], THREAD_AWAKENED, LCK_WAKE_DO_NOT_TRANSFER_PUSH, &woken_up);
+ T_ASSERT(ret == KERN_SUCCESS, "wakeup_one_with_inheritor woke next");
+ T_ASSERT(woken_up == info->head.threads[1], "thread woken up");
+
+ // i am still the inheritor, wake all to drop inheritership
+ ret = wakeup_all_with_inheritor((event_t) &info->head.threads[0], LCK_WAKE_DEFAULT);
+ T_ASSERT(ret == KERN_NOT_WAITING, "waiters on event");
+ } else {
+ wait_event = NULL;
+ wake_event = NULL;
+ for (i = 0; i < info->head.nthreads; i++) {
+ if (info->head.threads[i] == self) {
+ inheritor = info->head.threads[i - 1];
+ wait_event = (event_t) &info->head.threads[i - 1];
+ wake_event = (event_t) &info->head.threads[i];
+ break;
+ }
+ }
+
+ assert(wait_event != NULL);
+ lck_mtx_lock(&info->mtx_lock);
+ wake_threads(&info->synch);
+
+ lck_mtx_sleep_with_inheritor(&info->mtx_lock, LCK_SLEEP_UNLOCK, wait_event, inheritor, THREAD_UNINT | THREAD_WAIT_NOREPORT_USER, TIMEOUT_WAIT_FOREVER);
+
+ T_ASSERT((uint) self->sched_pri == max_pri, "sleep_inheritor inheritor priority current is %d, should be %d", self->sched_pri, max_pri);
+
+ ret = wakeup_one_with_inheritor(wake_event, THREAD_AWAKENED, LCK_WAKE_DO_NOT_TRANSFER_PUSH, &woken_up);
+ if (ret == KERN_SUCCESS) {
+ T_ASSERT(i != (info->head.nthreads - 1), "thread id");
+ T_ASSERT(woken_up == info->head.threads[i + 1], "wakeup_one_with_inheritor woke next");
+ } else {
+ T_ASSERT(i == (info->head.nthreads - 1), "thread id");
+ }
+
+ // i am still the inheritor, wake all to drop inheritership
+ ret = wakeup_all_with_inheritor(wake_event, LCK_WAKE_DEFAULT);
+ T_ASSERT(ret == KERN_NOT_WAITING, "waiters on event");
+ }
+
+ assert(current_thread()->kern_promotion_schedpri == 0);
+ notify_waiter((struct synch_test_common *)info);
+
+ thread_terminate_self();
+}
+
+static void
+test_sleep_chain(struct turnstile_chain_test *info)
+{
+ info->synch = 0;
+ info->synch_value = info->head.nthreads;
+
+ start_threads((thread_continue_t)thread_sleep_chain_work, (struct synch_test_common *)info, FALSE);
+ wait_all_thread((struct synch_test_common *)info);
+}
+
+static void
+test_gate_chain(struct turnstile_chain_test *info)
+{
+ info->synch = 0;
+ info->synch2 = 0;
+ info->synch_value = info->head.nthreads;
+
+ start_threads((thread_continue_t)thread_gate_chain_work, (struct synch_test_common *)info, FALSE);
+ wait_all_thread((struct synch_test_common *)info);
+}
+
+static void
+test_sleep_gate_chain(struct turnstile_chain_test *info)
+{
+ info->synch = 0;
+ info->synch2 = 0;
+ info->synch_value = info->head.nthreads;
+
+ start_threads((thread_continue_t)thread_sleep_gate_chain_work, (struct synch_test_common *)info, FALSE);
+ wait_all_thread((struct synch_test_common *)info);
+}
+
+kern_return_t
+ts_kernel_turnstile_chain_test(void)
+{
+ struct turnstile_chain_test info = {};
+ int i;
+
+ init_synch_test_common((struct synch_test_common *)&info, NUM_THREAD_CHAIN);
+ lck_attr_t* lck_attr = lck_attr_alloc_init();
+ lck_grp_attr_t* lck_grp_attr = lck_grp_attr_alloc_init();
+ lck_grp_t* lck_grp = lck_grp_alloc_init("test gate", lck_grp_attr);
+
+ lck_mtx_init(&info.mtx_lock, lck_grp, lck_attr);
+ for (i = 0; i < NUM_THREAD_CHAIN; i++) {
+ lck_mtx_gate_init(&info.mtx_lock, &info.gates[i]);
+ }
+
+ T_LOG("Testing sleep chain, lck");
+ test_sleep_chain(&info);
+
+ T_LOG("Testing gate chain, lck");
+ test_gate_chain(&info);
+
+ T_LOG("Testing sleep and gate chain, lck");
+ test_sleep_gate_chain(&info);
+
+ destroy_synch_test_common((struct synch_test_common *)&info);
+ for (i = 0; i < NUM_THREAD_CHAIN; i++) {
+ lck_mtx_gate_destroy(&info.mtx_lock, &info.gates[i]);
+ }
+ lck_attr_free(lck_attr);
+ lck_grp_attr_free(lck_grp_attr);
+ lck_mtx_destroy(&info.mtx_lock, lck_grp);
+ lck_grp_free(lck_grp);
+
+ return KERN_SUCCESS;
+}
+
+kern_return_t
+ts_kernel_timingsafe_bcmp_test(void)
+{
+ int i, buf_size;
+ char *buf = NULL;
+
+ // empty
+ T_ASSERT(timingsafe_bcmp(NULL, NULL, 0) == 0, NULL);
+ T_ASSERT(timingsafe_bcmp("foo", "foo", 0) == 0, NULL);
+ T_ASSERT(timingsafe_bcmp("foo", "bar", 0) == 0, NULL);
+
+ // equal
+ T_ASSERT(timingsafe_bcmp("foo", "foo", strlen("foo")) == 0, NULL);
+
+ // unequal
+ T_ASSERT(timingsafe_bcmp("foo", "bar", strlen("foo")) == 1, NULL);
+ T_ASSERT(timingsafe_bcmp("foo", "goo", strlen("foo")) == 1, NULL);
+ T_ASSERT(timingsafe_bcmp("foo", "fpo", strlen("foo")) == 1, NULL);
+ T_ASSERT(timingsafe_bcmp("foo", "fop", strlen("foo")) == 1, NULL);
+
+ // all possible bitwise differences
+ for (i = 1; i < 256; i += 1) {
+ unsigned char a = 0;
+ unsigned char b = (unsigned char)i;
+
+ T_ASSERT(timingsafe_bcmp(&a, &b, sizeof(a)) == 1, NULL);
+ }
+
+ // large
+ buf_size = 1024 * 16;
+ buf = kalloc(buf_size);
+ T_EXPECT_NOTNULL(buf, "kalloc of buf");
+
+ read_random(buf, buf_size);
+ T_ASSERT(timingsafe_bcmp(buf, buf, buf_size) == 0, NULL);
+ T_ASSERT(timingsafe_bcmp(buf, buf + 1, buf_size - 1) == 1, NULL);
+ T_ASSERT(timingsafe_bcmp(buf, buf + 128, 128) == 1, NULL);
+
+ memcpy(buf + 128, buf, 128);
+ T_ASSERT(timingsafe_bcmp(buf, buf + 128, 128) == 0, NULL);
+
+ kfree(buf, buf_size);
+
+ return KERN_SUCCESS;
+}
+
+kern_return_t
+kprintf_hhx_test(void)
+{
+ printf("POST hhx test %hx%hx%hx%hx %hhx%hhx%hhx%hhx - %llx",
+ (unsigned short)0xfeed, (unsigned short)0xface,
+ (unsigned short)0xabad, (unsigned short)0xcafe,
+ (unsigned char)'h', (unsigned char)'h', (unsigned char)'x',
+ (unsigned char)'!',
+ 0xfeedfaceULL);
+ return KERN_SUCCESS;
+}
projects / apple / xnu.git / blobdiff