X-Git-Url: https://git.saurik.com/apple/xnu.git/blobdiff_plain/cb3231590a3c94ab4375e2228bd5e86b0cf1ad7e..refs/heads/master:/tools/tests/zero-to-n/zero-to-n.c diff --git a/tools/tests/zero-to-n/zero-to-n.c b/tools/tests/zero-to-n/zero-to-n.c index cd1963c56..f8cccbefa 100644 --- a/tools/tests/zero-to-n/zero-to-n.c +++ b/tools/tests/zero-to-n/zero-to-n.c @@ -38,6 +38,7 @@ #include #include #include +#include #include #include @@ -56,9 +57,11 @@ #include #include +#include +#include typedef enum wake_type { WAKE_BROADCAST_ONESEM, WAKE_BROADCAST_PERTHREAD, WAKE_CHAIN, WAKE_HOP } wake_type_t; -typedef enum my_policy_type { MY_POLICY_REALTIME, MY_POLICY_TIMESHARE, MY_POLICY_FIXEDPRI } my_policy_type_t; +typedef enum my_policy_type { MY_POLICY_REALTIME, MY_POLICY_TIMESHARE, MY_POLICY_TIMESHARE_NO_SMT, MY_POLICY_FIXEDPRI } my_policy_type_t; #define mach_assert_zero(error) do { if ((error) != 0) { fprintf(stderr, "[FAIL] error %d (%s) ", (error), mach_error_string(error)); assert(error == 0); } } while (0) #define mach_assert_zero_t(tid, error) do { if ((error) != 0) { fprintf(stderr, "[FAIL] Thread %d error %d (%s) ", (tid), (error), mach_error_string(error)); assert(error == 0); } } while (0) @@ -66,6 +69,9 @@ typedef enum my_policy_type { MY_POLICY_REALTIME, MY_POLICY_TIMESHARE, MY_POLICY #define CONSTRAINT_NANOS (20000000ll) /* 20 ms */ #define COMPUTATION_NANOS (10000000ll) /* 10 ms */ +#define LL_CONSTRAINT_NANOS ( 2000000ll) /* 2 ms */ +#define LL_COMPUTATION_NANOS ( 1000000ll) /* 1 ms */ +#define RT_CHURN_COMP_NANOS ( 1000000ll) /* 1 ms */ #define TRACEWORTHY_NANOS (10000000ll) /* 10 ms */ #define TRACEWORTHY_NANOS_TEST ( 2000000ll) /* 2 ms */ @@ -105,8 +111,13 @@ static uint32_t g_iteration_sleeptime_us = 0; static uint32_t g_priority = 0; static uint32_t g_churn_pri = 0; static uint32_t g_churn_count = 0; +static uint32_t g_rt_churn_count = 0; static pthread_t* g_churn_threads = NULL; +static pthread_t* g_rt_churn_threads = NULL; + +/* should we skip test if run on non-intel */ +static boolean_t g_run_on_intel_only = FALSE; /* Threshold for dropping a 'bad run' tracepoint */ static uint64_t g_traceworthy_latency_ns = TRACEWORTHY_NANOS; @@ -123,9 +134,14 @@ static boolean_t g_do_all_spin = FALSE; /* Every thread backgrounds temporarily before parking */ static boolean_t g_drop_priority = FALSE; +/* Use low-latency (sub 4ms deadline) realtime threads */ +static boolean_t g_rt_ll = FALSE; + /* Test whether realtime threads are scheduled on the separate CPUs */ static boolean_t g_test_rt = FALSE; +static boolean_t g_rt_churn = FALSE; + /* On SMT machines, test whether realtime threads are scheduled on the correct CPUs */ static boolean_t g_test_rt_smt = FALSE; @@ -151,6 +167,8 @@ static semaphore_t g_broadcastsem; static semaphore_t g_leadersem; static semaphore_t g_readysem; static semaphore_t g_donesem; +static semaphore_t g_rt_churn_sem; +static semaphore_t g_rt_churn_start_sem; /* Global variables (chain) */ static semaphore_t *g_semarr; @@ -213,7 +231,7 @@ static void create_churn_threads() { if (g_churn_count == 0) { - g_churn_count = g_numcpus - 1; + g_churn_count = g_test_rt_smt ? g_numcpus : g_numcpus - 1; } errno_t err; @@ -270,6 +288,129 @@ join_churn_threads(void) } } +/* + * Set policy + */ +static int +rt_churn_thread_setup(void) +{ + kern_return_t kr; + thread_time_constraint_policy_data_t pol; + + /* Hard-coded realtime parameters (similar to what Digi uses) */ + pol.period = 100000; + pol.constraint = (uint32_t) nanos_to_abs(CONSTRAINT_NANOS * 2); + pol.computation = (uint32_t) nanos_to_abs(RT_CHURN_COMP_NANOS * 2); + pol.preemptible = 0; /* Ignored by OS */ + + kr = thread_policy_set(mach_thread_self(), THREAD_TIME_CONSTRAINT_POLICY, + (thread_policy_t) &pol, THREAD_TIME_CONSTRAINT_POLICY_COUNT); + mach_assert_zero_t(0, kr); + + return 0; +} + +static void * +rt_churn_thread(__unused void *arg) +{ + rt_churn_thread_setup(); + + for (uint32_t i = 0; i < g_iterations; i++) { + kern_return_t kr = semaphore_wait_signal(g_rt_churn_start_sem, g_rt_churn_sem); + mach_assert_zero_t(0, kr); + + volatile double x = 0.0; + volatile double y = 0.0; + + uint64_t endspin = mach_absolute_time() + nanos_to_abs(RT_CHURN_COMP_NANOS); + while (mach_absolute_time() < endspin) { + y = y + 1.5 + x; + x = sqrt(y); + } + } + + kern_return_t kr = semaphore_signal(g_rt_churn_sem); + mach_assert_zero_t(0, kr); + + return NULL; +} + +static void +wait_for_rt_churn_threads(void) +{ + for (uint32_t i = 0; i < g_rt_churn_count; i++) { + kern_return_t kr = semaphore_wait(g_rt_churn_sem); + mach_assert_zero_t(0, kr); + } +} + +static void +start_rt_churn_threads(void) +{ + for (uint32_t i = 0; i < g_rt_churn_count; i++) { + kern_return_t kr = semaphore_signal(g_rt_churn_start_sem); + mach_assert_zero_t(0, kr); + } +} + +static void +create_rt_churn_threads(void) +{ + if (g_rt_churn_count == 0) { + /* Leave 1 CPU to ensure that the main thread can make progress */ + g_rt_churn_count = g_numcpus - 1; + } + + errno_t err; + + struct sched_param param = { .sched_priority = (int)g_churn_pri }; + pthread_attr_t attr; + + /* Array for churn threads */ + g_rt_churn_threads = (pthread_t*) valloc(sizeof(pthread_t) * g_rt_churn_count); + assert(g_rt_churn_threads); + + if ((err = pthread_attr_init(&attr))) { + errc(EX_OSERR, err, "pthread_attr_init"); + } + + if ((err = pthread_attr_setschedparam(&attr, ¶m))) { + errc(EX_OSERR, err, "pthread_attr_setschedparam"); + } + + if ((err = pthread_attr_setschedpolicy(&attr, SCHED_RR))) { + errc(EX_OSERR, err, "pthread_attr_setschedpolicy"); + } + + for (uint32_t i = 0; i < g_rt_churn_count; i++) { + pthread_t new_thread; + + if ((err = pthread_create(&new_thread, &attr, rt_churn_thread, NULL))) { + errc(EX_OSERR, err, "pthread_create"); + } + g_rt_churn_threads[i] = new_thread; + } + + if ((err = pthread_attr_destroy(&attr))) { + errc(EX_OSERR, err, "pthread_attr_destroy"); + } + + /* Wait until all threads have checked in */ + wait_for_rt_churn_threads(); +} + +static void +join_rt_churn_threads(void) +{ + /* Rejoin rt churn threads */ + for (uint32_t i = 0; i < g_rt_churn_count; i++) { + errno_t err = pthread_join(g_rt_churn_threads[i], NULL); + if (err) { + errc(EX_OSERR, err, "pthread_join %d", i); + } + } +} + /* * Figure out what thread policy to use */ @@ -278,6 +419,8 @@ parse_thread_policy(const char *str) { if (strcmp(str, "timeshare") == 0) { return MY_POLICY_TIMESHARE; + } else if (strcmp(str, "timeshare_no_smt") == 0) { + return MY_POLICY_TIMESHARE_NO_SMT; } else if (strcmp(str, "realtime") == 0) { return MY_POLICY_REALTIME; } else if (strcmp(str, "fixed") == 0) { @@ -331,11 +474,19 @@ thread_setup(uint32_t my_id) switch (g_policy) { case MY_POLICY_TIMESHARE: break; + case MY_POLICY_TIMESHARE_NO_SMT: + proc_setthread_no_smt(); + break; case MY_POLICY_REALTIME: /* Hard-coded realtime parameters (similar to what Digi uses) */ pol.period = 100000; - pol.constraint = (uint32_t) nanos_to_abs(CONSTRAINT_NANOS); - pol.computation = (uint32_t) nanos_to_abs(COMPUTATION_NANOS); + if (g_rt_ll) { + pol.constraint = (uint32_t) nanos_to_abs(LL_CONSTRAINT_NANOS); + pol.computation = (uint32_t) nanos_to_abs(LL_COMPUTATION_NANOS); + } else { + pol.constraint = (uint32_t) nanos_to_abs(CONSTRAINT_NANOS); + pol.computation = (uint32_t) nanos_to_abs(COMPUTATION_NANOS); + } pol.preemptible = 0; /* Ignored by OS */ kr = thread_policy_set(mach_thread_self(), THREAD_TIME_CONSTRAINT_POLICY, @@ -365,6 +516,20 @@ thread_setup(uint32_t my_id) return 0; } +time_value_t +get_thread_runtime(void) +{ + thread_basic_info_data_t info; + mach_msg_type_number_t info_count = THREAD_BASIC_INFO_COUNT; + thread_info(pthread_mach_thread_np(pthread_self()), THREAD_BASIC_INFO, (thread_info_t)&info, &info_count); + + time_value_add(&info.user_time, &info.system_time); + + return info.user_time; +} + +time_value_t worker_threads_total_runtime = {}; + /* * Wait for a wakeup, potentially wake up another of the "0-N" threads, * and notify the main thread when done. @@ -372,6 +537,8 @@ thread_setup(uint32_t my_id) static void* worker_thread(void *arg) { + static os_unfair_lock runtime_lock = OS_UNFAIR_LOCK_INIT; + uint32_t my_id = (uint32_t)(uintptr_t)arg; kern_return_t kr; @@ -592,6 +759,11 @@ worker_thread(void *arg) mach_assert_zero_t(my_id, kr); } + time_value_t runtime = get_thread_runtime(); + os_unfair_lock_lock(&runtime_lock); + time_value_add(&worker_threads_total_runtime, &runtime); + os_unfair_lock_unlock(&runtime_lock); + return 0; } @@ -630,6 +802,29 @@ compute_stats(uint64_t *values, uint64_t count, float *averagep, uint64_t *maxp, *stddevp = _dev; } +typedef struct { + natural_t sys; + natural_t user; + natural_t idle; +} cpu_time_t; + +void +record_cpu_time(cpu_time_t *cpu_time) +{ + host_cpu_load_info_data_t load; + mach_msg_type_number_t count = HOST_CPU_LOAD_INFO_COUNT; + kern_return_t kr = host_statistics(mach_host_self(), HOST_CPU_LOAD_INFO, (int *)&load, &count); + mach_assert_zero_t(0, kr); + + natural_t total_system_time = load.cpu_ticks[CPU_STATE_SYSTEM]; + natural_t total_user_time = load.cpu_ticks[CPU_STATE_USER] + load.cpu_ticks[CPU_STATE_NICE]; + natural_t total_idle_time = load.cpu_ticks[CPU_STATE_IDLE]; + + cpu_time->sys = total_system_time; + cpu_time->user = total_user_time; + cpu_time->idle = total_idle_time; +} + int main(int argc, char **argv) { @@ -643,6 +838,7 @@ main(int argc, char **argv) float avg, stddev; bool test_fail = false; + bool test_warn = false; for (int i = 0; i < argc; i++) { if (strcmp(argv[i], "--switched_apptype") == 0) { @@ -660,6 +856,19 @@ main(int argc, char **argv) mach_timebase_info(&g_mti); +#if TARGET_OS_OSX + /* SKIP test if running on arm platform */ + if (g_run_on_intel_only) { + int is_arm = 0; + size_t is_arm_size = sizeof(is_arm); + ret = sysctlbyname("hw.optional.arm64", &is_arm, &is_arm_size, NULL, 0); + if (ret == 0 && is_arm) { + printf("Unsupported platform. Skipping test.\n"); + exit(0); + } + } +#endif /* TARGET_OS_OSX */ + size_t ncpu_size = sizeof(g_numcpus); ret = sysctlbyname("hw.ncpu", &g_numcpus, &ncpu_size, NULL, 0); if (ret) { @@ -828,6 +1037,12 @@ main(int argc, char **argv) kr = semaphore_create(mach_task_self(), &g_readysem, SYNC_POLICY_FIFO, 0); mach_assert_zero(kr); + kr = semaphore_create(mach_task_self(), &g_rt_churn_sem, SYNC_POLICY_FIFO, 0); + mach_assert_zero(kr); + + kr = semaphore_create(mach_task_self(), &g_rt_churn_start_sem, SYNC_POLICY_FIFO, 0); + mach_assert_zero(kr); + atomic_store_explicit(&g_done_threads, 0, memory_order_relaxed); /* Create the threads */ @@ -850,6 +1065,9 @@ main(int argc, char **argv) if (g_churn_pri) { create_churn_threads(); } + if (g_rt_churn) { + create_rt_churn_threads(); + } /* Let everyone get settled */ kr = semaphore_wait(g_main_sem); @@ -860,6 +1078,11 @@ main(int argc, char **argv) usleep(g_iteration_sleeptime_us); } + cpu_time_t start_time; + cpu_time_t finish_time; + + record_cpu_time(&start_time); + /* Go! */ for (uint32_t i = 0; i < g_iterations; i++) { uint32_t j; @@ -869,6 +1092,11 @@ main(int argc, char **argv) g_one_long_spin_id = (uint32_t)rand() % g_numthreads; } + if (g_rt_churn) { + start_rt_churn_threads(); + usleep(100); + } + debug_log("%d Main thread reset\n", i); atomic_store_explicit(&g_done_threads, 0, memory_order_seq_cst); @@ -883,6 +1111,10 @@ main(int argc, char **argv) assert(atomic_load_explicit(&g_done_threads, memory_order_relaxed) == g_numthreads); + if (g_rt_churn) { + wait_for_rt_churn_threads(); + } + /* * We report the worst latencies relative to start time * and relative to the lead worker thread. @@ -925,6 +1157,8 @@ main(int argc, char **argv) } } + record_cpu_time(&finish_time); + /* Rejoin threads */ for (uint32_t i = 0; i < g_numthreads; i++) { ret = pthread_join(threads[i], NULL); @@ -933,10 +1167,17 @@ main(int argc, char **argv) } } + if (g_rt_churn) { + join_rt_churn_threads(); + } + if (g_churn_pri) { join_churn_threads(); } + uint32_t cpu_idle_time = (finish_time.idle - start_time.idle) * 10; + uint32_t worker_threads_runtime = worker_threads_total_runtime.seconds * 1000 + worker_threads_total_runtime.microseconds / 1000; + compute_stats(worst_latencies_ns, g_iterations, &avg, &max, &min, &stddev); printf("Results (from a stop):\n"); printf("Max:\t\t%.2f us\n", ((float)max) / 1000.0); @@ -992,6 +1233,7 @@ main(int argc, char **argv) secondary ? " SECONDARY" : "", fail ? " FAIL" : ""); } + test_warn |= (secondary || fail); test_fail |= fail; fail_count += fail; } @@ -1002,6 +1244,17 @@ main(int argc, char **argv) } } + if (g_test_rt_smt && (g_each_spin_duration_ns >= 200000) && !test_warn) { + printf("cpu_idle_time=%dms worker_threads_runtime=%dms\n", cpu_idle_time, worker_threads_runtime); + if (cpu_idle_time < worker_threads_runtime / 4) { + printf("FAIL cpu_idle_time unexpectedly small\n"); + test_fail = 1; + } else if (cpu_idle_time > worker_threads_runtime * 2) { + printf("FAIL cpu_idle_time unexpectedly large\n"); + test_fail = 1; + } + } + free(threads); free(g_thread_endtimes_abs); free(worst_latencies_ns); @@ -1068,10 +1321,11 @@ static void __attribute__((noreturn)) usage() { errx(EX_USAGE, "Usage: %s " - " \n\t\t" + " \n\t\t" "[--trace ] " "[--verbose] [--spin-one] [--spin-all] [--spin-time ] [--affinity]\n\t\t" - "[--no-sleep] [--drop-priority] [--churn-pri ] [--churn-count ]", + "[--no-sleep] [--drop-priority] [--churn-pri ] [--churn-count ]\n\t\t" + "[--rt-churn] [--rt-churn-count ] [--rt-ll] [--test-rt] [--test-rt-smt] [--test-rt-avoid0]", getprogname()); } @@ -1104,6 +1358,7 @@ parse_args(int argc, char *argv[]) OPT_PRIORITY, OPT_CHURN_PRI, OPT_CHURN_COUNT, + OPT_RT_CHURN_COUNT, }; static struct option longopts[] = { @@ -1113,8 +1368,10 @@ parse_args(int argc, char *argv[]) { "priority", required_argument, NULL, OPT_PRIORITY }, { "churn-pri", required_argument, NULL, OPT_CHURN_PRI }, { "churn-count", required_argument, NULL, OPT_CHURN_COUNT }, + { "rt-churn-count", required_argument, NULL, OPT_RT_CHURN_COUNT }, { "switched_apptype", no_argument, (int*)&g_seen_apptype, TRUE }, { "spin-one", no_argument, (int*)&g_do_one_long_spin, TRUE }, + { "intel-only", no_argument, (int*)&g_run_on_intel_only, TRUE }, { "spin-all", no_argument, (int*)&g_do_all_spin, TRUE }, { "affinity", no_argument, (int*)&g_do_affinity, TRUE }, { "no-sleep", no_argument, (int*)&g_do_sleep, FALSE }, @@ -1122,6 +1379,8 @@ parse_args(int argc, char *argv[]) { "test-rt", no_argument, (int*)&g_test_rt, TRUE }, { "test-rt-smt", no_argument, (int*)&g_test_rt_smt, TRUE }, { "test-rt-avoid0", no_argument, (int*)&g_test_rt_avoid0, TRUE }, + { "rt-churn", no_argument, (int*)&g_rt_churn, TRUE }, + { "rt-ll", no_argument, (int*)&g_rt_ll, TRUE }, { "histogram", no_argument, (int*)&g_histogram, TRUE }, { "verbose", no_argument, (int*)&g_verbose, TRUE }, { "help", no_argument, NULL, 'h' }, @@ -1153,6 +1412,9 @@ parse_args(int argc, char *argv[]) case OPT_CHURN_COUNT: g_churn_count = read_dec_arg(); break; + case OPT_RT_CHURN_COUNT: + g_rt_churn_count = read_dec_arg(); + break; case '?': case 'h': default: