X-Git-Url: https://git.saurik.com/apple/xnu.git/blobdiff_plain/3e170ce000f1506b7b5d2c5c7faec85ceabb573d..b226f5e54a60dc81db17b1260381d7dbfea3cdf1:/tools/tests/zero-to-n/zero-to-n.c?ds=sidebyside diff --git a/tools/tests/zero-to-n/zero-to-n.c b/tools/tests/zero-to-n/zero-to-n.c index 2f36c2635..f31b58df5 100644 --- a/tools/tests/zero-to-n/zero-to-n.c +++ b/tools/tests/zero-to-n/zero-to-n.c @@ -44,8 +44,6 @@ #include #include -#include - #include #include #include @@ -53,6 +51,10 @@ #include +#include + +#include + typedef enum wake_type { WAKE_BROADCAST_ONESEM, WAKE_BROADCAST_PERTHREAD, WAKE_CHAIN, WAKE_HOP } wake_type_t; typedef enum my_policy_type { MY_POLICY_REALTIME, MY_POLICY_TIMESHARE, MY_POLICY_FIXEDPRI } my_policy_type_t; @@ -64,6 +66,8 @@ typedef enum my_policy_type { MY_POLICY_REALTIME, MY_POLICY_TIMESHARE, MY_POLICY #define COMPUTATION_NANOS (10000000ll) /* 10 ms */ #define TRACEWORTHY_NANOS (10000000ll) /* 10 ms */ +#define DEBUG 0 + #if DEBUG #define debug_log(args...) printf(args) #else @@ -78,6 +82,10 @@ static my_policy_type_t parse_thread_policy(const char *str); static void selfexec_with_apptype(int argc, char *argv[]); static void parse_args(int argc, char *argv[]); +static __attribute__((aligned(128))) _Atomic uint32_t g_done_threads; +static __attribute__((aligned(128))) _Atomic boolean_t g_churn_stop = FALSE; +static __attribute__((aligned(128))) _Atomic uint64_t g_churn_stopped_at = 0; + /* Global variables (general) */ static uint32_t g_numcpus; static uint32_t g_numthreads; @@ -87,11 +95,15 @@ static uint32_t g_iterations; static struct mach_timebase_info g_mti; static semaphore_t g_main_sem; static uint64_t *g_thread_endtimes_abs; -static volatile uint32_t g_done_threads; static boolean_t g_verbose = FALSE; static boolean_t g_do_affinity = FALSE; static uint64_t g_starttime_abs; static uint32_t g_iteration_sleeptime_us = 0; +static uint32_t g_priority = 0; +static uint32_t g_churn_pri = 0; +static uint32_t g_churn_count = 0; + +static pthread_t* g_churn_threads = NULL; /* Threshold for dropping a 'bad run' tracepoint */ static uint64_t g_traceworthy_latency_ns = TRACEWORTHY_NANOS; @@ -105,6 +117,9 @@ static boolean_t g_do_sleep = TRUE; /* Every thread spins until all threads have checked in */ static boolean_t g_do_all_spin = FALSE; +/* Every thread backgrounds temporarily before parking */ +static boolean_t g_drop_priority = FALSE; + /* One randomly chosen thread holds up the train for a certain duration. */ static boolean_t g_do_one_long_spin = FALSE; static uint32_t g_one_long_spin_id = 0; @@ -137,6 +152,92 @@ nanos_to_abs(uint64_t ns) return (uint64_t)(ns * (((double)g_mti.denom) / ((double)g_mti.numer))); } +inline static void +yield(void) +{ +#if defined(__arm__) || defined(__arm64__) + asm volatile("yield"); +#elif defined(__x86_64__) || defined(__i386__) + asm volatile("pause"); +#else +#error Unrecognized architecture +#endif +} + +static void * +churn_thread(__unused void *arg) +{ + uint64_t spin_count = 0; + + /* + * As a safety measure to avoid wedging, we will bail on the spin if + * it's been more than 1s after the most recent run start + */ + + while (g_churn_stop == FALSE && + mach_absolute_time() < (g_starttime_abs + NSEC_PER_SEC)) { + spin_count++; + yield(); + } + + /* This is totally racy, but only here to detect if anyone stops early */ + atomic_fetch_add_explicit(&g_churn_stopped_at, spin_count, memory_order_relaxed); + + return NULL; +} + +static void +create_churn_threads() +{ + if (g_churn_count == 0) + g_churn_count = g_numcpus - 1; + + errno_t err; + + struct sched_param param = { .sched_priority = (int)g_churn_pri }; + pthread_attr_t attr; + + /* Array for churn threads */ + g_churn_threads = (pthread_t*) valloc(sizeof(pthread_t) * g_churn_count); + assert(g_churn_threads); + + if ((err = pthread_attr_init(&attr))) + errc(EX_OSERR, err, "pthread_attr_init"); + + if ((err = pthread_attr_setschedparam(&attr, ¶m))) + errc(EX_OSERR, err, "pthread_attr_setschedparam"); + + if ((err = pthread_attr_setschedpolicy(&attr, SCHED_RR))) + errc(EX_OSERR, err, "pthread_attr_setschedpolicy"); + + for (uint32_t i = 0 ; i < g_churn_count ; i++) { + pthread_t new_thread; + + if ((err = pthread_create(&new_thread, &attr, churn_thread, NULL))) + errc(EX_OSERR, err, "pthread_create"); + g_churn_threads[i] = new_thread; + } + + if ((err = pthread_attr_destroy(&attr))) + errc(EX_OSERR, err, "pthread_attr_destroy"); +} + +static void +join_churn_threads(void) +{ + if (atomic_load_explicit(&g_churn_stopped_at, memory_order_seq_cst) != 0) + printf("Warning: Some of the churn threads may have stopped early: %lld\n", + g_churn_stopped_at); + + atomic_store_explicit(&g_churn_stop, TRUE, memory_order_seq_cst); + + /* Rejoin churn threads */ + for (uint32_t i = 0; i < g_churn_count; i++) { + errno_t err = pthread_join(g_churn_threads[i], NULL); + if (err) errc(EX_OSERR, err, "pthread_join %d", i); + } +} + /* * Figure out what thread policy to use */ @@ -183,6 +284,16 @@ thread_setup(uint32_t my_id) errno_t ret; thread_time_constraint_policy_data_t pol; + if (g_priority) { + int policy = SCHED_OTHER; + if (g_policy == MY_POLICY_FIXEDPRI) + policy = SCHED_RR; + + struct sched_param param = {.sched_priority = (int)g_priority}; + if ((ret = pthread_setschedparam(pthread_self(), policy, ¶m))) + errc(EX_OSERR, ret, "pthread_setschedparam: %d", my_id); + } + switch (g_policy) { case MY_POLICY_TIMESHARE: break; @@ -268,7 +379,7 @@ worker_thread(void *arg) debug_log("%d Leader thread go\n", i); - assert_zero_t(my_id, g_done_threads); + assert_zero_t(my_id, atomic_load_explicit(&g_done_threads, memory_order_relaxed)); switch (g_waketype) { case WAKE_BROADCAST_ONESEM: @@ -368,20 +479,32 @@ worker_thread(void *arg) } } - int32_t new = OSAtomicIncrement32((volatile int32_t *)&g_done_threads); - (void)new; + uint32_t done_threads; + done_threads = atomic_fetch_add_explicit(&g_done_threads, 1, memory_order_relaxed) + 1; - debug_log("Thread %p new value is %d, iteration %d\n", pthread_self(), new, i); + debug_log("Thread %p new value is %d, iteration %d\n", pthread_self(), done_threads, i); + + if (g_drop_priority) { + /* Drop priority to BG momentarily */ + errno_t ret = setpriority(PRIO_DARWIN_THREAD, 0, PRIO_DARWIN_BG); + if (ret) errc(EX_OSERR, ret, "setpriority PRIO_DARWIN_BG"); + } if (g_do_all_spin) { /* Everyone spins until the last thread checks in. */ - while (g_done_threads < g_numthreads) { + while (atomic_load_explicit(&g_done_threads, memory_order_relaxed) < g_numthreads) { y = y + 1.5 + x; x = sqrt(y); } } + if (g_drop_priority) { + /* Restore normal priority */ + errno_t ret = setpriority(PRIO_DARWIN_THREAD, 0, 0); + if (ret) errc(EX_OSERR, ret, "setpriority 0"); + } + debug_log("Thread %p done spinning, iteration %d\n", pthread_self(), i); } @@ -553,8 +676,9 @@ main(int argc, char **argv) kr = semaphore_create(mach_task_self(), &g_readysem, SYNC_POLICY_FIFO, 0); mach_assert_zero(kr); + atomic_store_explicit(&g_done_threads, 0, memory_order_relaxed); + /* Create the threads */ - g_done_threads = 0; for (uint32_t i = 0; i < g_numthreads; i++) { ret = pthread_create(&threads[i], NULL, worker_thread, (void*)(uintptr_t)i); if (ret) errc(EX_OSERR, ret, "pthread_create %d", i); @@ -565,6 +689,11 @@ main(int argc, char **argv) thread_setup(0); + g_starttime_abs = mach_absolute_time(); + + if (g_churn_pri) + create_churn_threads(); + /* Let everyone get settled */ kr = semaphore_wait(g_main_sem); mach_assert_zero(kr); @@ -583,8 +712,7 @@ main(int argc, char **argv) debug_log("%d Main thread reset\n", i); - g_done_threads = 0; - OSMemoryBarrier(); + atomic_store_explicit(&g_done_threads, 0, memory_order_seq_cst); g_starttime_abs = mach_absolute_time(); @@ -594,6 +722,8 @@ main(int argc, char **argv) debug_log("%d Main thread return\n", i); + assert(atomic_load_explicit(&g_done_threads, memory_order_relaxed) == g_numthreads); + /* * We report the worst latencies relative to start time * and relative to the lead worker thread. @@ -640,6 +770,9 @@ main(int argc, char **argv) if (ret) errc(EX_OSERR, ret, "pthread_join %d", i); } + if (g_churn_pri) + join_churn_threads(); + compute_stats(worst_latencies_ns, g_iterations, &avg, &max, &min, &stddev); printf("Results (from a stop):\n"); printf("Max:\t\t%.2f us\n", ((float)max) / 1000.0); @@ -715,51 +848,84 @@ selfexec_with_apptype(int argc, char *argv[]) static void __attribute__((noreturn)) usage() { - errx(EX_USAGE, "Usage: zn " - " [--trace ] " - "[--spin-one] [--spin-all] [--spin-time ] [--affinity] [--no-sleep] [--verbose]"); + errx(EX_USAGE, "Usage: %s " + " \n\t\t" + "[--trace ] " + "[--verbose] [--spin-one] [--spin-all] [--spin-time ] [--affinity]\n\t\t" + "[--no-sleep] [--drop-priority] [--churn-pri ] [--churn-count ]", + getprogname()); +} + +static struct option* g_longopts; +static int option_index; + +static uint32_t +read_dec_arg() +{ + char *cp; + /* char* optarg is a magic global */ + + uint32_t arg_val = (uint32_t)strtoull(optarg, &cp, 10); + + if (cp == optarg || *cp) + errx(EX_USAGE, "arg --%s requires a decimal number, found \"%s\"", + g_longopts[option_index].name, optarg); + + return arg_val; } static void parse_args(int argc, char *argv[]) { - int ch, option_index = 0; - char *cp; + enum { + OPT_GETOPT = 0, + OPT_SPIN_TIME, + OPT_TRACE, + OPT_PRIORITY, + OPT_CHURN_PRI, + OPT_CHURN_COUNT, + }; static struct option longopts[] = { - { "spin-time", required_argument, NULL, 2 }, - { "trace", required_argument, NULL, 3 }, + { "spin-time", required_argument, NULL, OPT_SPIN_TIME }, + { "trace", required_argument, NULL, OPT_TRACE }, + { "priority", required_argument, NULL, OPT_PRIORITY }, + { "churn-pri", required_argument, NULL, OPT_CHURN_PRI }, + { "churn-count", required_argument, NULL, OPT_CHURN_COUNT }, { "switched_apptype", no_argument, (int*)&g_seen_apptype, TRUE }, { "spin-one", no_argument, (int*)&g_do_one_long_spin, TRUE }, { "spin-all", no_argument, (int*)&g_do_all_spin, TRUE }, { "affinity", no_argument, (int*)&g_do_affinity, TRUE }, { "no-sleep", no_argument, (int*)&g_do_sleep, FALSE }, + { "drop-priority", no_argument, (int*)&g_drop_priority, TRUE }, { "verbose", no_argument, (int*)&g_verbose, TRUE }, { "help", no_argument, NULL, 'h' }, { NULL, 0, NULL, 0 } }; + g_longopts = longopts; + int ch = 0; + while ((ch = getopt_long(argc, argv, "h", longopts, &option_index)) != -1) { switch (ch) { - case 0: + case OPT_GETOPT: /* getopt_long set a variable */ break; - case 2: - /* spin-time */ + case OPT_SPIN_TIME: g_do_each_spin = TRUE; - g_each_spin_duration_ns = strtoull(optarg, &cp, 10); - - if (cp == optarg || *cp) - errx(EX_USAGE, "arg --%s requires a decimal number, found \"%s\"", - longopts[option_index].name, optarg); + g_each_spin_duration_ns = read_dec_arg(); break; - case 3: - /* trace */ - g_traceworthy_latency_ns = strtoull(optarg, &cp, 10); - - if (cp == optarg || *cp) - errx(EX_USAGE, "arg --%s requires a decimal number, found \"%s\"", - longopts[option_index].name, optarg); + case OPT_TRACE: + g_traceworthy_latency_ns = read_dec_arg(); + break; + case OPT_PRIORITY: + g_priority = read_dec_arg(); + break; + case OPT_CHURN_PRI: + g_churn_pri = read_dec_arg(); + break; + case OPT_CHURN_COUNT: + g_churn_count = read_dec_arg(); break; case '?': case 'h': @@ -787,6 +953,8 @@ parse_args(int argc, char *argv[]) usage(); } + char *cp; + /* How many threads? */ g_numthreads = (uint32_t)strtoull(argv[0], &cp, 10);