#include <sysexits.h>
#include <sys/sysctl.h>
#include <getopt.h>
+#include <libproc.h>
#include <spawn.h>
#include <spawn_private.h>
#include <stdatomic.h>
#include <os/tsd.h>
+#include <os/lock.h>
+#include <TargetConditionals.h>
typedef enum wake_type { WAKE_BROADCAST_ONESEM, WAKE_BROADCAST_PERTHREAD, WAKE_CHAIN, WAKE_HOP } wake_type_t;
-typedef enum my_policy_type { MY_POLICY_REALTIME, MY_POLICY_TIMESHARE, MY_POLICY_FIXEDPRI } my_policy_type_t;
+typedef enum my_policy_type { MY_POLICY_REALTIME, MY_POLICY_TIMESHARE, MY_POLICY_TIMESHARE_NO_SMT, MY_POLICY_FIXEDPRI } my_policy_type_t;
#define mach_assert_zero(error) do { if ((error) != 0) { fprintf(stderr, "[FAIL] error %d (%s) ", (error), mach_error_string(error)); assert(error == 0); } } while (0)
#define mach_assert_zero_t(tid, error) do { if ((error) != 0) { fprintf(stderr, "[FAIL] Thread %d error %d (%s) ", (tid), (error), mach_error_string(error)); assert(error == 0); } } while (0)
#define CONSTRAINT_NANOS (20000000ll) /* 20 ms */
#define COMPUTATION_NANOS (10000000ll) /* 10 ms */
+#define LL_CONSTRAINT_NANOS ( 2000000ll) /* 2 ms */
+#define LL_COMPUTATION_NANOS ( 1000000ll) /* 1 ms */
+#define RT_CHURN_COMP_NANOS ( 1000000ll) /* 1 ms */
#define TRACEWORTHY_NANOS (10000000ll) /* 10 ms */
#define TRACEWORTHY_NANOS_TEST ( 2000000ll) /* 2 ms */
static uint32_t g_priority = 0;
static uint32_t g_churn_pri = 0;
static uint32_t g_churn_count = 0;
+static uint32_t g_rt_churn_count = 0;
static pthread_t* g_churn_threads = NULL;
+static pthread_t* g_rt_churn_threads = NULL;
+
+/* should we skip test if run on non-intel */
+static boolean_t g_run_on_intel_only = FALSE;
/* Threshold for dropping a 'bad run' tracepoint */
static uint64_t g_traceworthy_latency_ns = TRACEWORTHY_NANOS;
/* Every thread backgrounds temporarily before parking */
static boolean_t g_drop_priority = FALSE;
+/* Use low-latency (sub 4ms deadline) realtime threads */
+static boolean_t g_rt_ll = FALSE;
+
/* Test whether realtime threads are scheduled on the separate CPUs */
static boolean_t g_test_rt = FALSE;
+static boolean_t g_rt_churn = FALSE;
+
/* On SMT machines, test whether realtime threads are scheduled on the correct CPUs */
static boolean_t g_test_rt_smt = FALSE;
static semaphore_t g_leadersem;
static semaphore_t g_readysem;
static semaphore_t g_donesem;
+static semaphore_t g_rt_churn_sem;
+static semaphore_t g_rt_churn_start_sem;
/* Global variables (chain) */
static semaphore_t *g_semarr;
create_churn_threads()
{
if (g_churn_count == 0) {
- g_churn_count = g_numcpus - 1;
+ g_churn_count = g_test_rt_smt ? g_numcpus : g_numcpus - 1;
}
errno_t err;
}
}
+/*
+ * Set policy
+ */
+static int
+rt_churn_thread_setup(void)
+{
+ kern_return_t kr;
+ thread_time_constraint_policy_data_t pol;
+
+ /* Hard-coded realtime parameters (similar to what Digi uses) */
+ pol.period = 100000;
+ pol.constraint = (uint32_t) nanos_to_abs(CONSTRAINT_NANOS * 2);
+ pol.computation = (uint32_t) nanos_to_abs(RT_CHURN_COMP_NANOS * 2);
+ pol.preemptible = 0; /* Ignored by OS */
+
+ kr = thread_policy_set(mach_thread_self(), THREAD_TIME_CONSTRAINT_POLICY,
+ (thread_policy_t) &pol, THREAD_TIME_CONSTRAINT_POLICY_COUNT);
+ mach_assert_zero_t(0, kr);
+
+ return 0;
+}
+
+static void *
+rt_churn_thread(__unused void *arg)
+{
+ rt_churn_thread_setup();
+
+ for (uint32_t i = 0; i < g_iterations; i++) {
+ kern_return_t kr = semaphore_wait_signal(g_rt_churn_start_sem, g_rt_churn_sem);
+ mach_assert_zero_t(0, kr);
+
+ volatile double x = 0.0;
+ volatile double y = 0.0;
+
+ uint64_t endspin = mach_absolute_time() + nanos_to_abs(RT_CHURN_COMP_NANOS);
+ while (mach_absolute_time() < endspin) {
+ y = y + 1.5 + x;
+ x = sqrt(y);
+ }
+ }
+
+ kern_return_t kr = semaphore_signal(g_rt_churn_sem);
+ mach_assert_zero_t(0, kr);
+
+ return NULL;
+}
+
+static void
+wait_for_rt_churn_threads(void)
+{
+ for (uint32_t i = 0; i < g_rt_churn_count; i++) {
+ kern_return_t kr = semaphore_wait(g_rt_churn_sem);
+ mach_assert_zero_t(0, kr);
+ }
+}
+
+static void
+start_rt_churn_threads(void)
+{
+ for (uint32_t i = 0; i < g_rt_churn_count; i++) {
+ kern_return_t kr = semaphore_signal(g_rt_churn_start_sem);
+ mach_assert_zero_t(0, kr);
+ }
+}
+
+static void
+create_rt_churn_threads(void)
+{
+ if (g_rt_churn_count == 0) {
+ /* Leave 1 CPU to ensure that the main thread can make progress */
+ g_rt_churn_count = g_numcpus - 1;
+ }
+
+ errno_t err;
+
+ struct sched_param param = { .sched_priority = (int)g_churn_pri };
+ pthread_attr_t attr;
+
+ /* Array for churn threads */
+ g_rt_churn_threads = (pthread_t*) valloc(sizeof(pthread_t) * g_rt_churn_count);
+ assert(g_rt_churn_threads);
+
+ if ((err = pthread_attr_init(&attr))) {
+ errc(EX_OSERR, err, "pthread_attr_init");
+ }
+
+ if ((err = pthread_attr_setschedparam(&attr, ¶m))) {
+ errc(EX_OSERR, err, "pthread_attr_setschedparam");
+ }
+
+ if ((err = pthread_attr_setschedpolicy(&attr, SCHED_RR))) {
+ errc(EX_OSERR, err, "pthread_attr_setschedpolicy");
+ }
+
+ for (uint32_t i = 0; i < g_rt_churn_count; i++) {
+ pthread_t new_thread;
+
+ if ((err = pthread_create(&new_thread, &attr, rt_churn_thread, NULL))) {
+ errc(EX_OSERR, err, "pthread_create");
+ }
+ g_rt_churn_threads[i] = new_thread;
+ }
+
+ if ((err = pthread_attr_destroy(&attr))) {
+ errc(EX_OSERR, err, "pthread_attr_destroy");
+ }
+
+ /* Wait until all threads have checked in */
+ wait_for_rt_churn_threads();
+}
+
+static void
+join_rt_churn_threads(void)
+{
+ /* Rejoin rt churn threads */
+ for (uint32_t i = 0; i < g_rt_churn_count; i++) {
+ errno_t err = pthread_join(g_rt_churn_threads[i], NULL);
+ if (err) {
+ errc(EX_OSERR, err, "pthread_join %d", i);
+ }
+ }
+}
+
/*
* Figure out what thread policy to use
*/
{
if (strcmp(str, "timeshare") == 0) {
return MY_POLICY_TIMESHARE;
+ } else if (strcmp(str, "timeshare_no_smt") == 0) {
+ return MY_POLICY_TIMESHARE_NO_SMT;
} else if (strcmp(str, "realtime") == 0) {
return MY_POLICY_REALTIME;
} else if (strcmp(str, "fixed") == 0) {
switch (g_policy) {
case MY_POLICY_TIMESHARE:
break;
+ case MY_POLICY_TIMESHARE_NO_SMT:
+ proc_setthread_no_smt();
+ break;
case MY_POLICY_REALTIME:
/* Hard-coded realtime parameters (similar to what Digi uses) */
pol.period = 100000;
- pol.constraint = (uint32_t) nanos_to_abs(CONSTRAINT_NANOS);
- pol.computation = (uint32_t) nanos_to_abs(COMPUTATION_NANOS);
+ if (g_rt_ll) {
+ pol.constraint = (uint32_t) nanos_to_abs(LL_CONSTRAINT_NANOS);
+ pol.computation = (uint32_t) nanos_to_abs(LL_COMPUTATION_NANOS);
+ } else {
+ pol.constraint = (uint32_t) nanos_to_abs(CONSTRAINT_NANOS);
+ pol.computation = (uint32_t) nanos_to_abs(COMPUTATION_NANOS);
+ }
pol.preemptible = 0; /* Ignored by OS */
kr = thread_policy_set(mach_thread_self(), THREAD_TIME_CONSTRAINT_POLICY,
return 0;
}
+time_value_t
+get_thread_runtime(void)
+{
+ thread_basic_info_data_t info;
+ mach_msg_type_number_t info_count = THREAD_BASIC_INFO_COUNT;
+ thread_info(pthread_mach_thread_np(pthread_self()), THREAD_BASIC_INFO, (thread_info_t)&info, &info_count);
+
+ time_value_add(&info.user_time, &info.system_time);
+
+ return info.user_time;
+}
+
+time_value_t worker_threads_total_runtime = {};
+
/*
* Wait for a wakeup, potentially wake up another of the "0-N" threads,
* and notify the main thread when done.
static void*
worker_thread(void *arg)
{
+ static os_unfair_lock runtime_lock = OS_UNFAIR_LOCK_INIT;
+
uint32_t my_id = (uint32_t)(uintptr_t)arg;
kern_return_t kr;
mach_assert_zero_t(my_id, kr);
}
+ time_value_t runtime = get_thread_runtime();
+ os_unfair_lock_lock(&runtime_lock);
+ time_value_add(&worker_threads_total_runtime, &runtime);
+ os_unfair_lock_unlock(&runtime_lock);
+
return 0;
}
*stddevp = _dev;
}
+typedef struct {
+ natural_t sys;
+ natural_t user;
+ natural_t idle;
+} cpu_time_t;
+
+void
+record_cpu_time(cpu_time_t *cpu_time)
+{
+ host_cpu_load_info_data_t load;
+ mach_msg_type_number_t count = HOST_CPU_LOAD_INFO_COUNT;
+ kern_return_t kr = host_statistics(mach_host_self(), HOST_CPU_LOAD_INFO, (int *)&load, &count);
+ mach_assert_zero_t(0, kr);
+
+ natural_t total_system_time = load.cpu_ticks[CPU_STATE_SYSTEM];
+ natural_t total_user_time = load.cpu_ticks[CPU_STATE_USER] + load.cpu_ticks[CPU_STATE_NICE];
+ natural_t total_idle_time = load.cpu_ticks[CPU_STATE_IDLE];
+
+ cpu_time->sys = total_system_time;
+ cpu_time->user = total_user_time;
+ cpu_time->idle = total_idle_time;
+}
+
int
main(int argc, char **argv)
{
float avg, stddev;
bool test_fail = false;
+ bool test_warn = false;
for (int i = 0; i < argc; i++) {
if (strcmp(argv[i], "--switched_apptype") == 0) {
mach_timebase_info(&g_mti);
+#if TARGET_OS_OSX
+ /* SKIP test if running on arm platform */
+ if (g_run_on_intel_only) {
+ int is_arm = 0;
+ size_t is_arm_size = sizeof(is_arm);
+ ret = sysctlbyname("hw.optional.arm64", &is_arm, &is_arm_size, NULL, 0);
+ if (ret == 0 && is_arm) {
+ printf("Unsupported platform. Skipping test.\n");
+ exit(0);
+ }
+ }
+#endif /* TARGET_OS_OSX */
+
size_t ncpu_size = sizeof(g_numcpus);
ret = sysctlbyname("hw.ncpu", &g_numcpus, &ncpu_size, NULL, 0);
if (ret) {
kr = semaphore_create(mach_task_self(), &g_readysem, SYNC_POLICY_FIFO, 0);
mach_assert_zero(kr);
+ kr = semaphore_create(mach_task_self(), &g_rt_churn_sem, SYNC_POLICY_FIFO, 0);
+ mach_assert_zero(kr);
+
+ kr = semaphore_create(mach_task_self(), &g_rt_churn_start_sem, SYNC_POLICY_FIFO, 0);
+ mach_assert_zero(kr);
+
atomic_store_explicit(&g_done_threads, 0, memory_order_relaxed);
/* Create the threads */
if (g_churn_pri) {
create_churn_threads();
}
+ if (g_rt_churn) {
+ create_rt_churn_threads();
+ }
/* Let everyone get settled */
kr = semaphore_wait(g_main_sem);
usleep(g_iteration_sleeptime_us);
}
+ cpu_time_t start_time;
+ cpu_time_t finish_time;
+
+ record_cpu_time(&start_time);
+
/* Go! */
for (uint32_t i = 0; i < g_iterations; i++) {
uint32_t j;
g_one_long_spin_id = (uint32_t)rand() % g_numthreads;
}
+ if (g_rt_churn) {
+ start_rt_churn_threads();
+ usleep(100);
+ }
+
debug_log("%d Main thread reset\n", i);
atomic_store_explicit(&g_done_threads, 0, memory_order_seq_cst);
assert(atomic_load_explicit(&g_done_threads, memory_order_relaxed) == g_numthreads);
+ if (g_rt_churn) {
+ wait_for_rt_churn_threads();
+ }
+
/*
* We report the worst latencies relative to start time
* and relative to the lead worker thread.
}
}
+ record_cpu_time(&finish_time);
+
/* Rejoin threads */
for (uint32_t i = 0; i < g_numthreads; i++) {
ret = pthread_join(threads[i], NULL);
}
}
+ if (g_rt_churn) {
+ join_rt_churn_threads();
+ }
+
if (g_churn_pri) {
join_churn_threads();
}
+ uint32_t cpu_idle_time = (finish_time.idle - start_time.idle) * 10;
+ uint32_t worker_threads_runtime = worker_threads_total_runtime.seconds * 1000 + worker_threads_total_runtime.microseconds / 1000;
+
compute_stats(worst_latencies_ns, g_iterations, &avg, &max, &min, &stddev);
printf("Results (from a stop):\n");
printf("Max:\t\t%.2f us\n", ((float)max) / 1000.0);
secondary ? " SECONDARY" : "",
fail ? " FAIL" : "");
}
+ test_warn |= (secondary || fail);
test_fail |= fail;
fail_count += fail;
}
}
}
+ if (g_test_rt_smt && (g_each_spin_duration_ns >= 200000) && !test_warn) {
+ printf("cpu_idle_time=%dms worker_threads_runtime=%dms\n", cpu_idle_time, worker_threads_runtime);
+ if (cpu_idle_time < worker_threads_runtime / 4) {
+ printf("FAIL cpu_idle_time unexpectedly small\n");
+ test_fail = 1;
+ } else if (cpu_idle_time > worker_threads_runtime * 2) {
+ printf("FAIL cpu_idle_time unexpectedly large\n");
+ test_fail = 1;
+ }
+ }
+
free(threads);
free(g_thread_endtimes_abs);
free(worst_latencies_ns);
usage()
{
errx(EX_USAGE, "Usage: %s <threads> <chain | hop | broadcast-single-sem | broadcast-per-thread> "
- "<realtime | timeshare | fixed> <iterations>\n\t\t"
+ "<realtime | timeshare | timeshare_no_smt | fixed> <iterations>\n\t\t"
"[--trace <traceworthy latency in ns>] "
"[--verbose] [--spin-one] [--spin-all] [--spin-time <nanos>] [--affinity]\n\t\t"
- "[--no-sleep] [--drop-priority] [--churn-pri <pri>] [--churn-count <n>]",
+ "[--no-sleep] [--drop-priority] [--churn-pri <pri>] [--churn-count <n>]\n\t\t"
+ "[--rt-churn] [--rt-churn-count <n>] [--rt-ll] [--test-rt] [--test-rt-smt] [--test-rt-avoid0]",
getprogname());
}
OPT_PRIORITY,
OPT_CHURN_PRI,
OPT_CHURN_COUNT,
+ OPT_RT_CHURN_COUNT,
};
static struct option longopts[] = {
{ "priority", required_argument, NULL, OPT_PRIORITY },
{ "churn-pri", required_argument, NULL, OPT_CHURN_PRI },
{ "churn-count", required_argument, NULL, OPT_CHURN_COUNT },
+ { "rt-churn-count", required_argument, NULL, OPT_RT_CHURN_COUNT },
{ "switched_apptype", no_argument, (int*)&g_seen_apptype, TRUE },
{ "spin-one", no_argument, (int*)&g_do_one_long_spin, TRUE },
+ { "intel-only", no_argument, (int*)&g_run_on_intel_only, TRUE },
{ "spin-all", no_argument, (int*)&g_do_all_spin, TRUE },
{ "affinity", no_argument, (int*)&g_do_affinity, TRUE },
{ "no-sleep", no_argument, (int*)&g_do_sleep, FALSE },
{ "test-rt", no_argument, (int*)&g_test_rt, TRUE },
{ "test-rt-smt", no_argument, (int*)&g_test_rt_smt, TRUE },
{ "test-rt-avoid0", no_argument, (int*)&g_test_rt_avoid0, TRUE },
+ { "rt-churn", no_argument, (int*)&g_rt_churn, TRUE },
+ { "rt-ll", no_argument, (int*)&g_rt_ll, TRUE },
{ "histogram", no_argument, (int*)&g_histogram, TRUE },
{ "verbose", no_argument, (int*)&g_verbose, TRUE },
{ "help", no_argument, NULL, 'h' },
case OPT_CHURN_COUNT:
g_churn_count = read_dec_arg();
break;
+ case OPT_RT_CHURN_COUNT:
+ g_rt_churn_count = read_dec_arg();
+ break;
case '?':
case 'h':
default:
projects / apple / xnu.git / blobdiff