[apple/xnu.git] / tests / perf_compressor.c

#include <stdio.h>
#include <signal.h>
#include <sys/sysctl.h>
#include <sys/kern_memorystatus.h>
#include <mach-o/dyld.h>
#include <perfcheck_keys.h>

#ifdef T_NAMESPACE
#undef T_NAMESPACE
#endif
#include <darwintest.h>
#include <darwintest_utils.h>

T_GLOBAL_META(
	T_META_NAMESPACE("xnu.vm.perf"),
	T_META_CHECK_LEAKS(false),
	T_META_TAG_PERF
	);

enum {
	ALL_ZEROS,
	MOSTLY_ZEROS,
	RANDOM,
	TYPICAL
};

#define CREATE_LIST(X) \
	X(SUCCESS) \
	X(TOO_FEW_ARGUMENTS) \
	X(SYSCTL_VM_PAGESIZE_FAILED) \
	X(VM_PAGESIZE_IS_ZERO) \
	X(UNKNOWN_PAGE_TYPE) \
	X(DISPATCH_SOURCE_CREATE_FAILED) \
	X(INITIAL_SIGNAL_TO_PARENT_FAILED) \
	X(SIGNAL_TO_PARENT_FAILED) \
	X(MEMORYSTATUS_CONTROL_FAILED) \
	X(IS_FREEZABLE_NOT_AS_EXPECTED) \
	X(EXIT_CODE_MAX)

#define EXIT_CODES_ENUM(VAR) VAR,
enum exit_codes_num {
	CREATE_LIST(EXIT_CODES_ENUM)
};

#define EXIT_CODES_STRING(VAR) #VAR,
static const char *exit_codes_str[] = {
	CREATE_LIST(EXIT_CODES_STRING)
};

#define SYSCTL_FREEZE_TO_MEMORY         "kern.memorystatus_freeze_to_memory=1"

static pid_t pid = -1;
static dt_stat_t ratio;
static dt_stat_time_t compr_time;
static dt_stat_time_t decompr_time;

void allocate_zero_pages(char **buf, int num_pages, int vmpgsize);
void allocate_mostly_zero_pages(char **buf, int num_pages, int vmpgsize);
void allocate_random_pages(char **buf, int num_pages, int vmpgsize);
void allocate_representative_pages(char **buf, int num_pages, int vmpgsize);
void run_compressor_test(int size_mb, int page_type);
void freeze_helper_process(void);
void cleanup(void);

void
allocate_zero_pages(char **buf, int num_pages, int vmpgsize)
{
	int i;

	for (i = 0; i < num_pages; i++) {
		buf[i] = (char*)malloc((size_t)vmpgsize * sizeof(char));
		memset(buf[i], 0, vmpgsize);
	}
}

void
allocate_mostly_zero_pages(char **buf, int num_pages, int vmpgsize)
{
	int i, j;

	for (i = 0; i < num_pages; i++) {
		buf[i] = (char*)malloc((size_t)vmpgsize * sizeof(char));
		memset(buf[i], 0, vmpgsize);
		for (j = 0; j < 40; j++) {
			buf[i][j] = (char)(j + 1);
		}
	}
}

void
allocate_random_pages(char **buf, int num_pages, int vmpgsize)
{
	int i;

	for (i = 0; i < num_pages; i++) {
		buf[i] = (char*)malloc((size_t)vmpgsize * sizeof(char));
		arc4random_buf((void*)buf[i], (size_t)vmpgsize);
	}
}

// Gives us the compression ratio we see in the typical case (~2.7)
void
allocate_representative_pages(char **buf, int num_pages, int vmpgsize)
{
	int i, j;
	char val;

	for (j = 0; j < num_pages; j++) {
		buf[j] = (char*)malloc((size_t)vmpgsize * sizeof(char));
		val = 0;
		for (i = 0; i < vmpgsize; i += 16) {
			memset(&buf[j][i], val, 16);
			if (i < 3400 * (vmpgsize / 4096)) {
				val++;
			}
		}
	}
}

void
freeze_helper_process(void)
{
	int ret, freeze_enabled;
	int64_t compressed_before, compressed_after, input_before, input_after;
	size_t length;
	int errno_sysctl_freeze;

	length = sizeof(compressed_before);
	T_QUIET; T_ASSERT_POSIX_SUCCESS(sysctlbyname("vm.compressor_compressed_bytes", &compressed_before, &length, NULL, 0),
	    "failed to query vm.compressor_compressed_bytes");
	length = sizeof(input_before);
	T_QUIET; T_ASSERT_POSIX_SUCCESS(sysctlbyname("vm.compressor_input_bytes", &input_before, &length, NULL, 0),
	    "failed to query vm.compressor_input_bytes");

	T_STAT_MEASURE(compr_time) {
		ret = sysctlbyname("kern.memorystatus_freeze", NULL, NULL, &pid, sizeof(pid));
		errno_sysctl_freeze = errno;
	};

	length = sizeof(compressed_after);
	T_QUIET; T_ASSERT_POSIX_SUCCESS(sysctlbyname("vm.compressor_compressed_bytes", &compressed_after, &length, NULL, 0),
	    "failed to query vm.compressor_compressed_bytes");
	length = sizeof(input_after);
	T_QUIET; T_ASSERT_POSIX_SUCCESS(sysctlbyname("vm.compressor_input_bytes", &input_after, &length, NULL, 0),
	    "failed to query vm.compressor_input_bytes");

	length = sizeof(freeze_enabled);
	T_QUIET; T_ASSERT_POSIX_SUCCESS(sysctlbyname("vm.freeze_enabled", &freeze_enabled, &length, NULL, 0),
	    "failed to query vm.freeze_enabled");
	if (freeze_enabled) {
		errno = errno_sysctl_freeze;
		T_QUIET; T_ASSERT_POSIX_SUCCESS(ret, "sysctl kern.memorystatus_freeze failed");
	} else {
		/* If freezer is disabled, skip the test. This can happen due to disk space shortage. */
		T_LOG("Freeze has been disabled. Terminating early.");
		T_END;
	}

	dt_stat_add(ratio, (double)(input_after - input_before) / (double)(compressed_after - compressed_before));

	ret = sysctlbyname("kern.memorystatus_thaw", NULL, NULL, &pid, sizeof(pid));
	T_QUIET; T_ASSERT_POSIX_SUCCESS(ret, "sysctl kern.memorystatus_thaw failed");

	T_QUIET; T_ASSERT_POSIX_SUCCESS(kill(pid, SIGUSR1), "failed to send SIGUSR1 to child process");
}

void
cleanup(void)
{
	/* No helper process. */
	if (pid == -1) {
		return;
	}
	/* Kill the helper process. */
	kill(pid, SIGKILL);
}

void
run_compressor_test(int size_mb, int page_type)
{
	int ret;
	char sz_str[50];
	char pt_str[50];
	char **launch_tool_args;
	char testpath[PATH_MAX];
	uint32_t testpath_buf_size;
	dispatch_source_t ds_freeze, ds_proc, ds_decompr;
	int freeze_enabled;
	size_t length;
	__block bool decompr_latency_is_stable = false;

	length = sizeof(freeze_enabled);
	T_QUIET; T_ASSERT_POSIX_SUCCESS(sysctlbyname("vm.freeze_enabled", &freeze_enabled, &length, NULL, 0),
	    "failed to query vm.freeze_enabled");
	if (!freeze_enabled) {
		/* If freezer is disabled, skip the test. This can happen due to disk space shortage. */
		T_SKIP("Freeze has been disabled. Skipping test.");
	}

	T_ATEND(cleanup);

	ratio = dt_stat_create("(input bytes / compressed bytes)", "compression_ratio");
	compr_time = dt_stat_time_create("compressor_latency");

	// This sets the A/B failure threshold at 50% of baseline for compressor_latency
	dt_stat_set_variable((struct dt_stat *)compr_time, kPCFailureThresholdPctVar, 50.0);

	signal(SIGUSR2, SIG_IGN);
	ds_decompr = dispatch_source_create(DISPATCH_SOURCE_TYPE_SIGNAL, SIGUSR2, 0, dispatch_get_main_queue());
	T_QUIET; T_ASSERT_NOTNULL(ds_decompr, "dispatch_source_create (ds_decompr)");

	dispatch_source_set_event_handler(ds_decompr, ^{
		decompr_latency_is_stable = true;
	});
	dispatch_activate(ds_decompr);

	signal(SIGUSR1, SIG_IGN);
	ds_freeze = dispatch_source_create(DISPATCH_SOURCE_TYPE_SIGNAL, SIGUSR1, 0, dispatch_get_main_queue());
	T_QUIET; T_ASSERT_NOTNULL(ds_freeze, "dispatch_source_create (ds_freeze)");

	dispatch_source_set_event_handler(ds_freeze, ^{
		if (!(dt_stat_stable(compr_time) && decompr_latency_is_stable)) {
		        freeze_helper_process();
		} else {
		        dt_stat_finalize(compr_time);
		        dt_stat_finalize(ratio);

		        kill(pid, SIGKILL);
		        dispatch_source_cancel(ds_freeze);
		        dispatch_source_cancel(ds_decompr);
		}
	});
	dispatch_activate(ds_freeze);

	testpath_buf_size = sizeof(testpath);
	ret = _NSGetExecutablePath(testpath, &testpath_buf_size);
	T_QUIET; T_ASSERT_POSIX_ZERO(ret, "_NSGetExecutablePath");
	T_LOG("Executable path: %s", testpath);

	sprintf(sz_str, "%d", size_mb);
	sprintf(pt_str, "%d", page_type);
	launch_tool_args = (char *[]){
		testpath,
		"-n",
		"allocate_pages",
		"--",
		sz_str,
		pt_str,
		NULL
	};

	/* Spawn the child process. Suspend after launch until the exit proc handler has been set up. */
	ret = dt_launch_tool(&pid, launch_tool_args, true, NULL, NULL);
	if (ret != 0) {
		T_LOG("dt_launch tool returned %d with error code %d", ret, errno);
	}
	T_QUIET; T_ASSERT_POSIX_SUCCESS(pid, "dt_launch_tool");

	ds_proc = dispatch_source_create(DISPATCH_SOURCE_TYPE_PROC, (uintptr_t)pid, DISPATCH_PROC_EXIT, dispatch_get_main_queue());
	T_QUIET; T_ASSERT_NOTNULL(ds_proc, "dispatch_source_create (ds_proc)");

	dispatch_source_set_event_handler(ds_proc, ^{
		int status = 0, code = 0;
		pid_t rc = waitpid(pid, &status, 0);
		T_QUIET; T_ASSERT_EQ(rc, pid, "waitpid");
		code = WEXITSTATUS(status);

		if (code == 0) {
		        T_END;
		} else if (code > 0 && code < EXIT_CODE_MAX) {
		        T_ASSERT_FAIL("Child exited with %s", exit_codes_str[code]);
		} else {
		        T_ASSERT_FAIL("Child exited with unknown exit code %d", code);
		}
	});
	dispatch_activate(ds_proc);

	T_QUIET; T_ASSERT_POSIX_SUCCESS(kill(pid, SIGCONT), "failed to send SIGCONT to child process");
	dispatch_main();
}

T_HELPER_DECL(allocate_pages, "allocates pages to compress") {
	int i, j, ret, size_mb, page_type, vmpgsize, freezable_state;
	size_t vmpgsize_length;
	__block int num_pages;
	__block char **buf;
	dispatch_source_t ds_signal;

	vmpgsize_length = sizeof(vmpgsize);
	ret = sysctlbyname("vm.pagesize", &vmpgsize, &vmpgsize_length, NULL, 0);
	if (ret != 0) {
		exit(SYSCTL_VM_PAGESIZE_FAILED);
	}
	if (vmpgsize == 0) {
		exit(VM_PAGESIZE_IS_ZERO);
	}

	if (argc < 2) {
		exit(TOO_FEW_ARGUMENTS);
	}

	size_mb = atoi(argv[0]);
	page_type = atoi(argv[1]);
	num_pages = size_mb * 1024 * 1024 / vmpgsize;
	buf = (char**)malloc(sizeof(char*) * (size_t)num_pages);

	// Switch on the type of page requested
	switch (page_type) {
	case ALL_ZEROS:
		allocate_zero_pages(buf, num_pages, vmpgsize);
		break;
	case MOSTLY_ZEROS:
		allocate_mostly_zero_pages(buf, num_pages, vmpgsize);
		break;
	case RANDOM:
		allocate_random_pages(buf, num_pages, vmpgsize);
		break;
	case TYPICAL:
		allocate_representative_pages(buf, num_pages, vmpgsize);
		break;
	default:
		exit(UNKNOWN_PAGE_TYPE);
	}

	for (j = 0; j < num_pages; j++) {
		i = buf[j][0];
	}

	decompr_time = dt_stat_time_create("decompression_latency");

	/* Opt in to freezing. */
	printf("[%d] Setting state to freezable\n", getpid());
	if (memorystatus_control(MEMORYSTATUS_CMD_SET_PROCESS_IS_FREEZABLE, getpid(), 1, NULL, 0) != KERN_SUCCESS) {
		exit(MEMORYSTATUS_CONTROL_FAILED);
	}

	/* Verify that the state has been set correctly */
	freezable_state = memorystatus_control(MEMORYSTATUS_CMD_GET_PROCESS_IS_FREEZABLE, getpid(), 0, NULL, 0);
	if (freezable_state != 1) {
		exit(IS_FREEZABLE_NOT_AS_EXPECTED);
	}

	dispatch_after(dispatch_time(DISPATCH_TIME_NOW, NSEC_PER_SEC), dispatch_get_main_queue(), ^{
		/* Signal to the parent that we're done allocating and it's ok to freeze us */
		printf("[%d] Sending initial signal to parent to begin freezing\n", getpid());
		if (kill(getppid(), SIGUSR1) != 0) {
		        exit(INITIAL_SIGNAL_TO_PARENT_FAILED);
		}
	});

	signal(SIGUSR1, SIG_IGN);
	ds_signal = dispatch_source_create(DISPATCH_SOURCE_TYPE_SIGNAL, SIGUSR1, 0, dispatch_get_main_queue());
	if (ds_signal == NULL) {
		exit(DISPATCH_SOURCE_CREATE_FAILED);
	}

	__block bool collect_dt_stat_measurements = true;

	dispatch_source_set_event_handler(ds_signal, ^{
		volatile int tmp;
		uint64_t decompr_start_time, decompr_end_time;

		decompr_start_time = mach_absolute_time();

		/* Make sure all the pages are accessed before trying to freeze again */
		for (int x = 0; x < num_pages; x++) {
		        tmp = buf[x][0];
		}

		decompr_end_time = mach_absolute_time();

		if (collect_dt_stat_measurements) {
			if (dt_stat_stable(decompr_time)) {
				collect_dt_stat_measurements = false;
				dt_stat_finalize(decompr_time);
				if (kill(getppid(), SIGUSR2) != 0) {
					exit(SIGNAL_TO_PARENT_FAILED);
				}
			} else {
				dt_stat_mach_time_add(decompr_time, decompr_end_time - decompr_start_time);
			}
		}

		if (kill(getppid(), SIGUSR1) != 0) {
		        exit(SIGNAL_TO_PARENT_FAILED);
		}
	});
	dispatch_activate(ds_signal);

	dispatch_main();
}

// Numbers for 10MB and above are fairly reproducible. Anything smaller shows a lot of variation.

// Keeping just the 100MB version for iOSMark
#ifndef DT_IOSMARK
T_DECL(compr_10MB_zero,
    "Compression latency for 10MB - zero pages",
    T_META_ASROOT(true),
    T_META_SYSCTL_INT(SYSCTL_FREEZE_TO_MEMORY)) {
	run_compressor_test(10, ALL_ZEROS);
}

T_DECL(compr_10MB_mostly_zero,
    "Compression latency for 10MB - mostly zero pages",
    T_META_ASROOT(true),
    T_META_SYSCTL_INT(SYSCTL_FREEZE_TO_MEMORY)) {
	run_compressor_test(10, MOSTLY_ZEROS);
}

T_DECL(compr_10MB_random,
    "Compression latency for 10MB - random pages",
    T_META_ASROOT(true),
    T_META_SYSCTL_INT(SYSCTL_FREEZE_TO_MEMORY)) {
	run_compressor_test(10, RANDOM);
}

T_DECL(compr_10MB_typical,
    "Compression latency for 10MB - typical pages",
    T_META_ASROOT(true),
    T_META_SYSCTL_INT(SYSCTL_FREEZE_TO_MEMORY)) {
	run_compressor_test(10, TYPICAL);
}

T_DECL(compr_100MB_zero,
    "Compression latency for 100MB - zero pages",
    T_META_ASROOT(true),
    T_META_SYSCTL_INT(SYSCTL_FREEZE_TO_MEMORY)) {
	run_compressor_test(100, ALL_ZEROS);
}

T_DECL(compr_100MB_mostly_zero,
    "Compression latency for 100MB - mostly zero pages",
    T_META_ASROOT(true),
    T_META_SYSCTL_INT(SYSCTL_FREEZE_TO_MEMORY)) {
	run_compressor_test(100, MOSTLY_ZEROS);
}

T_DECL(compr_100MB_random,
    "Compression latency for 100MB - random pages",
    T_META_ASROOT(true),
    T_META_SYSCTL_INT(SYSCTL_FREEZE_TO_MEMORY)) {
	run_compressor_test(100, RANDOM);
}
#endif

T_DECL(compr_100MB_typical,
    "Compression latency for 100MB - typical pages",
    T_META_ASROOT(true),
    T_META_SYSCTL_INT(SYSCTL_FREEZE_TO_MEMORY)) {
	run_compressor_test(100, TYPICAL);
}
Commit	Line	Data
a39ff7e2 A	1	#include <stdio.h>
	2	#include <signal.h>
	3	#include <sys/sysctl.h>
cb323159	4	#include <sys/kern_memorystatus.h>
a39ff7e2	5	#include <mach-o/dyld.h>
d9a64523	6	#include <perfcheck_keys.h>
a39ff7e2	7
813fb2f6 A	8	#ifdef T_NAMESPACE
	9	#undef T_NAMESPACE
	10	#endif
	11	#include <darwintest.h>
a39ff7e2	12	#include <darwintest_utils.h>
813fb2f6 A	13
	14	T_GLOBAL_META(
	15	T_META_NAMESPACE("xnu.vm.perf"),
d9a64523 A	16	T_META_CHECK_LEAKS(false),
d9a64523 A	17	T_META_TAG_PERF
0a7de745	18	);
813fb2f6 A	19
	20	enum {
	21	ALL_ZEROS,
	22	MOSTLY_ZEROS,
	23	RANDOM,
	24	TYPICAL
	25	};
	26
a39ff7e2 A	27	#define CREATE_LIST(X) \
	28	X(SUCCESS) \
	29	X(TOO_FEW_ARGUMENTS) \
	30	X(SYSCTL_VM_PAGESIZE_FAILED) \
	31	X(VM_PAGESIZE_IS_ZERO) \
	32	X(UNKNOWN_PAGE_TYPE) \
	33	X(DISPATCH_SOURCE_CREATE_FAILED) \
	34	X(INITIAL_SIGNAL_TO_PARENT_FAILED) \
	35	X(SIGNAL_TO_PARENT_FAILED) \
cb323159 A	36	X(MEMORYSTATUS_CONTROL_FAILED) \
cb323159 A	37	X(IS_FREEZABLE_NOT_AS_EXPECTED) \
a39ff7e2 A	38	X(EXIT_CODE_MAX)
	39
	40	#define EXIT_CODES_ENUM(VAR) VAR,
	41	enum exit_codes_num {
	42	CREATE_LIST(EXIT_CODES_ENUM)
	43	};
	44
	45	#define EXIT_CODES_STRING(VAR) #VAR,
	46	static const char *exit_codes_str[] = {
	47	CREATE_LIST(EXIT_CODES_STRING)
	48	};
	49
0a7de745	50	#define SYSCTL_FREEZE_TO_MEMORY "kern.memorystatus_freeze_to_memory=1"
a39ff7e2 A	51
a39ff7e2 A	52	static pid_t pid = -1;
cb323159 A	53	static dt_stat_t ratio;
	54	static dt_stat_time_t compr_time;
	55	static dt_stat_time_t decompr_time;
a39ff7e2	56
813fb2f6 A	57	void allocate_zero_pages(char **buf, int num_pages, int vmpgsize);
	58	void allocate_mostly_zero_pages(char **buf, int num_pages, int vmpgsize);
	59	void allocate_random_pages(char **buf, int num_pages, int vmpgsize);
	60	void allocate_representative_pages(char **buf, int num_pages, int vmpgsize);
813fb2f6	61	void run_compressor_test(int size_mb, int page_type);
a39ff7e2	62	void freeze_helper_process(void);
0a7de745	63	void cleanup(void);
813fb2f6	64
0a7de745 A	65	void
	66	allocate_zero_pages(char **buf, int num_pages, int vmpgsize)
	67	{
813fb2f6 A	68	int i;
	69
	70	for (i = 0; i < num_pages; i++) {
	71	buf[i] = (char)malloc((size_t)vmpgsize sizeof(char));
	72	memset(buf[i], 0, vmpgsize);
	73	}
	74	}
	75
0a7de745 A	76	void
	77	allocate_mostly_zero_pages(char **buf, int num_pages, int vmpgsize)
	78	{
813fb2f6 A	79	int i, j;
	80
	81	for (i = 0; i < num_pages; i++) {
	82	buf[i] = (char)malloc((size_t)vmpgsize sizeof(char));
	83	memset(buf[i], 0, vmpgsize);
	84	for (j = 0; j < 40; j++) {
0a7de745	85	buf[i][j] = (char)(j + 1);
813fb2f6 A	86	}
	87	}
	88	}
	89
0a7de745 A	90	void
	91	allocate_random_pages(char **buf, int num_pages, int vmpgsize)
	92	{
5ba3f43e	93	int i;
813fb2f6 A	94
	95	for (i = 0; i < num_pages; i++) {
	96	buf[i] = (char)malloc((size_t)vmpgsize sizeof(char));
5ba3f43e	97	arc4random_buf((void*)buf[i], (size_t)vmpgsize);
813fb2f6	98	}
813fb2f6 A	99	}
	100
	101	// Gives us the compression ratio we see in the typical case (~2.7)
0a7de745 A	102	void
	103	allocate_representative_pages(char **buf, int num_pages, int vmpgsize)
	104	{
813fb2f6 A	105	int i, j;
	106	char val;
	107
	108	for (j = 0; j < num_pages; j++) {
	109	buf[j] = (char)malloc((size_t)vmpgsize sizeof(char));
	110	val = 0;
	111	for (i = 0; i < vmpgsize; i += 16) {
	112	memset(&buf[j][i], val, 16);
a39ff7e2	113	if (i < 3400 * (vmpgsize / 4096)) {
813fb2f6 A	114	val++;
	115	}
	116	}
	117	}
	118	}
	119
0a7de745 A	120	void
	121	freeze_helper_process(void)
	122	{
	123	int ret, freeze_enabled;
a39ff7e2 A	124	int64_t compressed_before, compressed_after, input_before, input_after;
a39ff7e2 A	125	size_t length;
0a7de745	126	int errno_sysctl_freeze;
a39ff7e2	127
a39ff7e2 A	128	length = sizeof(compressed_before);
a39ff7e2 A	129	T_QUIET; T_ASSERT_POSIX_SUCCESS(sysctlbyname("vm.compressor_compressed_bytes", &compressed_before, &length, NULL, 0),
0a7de745	130	"failed to query vm.compressor_compressed_bytes");
a39ff7e2 A	131	length = sizeof(input_before);
a39ff7e2 A	132	T_QUIET; T_ASSERT_POSIX_SUCCESS(sysctlbyname("vm.compressor_input_bytes", &input_before, &length, NULL, 0),
0a7de745	133	"failed to query vm.compressor_input_bytes");
a39ff7e2	134
cb323159	135	T_STAT_MEASURE(compr_time) {
a39ff7e2	136	ret = sysctlbyname("kern.memorystatus_freeze", NULL, NULL, &pid, sizeof(pid));
0a7de745	137	errno_sysctl_freeze = errno;
a39ff7e2 A	138	};
	139
	140	length = sizeof(compressed_after);
	141	T_QUIET; T_ASSERT_POSIX_SUCCESS(sysctlbyname("vm.compressor_compressed_bytes", &compressed_after, &length, NULL, 0),
0a7de745	142	"failed to query vm.compressor_compressed_bytes");
a39ff7e2 A	143	length = sizeof(input_after);
a39ff7e2 A	144	T_QUIET; T_ASSERT_POSIX_SUCCESS(sysctlbyname("vm.compressor_input_bytes", &input_after, &length, NULL, 0),
0a7de745 A	145	"failed to query vm.compressor_input_bytes");
	146
	147	length = sizeof(freeze_enabled);
	148	T_QUIET; T_ASSERT_POSIX_SUCCESS(sysctlbyname("vm.freeze_enabled", &freeze_enabled, &length, NULL, 0),
	149	"failed to query vm.freeze_enabled");
	150	if (freeze_enabled) {
	151	errno = errno_sysctl_freeze;
	152	T_QUIET; T_ASSERT_POSIX_SUCCESS(ret, "sysctl kern.memorystatus_freeze failed");
	153	} else {
	154	/* If freezer is disabled, skip the test. This can happen due to disk space shortage. */
	155	T_LOG("Freeze has been disabled. Terminating early.");
	156	T_END;
	157	}
a39ff7e2	158
cb323159	159	dt_stat_add(ratio, (double)(input_after - input_before) / (double)(compressed_after - compressed_before));
a39ff7e2	160
a39ff7e2	161	ret = sysctlbyname("kern.memorystatus_thaw", NULL, NULL, &pid, sizeof(pid));
d9a64523	162	T_QUIET; T_ASSERT_POSIX_SUCCESS(ret, "sysctl kern.memorystatus_thaw failed");
a39ff7e2	163
d9a64523	164	T_QUIET; T_ASSERT_POSIX_SUCCESS(kill(pid, SIGUSR1), "failed to send SIGUSR1 to child process");
a39ff7e2 A	165	}
a39ff7e2 A	166
0a7de745 A	167	void
	168	cleanup(void)
	169	{
0a7de745 A	170	/* No helper process. */
	171	if (pid == -1) {
	172	return;
	173	}
	174	/* Kill the helper process. */
	175	kill(pid, SIGKILL);
	176	}
	177
	178	void
	179	run_compressor_test(int size_mb, int page_type)
	180	{
a39ff7e2 A	181	int ret;
	182	char sz_str[50];
	183	char pt_str[50];
	184	char **launch_tool_args;
	185	char testpath[PATH_MAX];
	186	uint32_t testpath_buf_size;
cb323159	187	dispatch_source_t ds_freeze, ds_proc, ds_decompr;
0a7de745 A	188	int freeze_enabled;
0a7de745 A	189	size_t length;
cb323159	190	__block bool decompr_latency_is_stable = false;
a39ff7e2	191
0a7de745 A	192	length = sizeof(freeze_enabled);
	193	T_QUIET; T_ASSERT_POSIX_SUCCESS(sysctlbyname("vm.freeze_enabled", &freeze_enabled, &length, NULL, 0),
	194	"failed to query vm.freeze_enabled");
	195	if (!freeze_enabled) {
	196	/* If freezer is disabled, skip the test. This can happen due to disk space shortage. */
	197	T_SKIP("Freeze has been disabled. Skipping test.");
	198	}
	199
	200	T_ATEND(cleanup);
a39ff7e2	201
cb323159 A	202	ratio = dt_stat_create("(input bytes / compressed bytes)", "compression_ratio");
	203	compr_time = dt_stat_time_create("compressor_latency");
	204
d9a64523	205	// This sets the A/B failure threshold at 50% of baseline for compressor_latency
cb323159 A	206	dt_stat_set_variable((struct dt_stat *)compr_time, kPCFailureThresholdPctVar, 50.0);
	207
	208	signal(SIGUSR2, SIG_IGN);
	209	ds_decompr = dispatch_source_create(DISPATCH_SOURCE_TYPE_SIGNAL, SIGUSR2, 0, dispatch_get_main_queue());
	210	T_QUIET; T_ASSERT_NOTNULL(ds_decompr, "dispatch_source_create (ds_decompr)");
	211
	212	dispatch_source_set_event_handler(ds_decompr, ^{
	213	decompr_latency_is_stable = true;
	214	});
	215	dispatch_activate(ds_decompr);
a39ff7e2 A	216
	217	signal(SIGUSR1, SIG_IGN);
	218	ds_freeze = dispatch_source_create(DISPATCH_SOURCE_TYPE_SIGNAL, SIGUSR1, 0, dispatch_get_main_queue());
	219	T_QUIET; T_ASSERT_NOTNULL(ds_freeze, "dispatch_source_create (ds_freeze)");
	220
	221	dispatch_source_set_event_handler(ds_freeze, ^{
cb323159	222	if (!(dt_stat_stable(compr_time) && decompr_latency_is_stable)) {
0a7de745	223	freeze_helper_process();
a39ff7e2	224	} else {
cb323159 A	225	dt_stat_finalize(compr_time);
cb323159 A	226	dt_stat_finalize(ratio);
a39ff7e2	227
0a7de745 A	228	kill(pid, SIGKILL);
0a7de745 A	229	dispatch_source_cancel(ds_freeze);
cb323159	230	dispatch_source_cancel(ds_decompr);
a39ff7e2 A	231	}
	232	});
	233	dispatch_activate(ds_freeze);
	234
	235	testpath_buf_size = sizeof(testpath);
	236	ret = _NSGetExecutablePath(testpath, &testpath_buf_size);
	237	T_QUIET; T_ASSERT_POSIX_ZERO(ret, "_NSGetExecutablePath");
	238	T_LOG("Executable path: %s", testpath);
	239
	240	sprintf(sz_str, "%d", size_mb);
	241	sprintf(pt_str, "%d", page_type);
	242	launch_tool_args = (char *[]){
	243	testpath,
	244	"-n",
	245	"allocate_pages",
	246	"--",
	247	sz_str,
	248	pt_str,
	249	NULL
	250	};
	251
	252	/* Spawn the child process. Suspend after launch until the exit proc handler has been set up. */
	253	ret = dt_launch_tool(&pid, launch_tool_args, true, NULL, NULL);
	254	if (ret != 0) {
	255	T_LOG("dt_launch tool returned %d with error code %d", ret, errno);
	256	}
	257	T_QUIET; T_ASSERT_POSIX_SUCCESS(pid, "dt_launch_tool");
	258
	259	ds_proc = dispatch_source_create(DISPATCH_SOURCE_TYPE_PROC, (uintptr_t)pid, DISPATCH_PROC_EXIT, dispatch_get_main_queue());
	260	T_QUIET; T_ASSERT_NOTNULL(ds_proc, "dispatch_source_create (ds_proc)");
	261
	262	dispatch_source_set_event_handler(ds_proc, ^{
	263	int status = 0, code = 0;
	264	pid_t rc = waitpid(pid, &status, 0);
	265	T_QUIET; T_ASSERT_EQ(rc, pid, "waitpid");
	266	code = WEXITSTATUS(status);
	267
	268	if (code == 0) {
0a7de745	269	T_END;
a39ff7e2	270	} else if (code > 0 && code < EXIT_CODE_MAX) {
0a7de745	271	T_ASSERT_FAIL("Child exited with %s", exit_codes_str[code]);
a39ff7e2	272	} else {
0a7de745	273	T_ASSERT_FAIL("Child exited with unknown exit code %d", code);
a39ff7e2 A	274	}
	275	});
	276	dispatch_activate(ds_proc);
	277
d9a64523	278	T_QUIET; T_ASSERT_POSIX_SUCCESS(kill(pid, SIGCONT), "failed to send SIGCONT to child process");
a39ff7e2 A	279	dispatch_main();
	280	}
	281
	282	T_HELPER_DECL(allocate_pages, "allocates pages to compress") {
cb323159	283	int i, j, ret, size_mb, page_type, vmpgsize, freezable_state;
813fb2f6	284	size_t vmpgsize_length;
a39ff7e2 A	285	__block int num_pages;
	286	__block char **buf;
	287	dispatch_source_t ds_signal;
813fb2f6 A	288
813fb2f6 A	289	vmpgsize_length = sizeof(vmpgsize);
a39ff7e2 A	290	ret = sysctlbyname("vm.pagesize", &vmpgsize, &vmpgsize_length, NULL, 0);
	291	if (ret != 0) {
	292	exit(SYSCTL_VM_PAGESIZE_FAILED);
	293	}
813fb2f6	294	if (vmpgsize == 0) {
a39ff7e2 A	295	exit(VM_PAGESIZE_IS_ZERO);
	296	}
	297
	298	if (argc < 2) {
	299	exit(TOO_FEW_ARGUMENTS);
813fb2f6 A	300	}
813fb2f6 A	301
a39ff7e2 A	302	size_mb = atoi(argv[0]);
a39ff7e2 A	303	page_type = atoi(argv[1]);
813fb2f6 A	304	num_pages = size_mb * 1024 * 1024 / vmpgsize;
	305	buf = (char*)malloc(sizeof(char) * (size_t)num_pages);
	306
	307	// Switch on the type of page requested
0a7de745 A	308	switch (page_type) {
	309	case ALL_ZEROS:
	310	allocate_zero_pages(buf, num_pages, vmpgsize);
	311	break;
	312	case MOSTLY_ZEROS:
	313	allocate_mostly_zero_pages(buf, num_pages, vmpgsize);
	314	break;
	315	case RANDOM:
	316	allocate_random_pages(buf, num_pages, vmpgsize);
	317	break;
	318	case TYPICAL:
	319	allocate_representative_pages(buf, num_pages, vmpgsize);
	320	break;
	321	default:
	322	exit(UNKNOWN_PAGE_TYPE);
813fb2f6 A	323	}
813fb2f6 A	324
a39ff7e2 A	325	for (j = 0; j < num_pages; j++) {
a39ff7e2 A	326	i = buf[j][0];
813fb2f6	327	}
813fb2f6	328
cb323159 A	329	decompr_time = dt_stat_time_create("decompression_latency");
	330
	331	/* Opt in to freezing. */
	332	printf("[%d] Setting state to freezable\n", getpid());
	333	if (memorystatus_control(MEMORYSTATUS_CMD_SET_PROCESS_IS_FREEZABLE, getpid(), 1, NULL, 0) != KERN_SUCCESS) {
	334	exit(MEMORYSTATUS_CONTROL_FAILED);
	335	}
	336
	337	/* Verify that the state has been set correctly */
	338	freezable_state = memorystatus_control(MEMORYSTATUS_CMD_GET_PROCESS_IS_FREEZABLE, getpid(), 0, NULL, 0);
	339	if (freezable_state != 1) {
	340	exit(IS_FREEZABLE_NOT_AS_EXPECTED);
	341	}
	342
a39ff7e2 A	343	dispatch_after(dispatch_time(DISPATCH_TIME_NOW, NSEC_PER_SEC), dispatch_get_main_queue(), ^{
a39ff7e2 A	344	/* Signal to the parent that we're done allocating and it's ok to freeze us */
0a7de745	345	printf("[%d] Sending initial signal to parent to begin freezing\n", getpid());
a39ff7e2	346	if (kill(getppid(), SIGUSR1) != 0) {
0a7de745	347	exit(INITIAL_SIGNAL_TO_PARENT_FAILED);
a39ff7e2 A	348	}
a39ff7e2 A	349	});
813fb2f6	350
a39ff7e2 A	351	signal(SIGUSR1, SIG_IGN);
	352	ds_signal = dispatch_source_create(DISPATCH_SOURCE_TYPE_SIGNAL, SIGUSR1, 0, dispatch_get_main_queue());
	353	if (ds_signal == NULL) {
	354	exit(DISPATCH_SOURCE_CREATE_FAILED);
	355	}
813fb2f6	356
cb323159 A	357	__block bool collect_dt_stat_measurements = true;
cb323159 A	358
a39ff7e2 A	359	dispatch_source_set_event_handler(ds_signal, ^{
a39ff7e2 A	360	volatile int tmp;
cb323159 A	361	uint64_t decompr_start_time, decompr_end_time;
	362
	363	decompr_start_time = mach_absolute_time();
813fb2f6	364
a39ff7e2 A	365	/* Make sure all the pages are accessed before trying to freeze again */
a39ff7e2 A	366	for (int x = 0; x < num_pages; x++) {
0a7de745	367	tmp = buf[x][0];
813fb2f6	368	}
cb323159 A	369
	370	decompr_end_time = mach_absolute_time();
	371
	372	if (collect_dt_stat_measurements) {
	373	if (dt_stat_stable(decompr_time)) {
	374	collect_dt_stat_measurements = false;
	375	dt_stat_finalize(decompr_time);
	376	if (kill(getppid(), SIGUSR2) != 0) {
	377	exit(SIGNAL_TO_PARENT_FAILED);
	378	}
	379	} else {
	380	dt_stat_mach_time_add(decompr_time, decompr_end_time - decompr_start_time);
	381	}
	382	}
	383
a39ff7e2	384	if (kill(getppid(), SIGUSR1) != 0) {
0a7de745	385	exit(SIGNAL_TO_PARENT_FAILED);
a39ff7e2 A	386	}
	387	});
	388	dispatch_activate(ds_signal);
813fb2f6	389
a39ff7e2	390	dispatch_main();
813fb2f6 A	391	}
	392
	393	// Numbers for 10MB and above are fairly reproducible. Anything smaller shows a lot of variation.
d9a64523 A	394
	395	// Keeping just the 100MB version for iOSMark
	396	#ifndef DT_IOSMARK
0a7de745 A	397	T_DECL(compr_10MB_zero,
0a7de745 A	398	"Compression latency for 10MB - zero pages",
cb323159	399	T_META_ASROOT(true),
0a7de745	400	T_META_SYSCTL_INT(SYSCTL_FREEZE_TO_MEMORY)) {
813fb2f6 A	401	run_compressor_test(10, ALL_ZEROS);
	402	}
	403
0a7de745 A	404	T_DECL(compr_10MB_mostly_zero,
0a7de745 A	405	"Compression latency for 10MB - mostly zero pages",
cb323159	406	T_META_ASROOT(true),
0a7de745	407	T_META_SYSCTL_INT(SYSCTL_FREEZE_TO_MEMORY)) {
813fb2f6 A	408	run_compressor_test(10, MOSTLY_ZEROS);
	409	}
	410
0a7de745 A	411	T_DECL(compr_10MB_random,
0a7de745 A	412	"Compression latency for 10MB - random pages",
cb323159	413	T_META_ASROOT(true),
0a7de745	414	T_META_SYSCTL_INT(SYSCTL_FREEZE_TO_MEMORY)) {
813fb2f6 A	415	run_compressor_test(10, RANDOM);
	416	}
	417
0a7de745 A	418	T_DECL(compr_10MB_typical,
0a7de745 A	419	"Compression latency for 10MB - typical pages",
cb323159	420	T_META_ASROOT(true),
0a7de745	421	T_META_SYSCTL_INT(SYSCTL_FREEZE_TO_MEMORY)) {
813fb2f6 A	422	run_compressor_test(10, TYPICAL);
	423	}
	424
0a7de745 A	425	T_DECL(compr_100MB_zero,
0a7de745 A	426	"Compression latency for 100MB - zero pages",
cb323159	427	T_META_ASROOT(true),
0a7de745	428	T_META_SYSCTL_INT(SYSCTL_FREEZE_TO_MEMORY)) {
813fb2f6 A	429	run_compressor_test(100, ALL_ZEROS);
	430	}
	431
0a7de745 A	432	T_DECL(compr_100MB_mostly_zero,
0a7de745 A	433	"Compression latency for 100MB - mostly zero pages",
cb323159	434	T_META_ASROOT(true),
0a7de745	435	T_META_SYSCTL_INT(SYSCTL_FREEZE_TO_MEMORY)) {
813fb2f6 A	436	run_compressor_test(100, MOSTLY_ZEROS);
	437	}
	438
0a7de745 A	439	T_DECL(compr_100MB_random,
0a7de745 A	440	"Compression latency for 100MB - random pages",
cb323159	441	T_META_ASROOT(true),
0a7de745	442	T_META_SYSCTL_INT(SYSCTL_FREEZE_TO_MEMORY)) {
813fb2f6 A	443	run_compressor_test(100, RANDOM);
813fb2f6 A	444	}
d9a64523	445	#endif
813fb2f6	446
0a7de745 A	447	T_DECL(compr_100MB_typical,
0a7de745 A	448	"Compression latency for 100MB - typical pages",
cb323159	449	T_META_ASROOT(true),
0a7de745	450	T_META_SYSCTL_INT(SYSCTL_FREEZE_TO_MEMORY)) {
813fb2f6 A	451	run_compressor_test(100, TYPICAL);
813fb2f6 A	452	}