[apple/xnu.git] / tests / perf_vmfault.c

#include <unistd.h>
#include <stdlib.h>
#include <sys/mman.h>
#include <sys/sysctl.h>
#include <mach/mach.h>
#include <mach/vm_map.h>
#include <darwintest.h>
#include <TargetConditionals.h>
#include <perfcheck_keys.h>

T_GLOBAL_META(
	T_META_NAMESPACE("xnu.vm.perf"),
	T_META_CHECK_LEAKS(false),
	T_META_TAG_PERF
	);

#ifdef DT_IOSMARK
#define MEMSIZE                 (1UL<<29)       /* 512 MB */
#else
#define MEMSIZE                 (1UL<<27)       /* 128 MB */
#endif

#define VM_TAG1                 100
#define VM_TAG2                 101

enum {
	SOFT_FAULT,
	ZERO_FILL,
	NUM_FAULT_TYPES
};

enum {
	VARIANT_DEFAULT = 1,
	VARIANT_SINGLE_REGION,
	VARIANT_MULTIPLE_REGIONS,
	NUM_MAPPING_VARIANTS
};

static char *variant_str[] = {
	"none",
	"default",
	"single-region",
	"multiple-regions"
};


typedef struct {
	char *region_addr;
	char *shared_region_addr;
	size_t region_len;
} memregion_config;

static memregion_config *memregion_config_per_thread;

static size_t pgsize;
static int num_threads;
static int ready_thread_count;
static int finished_thread_count;
static dt_stat_time_t runtime;
static pthread_cond_t start_cvar;
static pthread_cond_t threads_ready_cvar;
static pthread_cond_t threads_finished_cvar;
static pthread_mutex_t ready_thread_count_lock;
static pthread_mutex_t finished_thread_count_lock;

static void map_mem_regions_default(int fault_type, size_t memsize);
static void map_mem_regions_single(int fault_type, size_t memsize);
static void map_mem_regions_multiple(int fault_type, size_t memsize);
static void map_mem_regions(int fault_type, int mapping_variant, size_t memsize);
static void unmap_mem_regions(int mapping_variant, size_t memsize);
static void setup_per_thread_regions(char *memblock, char *memblock_share, int fault_type, size_t memsize);
static void fault_pages(int thread_id);
static void execute_threads(void);
static void *thread_setup(void *arg);
static void run_test(int fault_type, int mapping_variant, size_t memsize);
static void setup_and_run_test(int test, int threads);
static int get_ncpu(void);

/* Allocates memory using the default mmap behavior. Each VM region created is capped at 128 MB. */
static void
map_mem_regions_default(int fault_type, size_t memsize)
{
	volatile char val;
	vm_prot_t curprot, maxprot;
	char *ptr, *memblock, *memblock_share = NULL;

	memblock = (char *)mmap(NULL, memsize, PROT_READ | PROT_WRITE, MAP_ANON | MAP_PRIVATE, -1, 0);
	T_QUIET; T_ASSERT_NE((void *)memblock, MAP_FAILED, "mmap");

	if (fault_type == SOFT_FAULT) {
		/* Fault in all the pages of the original region. */
		for (ptr = memblock; ptr < memblock + memsize; ptr += pgsize) {
			val = *ptr;
		}
		/* Remap the region so that subsequent accesses result in read soft faults. */
		T_QUIET; T_ASSERT_MACH_SUCCESS(vm_remap(mach_task_self(), (vm_address_t *)&memblock_share,
		    memsize, 0, VM_FLAGS_ANYWHERE, mach_task_self(), (vm_address_t)memblock, FALSE,
		    &curprot, &maxprot, VM_INHERIT_DEFAULT), "vm_remap");
	}
	setup_per_thread_regions(memblock, memblock_share, fault_type, memsize);
}

/* Creates a single VM region by mapping in a named memory entry. */
static void
map_mem_regions_single(int fault_type, size_t memsize)
{
	volatile char val;
	vm_prot_t curprot, maxprot;
	char *ptr, *memblock = NULL, *memblock_share = NULL;
	vm_size_t size = memsize;
	vm_offset_t addr1 = 0;
	mach_port_t mem_handle = MACH_PORT_NULL;

	/* Allocate a region and fault in all the pages. */
	T_QUIET; T_ASSERT_MACH_SUCCESS(vm_allocate(mach_task_self(), &addr1, size, VM_FLAGS_ANYWHERE), "vm_allocate");
	for (ptr = (char *)addr1; ptr < (char *)addr1 + memsize; ptr += pgsize) {
		val = *ptr;
	}

	/* Create a named memory entry from the region allocated above, and de-allocate said region. */
	T_QUIET; T_ASSERT_MACH_SUCCESS(mach_make_memory_entry(mach_task_self(), &size, addr1, VM_PROT_ALL | MAP_MEM_NAMED_CREATE,
	    &mem_handle, MACH_PORT_NULL), "mach_make_memory_entry");
	T_QUIET; T_ASSERT_MACH_SUCCESS(vm_deallocate(mach_task_self(), addr1, size), "vm_deallocate");

	/* Map in the named entry and deallocate it. */
	T_QUIET; T_ASSERT_MACH_SUCCESS(vm_map(mach_task_self(), (vm_address_t *)&memblock, size, 0, VM_FLAGS_ANYWHERE, mem_handle, 0,
	    FALSE, VM_PROT_DEFAULT, VM_PROT_ALL, VM_INHERIT_NONE), "vm_map");
	T_QUIET; T_ASSERT_MACH_SUCCESS(mach_port_deallocate(mach_task_self(), mem_handle), "mach_port_deallocate");

	if (fault_type == SOFT_FAULT) {
		/* Fault in all the pages of the original region. */
		for (ptr = memblock; ptr < memblock + memsize; ptr += pgsize) {
			val = *ptr;
		}
		/* Remap the region so that subsequent accesses result in read soft faults. */
		T_QUIET; T_ASSERT_MACH_SUCCESS(vm_remap(mach_task_self(), (vm_address_t *)&memblock_share,
		    memsize, 0, VM_FLAGS_ANYWHERE, mach_task_self(), (vm_address_t)memblock, FALSE,
		    &curprot, &maxprot, VM_INHERIT_DEFAULT), "vm_remap");
	}
	setup_per_thread_regions(memblock, memblock_share, fault_type, memsize);
}

/* Allocates a separate VM region for each thread. */
static void
map_mem_regions_multiple(int fault_type, size_t memsize)
{
	int i;
	size_t region_len, num_pages;
	volatile char val;
	char *ptr, *memblock, *memblock_share;
	vm_prot_t curprot, maxprot;

	num_pages = memsize / pgsize;

	for (i = 0; i < num_threads; i++) {
		memblock = NULL;

		region_len = num_pages / (size_t)num_threads;
		if ((size_t)i < num_pages % (size_t)num_threads) {
			region_len++;
		}
		region_len *= pgsize;

		int flags = VM_MAKE_TAG((i % 2)? VM_TAG1 : VM_TAG2) | MAP_ANON | MAP_PRIVATE;

		memblock = (char *)mmap(NULL, region_len, PROT_READ | PROT_WRITE, flags, -1, 0);
		T_QUIET; T_ASSERT_NE((void *)memblock, MAP_FAILED, "mmap");
		memregion_config_per_thread[i].region_addr = memblock;
		memregion_config_per_thread[i].shared_region_addr = 0;
		memregion_config_per_thread[i].region_len = region_len;

		if (fault_type == SOFT_FAULT) {
			/* Fault in all the pages of the original region. */
			for (ptr = memblock; ptr < memblock + region_len; ptr += pgsize) {
				val = *ptr;
			}
			memblock_share = NULL;
			/* Remap the region so that subsequent accesses result in read soft faults. */
			T_QUIET; T_ASSERT_MACH_SUCCESS(vm_remap(mach_task_self(), (vm_address_t *)&memblock_share,
			    region_len, 0, VM_FLAGS_ANYWHERE, mach_task_self(), (vm_address_t)memblock, FALSE,
			    &curprot, &maxprot, VM_INHERIT_DEFAULT), "vm_remap");
			memregion_config_per_thread[i].shared_region_addr = memblock_share;
		}
	}
}

static void
map_mem_regions(int fault_type, int mapping_variant, size_t memsize)
{
	memregion_config_per_thread = (memregion_config *)malloc(sizeof(*memregion_config_per_thread) * (size_t)num_threads);
	switch (mapping_variant) {
	case VARIANT_SINGLE_REGION:
		map_mem_regions_single(fault_type, memsize);
		break;
	case VARIANT_MULTIPLE_REGIONS:
		map_mem_regions_multiple(fault_type, memsize);
		break;
	case VARIANT_DEFAULT:
	default:
		map_mem_regions_default(fault_type, memsize);
	}
}

static void
setup_per_thread_regions(char *memblock, char *memblock_share, int fault_type, size_t memsize)
{
	int i;
	size_t region_len, region_start, num_pages;

	num_pages = memsize / pgsize;
	for (i = 0; i < num_threads; i++) {
		region_len = num_pages / (size_t)num_threads;
		region_start = region_len * (size_t)i;

		if ((size_t)i < num_pages % (size_t)num_threads) {
			region_start += (size_t)i;
			region_len++;
		} else {
			region_start += num_pages % (size_t)num_threads;
		}

		region_start *= pgsize;
		region_len *= pgsize;

		memregion_config_per_thread[i].region_addr = memblock + region_start;
		memregion_config_per_thread[i].shared_region_addr = ((fault_type == SOFT_FAULT) ?
		    memblock_share + region_start : 0);
		memregion_config_per_thread[i].region_len = region_len;
	}
}

static void
unmap_mem_regions(int mapping_variant, size_t memsize)
{
	if (mapping_variant == VARIANT_MULTIPLE_REGIONS) {
		int i;
		for (i = 0; i < num_threads; i++) {
			if (memregion_config_per_thread[i].shared_region_addr != 0) {
				T_QUIET; T_ASSERT_MACH_SUCCESS(munmap(memregion_config_per_thread[i].shared_region_addr,
				    memregion_config_per_thread[i].region_len), "munmap");
			}
			T_QUIET; T_ASSERT_MACH_SUCCESS(munmap(memregion_config_per_thread[i].region_addr,
			    memregion_config_per_thread[i].region_len), "munmap");
		}
	} else {
		if (memregion_config_per_thread[0].shared_region_addr != 0) {
			T_QUIET; T_ASSERT_MACH_SUCCESS(munmap(memregion_config_per_thread[0].shared_region_addr, memsize), "munmap");
		}
		T_QUIET; T_ASSERT_MACH_SUCCESS(munmap(memregion_config_per_thread[0].region_addr, memsize), "munmap");
	}
}

static void
fault_pages(int thread_id)
{
	char *ptr, *block;
	volatile char val;

	block = memregion_config_per_thread[thread_id].shared_region_addr ?
	    memregion_config_per_thread[thread_id].shared_region_addr :
	    memregion_config_per_thread[thread_id].region_addr;
	for (ptr = block; ptr < block + memregion_config_per_thread[thread_id].region_len; ptr += pgsize) {
		val = *ptr;
	}
}

static void *
thread_setup(void *arg)
{
	int my_index = *((int *)arg);

	T_QUIET; T_ASSERT_POSIX_SUCCESS(pthread_mutex_lock(&ready_thread_count_lock), "pthread_mutex_lock");
	ready_thread_count++;
	if (ready_thread_count == num_threads) {
		T_QUIET; T_ASSERT_POSIX_SUCCESS(pthread_cond_signal(&threads_ready_cvar), "pthread_cond_signal");
	}
	T_QUIET; T_ASSERT_POSIX_SUCCESS(pthread_cond_wait(&start_cvar, &ready_thread_count_lock), "pthread_cond_wait");
	T_QUIET; T_ASSERT_POSIX_SUCCESS(pthread_mutex_unlock(&ready_thread_count_lock), "pthread_mutex_unlock");

	fault_pages(my_index);

	/* Up the finished count */
	T_QUIET; T_ASSERT_POSIX_SUCCESS(pthread_mutex_lock(&finished_thread_count_lock), "pthread_mutex_lock");
	finished_thread_count++;
	if (finished_thread_count == num_threads) {
		/* All the threads are done. Wake up the main thread */
		T_QUIET; T_ASSERT_POSIX_SUCCESS(pthread_cond_signal(&threads_finished_cvar), "pthread_cond_signal");
	}
	T_QUIET; T_ASSERT_POSIX_SUCCESS(pthread_mutex_unlock(&finished_thread_count_lock), "pthread_mutex_unlock");
	return NULL;
}

static void
execute_threads(void)
{
	int thread_index, thread_retval;
	int *thread_indices;
	void *thread_retval_ptr = &thread_retval;
	pthread_t* threads;

	T_QUIET; T_ASSERT_POSIX_SUCCESS(pthread_cond_init(&threads_ready_cvar, NULL), "pthread_cond_init");
	T_QUIET; T_ASSERT_POSIX_SUCCESS(pthread_cond_init(&start_cvar, NULL), "pthread_cond_init");
	T_QUIET; T_ASSERT_POSIX_SUCCESS(pthread_mutex_init(&ready_thread_count_lock, NULL), "pthread_mutex_init");
	T_QUIET; T_ASSERT_POSIX_SUCCESS(pthread_cond_init(&threads_finished_cvar, NULL), "pthread_cond_init");
	T_QUIET; T_ASSERT_POSIX_SUCCESS(pthread_mutex_init(&finished_thread_count_lock, NULL), "pthread_mutex_init");
	ready_thread_count = 0;
	finished_thread_count = 0;

	threads = (pthread_t *)malloc(sizeof(*threads) * (size_t)num_threads);
	thread_indices = (int *)malloc(sizeof(*thread_indices) * (size_t)num_threads);
	for (thread_index = 0; thread_index < num_threads; thread_index++) {
		thread_indices[thread_index] = thread_index;
		T_QUIET; T_ASSERT_POSIX_SUCCESS(pthread_create(&threads[thread_index], NULL,
		    thread_setup, (void *)&thread_indices[thread_index]), "pthread_create");
	}

	T_QUIET; T_ASSERT_POSIX_SUCCESS(pthread_mutex_lock(&ready_thread_count_lock), "pthread_mutex_lock");
	while (ready_thread_count != num_threads) {
		T_QUIET; T_ASSERT_POSIX_SUCCESS(pthread_cond_wait(&threads_ready_cvar, &ready_thread_count_lock),
		    "pthread_cond_wait");
	}
	T_QUIET; T_ASSERT_POSIX_SUCCESS(pthread_mutex_unlock(&ready_thread_count_lock), "pthread_mutex_unlock");

	T_STAT_MEASURE(runtime) {
		/* Ungate the threads */
		T_QUIET; T_ASSERT_POSIX_SUCCESS(pthread_cond_broadcast(&start_cvar), "pthread_cond_broadcast");
		/* Wait for the threads to finish */
		T_QUIET; T_ASSERT_POSIX_SUCCESS(pthread_mutex_lock(&finished_thread_count_lock), "pthread_mutex_lock");
		while (finished_thread_count != num_threads) {
			T_QUIET; T_ASSERT_POSIX_SUCCESS(pthread_cond_wait(&threads_finished_cvar, &finished_thread_count_lock), "pthread_cond_wait");
		}
	};

	/* Join the threads */
	for (thread_index = 0; thread_index < num_threads; thread_index++) {
		T_QUIET; T_ASSERT_POSIX_SUCCESS(pthread_join(threads[thread_index], &thread_retval_ptr),
		    "pthread_join");
	}

	free(threads);
	free(thread_indices);
}

static void
run_test(int fault_type, int mapping_variant, size_t memsize)
{
	char metric_str[32];
	size_t num_pages;
	size_t sysctl_size = sizeof(pgsize);
	int ret = sysctlbyname("vm.pagesize", &pgsize, &sysctl_size, NULL, 0);
	T_QUIET; T_ASSERT_POSIX_SUCCESS(ret, "sysctl vm.pagesize failed");

	num_pages = memsize / pgsize;

	T_QUIET; T_ASSERT_LT(fault_type, NUM_FAULT_TYPES, "invalid test type");
	T_QUIET; T_ASSERT_LT(mapping_variant, NUM_MAPPING_VARIANTS, "invalid mapping variant");
	T_QUIET; T_ASSERT_GT(num_threads, 0, "num_threads <= 0");
	T_QUIET; T_ASSERT_GT((int)num_pages / num_threads, 0, "num_pages/num_threads <= 0");

	T_LOG("No. of cpus:     %d", get_ncpu());
	T_LOG("No. of threads:  %d", num_threads);
	T_LOG("No. of pages:    %ld", num_pages);
	T_LOG("Pagesize:        %ld", pgsize);
	T_LOG("Allocation size: %ld MB", memsize / (1024 * 1024));
	T_LOG("Mapping variant: %s", variant_str[mapping_variant]);

	snprintf(metric_str, 32, "Runtime-%s", variant_str[mapping_variant]);
	runtime = dt_stat_time_create(metric_str);

	while (!dt_stat_stable(runtime)) {
		map_mem_regions(fault_type, mapping_variant, memsize);
		execute_threads();
		unmap_mem_regions(mapping_variant, memsize);
	}

	dt_stat_finalize(runtime);
	T_LOG("Throughput-%s (MB/s): %lf\n\n", variant_str[mapping_variant], (double)memsize / (1024 * 1024) / dt_stat_mean((dt_stat_t)runtime));
}

static void
setup_and_run_test(int fault_type, int threads)
{
	int i, mapping_variant;
	size_t memsize;
	char *e;

	mapping_variant = VARIANT_DEFAULT;
	memsize = MEMSIZE;
	num_threads = threads;

	if ((e = getenv("NTHREADS"))) {
		if (threads == 1) {
			T_SKIP("Custom environment variables specified. Skipping single threaded version.");
		}
		num_threads = (int)strtol(e, NULL, 0);
	}

	if ((e = getenv("MEMSIZEMB"))) {
		memsize = (size_t)strtol(e, NULL, 0) * 1024 * 1024;
	}

	if ((e = getenv("VARIANT"))) {
		mapping_variant = (int)strtol(e, NULL, 0);
		run_test(fault_type, mapping_variant, memsize);
	} else {
		for (i = VARIANT_DEFAULT; i < NUM_MAPPING_VARIANTS; i++) {
			run_test(fault_type, i, memsize);
		}
	}

	T_END;
}

static int
get_ncpu(void)
{
	int ncpu;
	size_t length = sizeof(ncpu);

	T_QUIET; T_ASSERT_POSIX_SUCCESS(sysctlbyname("hw.ncpu", &ncpu, &length, NULL, 0),
	    "failed to query hw.ncpu");
	return ncpu;
}

T_DECL(read_soft_fault,
    "Read soft faults (single thread)")
{
	setup_and_run_test(SOFT_FAULT, 1);
}

T_DECL(read_soft_fault_multithreaded,
    "Read soft faults (multi-threaded)")
{
	char *e;
	int nthreads;

	/* iOSMark passes in the no. of threads via an env. variable */
	if ((e = getenv("DT_STAT_NTHREADS"))) {
		nthreads = (int)strtol(e, NULL, 0);
	} else {
		nthreads = get_ncpu();
		if (nthreads == 1) {
			T_SKIP("Skipping multi-threaded test on single core device.");
		}
	}
	setup_and_run_test(SOFT_FAULT, nthreads);
}

T_DECL(zero_fill_fault,
    "Zero fill faults (single thread)")
{
	setup_and_run_test(ZERO_FILL, 1);
}

T_DECL(zero_fill_fault_multithreaded,
    "Zero fill faults (multi-threaded)")
{
	char *e;
	int nthreads;

	/* iOSMark passes in the no. of threads via an env. variable */
	if ((e = getenv("DT_STAT_NTHREADS"))) {
		nthreads = (int)strtol(e, NULL, 0);
	} else {
		nthreads = get_ncpu();
		if (nthreads == 1) {
			T_SKIP("Skipping multi-threaded test on single core device.");
		}
	}
	setup_and_run_test(ZERO_FILL, nthreads);
}
Commit	Line	Data
d9a64523 A	1	#include <unistd.h>
	2	#include <stdlib.h>
	3	#include <sys/mman.h>
	4	#include <sys/sysctl.h>
	5	#include <mach/mach.h>
	6	#include <mach/vm_map.h>
	7	#include <darwintest.h>
	8	#include <TargetConditionals.h>
	9	#include <perfcheck_keys.h>
	10
	11	T_GLOBAL_META(
	12	T_META_NAMESPACE("xnu.vm.perf"),
	13	T_META_CHECK_LEAKS(false),
	14	T_META_TAG_PERF
0a7de745	15	);
d9a64523 A	16
d9a64523 A	17	#ifdef DT_IOSMARK
0a7de745	18	#define MEMSIZE (1UL<<29) /* 512 MB */
d9a64523	19	#else
0a7de745	20	#define MEMSIZE (1UL<<27) /* 128 MB */
d9a64523 A	21	#endif
d9a64523 A	22
0a7de745 A	23	#define VM_TAG1 100
	24	#define VM_TAG2 101
	25
d9a64523 A	26	enum {
	27	SOFT_FAULT,
	28	ZERO_FILL,
0a7de745 A	29	NUM_FAULT_TYPES
	30	};
	31
	32	enum {
	33	VARIANT_DEFAULT = 1,
	34	VARIANT_SINGLE_REGION,
	35	VARIANT_MULTIPLE_REGIONS,
	36	NUM_MAPPING_VARIANTS
	37	};
	38
	39	static char *variant_str[] = {
	40	"none",
	41	"default",
	42	"single-region",
	43	"multiple-regions"
d9a64523 A	44	};
d9a64523 A	45
0a7de745 A	46
	47	typedef struct {
	48	char *region_addr;
	49	char *shared_region_addr;
	50	size_t region_len;
	51	} memregion_config;
	52
	53	static memregion_config *memregion_config_per_thread;
	54
	55	static size_t pgsize;
d9a64523 A	56	static int num_threads;
d9a64523 A	57	static int ready_thread_count;
cb323159	58	static int finished_thread_count;
0a7de745	59	static dt_stat_time_t runtime;
d9a64523 A	60	static pthread_cond_t start_cvar;
d9a64523 A	61	static pthread_cond_t threads_ready_cvar;
cb323159	62	static pthread_cond_t threads_finished_cvar;
d9a64523	63	static pthread_mutex_t ready_thread_count_lock;
cb323159	64	static pthread_mutex_t finished_thread_count_lock;
d9a64523	65
0a7de745 A	66	static void map_mem_regions_default(int fault_type, size_t memsize);
	67	static void map_mem_regions_single(int fault_type, size_t memsize);
	68	static void map_mem_regions_multiple(int fault_type, size_t memsize);
	69	static void map_mem_regions(int fault_type, int mapping_variant, size_t memsize);
	70	static void unmap_mem_regions(int mapping_variant, size_t memsize);
	71	static void setup_per_thread_regions(char memblock, char memblock_share, int fault_type, size_t memsize);
d9a64523 A	72	static void fault_pages(int thread_id);
	73	static void execute_threads(void);
	74	static void thread_setup(void arg);
0a7de745 A	75	static void run_test(int fault_type, int mapping_variant, size_t memsize);
0a7de745 A	76	static void setup_and_run_test(int test, int threads);
d9a64523 A	77	static int get_ncpu(void);
d9a64523 A	78
0a7de745 A	79	/* Allocates memory using the default mmap behavior. Each VM region created is capped at 128 MB. */
	80	static void
	81	map_mem_regions_default(int fault_type, size_t memsize)
d9a64523	82	{
d9a64523 A	83	volatile char val;
d9a64523 A	84	vm_prot_t curprot, maxprot;
0a7de745	85	char ptr, memblock, *memblock_share = NULL;
d9a64523	86
0a7de745	87	memblock = (char *)mmap(NULL, memsize, PROT_READ \| PROT_WRITE, MAP_ANON \| MAP_PRIVATE, -1, 0);
d9a64523 A	88	T_QUIET; T_ASSERT_NE((void *)memblock, MAP_FAILED, "mmap");
d9a64523 A	89
0a7de745	90	if (fault_type == SOFT_FAULT) {
d9a64523	91	/* Fault in all the pages of the original region. */
0a7de745	92	for (ptr = memblock; ptr < memblock + memsize; ptr += pgsize) {
d9a64523 A	93	val = *ptr;
	94	}
	95	/* Remap the region so that subsequent accesses result in read soft faults. */
	96	T_QUIET; T_ASSERT_MACH_SUCCESS(vm_remap(mach_task_self(), (vm_address_t *)&memblock_share,
0a7de745 A	97	memsize, 0, VM_FLAGS_ANYWHERE, mach_task_self(), (vm_address_t)memblock, FALSE,
0a7de745 A	98	&curprot, &maxprot, VM_INHERIT_DEFAULT), "vm_remap");
d9a64523	99	}
0a7de745	100	setup_per_thread_regions(memblock, memblock_share, fault_type, memsize);
d9a64523 A	101	}
d9a64523 A	102
0a7de745 A	103	/* Creates a single VM region by mapping in a named memory entry. */
	104	static void
	105	map_mem_regions_single(int fault_type, size_t memsize)
d9a64523	106	{
0a7de745 A	107	volatile char val;
	108	vm_prot_t curprot, maxprot;
	109	char ptr, memblock = NULL, *memblock_share = NULL;
	110	vm_size_t size = memsize;
	111	vm_offset_t addr1 = 0;
	112	mach_port_t mem_handle = MACH_PORT_NULL;
	113
	114	/* Allocate a region and fault in all the pages. */
	115	T_QUIET; T_ASSERT_MACH_SUCCESS(vm_allocate(mach_task_self(), &addr1, size, VM_FLAGS_ANYWHERE), "vm_allocate");
	116	for (ptr = (char )addr1; ptr < (char )addr1 + memsize; ptr += pgsize) {
	117	val = *ptr;
d9a64523	118	}
0a7de745 A	119
	120	/* Create a named memory entry from the region allocated above, and de-allocate said region. */
	121	T_QUIET; T_ASSERT_MACH_SUCCESS(mach_make_memory_entry(mach_task_self(), &size, addr1, VM_PROT_ALL \| MAP_MEM_NAMED_CREATE,
	122	&mem_handle, MACH_PORT_NULL), "mach_make_memory_entry");
	123	T_QUIET; T_ASSERT_MACH_SUCCESS(vm_deallocate(mach_task_self(), addr1, size), "vm_deallocate");
	124
	125	/* Map in the named entry and deallocate it. */
	126	T_QUIET; T_ASSERT_MACH_SUCCESS(vm_map(mach_task_self(), (vm_address_t *)&memblock, size, 0, VM_FLAGS_ANYWHERE, mem_handle, 0,
	127	FALSE, VM_PROT_DEFAULT, VM_PROT_ALL, VM_INHERIT_NONE), "vm_map");
	128	T_QUIET; T_ASSERT_MACH_SUCCESS(mach_port_deallocate(mach_task_self(), mem_handle), "mach_port_deallocate");
	129
	130	if (fault_type == SOFT_FAULT) {
	131	/* Fault in all the pages of the original region. */
	132	for (ptr = memblock; ptr < memblock + memsize; ptr += pgsize) {
	133	val = *ptr;
	134	}
	135	/* Remap the region so that subsequent accesses result in read soft faults. */
	136	T_QUIET; T_ASSERT_MACH_SUCCESS(vm_remap(mach_task_self(), (vm_address_t *)&memblock_share,
	137	memsize, 0, VM_FLAGS_ANYWHERE, mach_task_self(), (vm_address_t)memblock, FALSE,
	138	&curprot, &maxprot, VM_INHERIT_DEFAULT), "vm_remap");
	139	}
	140	setup_per_thread_regions(memblock, memblock_share, fault_type, memsize);
d9a64523 A	141	}
d9a64523 A	142
0a7de745 A	143	/* Allocates a separate VM region for each thread. */
	144	static void
	145	map_mem_regions_multiple(int fault_type, size_t memsize)
d9a64523	146	{
0a7de745 A	147	int i;
0a7de745 A	148	size_t region_len, num_pages;
d9a64523	149	volatile char val;
0a7de745 A	150	char ptr, memblock, *memblock_share;
0a7de745 A	151	vm_prot_t curprot, maxprot;
d9a64523	152
0a7de745	153	num_pages = memsize / pgsize;
d9a64523	154
0a7de745 A	155	for (i = 0; i < num_threads; i++) {
	156	memblock = NULL;
	157
	158	region_len = num_pages / (size_t)num_threads;
	159	if ((size_t)i < num_pages % (size_t)num_threads) {
	160	region_len++;
	161	}
	162	region_len *= pgsize;
	163
	164	int flags = VM_MAKE_TAG((i % 2)? VM_TAG1 : VM_TAG2) \| MAP_ANON \| MAP_PRIVATE;
	165
	166	memblock = (char *)mmap(NULL, region_len, PROT_READ \| PROT_WRITE, flags, -1, 0);
	167	T_QUIET; T_ASSERT_NE((void *)memblock, MAP_FAILED, "mmap");
	168	memregion_config_per_thread[i].region_addr = memblock;
	169	memregion_config_per_thread[i].shared_region_addr = 0;
	170	memregion_config_per_thread[i].region_len = region_len;
	171
	172	if (fault_type == SOFT_FAULT) {
	173	/* Fault in all the pages of the original region. */
	174	for (ptr = memblock; ptr < memblock + region_len; ptr += pgsize) {
	175	val = *ptr;
	176	}
	177	memblock_share = NULL;
	178	/* Remap the region so that subsequent accesses result in read soft faults. */
	179	T_QUIET; T_ASSERT_MACH_SUCCESS(vm_remap(mach_task_self(), (vm_address_t *)&memblock_share,
	180	region_len, 0, VM_FLAGS_ANYWHERE, mach_task_self(), (vm_address_t)memblock, FALSE,
	181	&curprot, &maxprot, VM_INHERIT_DEFAULT), "vm_remap");
	182	memregion_config_per_thread[i].shared_region_addr = memblock_share;
	183	}
d9a64523	184	}
0a7de745 A	185	}
	186
	187	static void
	188	map_mem_regions(int fault_type, int mapping_variant, size_t memsize)
	189	{
	190	memregion_config_per_thread = (memregion_config )malloc(sizeof(memregion_config_per_thread) * (size_t)num_threads);
	191	switch (mapping_variant) {
	192	case VARIANT_SINGLE_REGION:
	193	map_mem_regions_single(fault_type, memsize);
	194	break;
	195	case VARIANT_MULTIPLE_REGIONS:
	196	map_mem_regions_multiple(fault_type, memsize);
	197	break;
	198	case VARIANT_DEFAULT:
	199	default:
	200	map_mem_regions_default(fault_type, memsize);
d9a64523	201	}
0a7de745 A	202	}
	203
	204	static void
	205	setup_per_thread_regions(char memblock, char memblock_share, int fault_type, size_t memsize)
	206	{
	207	int i;
	208	size_t region_len, region_start, num_pages;
	209
	210	num_pages = memsize / pgsize;
	211	for (i = 0; i < num_threads; i++) {
	212	region_len = num_pages / (size_t)num_threads;
	213	region_start = region_len * (size_t)i;
	214
	215	if ((size_t)i < num_pages % (size_t)num_threads) {
	216	region_start += (size_t)i;
	217	region_len++;
	218	} else {
	219	region_start += num_pages % (size_t)num_threads;
	220	}
d9a64523	221
0a7de745 A	222	region_start *= pgsize;
0a7de745 A	223	region_len *= pgsize;
d9a64523	224
0a7de745 A	225	memregion_config_per_thread[i].region_addr = memblock + region_start;
	226	memregion_config_per_thread[i].shared_region_addr = ((fault_type == SOFT_FAULT) ?
	227	memblock_share + region_start : 0);
	228	memregion_config_per_thread[i].region_len = region_len;
	229	}
	230	}
	231
	232	static void
	233	unmap_mem_regions(int mapping_variant, size_t memsize)
	234	{
	235	if (mapping_variant == VARIANT_MULTIPLE_REGIONS) {
	236	int i;
	237	for (i = 0; i < num_threads; i++) {
	238	if (memregion_config_per_thread[i].shared_region_addr != 0) {
	239	T_QUIET; T_ASSERT_MACH_SUCCESS(munmap(memregion_config_per_thread[i].shared_region_addr,
	240	memregion_config_per_thread[i].region_len), "munmap");
	241	}
	242	T_QUIET; T_ASSERT_MACH_SUCCESS(munmap(memregion_config_per_thread[i].region_addr,
	243	memregion_config_per_thread[i].region_len), "munmap");
	244	}
	245	} else {
	246	if (memregion_config_per_thread[0].shared_region_addr != 0) {
	247	T_QUIET; T_ASSERT_MACH_SUCCESS(munmap(memregion_config_per_thread[0].shared_region_addr, memsize), "munmap");
	248	}
	249	T_QUIET; T_ASSERT_MACH_SUCCESS(munmap(memregion_config_per_thread[0].region_addr, memsize), "munmap");
	250	}
	251	}
	252
	253	static void
	254	fault_pages(int thread_id)
	255	{
	256	char ptr, block;
	257	volatile char val;
	258
	259	block = memregion_config_per_thread[thread_id].shared_region_addr ?
	260	memregion_config_per_thread[thread_id].shared_region_addr :
	261	memregion_config_per_thread[thread_id].region_addr;
	262	for (ptr = block; ptr < block + memregion_config_per_thread[thread_id].region_len; ptr += pgsize) {
d9a64523 A	263	val = *ptr;
	264	}
	265	}
	266
0a7de745 A	267	static void *
	268	thread_setup(void *arg)
	269	{
	270	int my_index = ((int )arg);
	271
	272	T_QUIET; T_ASSERT_POSIX_SUCCESS(pthread_mutex_lock(&ready_thread_count_lock), "pthread_mutex_lock");
	273	ready_thread_count++;
	274	if (ready_thread_count == num_threads) {
	275	T_QUIET; T_ASSERT_POSIX_SUCCESS(pthread_cond_signal(&threads_ready_cvar), "pthread_cond_signal");
	276	}
	277	T_QUIET; T_ASSERT_POSIX_SUCCESS(pthread_cond_wait(&start_cvar, &ready_thread_count_lock), "pthread_cond_wait");
	278	T_QUIET; T_ASSERT_POSIX_SUCCESS(pthread_mutex_unlock(&ready_thread_count_lock), "pthread_mutex_unlock");
	279
	280	fault_pages(my_index);
cb323159 A	281
	282	/* Up the finished count */
	283	T_QUIET; T_ASSERT_POSIX_SUCCESS(pthread_mutex_lock(&finished_thread_count_lock), "pthread_mutex_lock");
	284	finished_thread_count++;
	285	if (finished_thread_count == num_threads) {
	286	/* All the threads are done. Wake up the main thread */
	287	T_QUIET; T_ASSERT_POSIX_SUCCESS(pthread_cond_signal(&threads_finished_cvar), "pthread_cond_signal");
	288	}
	289	T_QUIET; T_ASSERT_POSIX_SUCCESS(pthread_mutex_unlock(&finished_thread_count_lock), "pthread_mutex_unlock");
0a7de745 A	290	return NULL;
	291	}
	292
	293	static void
	294	execute_threads(void)
d9a64523 A	295	{
	296	int thread_index, thread_retval;
	297	int *thread_indices;
0a7de745	298	void *thread_retval_ptr = &thread_retval;
d9a64523 A	299	pthread_t* threads;
	300
	301	T_QUIET; T_ASSERT_POSIX_SUCCESS(pthread_cond_init(&threads_ready_cvar, NULL), "pthread_cond_init");
	302	T_QUIET; T_ASSERT_POSIX_SUCCESS(pthread_cond_init(&start_cvar, NULL), "pthread_cond_init");
	303	T_QUIET; T_ASSERT_POSIX_SUCCESS(pthread_mutex_init(&ready_thread_count_lock, NULL), "pthread_mutex_init");
cb323159 A	304	T_QUIET; T_ASSERT_POSIX_SUCCESS(pthread_cond_init(&threads_finished_cvar, NULL), "pthread_cond_init");
cb323159 A	305	T_QUIET; T_ASSERT_POSIX_SUCCESS(pthread_mutex_init(&finished_thread_count_lock, NULL), "pthread_mutex_init");
d9a64523	306	ready_thread_count = 0;
cb323159	307	finished_thread_count = 0;
d9a64523 A	308
	309	threads = (pthread_t )malloc(sizeof(threads) * (size_t)num_threads);
	310	thread_indices = (int )malloc(sizeof(thread_indices) * (size_t)num_threads);
0a7de745	311	for (thread_index = 0; thread_index < num_threads; thread_index++) {
d9a64523 A	312	thread_indices[thread_index] = thread_index;
d9a64523 A	313	T_QUIET; T_ASSERT_POSIX_SUCCESS(pthread_create(&threads[thread_index], NULL,
0a7de745	314	thread_setup, (void *)&thread_indices[thread_index]), "pthread_create");
d9a64523 A	315	}
	316
	317	T_QUIET; T_ASSERT_POSIX_SUCCESS(pthread_mutex_lock(&ready_thread_count_lock), "pthread_mutex_lock");
cb323159	318	while (ready_thread_count != num_threads) {
d9a64523	319	T_QUIET; T_ASSERT_POSIX_SUCCESS(pthread_cond_wait(&threads_ready_cvar, &ready_thread_count_lock),
0a7de745	320	"pthread_cond_wait");
d9a64523 A	321	}
	322	T_QUIET; T_ASSERT_POSIX_SUCCESS(pthread_mutex_unlock(&ready_thread_count_lock), "pthread_mutex_unlock");
	323
0a7de745	324	T_STAT_MEASURE(runtime) {
cb323159	325	/* Ungate the threads */
d9a64523	326	T_QUIET; T_ASSERT_POSIX_SUCCESS(pthread_cond_broadcast(&start_cvar), "pthread_cond_broadcast");
cb323159 A	327	/* Wait for the threads to finish */
	328	T_QUIET; T_ASSERT_POSIX_SUCCESS(pthread_mutex_lock(&finished_thread_count_lock), "pthread_mutex_lock");
	329	while (finished_thread_count != num_threads) {
	330	T_QUIET; T_ASSERT_POSIX_SUCCESS(pthread_cond_wait(&threads_finished_cvar, &finished_thread_count_lock), "pthread_cond_wait");
d9a64523 A	331	}
	332	};
	333
cb323159 A	334	/* Join the threads */
	335	for (thread_index = 0; thread_index < num_threads; thread_index++) {
	336	T_QUIET; T_ASSERT_POSIX_SUCCESS(pthread_join(threads[thread_index], &thread_retval_ptr),
	337	"pthread_join");
	338	}
	339
d9a64523 A	340	free(threads);
	341	free(thread_indices);
	342	}
	343
0a7de745 A	344	static void
0a7de745 A	345	run_test(int fault_type, int mapping_variant, size_t memsize)
d9a64523	346	{
0a7de745 A	347	char metric_str[32];
0a7de745 A	348	size_t num_pages;
d9a64523 A	349	size_t sysctl_size = sizeof(pgsize);
	350	int ret = sysctlbyname("vm.pagesize", &pgsize, &sysctl_size, NULL, 0);
	351	T_QUIET; T_ASSERT_POSIX_SUCCESS(ret, "sysctl vm.pagesize failed");
	352
0a7de745	353	num_pages = memsize / pgsize;
d9a64523	354
0a7de745 A	355	T_QUIET; T_ASSERT_LT(fault_type, NUM_FAULT_TYPES, "invalid test type");
0a7de745 A	356	T_QUIET; T_ASSERT_LT(mapping_variant, NUM_MAPPING_VARIANTS, "invalid mapping variant");
d9a64523	357	T_QUIET; T_ASSERT_GT(num_threads, 0, "num_threads <= 0");
0a7de745	358	T_QUIET; T_ASSERT_GT((int)num_pages / num_threads, 0, "num_pages/num_threads <= 0");
d9a64523	359
0a7de745	360	T_LOG("No. of cpus: %d", get_ncpu());
d9a64523 A	361	T_LOG("No. of threads: %d", num_threads);
	362	T_LOG("No. of pages: %ld", num_pages);
	363	T_LOG("Pagesize: %ld", pgsize);
0a7de745 A	364	T_LOG("Allocation size: %ld MB", memsize / (1024 * 1024));
	365	T_LOG("Mapping variant: %s", variant_str[mapping_variant]);
	366
	367	snprintf(metric_str, 32, "Runtime-%s", variant_str[mapping_variant]);
	368	runtime = dt_stat_time_create(metric_str);
d9a64523	369
0a7de745 A	370	while (!dt_stat_stable(runtime)) {
0a7de745 A	371	map_mem_regions(fault_type, mapping_variant, memsize);
d9a64523	372	execute_threads();
0a7de745 A	373	unmap_mem_regions(mapping_variant, memsize);
	374	}
	375
	376	dt_stat_finalize(runtime);
	377	T_LOG("Throughput-%s (MB/s): %lf\n\n", variant_str[mapping_variant], (double)memsize / (1024 * 1024) / dt_stat_mean((dt_stat_t)runtime));
	378	}
	379
	380	static void
	381	setup_and_run_test(int fault_type, int threads)
	382	{
	383	int i, mapping_variant;
	384	size_t memsize;
	385	char *e;
	386
	387	mapping_variant = VARIANT_DEFAULT;
	388	memsize = MEMSIZE;
	389	num_threads = threads;
	390
	391	if ((e = getenv("NTHREADS"))) {
	392	if (threads == 1) {
	393	T_SKIP("Custom environment variables specified. Skipping single threaded version.");
	394	}
	395	num_threads = (int)strtol(e, NULL, 0);
	396	}
	397
	398	if ((e = getenv("MEMSIZEMB"))) {
	399	memsize = (size_t)strtol(e, NULL, 0) * 1024 * 1024;
	400	}
	401
	402	if ((e = getenv("VARIANT"))) {
	403	mapping_variant = (int)strtol(e, NULL, 0);
	404	run_test(fault_type, mapping_variant, memsize);
	405	} else {
	406	for (i = VARIANT_DEFAULT; i < NUM_MAPPING_VARIANTS; i++) {
	407	run_test(fault_type, i, memsize);
	408	}
d9a64523 A	409	}
d9a64523 A	410
d9a64523 A	411	T_END;
	412	}
	413
0a7de745 A	414	static int
0a7de745 A	415	get_ncpu(void)
d9a64523 A	416	{
	417	int ncpu;
	418	size_t length = sizeof(ncpu);
	419
	420	T_QUIET; T_ASSERT_POSIX_SUCCESS(sysctlbyname("hw.ncpu", &ncpu, &length, NULL, 0),
0a7de745	421	"failed to query hw.ncpu");
d9a64523 A	422	return ncpu;
	423	}
	424
	425	T_DECL(read_soft_fault,
0a7de745	426	"Read soft faults (single thread)")
d9a64523	427	{
0a7de745	428	setup_and_run_test(SOFT_FAULT, 1);
d9a64523 A	429	}
	430
	431	T_DECL(read_soft_fault_multithreaded,
0a7de745	432	"Read soft faults (multi-threaded)")
d9a64523 A	433	{
	434	char *e;
	435	int nthreads;
	436
	437	/* iOSMark passes in the no. of threads via an env. variable */
	438	if ((e = getenv("DT_STAT_NTHREADS"))) {
	439	nthreads = (int)strtol(e, NULL, 0);
	440	} else {
	441	nthreads = get_ncpu();
cb323159 A	442	if (nthreads == 1) {
	443	T_SKIP("Skipping multi-threaded test on single core device.");
	444	}
d9a64523	445	}
0a7de745	446	setup_and_run_test(SOFT_FAULT, nthreads);
d9a64523 A	447	}
	448
	449	T_DECL(zero_fill_fault,
0a7de745	450	"Zero fill faults (single thread)")
d9a64523	451	{
0a7de745	452	setup_and_run_test(ZERO_FILL, 1);
d9a64523 A	453	}
	454
	455	T_DECL(zero_fill_fault_multithreaded,
0a7de745	456	"Zero fill faults (multi-threaded)")
d9a64523 A	457	{
	458	char *e;
	459	int nthreads;
	460
	461	/* iOSMark passes in the no. of threads via an env. variable */
	462	if ((e = getenv("DT_STAT_NTHREADS"))) {
	463	nthreads = (int)strtol(e, NULL, 0);
	464	} else {
	465	nthreads = get_ncpu();
cb323159 A	466	if (nthreads == 1) {
	467	T_SKIP("Skipping multi-threaded test on single core device.");
	468	}
d9a64523	469	}
0a7de745	470	setup_and_run_test(ZERO_FILL, nthreads);
d9a64523	471	}