[apple/xnu.git] / tools / tests / TLBcoherency / TLBcoherency.c

/*
 * Copyright (c) 2011 Apple Inc. All rights reserved.
 *
 * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
 * 
 * This file contains Original Code and/or Modifications of Original Code
 * as defined in and that are subject to the Apple Public Source License
 * Version 2.0 (the 'License'). You may not use this file except in
 * compliance with the License. The rights granted to you under the License
 * may not be used to create, or enable the creation or redistribution of,
 * unlawful or unlicensed copies of an Apple operating system, or to
 * circumvent, violate, or enable the circumvention or violation of, any
 * terms of an Apple operating system software license agreement.
 * 
 * Please obtain a copy of the License at
 * http://www.opensource.apple.com/apsl/ and read it before using this file.
 * 
 * The Original Code and all software distributed under the License are
 * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
 * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
 * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
 * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
 * Please see the License for the specific language governing rights and
 * limitations under the License.
 * 
 * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
 */

/* A pool of threads which attempt to verify multiprocessor TLB coherency.
 * Creates -t threads, by default 4
 * Creates -s separate mmap(MAP_ANON) R/W mappings, sized at 1 page each but
 * alterable via -z <npages>
 * Initially read-faults each mapping in, verifying first-word zerofill--
 * The kernel typically uses the physical aperture to perform the zerofill
 * Writes map_address (page_aligned) | low 12 bits of the PID at the first word
 * This can help verify ASID related inconsistencies
 * Records a timestamp in a Structure associated with each mapping
 * With a custom kernel, it has the option of creating a remapping of the page in
 * the kernel's address space to exercise shared kernel mapping coherency.
 * Each thread subsequently loops around on the set of mappings. One thread is designated
 * the observer thread. The thread acquires a lock on the arena element,
 * verifies that the mapping has the expected pattern (Address | PID), if the
 * element is in the MAPPED state. Can optionally tell the kernel to check its
 * alias as well. If it notices a mismatch, it has the option to issue a syscall
 * to  stop kernel tracing. If the -f option is supplied, the test is terminated.
 * If the page has lingered beyond -l microseconds, non-observer threads will
 * unmap the page, optionally calling into the kernel to unmap its alias, and
 * repopulate the element.
 * After this sequence, the thread will optionally usleep for -p microseconds,
 * to allow for idle power management to engage if possible (errata might exist
 * in those areas), or context switches to occur.
 * Created Derek Kumar, 2011.
 */

#include <stdio.h>
#include <stdlib.h>
#include <unistd.h>
#include <sys/mman.h>
#include <pthread.h>
#include <string.h>
#include <mach/mach_time.h>
#include <libkern/OSAtomic.h>
#include <sys/syscall.h>
#include <sys/types.h>
#include <sys/sysctl.h>

typedef struct {
	OSSpinLock tlock;
	uintptr_t taddr;
	unsigned tstate;
	uint64_t tctime;
} cpage;

cpage *parray;

#define ARENASIZE (1024)
#define NTHREADS (4)
#define PAGE_LINGER_TIME (2000000)
#define MAX_THREADS (512)
#define MYSYS (215)
#define CONSISTENCY(...) fprintf(stderr, __VA_ARGS__ );

unsigned arenasize = ARENASIZE, mapping_size;
uint64_t page_linger_time = PAGE_LINGER_TIME;
enum arenastates {MTOUCHED = 1, UNMAPPED = 2, MAPPED = 4, WP =8};
enum syscaction {MDOMAP = 1, MDOUNMAP = 2, MDOCHECK = 4};
enum ttypes {OBSERVER = 1, LOOPER = 2};
bool trymode = true;
bool all_stop = false;
bool stop_on_failure = false;
bool reuse_addrs = true;
bool dosyscall = false;

pid_t cpid;
int sleepus;

pthread_t threads[MAX_THREADS];
uint32_t roles[MAX_THREADS];

void usage(char **a) {
	exit(1);
}

void set_enable(int val)
{
	int mib[6];
	size_t needed;

        mib[0] = CTL_KERN;
        mib[1] = KERN_KDEBUG;
        mib[2] = KERN_KDENABLE;
        mib[3] = val;
        mib[4] = 0;
        mib[5] = 0;

        if (sysctl(mib, 4, NULL, &needed, NULL, 0) < 0) {
                printf("trace facility failure, KERN_KDENABLE\n");
	}
}

void initialize_arena_element(int i) {
	__unused int sysret;
	void *hint = reuse_addrs ? (void *)0x1000 : NULL;
	parray[i].taddr = (uintptr_t)mmap(hint, mapping_size, PROT_READ | PROT_WRITE, MAP_ANON | MAP_SHARED, -1, 0);

	if (parray[i].taddr == (uintptr_t)MAP_FAILED) {
		perror("mmap");
		exit(2);
	}

#if	!defined(__LP64__)
	uint32_t pattern = parray[i].taddr;
	pattern |= cpid & 0xFFF;
//	memset_pattern4((void *)parray[i].taddr, &pattern, PAGE_SIZE); //
//	uncomment to fill the whole page, but a sufficiently unique first word
//	gets the job done without slowing down the test

#else
	uint64_t pattern = parray[i].taddr;
	pattern |= (cpid & 0xFFF);
//	memset_pattern8(parray[i].taddr, &pattern, PAGE_SIZE);
#endif

	uint64_t val = 	(*(uintptr_t *)parray[i].taddr);

	if (val != 0) {
		CONSISTENCY("Mismatch, actual: 0x%llx, expected: 0x%llx\n", (unsigned long long)val, 0ULL);
		if (stop_on_failure) {
			set_enable(0);
			exit(5);
		}
	}
	for (int k = 0; k < (mapping_size >> PAGE_SHIFT); k++) {
		*(uintptr_t *)(parray[i].taddr + k * PAGE_SIZE) = pattern;
	}

	parray[i].tctime = mach_absolute_time();
	parray[i].tstate = MTOUCHED;

	if (dosyscall) {
		sysret = syscall(MYSYS, MDOMAP, parray[i].taddr, pattern, i, mapping_size);
	}
}

void initialize_arena(void) {
	for (int i = 0; i < arenasize; i++) {
		initialize_arena_element(i);
	}
}

void *tlbexerciser(void *targs) {
	uint32_t role = *(uint32_t *)targs;
	__unused int sysret;
	printf("Starting thread %p, role: %u\n", pthread_self(), role);

	for(;;) {
		for (int i = 0; i < arenasize; i++) {
			if (all_stop)
				return NULL;

			if (trymode) {
				if (OSSpinLockTry(&parray[i].tlock) == false)
					continue;
			} else {
				OSSpinLockLock(&parray[i].tlock);
			}

			if (parray[i].tstate != UNMAPPED) {
				uintptr_t ad;
				ad = parray[i].taddr | (cpid & 0xFFF);
				uintptr_t val = *(uintptr_t *)parray[i].taddr;

				if (val != ad) {
					if (stop_on_failure)
						all_stop = true;
					syscall(180, 0x71BC0000, (ad >> 32), (ad & ~0), 0, 0, 0);
					CONSISTENCY("Mismatch, actual: 0x%llx, expected: 0x%llx\n", (unsigned long long)val, (unsigned long long)ad);
					if (stop_on_failure) {
						set_enable(0);
						exit(5);
					}
				}

				if (dosyscall) {
					sysret = syscall(MYSYS, MDOCHECK, parray[i].taddr, ad, i, 0);
				}

				if ((role != OBSERVER) && ((mach_absolute_time() - parray[i].tctime) > page_linger_time)) {
					parray[i].tstate = UNMAPPED;
					if (munmap((void *)parray[i].taddr, mapping_size) != 0) {
						perror("munmap");
					}

					if (dosyscall) {
						sysret = syscall(MYSYS, MDOUNMAP, parray[i].taddr, ad, i, mapping_size);
					}
				}
			} else {
				if (role != OBSERVER) {
					initialize_arena_element(i);
				}
			}

			parray[i].tlock = 0; //unlock

			if (sleepus)
				usleep(sleepus);
		}
	}

	return NULL;
}

int main(int argc, char **argv) {
	extern char *optarg;
	int arg;
	unsigned nthreads = NTHREADS;

	mapping_size = PAGE_SIZE;

	while ((arg = getopt(argc, argv, "l:t:h:s:p:z:fry")) != -1) {
		switch (arg) {
		case 'l':
			page_linger_time = strtoull(optarg, NULL, 0);
			break;
		case 't':
			nthreads = atoi(optarg);
			break;
		case 's':
			arenasize = atoi(optarg); // we typically want this to
						  // be sized < 2nd level TLB
			break;
		case 'f':
			stop_on_failure = true;
			break;
		case 'r':
			reuse_addrs = false;
			break;
		case 'p':
			sleepus = atoi(optarg);
			break;
		case 'y':
			dosyscall = true;
			break;
		case 'z':
			mapping_size = atoi(optarg) * PAGE_SIZE;
			break;
		case 'h':
			usage(argv);
		}
	}

	if(optind != argc) {
		usage(argv);
	}

	printf("page_linger_time: 0x%llx, nthreads: %u, arenasize: %u sleepus: %d reuse_addrs: %u, stop_on_failure: %u, dosyscall: %u, mappingsize: 0x%x\n", page_linger_time, nthreads, arenasize, sleepus, reuse_addrs, (unsigned) stop_on_failure, dosyscall, mapping_size);

	parray = calloc(arenasize, sizeof(cpage));
	cpid = getpid();

	initialize_arena();

	for (int dex = 0; dex < nthreads; dex++) {
		roles[dex] = LOOPER;
		if (dex == 0)
			roles[dex] = OBSERVER;
		int result = pthread_create(&threads[dex], NULL, tlbexerciser, &roles[dex]);
		if(result) {
			printf("pthread_create: %d starting worker thread; aborting.\n", result);
			return result;
		}
	}

	for(int dex = 0; dex < nthreads; dex++) {
		void *rtn;
		int result = pthread_join(threads[dex], &rtn);

		if(result) {
			printf("pthread_join(): %d, aborting\n", result);
			return result;
		}

		if(rtn) {
			printf("***Aborting on worker error\n");
			exit(1);
		}
	}
	return 0;
}
Commit	Line	Data
4bd07ac2 A	1	/*
	2	* Copyright (c) 2011 Apple Inc. All rights reserved.
	3	*
	4	* @APPLE_OSREFERENCE_LICENSE_HEADER_START@
	5	*
	6	* This file contains Original Code and/or Modifications of Original Code
	7	* as defined in and that are subject to the Apple Public Source License
	8	* Version 2.0 (the 'License'). You may not use this file except in
	9	* compliance with the License. The rights granted to you under the License
	10	* may not be used to create, or enable the creation or redistribution of,
	11	* unlawful or unlicensed copies of an Apple operating system, or to
	12	* circumvent, violate, or enable the circumvention or violation of, any
	13	* terms of an Apple operating system software license agreement.
	14	*
	15	* Please obtain a copy of the License at
	16	* http://www.opensource.apple.com/apsl/ and read it before using this file.
	17	*
	18	* The Original Code and all software distributed under the License are
	19	* distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
	20	* EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
	21	* INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
	22	* FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
	23	* Please see the License for the specific language governing rights and
	24	* limitations under the License.
	25	*
	26	* @APPLE_OSREFERENCE_LICENSE_HEADER_END@
	27	*/
	28
	29	/* A pool of threads which attempt to verify multiprocessor TLB coherency.
	30	* Creates -t threads, by default 4
	31	* Creates -s separate mmap(MAP_ANON) R/W mappings, sized at 1 page each but
	32	* alterable via -z <npages>
	33	* Initially read-faults each mapping in, verifying first-word zerofill--
	34	* The kernel typically uses the physical aperture to perform the zerofill
	35	* Writes map_address (page_aligned) \| low 12 bits of the PID at the first word
	36	* This can help verify ASID related inconsistencies
	37	* Records a timestamp in a Structure associated with each mapping
	38	* With a custom kernel, it has the option of creating a remapping of the page in
	39	* the kernel's address space to exercise shared kernel mapping coherency.
	40	* Each thread subsequently loops around on the set of mappings. One thread is designated
	41	* the observer thread. The thread acquires a lock on the arena element,
	42	* verifies that the mapping has the expected pattern (Address \| PID), if the
	43	* element is in the MAPPED state. Can optionally tell the kernel to check its
	44	* alias as well. If it notices a mismatch, it has the option to issue a syscall
	45	* to stop kernel tracing. If the -f option is supplied, the test is terminated.
	46	* If the page has lingered beyond -l microseconds, non-observer threads will
	47	* unmap the page, optionally calling into the kernel to unmap its alias, and
	48	* repopulate the element.
	49	* After this sequence, the thread will optionally usleep for -p microseconds,
	50	* to allow for idle power management to engage if possible (errata might exist
	51	* in those areas), or context switches to occur.
	52	* Created Derek Kumar, 2011.
	53	*/
	54
	55	#include <stdio.h>
	56	#include <stdlib.h>
	57	#include <unistd.h>
	58	#include <sys/mman.h>
	59	#include <pthread.h>
	60	#include <string.h>
	61	#include <mach/mach_time.h>
	62	#include <libkern/OSAtomic.h>
	63	#include <sys/syscall.h>
	64	#include <sys/types.h>
65	#include <sys/sysctl.h>
66
67	typedef struct {
68	OSSpinLock tlock;
69	uintptr_t taddr;
70	unsigned tstate;
71	uint64_t tctime;
72	} cpage;
73
74	cpage *parray;
75
76	#define ARENASIZE (1024)
77	#define NTHREADS (4)
78	#define PAGE_LINGER_TIME (2000000)
79	#define MAX_THREADS (512)
80	#define MYSYS (215)
81	#define CONSISTENCY(...) fprintf(stderr, __VA_ARGS__ );
82
83	unsigned arenasize = ARENASIZE, mapping_size;
84	uint64_t page_linger_time = PAGE_LINGER_TIME;
85	enum arenastates {MTOUCHED = 1, UNMAPPED = 2, MAPPED = 4, WP =8};
86	enum syscaction {MDOMAP = 1, MDOUNMAP = 2, MDOCHECK = 4};
87	enum ttypes {OBSERVER = 1, LOOPER = 2};
88	bool trymode = true;
89	bool all_stop = false;
90	bool stop_on_failure = false;
91	bool reuse_addrs = true;
92	bool dosyscall = false;
93
94	pid_t cpid;
95	int sleepus;
96
97	pthread_t threads[MAX_THREADS];
98	uint32_t roles[MAX_THREADS];
99
100	void usage(char **a) {
101	exit(1);
102	}
103
104	void set_enable(int val)
105	{
106	int mib[6];
107	size_t needed;
108
109	mib[0] = CTL_KERN;
110	mib[1] = KERN_KDEBUG;
111	mib[2] = KERN_KDENABLE;
112	mib[3] = val;
113	mib[4] = 0;
114	mib[5] = 0;
115
116	if (sysctl(mib, 4, NULL, &needed, NULL, 0) < 0) {
117	printf("trace facility failure, KERN_KDENABLE\n");
118	}
119	}
120
121	void initialize_arena_element(int i) {
122	__unused int sysret;
123	void hint = reuse_addrs ? (void )0x1000 : NULL;
124	parray[i].taddr = (uintptr_t)mmap(hint, mapping_size, PROT_READ \| PROT_WRITE, MAP_ANON \| MAP_SHARED, -1, 0);
125
126	if (parray[i].taddr == (uintptr_t)MAP_FAILED) {
127	perror("mmap");
128	exit(2);
129	}
130
131	#if !defined(__LP64__)
132	uint32_t pattern = parray[i].taddr;
133	pattern \|= cpid & 0xFFF;
134	// memset_pattern4((void *)parray[i].taddr, &pattern, PAGE_SIZE); //
135	// uncomment to fill the whole page, but a sufficiently unique first word
136	// gets the job done without slowing down the test
137
138	#else
139	uint64_t pattern = parray[i].taddr;
140	pattern \|= (cpid & 0xFFF);
141	// memset_pattern8(parray[i].taddr, &pattern, PAGE_SIZE);
142	#endif
143
144	uint64_t val = ((uintptr_t )parray[i].taddr);
145
146	if (val != 0) {
147	CONSISTENCY("Mismatch, actual: 0x%llx, expected: 0x%llx\n", (unsigned long long)val, 0ULL);
148	if (stop_on_failure) {
149	set_enable(0);
150	exit(5);
151	}
152	}
153	for (int k = 0; k < (mapping_size >> PAGE_SHIFT); k++) {
154	(uintptr_t )(parray[i].taddr + k * PAGE_SIZE) = pattern;
155	}
156
157	parray[i].tctime = mach_absolute_time();
158	parray[i].tstate = MTOUCHED;
159
160	if (dosyscall) {
161	sysret = syscall(MYSYS, MDOMAP, parray[i].taddr, pattern, i, mapping_size);
162	}
163	}
164
165	void initialize_arena(void) {
166	for (int i = 0; i < arenasize; i++) {
167	initialize_arena_element(i);
168	}
169	}
170
171	void tlbexerciser(void targs) {
172	uint32_t role = (uint32_t )targs;
173	__unused int sysret;
174	printf("Starting thread %p, role: %u\n", pthread_self(), role);
175
176	for(;;) {
177	for (int i = 0; i < arenasize; i++) {
178	if (all_stop)
179	return NULL;
180
181	if (trymode) {
182	if (OSSpinLockTry(&parray[i].tlock) == false)
183	continue;
184	} else {
185	OSSpinLockLock(&parray[i].tlock);
186	}
187
188	if (parray[i].tstate != UNMAPPED) {
189	uintptr_t ad;
190	ad = parray[i].taddr \| (cpid & 0xFFF);
191	uintptr_t val = (uintptr_t )parray[i].taddr;
192
193	if (val != ad) {
194	if (stop_on_failure)
195	all_stop = true;
196	syscall(180, 0x71BC0000, (ad >> 32), (ad & ~0), 0, 0, 0);
197	CONSISTENCY("Mismatch, actual: 0x%llx, expected: 0x%llx\n", (unsigned long long)val, (unsigned long long)ad);
198	if (stop_on_failure) {
199	set_enable(0);
200	exit(5);
201	}
202	}
203
204	if (dosyscall) {
205	sysret = syscall(MYSYS, MDOCHECK, parray[i].taddr, ad, i, 0);
206	}
207
208	if ((role != OBSERVER) && ((mach_absolute_time() - parray[i].tctime) > page_linger_time)) {
209	parray[i].tstate = UNMAPPED;
210	if (munmap((void *)parray[i].taddr, mapping_size) != 0) {
211	perror("munmap");
212	}
213
214	if (dosyscall) {
215	sysret = syscall(MYSYS, MDOUNMAP, parray[i].taddr, ad, i, mapping_size);
216	}
217	}
218	} else {
219	if (role != OBSERVER) {
220	initialize_arena_element(i);
221	}
222	}
223
224	parray[i].tlock = 0; //unlock
225
226	if (sleepus)
227	usleep(sleepus);
228	}
229	}
230
231	return NULL;
232	}
233
234	int main(int argc, char **argv) {
235	extern char *optarg;
236	int arg;
237	unsigned nthreads = NTHREADS;
238
239	mapping_size = PAGE_SIZE;
240
241	while ((arg = getopt(argc, argv, "l:t:h:s:p:z:fry")) != -1) {
242	switch (arg) {
243	case 'l':
244	page_linger_time = strtoull(optarg, NULL, 0);
245	break;
246	case 't':
247	nthreads = atoi(optarg);
248	break;
249	case 's':
250	arenasize = atoi(optarg); // we typically want this to
251	// be sized < 2nd level TLB
252	break;
253	case 'f':
254	stop_on_failure = true;
255	break;
256	case 'r':
257	reuse_addrs = false;
258	break;
259	case 'p':
260	sleepus = atoi(optarg);
261	break;
262	case 'y':
263	dosyscall = true;
264	break;
265	case 'z':
266	mapping_size = atoi(optarg) * PAGE_SIZE;
267	break;
268	case 'h':
269	usage(argv);
270	}
271	}
272
273	if(optind != argc) {
274	usage(argv);
275	}
276
277	printf("page_linger_time: 0x%llx, nthreads: %u, arenasize: %u sleepus: %d reuse_addrs: %u, stop_on_failure: %u, dosyscall: %u, mappingsize: 0x%x\n", page_linger_time, nthreads, arenasize, sleepus, reuse_addrs, (unsigned) stop_on_failure, dosyscall, mapping_size);
278
279	parray = calloc(arenasize, sizeof(cpage));
280	cpid = getpid();
281
282	initialize_arena();
283
284	for (int dex = 0; dex < nthreads; dex++) {
285	roles[dex] = LOOPER;
286	if (dex == 0)
287	roles[dex] = OBSERVER;
288	int result = pthread_create(&threads[dex], NULL, tlbexerciser, &roles[dex]);
289	if(result) {
290	printf("pthread_create: %d starting worker thread; aborting.\n", result);
291	return result;
292	}
293	}
294
295	for(int dex = 0; dex < nthreads; dex++) {
296	void *rtn;
297	int result = pthread_join(threads[dex], &rtn);
298
299	if(result) {
300	printf("pthread_join(): %d, aborting\n", result);
301	return result;
302	}
303
304	if(rtn) {
305	printf("***Aborting on worker error\n");
306	exit(1);
307	}
308	}
309	return 0;
310	}