4 #include <sys/sysctl.h>
6 #include <mach/vm_map.h>
7 #include <darwintest.h>
8 #include <TargetConditionals.h>
9 #include <perfcheck_keys.h>
12 T_META_NAMESPACE("xnu.vm.perf"),
13 T_META_CHECK_LEAKS(false),
18 #define MEMSIZE (1UL<<29) /* 512 MB */
20 #define MEMSIZE (1UL<<27) /* 128 MB */
34 VARIANT_SINGLE_REGION
,
35 VARIANT_MULTIPLE_REGIONS
,
39 static char *variant_str
[] = {
49 char *shared_region_addr
;
53 static memregion_config
*memregion_config_per_thread
;
56 static int num_threads
;
57 static int ready_thread_count
;
58 static int finished_thread_count
;
59 static dt_stat_time_t runtime
;
60 static pthread_cond_t start_cvar
;
61 static pthread_cond_t threads_ready_cvar
;
62 static pthread_cond_t threads_finished_cvar
;
63 static pthread_mutex_t ready_thread_count_lock
;
64 static pthread_mutex_t finished_thread_count_lock
;
66 static void map_mem_regions_default(int fault_type
, size_t memsize
);
67 static void map_mem_regions_single(int fault_type
, size_t memsize
);
68 static void map_mem_regions_multiple(int fault_type
, size_t memsize
);
69 static void map_mem_regions(int fault_type
, int mapping_variant
, size_t memsize
);
70 static void unmap_mem_regions(int mapping_variant
, size_t memsize
);
71 static void setup_per_thread_regions(char *memblock
, char *memblock_share
, int fault_type
, size_t memsize
);
72 static void fault_pages(int thread_id
);
73 static void execute_threads(void);
74 static void *thread_setup(void *arg
);
75 static void run_test(int fault_type
, int mapping_variant
, size_t memsize
);
76 static void setup_and_run_test(int test
, int threads
);
77 static int get_ncpu(void);
79 /* Allocates memory using the default mmap behavior. Each VM region created is capped at 128 MB. */
81 map_mem_regions_default(int fault_type
, size_t memsize
)
84 vm_prot_t curprot
, maxprot
;
85 char *ptr
, *memblock
, *memblock_share
= NULL
;
87 memblock
= (char *)mmap(NULL
, memsize
, PROT_READ
| PROT_WRITE
, MAP_ANON
| MAP_PRIVATE
, -1, 0);
88 T_QUIET
; T_ASSERT_NE((void *)memblock
, MAP_FAILED
, "mmap");
90 if (fault_type
== SOFT_FAULT
) {
91 /* Fault in all the pages of the original region. */
92 for (ptr
= memblock
; ptr
< memblock
+ memsize
; ptr
+= pgsize
) {
95 /* Remap the region so that subsequent accesses result in read soft faults. */
96 T_QUIET
; T_ASSERT_MACH_SUCCESS(vm_remap(mach_task_self(), (vm_address_t
*)&memblock_share
,
97 memsize
, 0, VM_FLAGS_ANYWHERE
, mach_task_self(), (vm_address_t
)memblock
, FALSE
,
98 &curprot
, &maxprot
, VM_INHERIT_DEFAULT
), "vm_remap");
100 setup_per_thread_regions(memblock
, memblock_share
, fault_type
, memsize
);
103 /* Creates a single VM region by mapping in a named memory entry. */
105 map_mem_regions_single(int fault_type
, size_t memsize
)
108 vm_prot_t curprot
, maxprot
;
109 char *ptr
, *memblock
= NULL
, *memblock_share
= NULL
;
110 vm_size_t size
= memsize
;
111 vm_offset_t addr1
= 0;
112 mach_port_t mem_handle
= MACH_PORT_NULL
;
114 /* Allocate a region and fault in all the pages. */
115 T_QUIET
; T_ASSERT_MACH_SUCCESS(vm_allocate(mach_task_self(), &addr1
, size
, VM_FLAGS_ANYWHERE
), "vm_allocate");
116 for (ptr
= (char *)addr1
; ptr
< (char *)addr1
+ memsize
; ptr
+= pgsize
) {
120 /* Create a named memory entry from the region allocated above, and de-allocate said region. */
121 T_QUIET
; T_ASSERT_MACH_SUCCESS(mach_make_memory_entry(mach_task_self(), &size
, addr1
, VM_PROT_ALL
| MAP_MEM_NAMED_CREATE
,
122 &mem_handle
, MACH_PORT_NULL
), "mach_make_memory_entry");
123 T_QUIET
; T_ASSERT_MACH_SUCCESS(vm_deallocate(mach_task_self(), addr1
, size
), "vm_deallocate");
125 /* Map in the named entry and deallocate it. */
126 T_QUIET
; T_ASSERT_MACH_SUCCESS(vm_map(mach_task_self(), (vm_address_t
*)&memblock
, size
, 0, VM_FLAGS_ANYWHERE
, mem_handle
, 0,
127 FALSE
, VM_PROT_DEFAULT
, VM_PROT_ALL
, VM_INHERIT_NONE
), "vm_map");
128 T_QUIET
; T_ASSERT_MACH_SUCCESS(mach_port_deallocate(mach_task_self(), mem_handle
), "mach_port_deallocate");
130 if (fault_type
== SOFT_FAULT
) {
131 /* Fault in all the pages of the original region. */
132 for (ptr
= memblock
; ptr
< memblock
+ memsize
; ptr
+= pgsize
) {
135 /* Remap the region so that subsequent accesses result in read soft faults. */
136 T_QUIET
; T_ASSERT_MACH_SUCCESS(vm_remap(mach_task_self(), (vm_address_t
*)&memblock_share
,
137 memsize
, 0, VM_FLAGS_ANYWHERE
, mach_task_self(), (vm_address_t
)memblock
, FALSE
,
138 &curprot
, &maxprot
, VM_INHERIT_DEFAULT
), "vm_remap");
140 setup_per_thread_regions(memblock
, memblock_share
, fault_type
, memsize
);
143 /* Allocates a separate VM region for each thread. */
145 map_mem_regions_multiple(int fault_type
, size_t memsize
)
148 size_t region_len
, num_pages
;
150 char *ptr
, *memblock
, *memblock_share
;
151 vm_prot_t curprot
, maxprot
;
153 num_pages
= memsize
/ pgsize
;
155 for (i
= 0; i
< num_threads
; i
++) {
158 region_len
= num_pages
/ (size_t)num_threads
;
159 if ((size_t)i
< num_pages
% (size_t)num_threads
) {
162 region_len
*= pgsize
;
164 int fd
= VM_MAKE_TAG((i
% 2)? VM_TAG1
: VM_TAG2
);
165 memblock
= (char *)mmap(NULL
, region_len
, PROT_READ
| PROT_WRITE
, MAP_ANON
| MAP_PRIVATE
, fd
, 0);
166 T_QUIET
; T_ASSERT_NE((void *)memblock
, MAP_FAILED
, "mmap");
167 memregion_config_per_thread
[i
].region_addr
= memblock
;
168 memregion_config_per_thread
[i
].shared_region_addr
= 0;
169 memregion_config_per_thread
[i
].region_len
= region_len
;
171 if (fault_type
== SOFT_FAULT
) {
172 /* Fault in all the pages of the original region. */
173 for (ptr
= memblock
; ptr
< memblock
+ region_len
; ptr
+= pgsize
) {
176 memblock_share
= NULL
;
177 /* Remap the region so that subsequent accesses result in read soft faults. */
178 T_QUIET
; T_ASSERT_MACH_SUCCESS(vm_remap(mach_task_self(), (vm_address_t
*)&memblock_share
,
179 region_len
, 0, VM_FLAGS_ANYWHERE
, mach_task_self(), (vm_address_t
)memblock
, FALSE
,
180 &curprot
, &maxprot
, VM_INHERIT_DEFAULT
), "vm_remap");
181 memregion_config_per_thread
[i
].shared_region_addr
= memblock_share
;
187 map_mem_regions(int fault_type
, int mapping_variant
, size_t memsize
)
189 memregion_config_per_thread
= (memregion_config
*)malloc(sizeof(*memregion_config_per_thread
) * (size_t)num_threads
);
190 switch (mapping_variant
) {
191 case VARIANT_SINGLE_REGION
:
192 map_mem_regions_single(fault_type
, memsize
);
194 case VARIANT_MULTIPLE_REGIONS
:
195 map_mem_regions_multiple(fault_type
, memsize
);
197 case VARIANT_DEFAULT
:
199 map_mem_regions_default(fault_type
, memsize
);
204 setup_per_thread_regions(char *memblock
, char *memblock_share
, int fault_type
, size_t memsize
)
207 size_t region_len
, region_start
, num_pages
;
209 num_pages
= memsize
/ pgsize
;
210 for (i
= 0; i
< num_threads
; i
++) {
211 region_len
= num_pages
/ (size_t)num_threads
;
212 region_start
= region_len
* (size_t)i
;
214 if ((size_t)i
< num_pages
% (size_t)num_threads
) {
215 region_start
+= (size_t)i
;
218 region_start
+= num_pages
% (size_t)num_threads
;
221 region_start
*= pgsize
;
222 region_len
*= pgsize
;
224 memregion_config_per_thread
[i
].region_addr
= memblock
+ region_start
;
225 memregion_config_per_thread
[i
].shared_region_addr
= ((fault_type
== SOFT_FAULT
) ?
226 memblock_share
+ region_start
: 0);
227 memregion_config_per_thread
[i
].region_len
= region_len
;
232 unmap_mem_regions(int mapping_variant
, size_t memsize
)
234 if (mapping_variant
== VARIANT_MULTIPLE_REGIONS
) {
236 for (i
= 0; i
< num_threads
; i
++) {
237 if (memregion_config_per_thread
[i
].shared_region_addr
!= 0) {
238 T_QUIET
; T_ASSERT_MACH_SUCCESS(munmap(memregion_config_per_thread
[i
].shared_region_addr
,
239 memregion_config_per_thread
[i
].region_len
), "munmap");
241 T_QUIET
; T_ASSERT_MACH_SUCCESS(munmap(memregion_config_per_thread
[i
].region_addr
,
242 memregion_config_per_thread
[i
].region_len
), "munmap");
245 if (memregion_config_per_thread
[0].shared_region_addr
!= 0) {
246 T_QUIET
; T_ASSERT_MACH_SUCCESS(munmap(memregion_config_per_thread
[0].shared_region_addr
, memsize
), "munmap");
248 T_QUIET
; T_ASSERT_MACH_SUCCESS(munmap(memregion_config_per_thread
[0].region_addr
, memsize
), "munmap");
253 fault_pages(int thread_id
)
258 block
= memregion_config_per_thread
[thread_id
].shared_region_addr
?
259 memregion_config_per_thread
[thread_id
].shared_region_addr
:
260 memregion_config_per_thread
[thread_id
].region_addr
;
261 for (ptr
= block
; ptr
< block
+ memregion_config_per_thread
[thread_id
].region_len
; ptr
+= pgsize
) {
267 thread_setup(void *arg
)
269 int my_index
= *((int *)arg
);
271 T_QUIET
; T_ASSERT_POSIX_SUCCESS(pthread_mutex_lock(&ready_thread_count_lock
), "pthread_mutex_lock");
272 ready_thread_count
++;
273 if (ready_thread_count
== num_threads
) {
274 T_QUIET
; T_ASSERT_POSIX_SUCCESS(pthread_cond_signal(&threads_ready_cvar
), "pthread_cond_signal");
276 T_QUIET
; T_ASSERT_POSIX_SUCCESS(pthread_cond_wait(&start_cvar
, &ready_thread_count_lock
), "pthread_cond_wait");
277 T_QUIET
; T_ASSERT_POSIX_SUCCESS(pthread_mutex_unlock(&ready_thread_count_lock
), "pthread_mutex_unlock");
279 fault_pages(my_index
);
281 /* Up the finished count */
282 T_QUIET
; T_ASSERT_POSIX_SUCCESS(pthread_mutex_lock(&finished_thread_count_lock
), "pthread_mutex_lock");
283 finished_thread_count
++;
284 if (finished_thread_count
== num_threads
) {
285 /* All the threads are done. Wake up the main thread */
286 T_QUIET
; T_ASSERT_POSIX_SUCCESS(pthread_cond_signal(&threads_finished_cvar
), "pthread_cond_signal");
288 T_QUIET
; T_ASSERT_POSIX_SUCCESS(pthread_mutex_unlock(&finished_thread_count_lock
), "pthread_mutex_unlock");
293 execute_threads(void)
295 int thread_index
, thread_retval
;
297 void *thread_retval_ptr
= &thread_retval
;
300 T_QUIET
; T_ASSERT_POSIX_SUCCESS(pthread_cond_init(&threads_ready_cvar
, NULL
), "pthread_cond_init");
301 T_QUIET
; T_ASSERT_POSIX_SUCCESS(pthread_cond_init(&start_cvar
, NULL
), "pthread_cond_init");
302 T_QUIET
; T_ASSERT_POSIX_SUCCESS(pthread_mutex_init(&ready_thread_count_lock
, NULL
), "pthread_mutex_init");
303 T_QUIET
; T_ASSERT_POSIX_SUCCESS(pthread_cond_init(&threads_finished_cvar
, NULL
), "pthread_cond_init");
304 T_QUIET
; T_ASSERT_POSIX_SUCCESS(pthread_mutex_init(&finished_thread_count_lock
, NULL
), "pthread_mutex_init");
305 ready_thread_count
= 0;
306 finished_thread_count
= 0;
308 threads
= (pthread_t
*)malloc(sizeof(*threads
) * (size_t)num_threads
);
309 thread_indices
= (int *)malloc(sizeof(*thread_indices
) * (size_t)num_threads
);
310 for (thread_index
= 0; thread_index
< num_threads
; thread_index
++) {
311 thread_indices
[thread_index
] = thread_index
;
312 T_QUIET
; T_ASSERT_POSIX_SUCCESS(pthread_create(&threads
[thread_index
], NULL
,
313 thread_setup
, (void *)&thread_indices
[thread_index
]), "pthread_create");
316 T_QUIET
; T_ASSERT_POSIX_SUCCESS(pthread_mutex_lock(&ready_thread_count_lock
), "pthread_mutex_lock");
317 while (ready_thread_count
!= num_threads
) {
318 T_QUIET
; T_ASSERT_POSIX_SUCCESS(pthread_cond_wait(&threads_ready_cvar
, &ready_thread_count_lock
),
319 "pthread_cond_wait");
321 T_QUIET
; T_ASSERT_POSIX_SUCCESS(pthread_mutex_unlock(&ready_thread_count_lock
), "pthread_mutex_unlock");
323 T_STAT_MEASURE(runtime
) {
324 /* Ungate the threads */
325 T_QUIET
; T_ASSERT_POSIX_SUCCESS(pthread_cond_broadcast(&start_cvar
), "pthread_cond_broadcast");
326 /* Wait for the threads to finish */
327 T_QUIET
; T_ASSERT_POSIX_SUCCESS(pthread_mutex_lock(&finished_thread_count_lock
), "pthread_mutex_lock");
328 while (finished_thread_count
!= num_threads
) {
329 T_QUIET
; T_ASSERT_POSIX_SUCCESS(pthread_cond_wait(&threads_finished_cvar
, &finished_thread_count_lock
), "pthread_cond_wait");
333 /* Join the threads */
334 for (thread_index
= 0; thread_index
< num_threads
; thread_index
++) {
335 T_QUIET
; T_ASSERT_POSIX_SUCCESS(pthread_join(threads
[thread_index
], &thread_retval_ptr
),
340 free(thread_indices
);
344 run_test(int fault_type
, int mapping_variant
, size_t memsize
)
348 size_t sysctl_size
= sizeof(pgsize
);
349 int ret
= sysctlbyname("vm.pagesize", &pgsize
, &sysctl_size
, NULL
, 0);
350 T_QUIET
; T_ASSERT_POSIX_SUCCESS(ret
, "sysctl vm.pagesize failed");
352 num_pages
= memsize
/ pgsize
;
354 T_QUIET
; T_ASSERT_LT(fault_type
, NUM_FAULT_TYPES
, "invalid test type");
355 T_QUIET
; T_ASSERT_LT(mapping_variant
, NUM_MAPPING_VARIANTS
, "invalid mapping variant");
356 T_QUIET
; T_ASSERT_GT(num_threads
, 0, "num_threads <= 0");
357 T_QUIET
; T_ASSERT_GT((int)num_pages
/ num_threads
, 0, "num_pages/num_threads <= 0");
359 T_LOG("No. of cpus: %d", get_ncpu());
360 T_LOG("No. of threads: %d", num_threads
);
361 T_LOG("No. of pages: %ld", num_pages
);
362 T_LOG("Pagesize: %ld", pgsize
);
363 T_LOG("Allocation size: %ld MB", memsize
/ (1024 * 1024));
364 T_LOG("Mapping variant: %s", variant_str
[mapping_variant
]);
366 snprintf(metric_str
, 32, "Runtime-%s", variant_str
[mapping_variant
]);
367 runtime
= dt_stat_time_create(metric_str
);
369 while (!dt_stat_stable(runtime
)) {
370 map_mem_regions(fault_type
, mapping_variant
, memsize
);
372 unmap_mem_regions(mapping_variant
, memsize
);
375 dt_stat_finalize(runtime
);
376 T_LOG("Throughput-%s (MB/s): %lf\n\n", variant_str
[mapping_variant
], (double)memsize
/ (1024 * 1024) / dt_stat_mean((dt_stat_t
)runtime
));
380 setup_and_run_test(int fault_type
, int threads
)
382 int i
, mapping_variant
;
386 mapping_variant
= VARIANT_DEFAULT
;
388 num_threads
= threads
;
390 if ((e
= getenv("NTHREADS"))) {
392 T_SKIP("Custom environment variables specified. Skipping single threaded version.");
394 num_threads
= (int)strtol(e
, NULL
, 0);
397 if ((e
= getenv("MEMSIZEMB"))) {
398 memsize
= (size_t)strtol(e
, NULL
, 0) * 1024 * 1024;
401 if ((e
= getenv("VARIANT"))) {
402 mapping_variant
= (int)strtol(e
, NULL
, 0);
403 run_test(fault_type
, mapping_variant
, memsize
);
405 for (i
= VARIANT_DEFAULT
; i
< NUM_MAPPING_VARIANTS
; i
++) {
406 run_test(fault_type
, i
, memsize
);
417 size_t length
= sizeof(ncpu
);
419 T_QUIET
; T_ASSERT_POSIX_SUCCESS(sysctlbyname("hw.ncpu", &ncpu
, &length
, NULL
, 0),
420 "failed to query hw.ncpu");
424 T_DECL(read_soft_fault
,
425 "Read soft faults (single thread)")
427 setup_and_run_test(SOFT_FAULT
, 1);
430 T_DECL(read_soft_fault_multithreaded
,
431 "Read soft faults (multi-threaded)")
436 /* iOSMark passes in the no. of threads via an env. variable */
437 if ((e
= getenv("DT_STAT_NTHREADS"))) {
438 nthreads
= (int)strtol(e
, NULL
, 0);
440 nthreads
= get_ncpu();
442 T_SKIP("Skipping multi-threaded test on single core device.");
445 setup_and_run_test(SOFT_FAULT
, nthreads
);
448 T_DECL(zero_fill_fault
,
449 "Zero fill faults (single thread)")
451 setup_and_run_test(ZERO_FILL
, 1);
454 T_DECL(zero_fill_fault_multithreaded
,
455 "Zero fill faults (multi-threaded)")
460 /* iOSMark passes in the no. of threads via an env. variable */
461 if ((e
= getenv("DT_STAT_NTHREADS"))) {
462 nthreads
= (int)strtol(e
, NULL
, 0);
464 nthreads
= get_ncpu();
466 T_SKIP("Skipping multi-threaded test on single core device.");
469 setup_and_run_test(ZERO_FILL
, nthreads
);