tools/tests/TLBcoherency/TLBcoherency.c

   1 /*
   2  * Copyright (c) 2011 Apple Inc. All rights reserved.
   3  *
   4  * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
   5  *
   6  * This file contains Original Code and/or Modifications of Original Code
   7  * as defined in and that are subject to the Apple Public Source License
   8  * Version 2.0 (the 'License'). You may not use this file except in
   9  * compliance with the License. The rights granted to you under the License
  10  * may not be used to create, or enable the creation or redistribution of,
  11  * unlawful or unlicensed copies of an Apple operating system, or to
  12  * circumvent, violate, or enable the circumvention or violation of, any
  13  * terms of an Apple operating system software license agreement.
  14  *
  15  * Please obtain a copy of the License at
  16  * http://www.opensource.apple.com/apsl/ and read it before using this file.
  17  *
  18  * The Original Code and all software distributed under the License are
  19  * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  20  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  21  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
  22  * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
  23  * Please see the License for the specific language governing rights and
  24  * limitations under the License.
  25  *
  26  * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  27  */
  28
  29 /* A pool of threads which attempt to verify multiprocessor TLB coherency.
  30  * Creates -t threads, by default 4
  31  * Creates -s separate mmap(MAP_ANON) R/W mappings, sized at 1 page each but
  32  * alterable via -z <npages>
  33  * Initially read-faults each mapping in, verifying first-word zerofill--
  34  * The kernel typically uses the physical aperture to perform the zerofill
  35  * Writes map_address (page_aligned) | low 12 bits of the PID at the first word
  36  * This can help verify ASID related inconsistencies
  37  * Records a timestamp in a Structure associated with each mapping
  38  * With a custom kernel, it has the option of creating a remapping of the page in
  39  * the kernel's address space to exercise shared kernel mapping coherency.
  40  * Each thread subsequently loops around on the set of mappings. One thread is designated
  41  * the observer thread. The thread acquires a lock on the arena element,
  42  * verifies that the mapping has the expected pattern (Address | PID), if the
  43  * element is in the MAPPED state. Can optionally tell the kernel to check its
  44  * alias as well. If it notices a mismatch, it has the option to issue a syscall
  45  * to  stop kernel tracing. If the -f option is supplied, the test is terminated.
  46  * If the page has lingered beyond -l microseconds, non-observer threads will
  47  * unmap the page, optionally calling into the kernel to unmap its alias, and
  48  * repopulate the element.
  49  * After this sequence, the thread will optionally usleep for -p microseconds,
  50  * to allow for idle power management to engage if possible (errata might exist
  51  * in those areas), or context switches to occur.
  52  * Created Derek Kumar, 2011.
  53  */
  54
  55 #include <stdio.h>
  56 #include <stdlib.h>
  57 #include <unistd.h>
  58 #include <sys/mman.h>
  59 #include <pthread.h>
  60 #include <string.h>
  61 #include <mach/mach_time.h>
  62 #include <libkern/OSAtomic.h>
  63 #include <sys/syscall.h>
  64 #include <sys/types.h>
  65 #include <sys/sysctl.h>
  66
  67 typedef struct {
  68         OSSpinLock tlock;
  69         uintptr_t taddr;
  70         unsigned tstate;
  71         uint64_t tctime;
  72 } cpage;
  73
  74 cpage *parray;
  75
  76 #define ARENASIZE (1024)
  77 #define NTHREADS (4)
  78 #define PAGE_LINGER_TIME (2000000)
  79 #define MAX_THREADS (512)
  80 #define MYSYS (215)
  81 #define CONSISTENCY(...) fprintf(stderr, __VA_ARGS__ );
  82
  83 unsigned arenasize = ARENASIZE, mapping_size;
  84 uint64_t page_linger_time = PAGE_LINGER_TIME;
  85 enum arenastates {MTOUCHED = 1, UNMAPPED = 2, MAPPED = 4, WP =8};
  86 enum syscaction {MDOMAP = 1, MDOUNMAP = 2, MDOCHECK = 4};
  87 enum ttypes {OBSERVER = 1, LOOPER = 2};
  88 bool trymode = true;
  89 bool all_stop = false;
  90 bool stop_on_failure = false;
  91 bool reuse_addrs = true;
  92 bool dosyscall = false;
  93
  94 pid_t cpid;
  95 int sleepus;
  96
  97 pthread_t threads[MAX_THREADS];
  98 uint32_t roles[MAX_THREADS];
  99
 100 void usage(char **a) {
 101         exit(1);
 102 }
 103
 104 void set_enable(int val)
 105 {
 106         int mib[6];
 107         size_t needed;
 108
 109         mib[0] = CTL_KERN;
 110         mib[1] = KERN_KDEBUG;
 111         mib[2] = KERN_KDENABLE;
 112         mib[3] = val;
 113         mib[4] = 0;
 114         mib[5] = 0;
 115
 116         if (sysctl(mib, 4, NULL, &needed, NULL, 0) < 0) {
 117                 printf("trace facility failure, KERN_KDENABLE\n");
 118         }
 119 }
 120
 121 void initialize_arena_element(int i) {
 122         __unused int sysret;
 123         void *hint = reuse_addrs ? (void *)0x1000 : NULL;
 124         parray[i].taddr = (uintptr_t)mmap(hint, mapping_size, PROT_READ | PROT_WRITE, MAP_ANON | MAP_SHARED, -1, 0);
 125
 126         if (parray[i].taddr == (uintptr_t)MAP_FAILED) {
 127                 perror("mmap");
 128                 exit(2);
 129         }
 130
 131 #if     !defined(__LP64__)
 132         uint32_t pattern = parray[i].taddr;
 133         pattern |= cpid & 0xFFF;
 134 //      memset_pattern4((void *)parray[i].taddr, &pattern, PAGE_SIZE); //
 135 //      uncomment to fill the whole page, but a sufficiently unique first word
 136 //      gets the job done without slowing down the test
 137
 138 #else
 139         uint64_t pattern = parray[i].taddr;
 140         pattern |= (cpid & 0xFFF);
 141 //      memset_pattern8(parray[i].taddr, &pattern, PAGE_SIZE);
 142 #endif
 143
 144         uint64_t val =  (*(uintptr_t *)parray[i].taddr);
 145
 146         if (val != 0) {
 147                 CONSISTENCY("Mismatch, actual: 0x%llx, expected: 0x%llx\n", (unsigned long long)val, 0ULL);
 148                 if (stop_on_failure) {
 149                         set_enable(0);
 150                         exit(5);
 151                 }
 152         }
 153         for (int k = 0; k < (mapping_size >> PAGE_SHIFT); k++) {
 154                 *(uintptr_t *)(parray[i].taddr + k * PAGE_SIZE) = pattern;
 155         }
 156
 157         parray[i].tctime = mach_absolute_time();
 158         parray[i].tstate = MTOUCHED;
 159
 160         if (dosyscall) {
 161                 sysret = syscall(MYSYS, MDOMAP, parray[i].taddr, pattern, i, mapping_size);
 162         }
 163 }
 164
 165 void initialize_arena(void) {
 166         for (int i = 0; i < arenasize; i++) {
 167                 initialize_arena_element(i);
 168         }
 169 }
 170
 171 void *tlbexerciser(void *targs) {
 172         uint32_t role = *(uint32_t *)targs;
 173         __unused int sysret;
 174         printf("Starting thread %p, role: %u\n", pthread_self(), role);
 175
 176         for(;;) {
 177                 for (int i = 0; i < arenasize; i++) {
 178                         if (all_stop)
 179                                 return NULL;
 180
 181                         if (trymode) {
 182                                 if (OSSpinLockTry(&parray[i].tlock) == false)
 183                                         continue;
 184                         } else {
 185                                 OSSpinLockLock(&parray[i].tlock);
 186                         }
 187
 188                         if (parray[i].tstate != UNMAPPED) {
 189                                 uintptr_t ad;
 190                                 ad = parray[i].taddr | (cpid & 0xFFF);
 191                                 uintptr_t val = *(uintptr_t *)parray[i].taddr;
 192
 193                                 if (val != ad) {
 194                                         if (stop_on_failure)
 195                                                 all_stop = true;
 196                                         syscall(180, 0x71BC0000, (ad >> 32), (ad & ~0), 0, 0, 0);
 197                                         CONSISTENCY("Mismatch, actual: 0x%llx, expected: 0x%llx\n", (unsigned long long)val, (unsigned long long)ad);
 198                                         if (stop_on_failure) {
 199                                                 set_enable(0);
 200                                                 exit(5);
 201                                         }
 202                                 }
 203
 204                                 if (dosyscall) {
 205                                         sysret = syscall(MYSYS, MDOCHECK, parray[i].taddr, ad, i, 0);
 206                                 }
 207
 208                                 if ((role != OBSERVER) && ((mach_absolute_time() - parray[i].tctime) > page_linger_time)) {
 209                                         parray[i].tstate = UNMAPPED;
 210                                         if (munmap((void *)parray[i].taddr, mapping_size) != 0) {
 211                                                 perror("munmap");
 212                                         }
 213
 214                                         if (dosyscall) {
 215                                                 sysret = syscall(MYSYS, MDOUNMAP, parray[i].taddr, ad, i, mapping_size);
 216                                         }
 217                                 }
 218                         } else {
 219                                 if (role != OBSERVER) {
 220                                         initialize_arena_element(i);
 221                                 }
 222                         }
 223
 224                         parray[i].tlock = 0; //unlock
 225
 226                         if (sleepus)
 227                                 usleep(sleepus);
 228                 }
 229         }
 230
 231         return NULL;
 232 }
 233
 234 int main(int argc, char **argv) {
 235         extern char *optarg;
 236         int arg;
 237         unsigned nthreads = NTHREADS;
 238
 239         mapping_size = PAGE_SIZE;
 240
 241         while ((arg = getopt(argc, argv, "l:t:h:s:p:z:fry")) != -1) {
 242                 switch (arg) {
 243                 case 'l':
 244                         page_linger_time = strtoull(optarg, NULL, 0);
 245                         break;
 246                 case 't':
 247                         nthreads = atoi(optarg);
 248                         break;
 249                 case 's':
 250                         arenasize = atoi(optarg); // we typically want this to
 251                                                   // be sized < 2nd level TLB
 252                         break;
 253                 case 'f':
 254                         stop_on_failure = true;
 255                         break;
 256                 case 'r':
 257                         reuse_addrs = false;
 258                         break;
 259                 case 'p':
 260                         sleepus = atoi(optarg);
 261                         break;
 262                 case 'y':
 263                         dosyscall = true;
 264                         break;
 265                 case 'z':
 266                         mapping_size = atoi(optarg) * PAGE_SIZE;
 267                         break;
 268                 case 'h':
 269                         usage(argv);
 270                 }
 271         }
 272
 273         if(optind != argc) {
 274                 usage(argv);
 275         }
 276
 277         printf("page_linger_time: 0x%llx, nthreads: %u, arenasize: %u sleepus: %d reuse_addrs: %u, stop_on_failure: %u, dosyscall: %u, mappingsize: 0x%x\n", page_linger_time, nthreads, arenasize, sleepus, reuse_addrs, (unsigned) stop_on_failure, dosyscall, mapping_size);
 278
 279         parray = calloc(arenasize, sizeof(cpage));
 280         cpid = getpid();
 281
 282         initialize_arena();
 283
 284         for (int dex = 0; dex < nthreads; dex++) {
 285                 roles[dex] = LOOPER;
 286                 if (dex == 0)
 287                         roles[dex] = OBSERVER;
 288                 int result = pthread_create(&threads[dex], NULL, tlbexerciser, &roles[dex]);
 289                 if(result) {
 290                         printf("pthread_create: %d starting worker thread; aborting.\n", result);
 291                         return result;
 292                 }
 293         }
 294
 295         for(int dex = 0; dex < nthreads; dex++) {
 296                 void *rtn;
 297                 int result = pthread_join(threads[dex], &rtn);
 298
 299                 if(result) {
 300                         printf("pthread_join(): %d, aborting\n", result);
 301                         return result;
 302                 }
 303
 304                 if(rtn) {
 305                         printf("***Aborting on worker error\n");
 306                         exit(1);
 307                 }
 308         }
 309         return 0;
 310 }