tests/perf_compressor.c

   1 #include <stdio.h>
   2 #include <signal.h>
   3 #include <sys/sysctl.h>
   4 #include <sys/kern_memorystatus.h>
   5 #include <mach-o/dyld.h>
   6 #include <perfcheck_keys.h>
   7
   8 #ifdef T_NAMESPACE
   9 #undef T_NAMESPACE
  10 #endif
  11 #include <darwintest.h>
  12 #include <darwintest_utils.h>
  13
  14 T_GLOBAL_META(
  15         T_META_NAMESPACE("xnu.vm.perf"),
  16         T_META_CHECK_LEAKS(false),
  17         T_META_TAG_PERF
  18         );
  19
  20 enum {
  21         ALL_ZEROS,
  22         MOSTLY_ZEROS,
  23         RANDOM,
  24         TYPICAL
  25 };
  26
  27 #define CREATE_LIST(X) \
  28         X(SUCCESS) \
  29         X(TOO_FEW_ARGUMENTS) \
  30         X(SYSCTL_VM_PAGESIZE_FAILED) \
  31         X(VM_PAGESIZE_IS_ZERO) \
  32         X(UNKNOWN_PAGE_TYPE) \
  33         X(DISPATCH_SOURCE_CREATE_FAILED) \
  34         X(INITIAL_SIGNAL_TO_PARENT_FAILED) \
  35         X(SIGNAL_TO_PARENT_FAILED) \
  36         X(MEMORYSTATUS_CONTROL_FAILED) \
  37         X(IS_FREEZABLE_NOT_AS_EXPECTED) \
  38         X(EXIT_CODE_MAX)
  39
  40 #define EXIT_CODES_ENUM(VAR) VAR,
  41 enum exit_codes_num {
  42         CREATE_LIST(EXIT_CODES_ENUM)
  43 };
  44
  45 #define EXIT_CODES_STRING(VAR) #VAR,
  46 static const char *exit_codes_str[] = {
  47         CREATE_LIST(EXIT_CODES_STRING)
  48 };
  49
  50 #define SYSCTL_FREEZE_TO_MEMORY         "kern.memorystatus_freeze_to_memory=1"
  51
  52 static pid_t pid = -1;
  53 static dt_stat_t ratio;
  54 static dt_stat_time_t compr_time;
  55 static dt_stat_time_t decompr_time;
  56
  57 void allocate_zero_pages(char **buf, int num_pages, int vmpgsize);
  58 void allocate_mostly_zero_pages(char **buf, int num_pages, int vmpgsize);
  59 void allocate_random_pages(char **buf, int num_pages, int vmpgsize);
  60 void allocate_representative_pages(char **buf, int num_pages, int vmpgsize);
  61 void run_compressor_test(int size_mb, int page_type);
  62 void freeze_helper_process(void);
  63 void cleanup(void);
  64
  65 void
  66 allocate_zero_pages(char **buf, int num_pages, int vmpgsize)
  67 {
  68         int i;
  69
  70         for (i = 0; i < num_pages; i++) {
  71                 buf[i] = (char*)malloc((size_t)vmpgsize * sizeof(char));
  72                 memset(buf[i], 0, vmpgsize);
  73         }
  74 }
  75
  76 void
  77 allocate_mostly_zero_pages(char **buf, int num_pages, int vmpgsize)
  78 {
  79         int i, j;
  80
  81         for (i = 0; i < num_pages; i++) {
  82                 buf[i] = (char*)malloc((size_t)vmpgsize * sizeof(char));
  83                 memset(buf[i], 0, vmpgsize);
  84                 for (j = 0; j < 40; j++) {
  85                         buf[i][j] = (char)(j + 1);
  86                 }
  87         }
  88 }
  89
  90 void
  91 allocate_random_pages(char **buf, int num_pages, int vmpgsize)
  92 {
  93         int i;
  94
  95         for (i = 0; i < num_pages; i++) {
  96                 buf[i] = (char*)malloc((size_t)vmpgsize * sizeof(char));
  97                 arc4random_buf((void*)buf[i], (size_t)vmpgsize);
  98         }
  99 }
 100
 101 // Gives us the compression ratio we see in the typical case (~2.7)
 102 void
 103 allocate_representative_pages(char **buf, int num_pages, int vmpgsize)
 104 {
 105         int i, j;
 106         char val;
 107
 108         for (j = 0; j < num_pages; j++) {
 109                 buf[j] = (char*)malloc((size_t)vmpgsize * sizeof(char));
 110                 val = 0;
 111                 for (i = 0; i < vmpgsize; i += 16) {
 112                         memset(&buf[j][i], val, 16);
 113                         if (i < 3400 * (vmpgsize / 4096)) {
 114                                 val++;
 115                         }
 116                 }
 117         }
 118 }
 119
 120 void
 121 freeze_helper_process(void)
 122 {
 123         int ret, freeze_enabled;
 124         int64_t compressed_before, compressed_after, input_before, input_after;
 125         size_t length;
 126         int errno_sysctl_freeze;
 127
 128         length = sizeof(compressed_before);
 129         T_QUIET; T_ASSERT_POSIX_SUCCESS(sysctlbyname("vm.compressor_compressed_bytes", &compressed_before, &length, NULL, 0),
 130             "failed to query vm.compressor_compressed_bytes");
 131         length = sizeof(input_before);
 132         T_QUIET; T_ASSERT_POSIX_SUCCESS(sysctlbyname("vm.compressor_input_bytes", &input_before, &length, NULL, 0),
 133             "failed to query vm.compressor_input_bytes");
 134
 135         T_STAT_MEASURE(compr_time) {
 136                 ret = sysctlbyname("kern.memorystatus_freeze", NULL, NULL, &pid, sizeof(pid));
 137                 errno_sysctl_freeze = errno;
 138         };
 139
 140         length = sizeof(compressed_after);
 141         T_QUIET; T_ASSERT_POSIX_SUCCESS(sysctlbyname("vm.compressor_compressed_bytes", &compressed_after, &length, NULL, 0),
 142             "failed to query vm.compressor_compressed_bytes");
 143         length = sizeof(input_after);
 144         T_QUIET; T_ASSERT_POSIX_SUCCESS(sysctlbyname("vm.compressor_input_bytes", &input_after, &length, NULL, 0),
 145             "failed to query vm.compressor_input_bytes");
 146
 147         length = sizeof(freeze_enabled);
 148         T_QUIET; T_ASSERT_POSIX_SUCCESS(sysctlbyname("vm.freeze_enabled", &freeze_enabled, &length, NULL, 0),
 149             "failed to query vm.freeze_enabled");
 150         if (freeze_enabled) {
 151                 errno = errno_sysctl_freeze;
 152                 T_QUIET; T_ASSERT_POSIX_SUCCESS(ret, "sysctl kern.memorystatus_freeze failed");
 153         } else {
 154                 /* If freezer is disabled, skip the test. This can happen due to disk space shortage. */
 155                 T_LOG("Freeze has been disabled. Terminating early.");
 156                 T_END;
 157         }
 158
 159         dt_stat_add(ratio, (double)(input_after - input_before) / (double)(compressed_after - compressed_before));
 160
 161         ret = sysctlbyname("kern.memorystatus_thaw", NULL, NULL, &pid, sizeof(pid));
 162         T_QUIET; T_ASSERT_POSIX_SUCCESS(ret, "sysctl kern.memorystatus_thaw failed");
 163
 164         T_QUIET; T_ASSERT_POSIX_SUCCESS(kill(pid, SIGUSR1), "failed to send SIGUSR1 to child process");
 165 }
 166
 167 void
 168 cleanup(void)
 169 {
 170         /* No helper process. */
 171         if (pid == -1) {
 172                 return;
 173         }
 174         /* Kill the helper process. */
 175         kill(pid, SIGKILL);
 176 }
 177
 178 void
 179 run_compressor_test(int size_mb, int page_type)
 180 {
 181         int ret;
 182         char sz_str[50];
 183         char pt_str[50];
 184         char **launch_tool_args;
 185         char testpath[PATH_MAX];
 186         uint32_t testpath_buf_size;
 187         dispatch_source_t ds_freeze, ds_proc, ds_decompr;
 188         int freeze_enabled;
 189         size_t length;
 190         __block bool decompr_latency_is_stable = false;
 191
 192         length = sizeof(freeze_enabled);
 193         T_QUIET; T_ASSERT_POSIX_SUCCESS(sysctlbyname("vm.freeze_enabled", &freeze_enabled, &length, NULL, 0),
 194             "failed to query vm.freeze_enabled");
 195         if (!freeze_enabled) {
 196                 /* If freezer is disabled, skip the test. This can happen due to disk space shortage. */
 197                 T_SKIP("Freeze has been disabled. Skipping test.");
 198         }
 199
 200         T_ATEND(cleanup);
 201
 202         ratio = dt_stat_create("(input bytes / compressed bytes)", "compression_ratio");
 203         compr_time = dt_stat_time_create("compressor_latency");
 204
 205         // This sets the A/B failure threshold at 50% of baseline for compressor_latency
 206         dt_stat_set_variable((struct dt_stat *)compr_time, kPCFailureThresholdPctVar, 50.0);
 207
 208         signal(SIGUSR2, SIG_IGN);
 209         ds_decompr = dispatch_source_create(DISPATCH_SOURCE_TYPE_SIGNAL, SIGUSR2, 0, dispatch_get_main_queue());
 210         T_QUIET; T_ASSERT_NOTNULL(ds_decompr, "dispatch_source_create (ds_decompr)");
 211
 212         dispatch_source_set_event_handler(ds_decompr, ^{
 213                 decompr_latency_is_stable = true;
 214         });
 215         dispatch_activate(ds_decompr);
 216
 217         signal(SIGUSR1, SIG_IGN);
 218         ds_freeze = dispatch_source_create(DISPATCH_SOURCE_TYPE_SIGNAL, SIGUSR1, 0, dispatch_get_main_queue());
 219         T_QUIET; T_ASSERT_NOTNULL(ds_freeze, "dispatch_source_create (ds_freeze)");
 220
 221         dispatch_source_set_event_handler(ds_freeze, ^{
 222                 if (!(dt_stat_stable(compr_time) && decompr_latency_is_stable)) {
 223                         freeze_helper_process();
 224                 } else {
 225                         dt_stat_finalize(compr_time);
 226                         dt_stat_finalize(ratio);
 227
 228                         kill(pid, SIGKILL);
 229                         dispatch_source_cancel(ds_freeze);
 230                         dispatch_source_cancel(ds_decompr);
 231                 }
 232         });
 233         dispatch_activate(ds_freeze);
 234
 235         testpath_buf_size = sizeof(testpath);
 236         ret = _NSGetExecutablePath(testpath, &testpath_buf_size);
 237         T_QUIET; T_ASSERT_POSIX_ZERO(ret, "_NSGetExecutablePath");
 238         T_LOG("Executable path: %s", testpath);
 239
 240         sprintf(sz_str, "%d", size_mb);
 241         sprintf(pt_str, "%d", page_type);
 242         launch_tool_args = (char *[]){
 243                 testpath,
 244                 "-n",
 245                 "allocate_pages",
 246                 "--",
 247                 sz_str,
 248                 pt_str,
 249                 NULL
 250         };
 251
 252         /* Spawn the child process. Suspend after launch until the exit proc handler has been set up. */
 253         ret = dt_launch_tool(&pid, launch_tool_args, true, NULL, NULL);
 254         if (ret != 0) {
 255                 T_LOG("dt_launch tool returned %d with error code %d", ret, errno);
 256         }
 257         T_QUIET; T_ASSERT_POSIX_SUCCESS(pid, "dt_launch_tool");
 258
 259         ds_proc = dispatch_source_create(DISPATCH_SOURCE_TYPE_PROC, (uintptr_t)pid, DISPATCH_PROC_EXIT, dispatch_get_main_queue());
 260         T_QUIET; T_ASSERT_NOTNULL(ds_proc, "dispatch_source_create (ds_proc)");
 261
 262         dispatch_source_set_event_handler(ds_proc, ^{
 263                 int status = 0, code = 0;
 264                 pid_t rc = waitpid(pid, &status, 0);
 265                 T_QUIET; T_ASSERT_EQ(rc, pid, "waitpid");
 266                 code = WEXITSTATUS(status);
 267
 268                 if (code == 0) {
 269                         T_END;
 270                 } else if (code > 0 && code < EXIT_CODE_MAX) {
 271                         T_ASSERT_FAIL("Child exited with %s", exit_codes_str[code]);
 272                 } else {
 273                         T_ASSERT_FAIL("Child exited with unknown exit code %d", code);
 274                 }
 275         });
 276         dispatch_activate(ds_proc);
 277
 278         T_QUIET; T_ASSERT_POSIX_SUCCESS(kill(pid, SIGCONT), "failed to send SIGCONT to child process");
 279         dispatch_main();
 280 }
 281
 282 T_HELPER_DECL(allocate_pages, "allocates pages to compress") {
 283         int i, j, ret, size_mb, page_type, vmpgsize, freezable_state;
 284         size_t vmpgsize_length;
 285         __block int num_pages;
 286         __block char **buf;
 287         dispatch_source_t ds_signal;
 288
 289         vmpgsize_length = sizeof(vmpgsize);
 290         ret = sysctlbyname("vm.pagesize", &vmpgsize, &vmpgsize_length, NULL, 0);
 291         if (ret != 0) {
 292                 exit(SYSCTL_VM_PAGESIZE_FAILED);
 293         }
 294         if (vmpgsize == 0) {
 295                 exit(VM_PAGESIZE_IS_ZERO);
 296         }
 297
 298         if (argc < 2) {
 299                 exit(TOO_FEW_ARGUMENTS);
 300         }
 301
 302         size_mb = atoi(argv[0]);
 303         page_type = atoi(argv[1]);
 304         num_pages = size_mb * 1024 * 1024 / vmpgsize;
 305         buf = (char**)malloc(sizeof(char*) * (size_t)num_pages);
 306
 307         // Switch on the type of page requested
 308         switch (page_type) {
 309         case ALL_ZEROS:
 310                 allocate_zero_pages(buf, num_pages, vmpgsize);
 311                 break;
 312         case MOSTLY_ZEROS:
 313                 allocate_mostly_zero_pages(buf, num_pages, vmpgsize);
 314                 break;
 315         case RANDOM:
 316                 allocate_random_pages(buf, num_pages, vmpgsize);
 317                 break;
 318         case TYPICAL:
 319                 allocate_representative_pages(buf, num_pages, vmpgsize);
 320                 break;
 321         default:
 322                 exit(UNKNOWN_PAGE_TYPE);
 323         }
 324
 325         for (j = 0; j < num_pages; j++) {
 326                 i = buf[j][0];
 327         }
 328
 329         decompr_time = dt_stat_time_create("decompression_latency");
 330
 331         /* Opt in to freezing. */
 332         printf("[%d] Setting state to freezable\n", getpid());
 333         if (memorystatus_control(MEMORYSTATUS_CMD_SET_PROCESS_IS_FREEZABLE, getpid(), 1, NULL, 0) != KERN_SUCCESS) {
 334                 exit(MEMORYSTATUS_CONTROL_FAILED);
 335         }
 336
 337         /* Verify that the state has been set correctly */
 338         freezable_state = memorystatus_control(MEMORYSTATUS_CMD_GET_PROCESS_IS_FREEZABLE, getpid(), 0, NULL, 0);
 339         if (freezable_state != 1) {
 340                 exit(IS_FREEZABLE_NOT_AS_EXPECTED);
 341         }
 342
 343         dispatch_after(dispatch_time(DISPATCH_TIME_NOW, NSEC_PER_SEC), dispatch_get_main_queue(), ^{
 344                 /* Signal to the parent that we're done allocating and it's ok to freeze us */
 345                 printf("[%d] Sending initial signal to parent to begin freezing\n", getpid());
 346                 if (kill(getppid(), SIGUSR1) != 0) {
 347                         exit(INITIAL_SIGNAL_TO_PARENT_FAILED);
 348                 }
 349         });
 350
 351         signal(SIGUSR1, SIG_IGN);
 352         ds_signal = dispatch_source_create(DISPATCH_SOURCE_TYPE_SIGNAL, SIGUSR1, 0, dispatch_get_main_queue());
 353         if (ds_signal == NULL) {
 354                 exit(DISPATCH_SOURCE_CREATE_FAILED);
 355         }
 356
 357         __block bool collect_dt_stat_measurements = true;
 358
 359         dispatch_source_set_event_handler(ds_signal, ^{
 360                 volatile int tmp;
 361                 uint64_t decompr_start_time, decompr_end_time;
 362
 363                 decompr_start_time = mach_absolute_time();
 364
 365                 /* Make sure all the pages are accessed before trying to freeze again */
 366                 for (int x = 0; x < num_pages; x++) {
 367                         tmp = buf[x][0];
 368                 }
 369
 370                 decompr_end_time = mach_absolute_time();
 371
 372                 if (collect_dt_stat_measurements) {
 373                         if (dt_stat_stable(decompr_time)) {
 374                                 collect_dt_stat_measurements = false;
 375                                 dt_stat_finalize(decompr_time);
 376                                 if (kill(getppid(), SIGUSR2) != 0) {
 377                                         exit(SIGNAL_TO_PARENT_FAILED);
 378                                 }
 379                         } else {
 380                                 dt_stat_mach_time_add(decompr_time, decompr_end_time - decompr_start_time);
 381                         }
 382                 }
 383
 384                 if (kill(getppid(), SIGUSR1) != 0) {
 385                         exit(SIGNAL_TO_PARENT_FAILED);
 386                 }
 387         });
 388         dispatch_activate(ds_signal);
 389
 390         dispatch_main();
 391 }
 392
 393 // Numbers for 10MB and above are fairly reproducible. Anything smaller shows a lot of variation.
 394
 395 // Keeping just the 100MB version for iOSMark
 396 #ifndef DT_IOSMARK
 397 T_DECL(compr_10MB_zero,
 398     "Compression latency for 10MB - zero pages",
 399     T_META_ASROOT(true),
 400     T_META_SYSCTL_INT(SYSCTL_FREEZE_TO_MEMORY)) {
 401         run_compressor_test(10, ALL_ZEROS);
 402 }
 403
 404 T_DECL(compr_10MB_mostly_zero,
 405     "Compression latency for 10MB - mostly zero pages",
 406     T_META_ASROOT(true),
 407     T_META_SYSCTL_INT(SYSCTL_FREEZE_TO_MEMORY)) {
 408         run_compressor_test(10, MOSTLY_ZEROS);
 409 }
 410
 411 T_DECL(compr_10MB_random,
 412     "Compression latency for 10MB - random pages",
 413     T_META_ASROOT(true),
 414     T_META_SYSCTL_INT(SYSCTL_FREEZE_TO_MEMORY)) {
 415         run_compressor_test(10, RANDOM);
 416 }
 417
 418 T_DECL(compr_10MB_typical,
 419     "Compression latency for 10MB - typical pages",
 420     T_META_ASROOT(true),
 421     T_META_SYSCTL_INT(SYSCTL_FREEZE_TO_MEMORY)) {
 422         run_compressor_test(10, TYPICAL);
 423 }
 424
 425 T_DECL(compr_100MB_zero,
 426     "Compression latency for 100MB - zero pages",
 427     T_META_ASROOT(true),
 428     T_META_SYSCTL_INT(SYSCTL_FREEZE_TO_MEMORY)) {
 429         run_compressor_test(100, ALL_ZEROS);
 430 }
 431
 432 T_DECL(compr_100MB_mostly_zero,
 433     "Compression latency for 100MB - mostly zero pages",
 434     T_META_ASROOT(true),
 435     T_META_SYSCTL_INT(SYSCTL_FREEZE_TO_MEMORY)) {
 436         run_compressor_test(100, MOSTLY_ZEROS);
 437 }
 438
 439 T_DECL(compr_100MB_random,
 440     "Compression latency for 100MB - random pages",
 441     T_META_ASROOT(true),
 442     T_META_SYSCTL_INT(SYSCTL_FREEZE_TO_MEMORY)) {
 443         run_compressor_test(100, RANDOM);
 444 }
 445 #endif
 446
 447 T_DECL(compr_100MB_typical,
 448     "Compression latency for 100MB - typical pages",
 449     T_META_ASROOT(true),
 450     T_META_SYSCTL_INT(SYSCTL_FREEZE_TO_MEMORY)) {
 451         run_compressor_test(100, TYPICAL);
 452 }