OSX/regressions/test/testenv.m

   1 /*
   2  * Copyright (c) 2005-2007,2009-2014 Apple Inc. All Rights Reserved.
   3  *
   4  * @APPLE_LICENSE_HEADER_START@
   5  *
   6  * This file contains Original Code and/or Modifications of Original Code
   7  * as defined in and that are subject to the Apple Public Source License
   8  * Version 2.0 (the 'License'). You may not use this file except in
   9  * compliance with the License. Please obtain a copy of the License at
  10  * http://www.opensource.apple.com/apsl/ and read it before using this
  11  * file.
  12  *
  13  * The Original Code and all software distributed under the License are
  14  * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  15  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  16  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
  17  * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
  18  * Please see the License for the specific language governing rights and
  19  * limitations under the License.
  20  *
  21  * @APPLE_LICENSE_HEADER_END@
  22  *
  23  * testenv.c
  24  */
  25
  26 /*
  27  * This is to fool os services to not provide the Keychain manager
  28  * interface tht doens't work since we don't have unified headers
  29  * between iOS and OS X. rdar://23405418/
  30  */
  31 #define __KEYCHAINCORE__ 1
  32
  33
  34 #import <Foundation/Foundation.h>
  35 #include <fcntl.h>
  36 #include <stdarg.h>
  37 #include <stdlib.h>
  38 #include <string.h>
  39 #include <sys/stat.h>
  40 #include <sys/types.h>
  41 #include <unistd.h>
  42 #include <stdbool.h>
  43 #include <time.h>
  44 #include <sys/time.h>
  45
  46 #include "testmore.h"
  47 #include "testenv.h"
  48
  49 #include <utilities/debugging.h>
  50 #include <utilities/SecCFRelease.h>
  51 #include <utilities/SecCFWrappers.h>
  52 #include <utilities/SecFileLocations.h>
  53
  54 #include <dispatch/dispatch.h>
  55
  56 #include "keychain/ckks/CKKS.h"
  57
  58 int test_strict_bats = 1;
  59 int test_verbose = 0;
  60 int test_onebatstest = 0;
  61 int test_check_leaks = 0;
  62 char **test_skip_leaks_test = NULL;
  63
  64 #ifdef NO_SERVER
  65 #include "keychain/securityd/spi.h"
  66
  67 static int current_dir = -1;
  68 static char scratch_dir[50];
  69 static char *home_var;
  70
  71 static int
  72 rmdir_recursive(const char *path)
  73 {
  74 #if (!TARGET_OS_SIMULATOR)
  75     char command_buf[256];
  76         if (strlen(path) + 10 > sizeof(command_buf) || strchr(path, '\''))
  77         {
  78                 fprintf(stdout, "# rmdir_recursive: invalid path: %s", path);
  79                 return -1;
  80         }
  81
  82         sprintf(command_buf, "/bin/rm -rf '%s'", path);
  83         return system(command_buf);
  84 #else
  85     fprintf(stdout, "# rmdir_recursive: simulator can't rmdir, leaving path: %s\n", path);
  86     return 0;
  87 #endif
  88 }
  89 #endif
  90
  91 static int tests_init(void) {
  92     int ok = 0;
  93 #ifdef NO_SERVER
  94
  95         setup("tests_init");
  96
  97     // Get TMP directory
  98     NSError* error = nil;
  99     NSString* pid = [NSString stringWithFormat: @"tst-%d", [[NSProcessInfo processInfo] processIdentifier]];
 100     NSURL* tmpDirURL = [[NSURL fileURLWithPath:NSTemporaryDirectory() isDirectory:YES] URLByAppendingPathComponent:pid];
 101
 102     ok = ok(tmpDirURL, "Got error setting up tmp dir: %@", error);
 103
 104     ok = ok && ok([[NSFileManager defaultManager] createDirectoryAtURL:tmpDirURL
 105                                            withIntermediateDirectories:NO
 106                                                             attributes:NULL
 107                                                                  error:&error],
 108                   "Failed to make %@: %@", tmpDirURL, error);
 109
 110     NSURL* libraryURL = [tmpDirURL URLByAppendingPathComponent:@"Library"];
 111     NSURL* preferencesURL = [tmpDirURL URLByAppendingPathComponent:@"Preferences"];
 112
 113     ok =  (ok && ok_unix(current_dir = open(".", O_RDONLY), "open")
 114               && ok_unix(chdir([tmpDirURL fileSystemRepresentation]), "chdir")
 115               && ok_unix(setenv("HOME", [tmpDirURL fileSystemRepresentation], 1), "setenv")
 116               && ok(home_var = getenv("HOME"), "getenv"));
 117
 118     ok = ok && ok([[NSFileManager defaultManager] createDirectoryAtURL:libraryURL
 119                                            withIntermediateDirectories:NO
 120                                                             attributes:NULL
 121                                                                  error:&error],
 122                   "Failed to make %@: %@", libraryURL, error);
 123
 124     ok = ok && ok([[NSFileManager defaultManager] createDirectoryAtURL:preferencesURL
 125                                            withIntermediateDirectories:NO
 126                                                             attributes:NULL
 127                                                                  error:&error],
 128                   "Failed to make %@: %@", preferencesURL, error);
 129
 130     if (ok > 0) {
 131         securityd_init((__bridge CFURLRef) tmpDirURL);
 132         SecCKKSDisable();
 133     }
 134
 135 #endif
 136
 137     return ok;
 138 }
 139
 140 static void
 141 tests_end(void)
 142 {
 143 #ifdef NO_SERVER
 144         setup("tests_end");
 145
 146         /* Restore previous cwd and remove scratch dir. */
 147     ok_unix(fchdir(current_dir), "fchdir");
 148     ok_unix(close(current_dir), "close");
 149     ok_unix(rmdir_recursive(scratch_dir), "rmdir_recursive");
 150 #endif
 151 }
 152
 153 static void usage(const char *progname)
 154 {
 155     fprintf(stderr, "usage: %s [-w][testname [testargs] ...]\n", progname);
 156     exit(1);
 157 }
 158
 159 /* run one test, described by test, return info in test struct */
 160 static int tests_run_test(struct one_test_s *test, int argc, char * const *argv)
 161 {
 162     int ch;
 163
 164     while ((ch = getopt(argc, argv, "v")) != -1)
 165     {
 166         switch  (ch)
 167         {
 168             case 'v':
 169                 test_verbose++;
 170                 break;
 171             default:
 172                 usage(argv[0]);
 173         }
 174     }
 175
 176     if (test_onebatstest)
 177         fprintf(stdout, "[TEST] %s\n", test->name);
 178     else
 179         fprintf(stdout, "[BEGIN] %s\n", test->name);
 180
 181     const char *token = "PASS";
 182     if (test->entry == NULL) {
 183         fprintf(stdout, "%s:%d: error, no entry\n", __FILE__, __LINE__);
 184         test->failed_tests = 1;
 185     } else {
 186         struct timeval start, stop;
 187         gettimeofday(&start, NULL);
 188         @autoreleasepool {
 189             test->entry(argc, argv);
 190         }
 191         gettimeofday(&stop, NULL);
 192         /* this may overflow... */
 193         test->duration = (stop.tv_sec-start.tv_sec) * 1000 + (stop.tv_usec / 1000) - (start.tv_usec / 1000);
 194         if (test_plan_ok()) {
 195             token = "WARN";
 196         }
 197     }
 198
 199     /*
 200      * Check if we have any leaks, allow skipping test
 201      */
 202
 203     if (test_check_leaks) {
 204         char *cmd = NULL;
 205         int ret = 0;
 206
 207         asprintf(&cmd, "leaks %d >/dev/null", getpid());
 208         if (cmd) {
 209             ret = system(cmd);
 210             free(cmd);
 211         }
 212         if (ret != 0) {
 213             unsigned n = 0;
 214             fprintf(stdout, "leaks found in test %s\n", test->name);
 215
 216             if (test_skip_leaks_test) {
 217                 while (test_skip_leaks_test[n]) {
 218                     if (strcmp(test_skip_leaks_test[n], test->name) == 0) {
 219                         fprintf(stdout, "test %s known to be leaky, skipping\n", test->name);
 220                         ret = 0;
 221                         break;
 222                     }
 223                 }
 224             }
 225             if (ret) {
 226                 token = "FAIL";
 227             }
 228         } else {
 229             if (test_skip_leaks_test) {
 230                 unsigned n = 0;
 231
 232                 while (test_skip_leaks_test[n]) {
 233                     if (strcmp(test_skip_leaks_test[n], test->name) == 0) {
 234                         fprintf(stdout, "leaks didn't find leak in test %s, yet it was ignore\n", test->name);
 235                         token = "FAIL";
 236                         break;
 237                     }
 238                 }
 239             }
 240
 241         }
 242     }
 243
 244     test_plan_final(&test->failed_tests, &test->todo_pass_tests, &test->todo_tests,
 245                     &test->actual_tests, &test->planned_tests,
 246                     &test->plan_file, &test->plan_line);
 247     if (test_verbose) {
 248         // TODO Use ccperf timing and  printing routines that use proper si scaling
 249         fprintf(stdout, "%s  took %lu ms\n", test->name, test->duration);
 250     }
 251     if (test->failed_tests) {
 252         token = "FAIL";
 253     }
 254     fprintf(stdout, "[%s] %s\n", token, test->name);
 255
 256     return test->failed_tests;
 257 }
 258
 259 static int strcmp_under_is_dash(const char *s, const char *t) {
 260     for (;;) {
 261         char a = *s++, b = *t++;
 262         if (a != b) {
 263             if (a != '_' || b != '-')
 264                 return a - b;
 265         } else if (a == 0) {
 266             return 0;
 267         }
 268     }
 269 }
 270
 271 static int tests_named_index(const char *testcase)
 272 {
 273     int i;
 274
 275     for (i = 0; testlist[i].name; ++i) {
 276         if (strcmp_under_is_dash(testlist[i].name, testcase) == 0) {
 277             return i;
 278         }
 279     }
 280
 281     return -1;
 282 }
 283
 284 static int tests_run_all(int argc, char * const *argv)
 285 {
 286     int curroptind = optind;
 287     int i;
 288     int failcount=0;
 289
 290     for (i = 0; testlist[i].name; ++i) {
 291         if(!testlist[i].off) {
 292             @autoreleasepool {
 293                 failcount+=tests_run_test(&testlist[i], argc, argv);
 294             }
 295             optind = curroptind;
 296         }
 297     }
 298
 299     return failcount;
 300 }
 301
 302 static void
 303 tests_summary(const char *progname) {
 304     int failed_tests = 0;
 305     int todo_pass_tests = 0;
 306     int todo_tests = 0;
 307     int actual_tests = 0;
 308     int planned_tests = 0;
 309
 310     // First compute the totals to help decide if we need to print headers or not.
 311     for (int i = 0; testlist[i].name; ++i) {
 312         if (!testlist[i].off) {
 313             failed_tests += testlist[i].failed_tests;
 314             todo_pass_tests += testlist[i].todo_pass_tests;
 315             todo_tests += testlist[i].todo_tests;
 316             actual_tests += testlist[i].actual_tests;
 317             planned_tests += testlist[i].planned_tests;
 318         }
 319     }
 320
 321     if (!test_onebatstest) {
 322         fprintf(stdout, "[%s] %s\n", failed_tests ? "FAIL" : (actual_tests == planned_tests ? "PASS" : "WARN"), progname);
 323     }
 324
 325     fprintf(stdout, "[SUMMARY]\n");
 326
 327     // -v -v makes the summary verbose as well.
 328     fprintf(stdout, "Test name                                                 failed !failed  todo  total   plan\n");
 329     if (test_verbose > 1 || failed_tests || todo_pass_tests || actual_tests != planned_tests || (test_verbose && todo_tests)) {
 330         fprintf(stdout, "============================================================================================\n");
 331     }
 332     for (int i = 0; testlist[i].name; ++i) {
 333         if (!testlist[i].off) {
 334             const char *token = NULL;
 335             if (testlist[i].failed_tests) {
 336                 token = "FAIL";
 337             } else if (testlist[i].actual_tests != testlist[i].planned_tests) {
 338                 token = "WARN";
 339             } else if (testlist[i].todo_pass_tests) {
 340                 token = "WARN";
 341             } else if (test_verbose > 1 || (test_verbose && testlist[i].todo_tests)) {
 342                 token = "PASS";
 343             }
 344             if (token) {
 345                 fprintf(stdout, "[%s] %-50s %6d %6d %6d %6d %6d\n", token, testlist[i].name, testlist[i].failed_tests, testlist[i].todo_pass_tests, testlist[i].todo_tests, testlist[i].actual_tests, testlist[i].planned_tests);
 346             }
 347         }
 348     }
 349     if (test_verbose > 1 || failed_tests || todo_pass_tests || actual_tests != planned_tests || (test_verbose && todo_tests)) {
 350         fprintf(stdout, "============================================================================================\n");
 351     }
 352     fprintf(stdout, "Totals                                                    %6d %6d %6d %6d %6d\n", failed_tests, todo_pass_tests, todo_tests, actual_tests, planned_tests);
 353
 354 }
 355
 356 #define ASYNC_LOGGING  0
 357 #define DATE_LOGGING   0
 358
 359 int
 360 tests_begin(int argc, char * const *argv) {
 361     const char *testcase = NULL;
 362     bool initialized = false;
 363     __block bool print_security_logs = false;
 364     int testix = -1;
 365     int failcount = 0;
 366         int ch;
 367     int loop = 0;
 368     int list = 0;
 369
 370 #if ASYNC_LOGGING
 371     dispatch_queue_t show_queue = dispatch_queue_create("sec log queue", DISPATCH_QUEUE_SERIAL);
 372 #endif
 373
 374
 375 #if USEOLDLOGGING
 376     security_log_handler handle_logs = ^(int level, CFStringRef scope, const char *function, const char *file, int line, CFStringRef message) {
 377         time_t now = time(NULL);
 378
 379 #if DATE_LOGGING
 380         char *date = ctime(&now);
 381         date[19] = '\0';
 382         CFStringRef logStr = CFStringCreateWithFormat(kCFAllocatorDefault, NULL,
 383                                                       CFSTR("%s %@{} %s %@\n"), date + 4,
 384                                                       scope ? scope : CFSTR(""), function, message);
 385 #else
 386         CFStringRef logStr = CFStringCreateWithFormat(kCFAllocatorDefault, NULL,
 387                                                       CFSTR("%lu %@{} %s %@\n"), now,
 388                                                       scope ? scope : CFSTR(""), function, message);
 389 #endif
 390 #if ASYNC_LOGGING
 391         dispatch_async(show_queue, ^{ CFShow(logStr); CFReleaseSafe(logStr); });
 392 #else
 393         CFShow(logStr);
 394         CFReleaseSafe(logStr);
 395 #endif
 396     };
 397 #endif
 398
 399
 400     for (;;) {
 401         while (!testcase && (ch = getopt(argc, argv, "blL1vwqs")) != -1)
 402         {
 403             switch  (ch)
 404             {
 405             case 's':
 406                 if (!print_security_logs) {
 407                     //add_security_log_handler(handle_logs);
 408                     print_security_logs = true;
 409                 }
 410                 break;
 411
 412             case 'b':
 413                 test_strict_bats = 0;
 414                 break;
 415
 416             case 'v':
 417                 test_verbose++;
 418                 break;
 419
 420             case 'w':
 421                 sleep(100);
 422                 break;
 423             case 'l':
 424                 loop=1;
 425                 break;
 426             case 'L':
 427                 list=1;
 428                 break;
 429             case '1':
 430                 test_onebatstest=1;
 431                 break;
 432             case '?':
 433             default:
 434                 printf("invalid option %c\n",ch);
 435                 usage(argv[0]);
 436             }
 437         }
 438
 439         if (optind < argc) {
 440             testix = tests_named_index(argv[optind]);
 441             if(testix<0) {
 442                 printf("invalid test %s\n",argv[optind]);
 443                 usage(argv[0]);
 444             }
 445         }
 446
 447         if(!print_security_logs) secLogDisable();
 448
 449
 450         if (!list && !initialized && !test_onebatstest)
 451             fprintf(stdout, "[TEST] %s\n", getprogname());
 452         if (testix < 0) {
 453             if (!initialized) {
 454                 tests_init();
 455                 initialized = true;
 456                 (void)initialized;
 457                 if (!list)
 458                     failcount+=tests_run_all(argc, argv);
 459             }
 460             break;
 461         } else {
 462             if (!initialized) {
 463                 tests_init();
 464                 initialized = true;
 465                 for (int i = 0; testlist[i].name; ++i) {
 466                     testlist[i].off = 1;
 467                 }
 468             }
 469             optind++;
 470             testlist[testix].off = 0;
 471             if (!list) {
 472                 @autoreleasepool {
 473                     failcount+=tests_run_test(&testlist[testix], argc, argv);
 474                 }
 475             }
 476             testix = -1;
 477         }
 478     }
 479     if (list) {
 480         for (int i = 0; testlist[i].name; ++i) {
 481             if (!testlist[i].off) {
 482                     fprintf(stdout, "%s\n", testlist[i].name);
 483             }
 484         }
 485     } else {
 486         tests_summary(getprogname());
 487     }
 488
 489     // remove_security_log_handler(handle_logs);
 490     fflush(stdout);
 491
 492
 493     /* Cleanups */
 494     tests_end();
 495
 496     if (loop) {
 497         printf("Looping until key press 'q'. You can run leaks now.\n");
 498         while(getchar()!='q');
 499     }
 500
 501     return failcount;
 502 }
 503