OSX/regressions/test/testenv.m

   1 /*
   2  * Copyright (c) 2005-2007,2009-2014 Apple Inc. All Rights Reserved.
   3  *
   4  * @APPLE_LICENSE_HEADER_START@
   5  *
   6  * This file contains Original Code and/or Modifications of Original Code
   7  * as defined in and that are subject to the Apple Public Source License
   8  * Version 2.0 (the 'License'). You may not use this file except in
   9  * compliance with the License. Please obtain a copy of the License at
  10  * http://www.opensource.apple.com/apsl/ and read it before using this
  11  * file.
  12  *
  13  * The Original Code and all software distributed under the License are
  14  * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  15  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  16  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
  17  * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
  18  * Please see the License for the specific language governing rights and
  19  * limitations under the License.
  20  *
  21  * @APPLE_LICENSE_HEADER_END@
  22  *
  23  * testenv.c
  24  */
  25
  26 /*
  27  * This is to fool os services to not provide the Keychain manager
  28  * interface tht doens't work since we don't have unified headers
  29  * between iOS and OS X. rdar://23405418/
  30  */
  31 #define __KEYCHAINCORE__ 1
  32
  33
  34 #import <Foundation/Foundation.h>
  35 #include <fcntl.h>
  36 #include <stdarg.h>
  37 #include <stdlib.h>
  38 #include <string.h>
  39 #include <sys/stat.h>
  40 #include <sys/types.h>
  41 #include <unistd.h>
  42 #include <stdbool.h>
  43 #include <time.h>
  44 #include <sys/time.h>
  45
  46 #include "testmore.h"
  47 #include "testenv.h"
  48
  49 #include <utilities/debugging.h>
  50 #include <utilities/SecCFRelease.h>
  51 #include <utilities/SecCFWrappers.h>
  52 #include <utilities/SecFileLocations.h>
  53
  54 #include <dispatch/dispatch.h>
  55
  56 #include "keychain/ckks/CKKS.h"
  57
  58 int test_strict_bats = 1;
  59 int test_verbose = 0;
  60 int test_onebatstest = 0;
  61 int test_check_leaks = 0;
  62
  63 #ifdef NO_SERVER
  64 #include "keychain/securityd/spi.h"
  65
  66 static int current_dir = -1;
  67 static char scratch_dir[50];
  68 static char *home_var;
  69
  70 static int
  71 rmdir_recursive(const char *path)
  72 {
  73 #if (!TARGET_OS_SIMULATOR)
  74     char command_buf[256];
  75         if (strlen(path) + 10 > sizeof(command_buf) || strchr(path, '\''))
  76         {
  77                 fprintf(stdout, "# rmdir_recursive: invalid path: %s", path);
  78                 return -1;
  79         }
  80
  81         sprintf(command_buf, "/bin/rm -rf '%s'", path);
  82         return system(command_buf);
  83 #else
  84     fprintf(stdout, "# rmdir_recursive: simulator can't rmdir, leaving path: %s\n", path);
  85     return 0;
  86 #endif
  87 }
  88 #endif
  89
  90 static int tests_init(void) {
  91     int ok = 0;
  92 #ifdef NO_SERVER
  93
  94         setup("tests_init");
  95
  96     // Get TMP directory
  97     NSError* error = nil;
  98     NSString* pid = [NSString stringWithFormat: @"tst-%d", [[NSProcessInfo processInfo] processIdentifier]];
  99     NSURL* tmpDirURL = [[NSURL fileURLWithPath:NSTemporaryDirectory() isDirectory:YES] URLByAppendingPathComponent:pid];
 100
 101     ok = ok(tmpDirURL, "Got error setting up tmp dir: %@", error);
 102
 103     ok = ok && ok([[NSFileManager defaultManager] createDirectoryAtURL:tmpDirURL
 104                                            withIntermediateDirectories:NO
 105                                                             attributes:NULL
 106                                                                  error:&error],
 107                   "Failed to make %@: %@", tmpDirURL, error);
 108
 109     NSURL* libraryURL = [tmpDirURL URLByAppendingPathComponent:@"Library"];
 110     NSURL* preferencesURL = [tmpDirURL URLByAppendingPathComponent:@"Preferences"];
 111
 112     ok =  (ok && ok_unix(current_dir = open(".", O_RDONLY), "open")
 113               && ok_unix(chdir([tmpDirURL fileSystemRepresentation]), "chdir")
 114               && ok_unix(setenv("HOME", [tmpDirURL fileSystemRepresentation], 1), "setenv")
 115               && ok(home_var = getenv("HOME"), "getenv"));
 116
 117     ok = ok && ok([[NSFileManager defaultManager] createDirectoryAtURL:libraryURL
 118                                            withIntermediateDirectories:NO
 119                                                             attributes:NULL
 120                                                                  error:&error],
 121                   "Failed to make %@: %@", libraryURL, error);
 122
 123     ok = ok && ok([[NSFileManager defaultManager] createDirectoryAtURL:preferencesURL
 124                                            withIntermediateDirectories:NO
 125                                                             attributes:NULL
 126                                                                  error:&error],
 127                   "Failed to make %@: %@", preferencesURL, error);
 128
 129     if (ok > 0) {
 130         securityd_init((__bridge CFURLRef) tmpDirURL);
 131         SecCKKSDisable();
 132     }
 133
 134 #endif
 135
 136     return ok;
 137 }
 138
 139 static void
 140 tests_end(void)
 141 {
 142 #ifdef NO_SERVER
 143         setup("tests_end");
 144
 145         /* Restore previous cwd and remove scratch dir. */
 146     ok_unix(fchdir(current_dir), "fchdir");
 147     ok_unix(close(current_dir), "close");
 148     ok_unix(rmdir_recursive(scratch_dir), "rmdir_recursive");
 149 #endif
 150 }
 151
 152 static void usage(const char *progname)
 153 {
 154     fprintf(stderr, "usage: %s [-w][testname [testargs] ...]\n", progname);
 155     exit(1);
 156 }
 157
 158 /* run one test, described by test, return info in test struct */
 159 static int tests_run_test(struct one_test_s *test, int argc, char * const *argv)
 160 {
 161     int ch;
 162
 163     while ((ch = getopt(argc, argv, "v")) != -1)
 164     {
 165         switch  (ch)
 166         {
 167             case 'v':
 168                 test_verbose++;
 169                 break;
 170             default:
 171                 usage(argv[0]);
 172         }
 173     }
 174
 175     if (test_onebatstest)
 176         fprintf(stdout, "[TEST] %s\n", test->name);
 177     else
 178         fprintf(stdout, "[BEGIN] %s\n", test->name);
 179
 180     const char *token = "PASS";
 181     if (test->entry == NULL) {
 182         fprintf(stdout, "%s:%d: error, no entry\n", __FILE__, __LINE__);
 183         test->failed_tests = 1;
 184     } else {
 185         struct timeval start, stop;
 186         gettimeofday(&start, NULL);
 187         @autoreleasepool {
 188             test->entry(argc, argv);
 189         }
 190         gettimeofday(&stop, NULL);
 191         /* this may overflow... */
 192         test->duration = (stop.tv_sec-start.tv_sec) * 1000 + (stop.tv_usec / 1000) - (start.tv_usec / 1000);
 193         if (test_plan_ok()) {
 194             token = "WARN";
 195         }
 196     }
 197
 198     /*
 199      * Check if we have any leaks, allow skipping test
 200      */
 201
 202     if (test_check_leaks) {
 203         char *cmd = NULL;
 204         int ret = 0;
 205
 206         asprintf(&cmd, "leaks %d >/dev/null", getpid());
 207         if (cmd) {
 208             ret = system(cmd);
 209             free(cmd);
 210         }
 211         if (ret != 0) {
 212             fprintf(stdout, "leaks found in test %s\n", test->name);
 213             token = "FAIL";
 214         }
 215     }
 216
 217     test_plan_final(&test->failed_tests, &test->todo_pass_tests, &test->todo_tests,
 218                     &test->actual_tests, &test->planned_tests,
 219                     &test->plan_file, &test->plan_line);
 220     if (test_verbose) {
 221         // TODO Use ccperf timing and  printing routines that use proper si scaling
 222         fprintf(stdout, "%s  took %lu ms\n", test->name, test->duration);
 223     }
 224     if (test->failed_tests) {
 225         token = "FAIL";
 226     }
 227     fprintf(stdout, "[%s] %s\n", token, test->name);
 228
 229     return test->failed_tests;
 230 }
 231
 232 static int strcmp_under_is_dash(const char *s, const char *t) {
 233     for (;;) {
 234         char a = *s++, b = *t++;
 235         if (a != b) {
 236             if (a != '_' || b != '-')
 237                 return a - b;
 238         } else if (a == 0) {
 239             return 0;
 240         }
 241     }
 242 }
 243
 244 static int tests_named_index(const char *testcase)
 245 {
 246     int i;
 247
 248     for (i = 0; testlist[i].name; ++i) {
 249         if (strcmp_under_is_dash(testlist[i].name, testcase) == 0) {
 250             return i;
 251         }
 252     }
 253
 254     return -1;
 255 }
 256
 257 static int tests_run_all(int argc, char * const *argv)
 258 {
 259     int curroptind = optind;
 260     int i;
 261     int failcount=0;
 262
 263     for (i = 0; testlist[i].name; ++i) {
 264         if(!testlist[i].off) {
 265             @autoreleasepool {
 266                 failcount+=tests_run_test(&testlist[i], argc, argv);
 267             }
 268             optind = curroptind;
 269         }
 270     }
 271
 272     return failcount;
 273 }
 274
 275 static void
 276 tests_summary(const char *progname) {
 277     int failed_tests = 0;
 278     int todo_pass_tests = 0;
 279     int todo_tests = 0;
 280     int actual_tests = 0;
 281     int planned_tests = 0;
 282
 283     // First compute the totals to help decide if we need to print headers or not.
 284     for (int i = 0; testlist[i].name; ++i) {
 285         if (!testlist[i].off) {
 286             failed_tests += testlist[i].failed_tests;
 287             todo_pass_tests += testlist[i].todo_pass_tests;
 288             todo_tests += testlist[i].todo_tests;
 289             actual_tests += testlist[i].actual_tests;
 290             planned_tests += testlist[i].planned_tests;
 291         }
 292     }
 293
 294     if (!test_onebatstest) {
 295         fprintf(stdout, "[%s] %s\n", failed_tests ? "FAIL" : (actual_tests == planned_tests ? "PASS" : "WARN"), progname);
 296     }
 297
 298     fprintf(stdout, "[SUMMARY]\n");
 299
 300     // -v -v makes the summary verbose as well.
 301     fprintf(stdout, "Test name                                                 failed !failed  todo  total   plan\n");
 302     if (test_verbose > 1 || failed_tests || todo_pass_tests || actual_tests != planned_tests || (test_verbose && todo_tests)) {
 303         fprintf(stdout, "============================================================================================\n");
 304     }
 305     for (int i = 0; testlist[i].name; ++i) {
 306         if (!testlist[i].off) {
 307             const char *token = NULL;
 308             if (testlist[i].failed_tests) {
 309                 token = "FAIL";
 310             } else if (testlist[i].actual_tests != testlist[i].planned_tests) {
 311                 token = "WARN";
 312             } else if (testlist[i].todo_pass_tests) {
 313                 token = "WARN";
 314             } else if (test_verbose > 1 || (test_verbose && testlist[i].todo_tests)) {
 315                 token = "PASS";
 316             }
 317             if (token) {
 318                 fprintf(stdout, "[%s] %-50s %6d %6d %6d %6d %6d\n", token, testlist[i].name, testlist[i].failed_tests, testlist[i].todo_pass_tests, testlist[i].todo_tests, testlist[i].actual_tests, testlist[i].planned_tests);
 319             }
 320         }
 321     }
 322     if (test_verbose > 1 || failed_tests || todo_pass_tests || actual_tests != planned_tests || (test_verbose && todo_tests)) {
 323         fprintf(stdout, "============================================================================================\n");
 324     }
 325     fprintf(stdout, "Totals                                                    %6d %6d %6d %6d %6d\n", failed_tests, todo_pass_tests, todo_tests, actual_tests, planned_tests);
 326
 327 }
 328
 329 #define ASYNC_LOGGING  0
 330 #define DATE_LOGGING   0
 331
 332 int
 333 tests_begin(int argc, char * const *argv) {
 334     const char *testcase = NULL;
 335     bool initialized = false;
 336     __block bool print_security_logs = false;
 337     int testix = -1;
 338     int failcount = 0;
 339         int ch;
 340     int loop = 0;
 341     int list = 0;
 342
 343 #if ASYNC_LOGGING
 344     dispatch_queue_t show_queue = dispatch_queue_create("sec log queue", DISPATCH_QUEUE_SERIAL);
 345 #endif
 346
 347
 348 #if USEOLDLOGGING
 349     security_log_handler handle_logs = ^(int level, CFStringRef scope, const char *function, const char *file, int line, CFStringRef message) {
 350         time_t now = time(NULL);
 351
 352 #if DATE_LOGGING
 353         char *date = ctime(&now);
 354         date[19] = '\0';
 355         CFStringRef logStr = CFStringCreateWithFormat(kCFAllocatorDefault, NULL,
 356                                                       CFSTR("%s %@{} %s %@\n"), date + 4,
 357                                                       scope ? scope : CFSTR(""), function, message);
 358 #else
 359         CFStringRef logStr = CFStringCreateWithFormat(kCFAllocatorDefault, NULL,
 360                                                       CFSTR("%lu %@{} %s %@\n"), now,
 361                                                       scope ? scope : CFSTR(""), function, message);
 362 #endif
 363 #if ASYNC_LOGGING
 364         dispatch_async(show_queue, ^{ CFShow(logStr); CFReleaseSafe(logStr); });
 365 #else
 366         CFShow(logStr);
 367         CFReleaseSafe(logStr);
 368 #endif
 369     };
 370 #endif
 371
 372
 373     for (;;) {
 374         while (!testcase && (ch = getopt(argc, argv, "blL1vwqs")) != -1)
 375         {
 376             switch  (ch)
 377             {
 378             case 's':
 379                 if (!print_security_logs) {
 380                     //add_security_log_handler(handle_logs);
 381                     print_security_logs = true;
 382                 }
 383                 break;
 384
 385             case 'b':
 386                 test_strict_bats = 0;
 387                 break;
 388
 389             case 'v':
 390                 test_verbose++;
 391                 break;
 392
 393             case 'w':
 394                 sleep(100);
 395                 break;
 396             case 'l':
 397                 loop=1;
 398                 break;
 399             case 'L':
 400                 list=1;
 401                 break;
 402             case '1':
 403                 test_onebatstest=1;
 404                 break;
 405             case '?':
 406             default:
 407                 printf("invalid option %c\n",ch);
 408                 usage(argv[0]);
 409             }
 410         }
 411
 412         if (optind < argc) {
 413             testix = tests_named_index(argv[optind]);
 414             if(testix<0) {
 415                 printf("invalid test %s\n",argv[optind]);
 416                 usage(argv[0]);
 417             }
 418         }
 419
 420         if(!print_security_logs) secLogDisable();
 421
 422
 423         if (!list && !initialized && !test_onebatstest)
 424             fprintf(stdout, "[TEST] %s\n", getprogname());
 425         if (testix < 0) {
 426             if (!initialized) {
 427                 tests_init();
 428                 initialized = true;
 429                 (void)initialized;
 430                 if (!list)
 431                     failcount+=tests_run_all(argc, argv);
 432             }
 433             break;
 434         } else {
 435             if (!initialized) {
 436                 tests_init();
 437                 initialized = true;
 438                 for (int i = 0; testlist[i].name; ++i) {
 439                     testlist[i].off = 1;
 440                 }
 441             }
 442             optind++;
 443             testlist[testix].off = 0;
 444             if (!list) {
 445                 @autoreleasepool {
 446                     failcount+=tests_run_test(&testlist[testix], argc, argv);
 447                 }
 448             }
 449             testix = -1;
 450         }
 451     }
 452     if (list) {
 453         for (int i = 0; testlist[i].name; ++i) {
 454             if (!testlist[i].off) {
 455                     fprintf(stdout, "%s\n", testlist[i].name);
 456             }
 457         }
 458     } else {
 459         tests_summary(getprogname());
 460     }
 461
 462     // remove_security_log_handler(handle_logs);
 463     fflush(stdout);
 464
 465
 466     /* Cleanups */
 467     tests_end();
 468
 469     if (loop) {
 470         printf("Looping until key press 'q'. You can run leaks now.\n");
 471         while(getchar()!='q');
 472     }
 473
 474     return failcount;
 475 }
 476