OSX/regressions/test/testenv.m

   1 /*
   2  * Copyright (c) 2005-2007,2009-2014 Apple Inc. All Rights Reserved.
   3  *
   4  * @APPLE_LICENSE_HEADER_START@
   5  *
   6  * This file contains Original Code and/or Modifications of Original Code
   7  * as defined in and that are subject to the Apple Public Source License
   8  * Version 2.0 (the 'License'). You may not use this file except in
   9  * compliance with the License. Please obtain a copy of the License at
  10  * http://www.opensource.apple.com/apsl/ and read it before using this
  11  * file.
  12  *
  13  * The Original Code and all software distributed under the License are
  14  * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  15  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  16  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
  17  * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
  18  * Please see the License for the specific language governing rights and
  19  * limitations under the License.
  20  *
  21  * @APPLE_LICENSE_HEADER_END@
  22  *
  23  * testenv.c
  24  */
  25
  26 /*
  27  * This is to fool os services to not provide the Keychain manager
  28  * interface tht doens't work since we don't have unified headers
  29  * between iOS and OS X. rdar://23405418/
  30  */
  31 #define __KEYCHAINCORE__ 1
  32
  33
  34 #import <Foundation/Foundation.h>
  35 #include <fcntl.h>
  36 #include <stdarg.h>
  37 #include <stdlib.h>
  38 #include <string.h>
  39 #include <sys/stat.h>
  40 #include <sys/types.h>
  41 #include <unistd.h>
  42 #include <stdbool.h>
  43 #include <time.h>
  44 #include <sys/time.h>
  45
  46 #include "testmore.h"
  47 #include "testenv.h"
  48
  49 #include <utilities/debugging.h>
  50 #include <utilities/SecCFRelease.h>
  51 #include <utilities/SecCFWrappers.h>
  52 #include <utilities/SecFileLocations.h>
  53
  54 #include <dispatch/dispatch.h>
  55
  56
  57 int test_strict_bats = 1;
  58 int test_verbose = 0;
  59 int test_onebatstest = 0;
  60
  61 #ifdef NO_SERVER
  62 #include <securityd/spi.h>
  63
  64 static int current_dir = -1;
  65 static char scratch_dir[50];
  66 static char *home_var;
  67 static bool keep_scratch_dir = false;
  68
  69 static int
  70 rmdir_recursive(const char *path)
  71 {
  72 #if (!TARGET_IPHONE_SIMULATOR)
  73     char command_buf[256];
  74         if (strlen(path) + 10 > sizeof(command_buf) || strchr(path, '\''))
  75         {
  76                 fprintf(stdout, "# rmdir_recursive: invalid path: %s", path);
  77                 return -1;
  78         }
  79
  80         sprintf(command_buf, "/bin/rm -rf '%s'", path);
  81         return system(command_buf);
  82 #else
  83     fprintf(stdout, "# rmdir_recursive: simulator can't rmdir, leaving path: %s\n", path);
  84     return 0;
  85 #endif
  86 }
  87 #endif
  88
  89 static int tests_init(void) {
  90     int ok = 0;
  91 #ifdef NO_SERVER
  92         char preferences_dir[80];
  93         char library_dir[70];
  94
  95         setup("tests_init");
  96
  97     /* Create scratch dir for tests to run in. */
  98     sprintf(scratch_dir, "/tmp/tst-%d", getpid());
  99     if (keep_scratch_dir) {
 100         printf("running tests with HOME=%s\n", scratch_dir);
 101     }
 102
 103     sprintf(library_dir, "%s/Library", scratch_dir);
 104     sprintf(preferences_dir, "%s/Preferences", library_dir);
 105     ok =  (ok_unix(mkdir(scratch_dir, 0755), "mkdir") &&
 106            ok_unix(current_dir = open(".", O_RDONLY), "open") &&
 107            ok_unix(chdir(scratch_dir), "chdir") &&
 108            ok_unix(setenv("HOME", scratch_dir, 1), "setenv") &&
 109            /* @@@ Work around a bug that the prefs code in
 110             libsecurity_keychain never creates the Library/Preferences
 111             dir. */
 112            ok_unix(mkdir(library_dir, 0755), "mkdir") &&
 113            ok_unix(mkdir(preferences_dir, 0755), "mkdir") &&
 114            ok(home_var = getenv("HOME"), "getenv"));
 115
 116     if (ok > 0)
 117         securityd_init(scratch_dir);
 118 #endif
 119
 120     return ok;
 121 }
 122
 123 static int
 124 tests_end(void)
 125 {
 126 #ifdef NO_SERVER
 127         setup("tests_end");
 128         /* Restore previous cwd and remove scratch dir. */
 129         int ok = ok_unix(fchdir(current_dir), "fchdir");
 130         if (ok)
 131                 ok = ok_unix(close(current_dir), "close");
 132         if (ok) {
 133                 if (!keep_scratch_dir) {
 134                         ok = ok_unix(rmdir_recursive(scratch_dir), "rmdir_recursive");
 135                 }
 136         }
 137
 138         return ok;
 139 #else
 140     return 0;
 141 #endif
 142 }
 143
 144 static void usage(const char *progname)
 145 {
 146     fprintf(stderr, "usage: %s [-k][-w][testname [testargs] ...]\n", progname);
 147     exit(1);
 148 }
 149
 150 /* run one test, described by test, return info in test struct */
 151 static int tests_run_test(struct one_test_s *test, int argc, char * const *argv)
 152 {
 153     int ch;
 154
 155     while ((ch = getopt(argc, argv, "v")) != -1)
 156     {
 157         switch  (ch)
 158         {
 159             case 'v':
 160                 test_verbose++;
 161                 break;
 162             default:
 163                 usage(argv[0]);
 164         }
 165     }
 166
 167     if (test_onebatstest)
 168         fprintf(stdout, "[TEST] %s\n", test->name);
 169     else
 170         fprintf(stdout, "[BEGIN] %s\n", test->name);
 171
 172     const char *token = "PASS";
 173     if (test->entry == NULL) {
 174         fprintf(stdout, "%s:%d: error, no entry\n", __FILE__, __LINE__);
 175         test->failed_tests = 1;
 176     } else {
 177         struct timeval start, stop;
 178         gettimeofday(&start, NULL);
 179         test->entry(argc, argv);
 180         gettimeofday(&stop, NULL);
 181         /* this may overflow... */
 182         test->duration = (stop.tv_sec-start.tv_sec) * 1000 + (stop.tv_usec / 1000) - (start.tv_usec / 1000);
 183         if (test_plan_ok()) {
 184             token = "WARN";
 185         }
 186     }
 187
 188     test_plan_final(&test->failed_tests, &test->todo_pass_tests, &test->todo_tests,
 189                     &test->actual_tests, &test->planned_tests,
 190                     &test->plan_file, &test->plan_line);
 191     if (test_verbose) {
 192         // TODO Use ccperf timing and  printing routines that use proper si scaling
 193         fprintf(stdout, "%s  took %lu ms\n", test->name, test->duration);
 194     }
 195     if (test->failed_tests) {
 196         token = "FAIL";
 197     }
 198     fprintf(stdout, "[%s] %s\n", token, test->name);
 199
 200     return test->failed_tests;
 201 }
 202
 203 static int strcmp_under_is_dash(const char *s, const char *t) {
 204     for (;;) {
 205         char a = *s++, b = *t++;
 206         if (a != b) {
 207             if (a != '_' || b != '-')
 208                 return a - b;
 209         } else if (a == 0) {
 210             return 0;
 211         }
 212     }
 213 }
 214
 215 static int tests_named_index(const char *testcase)
 216 {
 217     int i;
 218
 219     for (i = 0; testlist[i].name; ++i) {
 220         if (strcmp_under_is_dash(testlist[i].name, testcase) == 0) {
 221             return i;
 222         }
 223     }
 224
 225     return -1;
 226 }
 227
 228 static int tests_run_all(int argc, char * const *argv)
 229 {
 230     int curroptind = optind;
 231     int i;
 232     int failcount=0;
 233
 234     for (i = 0; testlist[i].name; ++i) {
 235         if(!testlist[i].off) {
 236             @autoreleasepool {
 237                 failcount+=tests_run_test(&testlist[i], argc, argv);
 238             }
 239             optind = curroptind;
 240         }
 241     }
 242
 243     return failcount;
 244 }
 245
 246 static void
 247 tests_summary(const char *progname) {
 248     int failed_tests = 0;
 249     int todo_pass_tests = 0;
 250     int todo_tests = 0;
 251     int actual_tests = 0;
 252     int planned_tests = 0;
 253
 254     // First compute the totals to help decide if we need to print headers or not.
 255     for (int i = 0; testlist[i].name; ++i) {
 256         if (!testlist[i].off) {
 257             failed_tests += testlist[i].failed_tests;
 258             todo_pass_tests += testlist[i].todo_pass_tests;
 259             todo_tests += testlist[i].todo_tests;
 260             actual_tests += testlist[i].actual_tests;
 261             planned_tests += testlist[i].planned_tests;
 262         }
 263     }
 264
 265     if (!test_onebatstest) {
 266         fprintf(stdout, "[%s] %s\n", failed_tests ? "FAIL" : (actual_tests == planned_tests ? "PASS" : "WARN"), progname);
 267     }
 268
 269     fprintf(stdout, "[SUMMARY]\n");
 270
 271     // -v -v makes the summary verbose as well.
 272     fprintf(stdout, "Test name                                                 failed !failed  todo  total   plan\n");
 273     if (test_verbose > 1 || failed_tests || todo_pass_tests || actual_tests != planned_tests || (test_verbose && todo_tests)) {
 274         fprintf(stdout, "============================================================================================\n");
 275     }
 276     for (int i = 0; testlist[i].name; ++i) {
 277         if (!testlist[i].off) {
 278             const char *token = NULL;
 279             if (testlist[i].failed_tests) {
 280                 token = "FAIL";
 281             } else if (testlist[i].actual_tests != testlist[i].planned_tests) {
 282                 token = "WARN";
 283             } else if (testlist[i].todo_pass_tests) {
 284                 token = "WARN";
 285             } else if (test_verbose > 1 || (test_verbose && testlist[i].todo_tests)) {
 286                 token = "PASS";
 287             }
 288             if (token) {
 289                 fprintf(stdout, "[%s] %-50s %6d %6d %6d %6d %6d\n", token, testlist[i].name, testlist[i].failed_tests, testlist[i].todo_pass_tests, testlist[i].todo_tests, testlist[i].actual_tests, testlist[i].planned_tests);
 290             }
 291         }
 292     }
 293     if (test_verbose > 1 || failed_tests || todo_pass_tests || actual_tests != planned_tests || (test_verbose && todo_tests)) {
 294         fprintf(stdout, "============================================================================================\n");
 295     }
 296     fprintf(stdout, "Totals                                                    %6d %6d %6d %6d %6d\n", failed_tests, todo_pass_tests, todo_tests, actual_tests, planned_tests);
 297
 298 }
 299
 300 #define ASYNC_LOGGING  0
 301 #define DATE_LOGGING   0
 302
 303 int
 304 tests_begin(int argc, char * const *argv) {
 305     const char *testcase = NULL;
 306     bool initialized = false;
 307     __block bool print_security_logs = false;
 308     int testix = -1;
 309     int failcount = 0;
 310         int ch;
 311     int loop = 0;
 312     int list = 0;
 313
 314 #if ASYNC_LOGGING
 315     dispatch_queue_t show_queue = dispatch_queue_create("sec log queue", DISPATCH_QUEUE_SERIAL);
 316 #endif
 317
 318     security_log_handler handle_logs = ^(int level, CFStringRef scope, const char *function, const char *file, int line, CFStringRef message) {
 319         time_t now = time(NULL);
 320 #if DATE_LOGGING
 321         char *date = ctime(&now);
 322         date[19] = '\0';
 323         CFStringRef logStr = CFStringCreateWithFormat(kCFAllocatorDefault, NULL,
 324                                                       CFSTR("%s %@{} %s %@\n"), date + 4,
 325                                                       scope ? scope : CFSTR(""), function, message);
 326 #else
 327         CFStringRef logStr = CFStringCreateWithFormat(kCFAllocatorDefault, NULL,
 328                                                       CFSTR("%lu %@{} %s %@\n"), now,
 329                                                       scope ? scope : CFSTR(""), function, message);
 330 #endif
 331 #if ASYNC_LOGGING
 332         dispatch_async(show_queue, ^{ CFShow(logStr); CFReleaseSafe(logStr); });
 333 #else
 334         CFShow(logStr);
 335         CFReleaseSafe(logStr);
 336 #endif
 337     };
 338
 339     for (;;) {
 340         while (!testcase && (ch = getopt(argc, argv, "bklL1vwqs")) != -1)
 341         {
 342             switch  (ch)
 343             {
 344 #ifdef NO_SERVER
 345             case 'k':
 346                 keep_scratch_dir = true;
 347                 break;
 348 #endif
 349             case 's':
 350                 if (!print_security_logs) {
 351                     add_security_log_handler(handle_logs);
 352                     print_security_logs = true;
 353                 }
 354                 break;
 355
 356             case 'b':
 357                 test_strict_bats = 0;
 358                 break;
 359
 360             case 'v':
 361                 test_verbose++;
 362                 break;
 363
 364             case 'w':
 365                 sleep(100);
 366                 break;
 367             case 'l':
 368                 loop=1;
 369                 break;
 370             case 'L':
 371                 list=1;
 372                 break;
 373             case '1':
 374                 test_onebatstest=1;
 375                 break;
 376             case '?':
 377             default:
 378                 printf("invalid option %c\n",ch);
 379                 usage(argv[0]);
 380             }
 381         }
 382
 383         if (optind < argc) {
 384             testix = tests_named_index(argv[optind]);
 385             if(testix<0) {
 386                 printf("invalid test %s\n",argv[optind]);
 387                 usage(argv[0]);
 388             }
 389         }
 390
 391         if (!list && !initialized && !test_onebatstest)
 392             fprintf(stdout, "[TEST] %s\n", getprogname());
 393         if (testix < 0) {
 394             if (!initialized) {
 395                 tests_init();
 396                 initialized = true;
 397                 (void)initialized;
 398                 if (!list)
 399                     failcount+=tests_run_all(argc, argv);
 400             }
 401             break;
 402         } else {
 403             if (!initialized) {
 404                 tests_init();
 405                 initialized = true;
 406                 for (int i = 0; testlist[i].name; ++i) {
 407                     testlist[i].off = 1;
 408                 }
 409             }
 410             optind++;
 411             testlist[testix].off = 0;
 412             if (!list) {
 413                 @autoreleasepool {
 414                     failcount+=tests_run_test(&testlist[testix], argc, argv);
 415                 }
 416             }
 417             testix = -1;
 418         }
 419     }
 420     if (list) {
 421         for (int i = 0; testlist[i].name; ++i) {
 422             if (!testlist[i].off) {
 423                     fprintf(stdout, "%s\n", testlist[i].name);
 424             }
 425         }
 426     } else {
 427         tests_summary(getprogname());
 428     }
 429
 430     remove_security_log_handler(handle_logs);
 431     fflush(stdout);
 432
 433
 434     /* Cleanups */
 435     tests_end();
 436
 437     if (loop) {
 438         printf("Looping until key press 'q'. You can run leaks now.\n");
 439         while(getchar()!='q');
 440     }
 441
 442     return failcount;
 443 }
 444