OSX/regressions/test/testenv.c

   1 /*
   2  * Copyright (c) 2005-2007,2009-2014 Apple Inc. All Rights Reserved.
   3  *
   4  * @APPLE_LICENSE_HEADER_START@
   5  *
   6  * This file contains Original Code and/or Modifications of Original Code
   7  * as defined in and that are subject to the Apple Public Source License
   8  * Version 2.0 (the 'License'). You may not use this file except in
   9  * compliance with the License. Please obtain a copy of the License at
  10  * http://www.opensource.apple.com/apsl/ and read it before using this
  11  * file.
  12  *
  13  * The Original Code and all software distributed under the License are
  14  * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  15  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  16  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
  17  * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
  18  * Please see the License for the specific language governing rights and
  19  * limitations under the License.
  20  *
  21  * @APPLE_LICENSE_HEADER_END@
  22  *
  23  * testenv.c
  24  */
  25
  26 #include <fcntl.h>
  27 #include <stdarg.h>
  28 #include <stdlib.h>
  29 #include <string.h>
  30 #include <sys/stat.h>
  31 #include <sys/types.h>
  32 #include <unistd.h>
  33 #include <stdbool.h>
  34 #include <time.h>
  35 #include <sys/time.h>
  36
  37 #include "testmore.h"
  38 #include "testenv.h"
  39
  40 #include <utilities/debugging.h>
  41 #include <utilities/SecCFRelease.h>
  42 #include <utilities/SecCFWrappers.h>
  43 #include <utilities/SecFileLocations.h>
  44
  45 #include <dispatch/dispatch.h>
  46
  47
  48 int test_strict_bats = 1;
  49 int test_verbose = 0;
  50 int test_onebatstest = 0;
  51
  52 #ifdef NO_SERVER
  53 #include <securityd/spi.h>
  54
  55 static int current_dir = -1;
  56 static char scratch_dir[50];
  57 static char *home_var;
  58 static bool keep_scratch_dir = false;
  59
  60 static int
  61 rmdir_recursive(const char *path)
  62 {
  63 #if (!TARGET_IPHONE_SIMULATOR)
  64     char command_buf[256];
  65         if (strlen(path) + 10 > sizeof(command_buf) || strchr(path, '\''))
  66         {
  67                 fprintf(stdout, "# rmdir_recursive: invalid path: %s", path);
  68                 return -1;
  69         }
  70
  71         sprintf(command_buf, "/bin/rm -rf '%s'", path);
  72         return system(command_buf);
  73 #else
  74     fprintf(stdout, "# rmdir_recursive: simulator can't rmdir, leaving path: %s\n", path);
  75     return 0;
  76 #endif
  77 }
  78 #endif
  79
  80 static int tests_init(void) {
  81     int ok = 0;
  82 #ifdef NO_SERVER
  83         char preferences_dir[80];
  84         char library_dir[70];
  85
  86         setup("tests_init");
  87
  88     /* Create scratch dir for tests to run in. */
  89     sprintf(scratch_dir, "/tmp/tst-%d", getpid());
  90     if (keep_scratch_dir) {
  91         printf("running tests with HOME=%s\n", scratch_dir);
  92     }
  93
  94     sprintf(library_dir, "%s/Library", scratch_dir);
  95     sprintf(preferences_dir, "%s/Preferences", library_dir);
  96     ok =  (ok_unix(mkdir(scratch_dir, 0755), "mkdir") &&
  97            ok_unix(current_dir = open(".", O_RDONLY), "open") &&
  98            ok_unix(chdir(scratch_dir), "chdir") &&
  99            ok_unix(setenv("HOME", scratch_dir, 1), "setenv") &&
 100            /* @@@ Work around a bug that the prefs code in
 101             libsecurity_keychain never creates the Library/Preferences
 102             dir. */
 103            ok_unix(mkdir(library_dir, 0755), "mkdir") &&
 104            ok_unix(mkdir(preferences_dir, 0755), "mkdir") &&
 105            ok(home_var = getenv("HOME"), "getenv"));
 106
 107     if (ok > 0)
 108         securityd_init(scratch_dir);
 109 #endif
 110
 111     return ok;
 112 }
 113
 114 static int
 115 tests_end(void)
 116 {
 117 #ifdef NO_SERVER
 118         setup("tests_end");
 119         /* Restore previous cwd and remove scratch dir. */
 120         int ok = ok_unix(fchdir(current_dir), "fchdir");
 121         if (ok)
 122                 ok = ok_unix(close(current_dir), "close");
 123         if (ok) {
 124                 if (!keep_scratch_dir) {
 125                         ok = ok_unix(rmdir_recursive(scratch_dir), "rmdir_recursive");
 126                 }
 127         }
 128
 129         return ok;
 130 #else
 131     return 0;
 132 #endif
 133 }
 134
 135 static void usage(const char *progname)
 136 {
 137     fprintf(stderr, "usage: %s [-k][-w][testname [testargs] ...]\n", progname);
 138     exit(1);
 139 }
 140
 141 /* run one test, described by test, return info in test struct */
 142 static int tests_run_test(struct one_test_s *test, int argc, char * const *argv)
 143 {
 144     int ch;
 145
 146     while ((ch = getopt(argc, argv, "v")) != -1)
 147     {
 148         switch  (ch)
 149         {
 150             case 'v':
 151                 test_verbose++;
 152                 break;
 153             default:
 154                 usage(argv[0]);
 155         }
 156     }
 157
 158     if (test_onebatstest)
 159         fprintf(stdout, "[TEST] %s\n", test->name);
 160     else
 161         fprintf(stdout, "[BEGIN] %s\n", test->name);
 162
 163     const char *token = "PASS";
 164     if (test->entry == NULL) {
 165         fprintf(stdout, "%s:%d: error, no entry\n", __FILE__, __LINE__);
 166         test->failed_tests = 1;
 167     } else {
 168         struct timeval start, stop;
 169         gettimeofday(&start, NULL);
 170         test->entry(argc, argv);
 171         gettimeofday(&stop, NULL);
 172         /* this may overflow... */
 173         test->duration = (stop.tv_sec-start.tv_sec) * 1000 + (stop.tv_usec / 1000) - (start.tv_usec / 1000);
 174         if (test_plan_ok()) {
 175             token = "WARN";
 176         }
 177     }
 178
 179     test_plan_final(&test->failed_tests, &test->todo_pass_tests, &test->todo_tests,
 180                     &test->actual_tests, &test->planned_tests,
 181                     &test->plan_file, &test->plan_line);
 182     if (test_verbose) {
 183         // TODO Use ccperf timing and  printing routines that use proper si scaling
 184         fprintf(stdout, "%s  took %lu ms\n", test->name, test->duration);
 185     }
 186     if (test->failed_tests) {
 187         token = "FAIL";
 188     }
 189     fprintf(stdout, "[%s] %s\n", token, test->name);
 190
 191     return test->failed_tests;
 192 }
 193
 194 static int strcmp_under_is_dash(const char *s, const char *t) {
 195     for (;;) {
 196         char a = *s++, b = *t++;
 197         if (a != b) {
 198             if (a != '_' || b != '-')
 199                 return a - b;
 200         } else if (a == 0) {
 201             return 0;
 202         }
 203     }
 204 }
 205
 206 static int tests_named_index(const char *testcase)
 207 {
 208     int i;
 209
 210     for (i = 0; testlist[i].name; ++i) {
 211         if (strcmp_under_is_dash(testlist[i].name, testcase) == 0) {
 212             return i;
 213         }
 214     }
 215
 216     return -1;
 217 }
 218
 219 static int tests_run_all(int argc, char * const *argv)
 220 {
 221     int curroptind = optind;
 222     int i;
 223     int failcount=0;
 224
 225     for (i = 0; testlist[i].name; ++i) {
 226         if(!testlist[i].off) {
 227             failcount+=tests_run_test(&testlist[i], argc, argv);
 228             optind = curroptind;
 229         }
 230     }
 231
 232     return failcount;
 233 }
 234
 235 static void
 236 tests_summary(const char *progname) {
 237     int failed_tests = 0;
 238     int todo_pass_tests = 0;
 239     int todo_tests = 0;
 240     int actual_tests = 0;
 241     int planned_tests = 0;
 242
 243     // First compute the totals to help decide if we need to print headers or not.
 244     for (int i = 0; testlist[i].name; ++i) {
 245         if (!testlist[i].off) {
 246             failed_tests += testlist[i].failed_tests;
 247             todo_pass_tests += testlist[i].todo_pass_tests;
 248             todo_tests += testlist[i].todo_tests;
 249             actual_tests += testlist[i].actual_tests;
 250             planned_tests += testlist[i].planned_tests;
 251         }
 252     }
 253
 254     if (!test_onebatstest) {
 255         fprintf(stdout, "[%s] %s\n", failed_tests ? "FAIL" : (actual_tests == planned_tests ? "PASS" : "WARN"), progname);
 256     }
 257
 258     fprintf(stdout, "[SUMMARY]\n");
 259
 260     // -v -v makes the summary verbose as well.
 261     fprintf(stdout, "Test name                                                 failed !failed  todo  total   plan\n");
 262     if (test_verbose > 1 || failed_tests || todo_pass_tests || actual_tests != planned_tests || (test_verbose && todo_tests)) {
 263         fprintf(stdout, "============================================================================================\n");
 264     }
 265     for (int i = 0; testlist[i].name; ++i) {
 266         if (!testlist[i].off) {
 267             const char *token = NULL;
 268             if (testlist[i].failed_tests) {
 269                 token = "FAIL";
 270             } else if (testlist[i].actual_tests != testlist[i].planned_tests) {
 271                 token = "WARN";
 272             } else if (testlist[i].todo_pass_tests) {
 273                 token = "WARN";
 274             } else if (test_verbose > 1 || (test_verbose && testlist[i].todo_tests)) {
 275                 token = "PASS";
 276             }
 277             if (token) {
 278                 fprintf(stdout, "[%s] %-50s %6d %6d %6d %6d %6d\n", token, testlist[i].name, testlist[i].failed_tests, testlist[i].todo_pass_tests, testlist[i].todo_tests, testlist[i].actual_tests, testlist[i].planned_tests);
 279             }
 280         }
 281     }
 282     if (test_verbose > 1 || failed_tests || todo_pass_tests || actual_tests != planned_tests || (test_verbose && todo_tests)) {
 283         fprintf(stdout, "============================================================================================\n");
 284     }
 285     fprintf(stdout, "Totals                                                    %6d %6d %6d %6d %6d\n", failed_tests, todo_pass_tests, todo_tests, actual_tests, planned_tests);
 286
 287 }
 288
 289 #define ASYNC_LOGGING  0
 290 #define DATE_LOGGING   0
 291
 292 int
 293 tests_begin(int argc, char * const *argv) {
 294     const char *testcase = NULL;
 295     bool initialized = false;
 296     __block bool print_security_logs = false;
 297     int testix = -1;
 298     int failcount = 0;
 299         int ch;
 300     int loop = 0;
 301     int list = 0;
 302
 303 #if ASYNC_LOGGING
 304     dispatch_queue_t show_queue = dispatch_queue_create("sec log queue", DISPATCH_QUEUE_SERIAL);
 305 #endif
 306
 307     security_log_handler handle_logs = ^(int level, CFStringRef scope, const char *function, const char *file, int line, CFStringRef message) {
 308         time_t now = time(NULL);
 309 #if DATE_LOGGING
 310         char *date = ctime(&now);
 311         date[19] = '\0';
 312         CFStringRef logStr = CFStringCreateWithFormat(kCFAllocatorDefault, NULL,
 313                                                       CFSTR("%s %@{} %s %@\n"), date + 4,
 314                                                       scope ? scope : CFSTR(""), function, message);
 315 #else
 316         CFStringRef logStr = CFStringCreateWithFormat(kCFAllocatorDefault, NULL,
 317                                                       CFSTR("%lu %@{} %s %@\n"), now,
 318                                                       scope ? scope : CFSTR(""), function, message);
 319 #endif
 320 #if ASYNC_LOGGING
 321         dispatch_async(show_queue, ^{ CFShow(logStr); CFReleaseSafe(logStr); });
 322 #else
 323         CFShow(logStr);
 324         CFReleaseSafe(logStr);
 325 #endif
 326     };
 327
 328     // TODO Currently our callers do this, but we can move this here together with the build date info.
 329     const char *progname = strrchr(argv[0], '/');
 330     progname = progname ? progname + 1 : argv[0];
 331
 332     for (;;) {
 333         while (!testcase && (ch = getopt(argc, argv, "bklL1vwqs")) != -1)
 334         {
 335             switch  (ch)
 336             {
 337 #ifdef NO_SERVER
 338             case 'k':
 339                 keep_scratch_dir = true;
 340                 break;
 341 #endif
 342             case 's':
 343                 if (!print_security_logs) {
 344                     add_security_log_handler(handle_logs);
 345                     print_security_logs = true;
 346                 }
 347                 break;
 348
 349             case 'b':
 350                 test_strict_bats = 0;
 351                 break;
 352
 353             case 'v':
 354                 test_verbose++;
 355                 break;
 356
 357             case 'w':
 358                 sleep(100);
 359                 break;
 360             case 'l':
 361                 loop=1;
 362                 break;
 363             case 'L':
 364                 list=1;
 365                 break;
 366             case '1':
 367                 test_onebatstest=1;
 368                 break;
 369             case '?':
 370             default:
 371                 printf("invalid option %c\n",ch);
 372                 usage(argv[0]);
 373             }
 374         }
 375
 376         if (optind < argc) {
 377             testix = tests_named_index(argv[optind]);
 378             if(testix<0) {
 379                 printf("invalid test %s\n",argv[optind]);
 380                 usage(argv[0]);
 381             }
 382         }
 383
 384         if (!list && !initialized && !test_onebatstest)
 385             fprintf(stdout, "[TEST] %s\n", progname);
 386         if (testix < 0) {
 387             if (!initialized) {
 388                 tests_init();
 389                 initialized = true;
 390                 (void)initialized;
 391                 if (!list)
 392                     failcount+=tests_run_all(argc, argv);
 393             }
 394             break;
 395         } else {
 396             if (!initialized) {
 397                 tests_init();
 398                 initialized = true;
 399                 for (int i = 0; testlist[i].name; ++i) {
 400                     testlist[i].off = 1;
 401                 }
 402             }
 403             optind++;
 404             testlist[testix].off = 0;
 405             if (!list)
 406                 failcount+=tests_run_test(&testlist[testix], argc, argv);
 407             testix = -1;
 408         }
 409     }
 410     if (list) {
 411         for (int i = 0; testlist[i].name; ++i) {
 412             if (!testlist[i].off) {
 413                     fprintf(stdout, "%s\n", testlist[i].name);
 414             }
 415         }
 416     } else {
 417         tests_summary(progname);
 418     }
 419
 420     remove_security_log_handler(handle_logs);
 421     fflush(stdout);
 422
 423
 424     /* Cleanups */
 425     tests_end();
 426
 427     if (loop) {
 428         printf("Looping until key press 'q'. You can run leaks now.\n");
 429         while(getchar()!='q');
 430     }
 431
 432     return failcount;
 433 }
 434