OSX/regressions/test/testenv.m

   1 /*
   2  * Copyright (c) 2005-2007,2009-2014 Apple Inc. All Rights Reserved.
   3  *
   4  * @APPLE_LICENSE_HEADER_START@
   5  *
   6  * This file contains Original Code and/or Modifications of Original Code
   7  * as defined in and that are subject to the Apple Public Source License
   8  * Version 2.0 (the 'License'). You may not use this file except in
   9  * compliance with the License. Please obtain a copy of the License at
  10  * http://www.opensource.apple.com/apsl/ and read it before using this
  11  * file.
  12  *
  13  * The Original Code and all software distributed under the License are
  14  * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  15  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  16  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
  17  * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
  18  * Please see the License for the specific language governing rights and
  19  * limitations under the License.
  20  *
  21  * @APPLE_LICENSE_HEADER_END@
  22  *
  23  * testenv.c
  24  */
  25
  26 /*
  27  * This is to fool os services to not provide the Keychain manager
  28  * interface tht doens't work since we don't have unified headers
  29  * between iOS and OS X. rdar://23405418/
  30  */
  31 #define __KEYCHAINCORE__ 1
  32
  33
  34 #import <Foundation/Foundation.h>
  35 #include <fcntl.h>
  36 #include <stdarg.h>
  37 #include <stdlib.h>
  38 #include <string.h>
  39 #include <sys/stat.h>
  40 #include <sys/types.h>
  41 #include <unistd.h>
  42 #include <stdbool.h>
  43 #include <time.h>
  44 #include <sys/time.h>
  45
  46 #include "testmore.h"
  47 #include "testenv.h"
  48
  49 #include <utilities/debugging.h>
  50 #include <utilities/SecCFRelease.h>
  51 #include <utilities/SecCFWrappers.h>
  52 #include <utilities/SecFileLocations.h>
  53
  54 #include <dispatch/dispatch.h>
  55
  56
  57 int test_strict_bats = 1;
  58 int test_verbose = 0;
  59 int test_onebatstest = 0;
  60 int test_check_leaks = 0;
  61 char **test_skip_leaks_test = NULL;
  62
  63 #ifdef NO_SERVER
  64 #include <securityd/spi.h>
  65
  66 static int current_dir = -1;
  67 static char scratch_dir[50];
  68 static char *home_var;
  69 static bool keep_scratch_dir = false;
  70
  71 static int
  72 rmdir_recursive(const char *path)
  73 {
  74 #if (!TARGET_IPHONE_SIMULATOR)
  75     char command_buf[256];
  76         if (strlen(path) + 10 > sizeof(command_buf) || strchr(path, '\''))
  77         {
  78                 fprintf(stdout, "# rmdir_recursive: invalid path: %s", path);
  79                 return -1;
  80         }
  81
  82         sprintf(command_buf, "/bin/rm -rf '%s'", path);
  83         return system(command_buf);
  84 #else
  85     fprintf(stdout, "# rmdir_recursive: simulator can't rmdir, leaving path: %s\n", path);
  86     return 0;
  87 #endif
  88 }
  89 #endif
  90
  91 static int tests_init(void) {
  92     int ok = 0;
  93 #ifdef NO_SERVER
  94         char preferences_dir[80];
  95         char library_dir[70];
  96
  97         setup("tests_init");
  98
  99     /* Create scratch dir for tests to run in. */
 100     sprintf(scratch_dir, "/tmp/tst-%d", getpid());
 101     if (keep_scratch_dir) {
 102         printf("running tests with HOME=%s\n", scratch_dir);
 103     }
 104
 105     sprintf(library_dir, "%s/Library", scratch_dir);
 106     sprintf(preferences_dir, "%s/Preferences", library_dir);
 107     ok =  (ok_unix(mkdir(scratch_dir, 0755), "mkdir") &&
 108            ok_unix(current_dir = open(".", O_RDONLY), "open") &&
 109            ok_unix(chdir(scratch_dir), "chdir") &&
 110            ok_unix(setenv("HOME", scratch_dir, 1), "setenv") &&
 111            /* @@@ Work around a bug that the prefs code in
 112             libsecurity_keychain never creates the Library/Preferences
 113             dir. */
 114            ok_unix(mkdir(library_dir, 0755), "mkdir") &&
 115            ok_unix(mkdir(preferences_dir, 0755), "mkdir") &&
 116            ok(home_var = getenv("HOME"), "getenv"));
 117
 118     if (ok > 0)
 119         securityd_init(scratch_dir);
 120 #endif
 121
 122     return ok;
 123 }
 124
 125 static int
 126 tests_end(void)
 127 {
 128 #ifdef NO_SERVER
 129         setup("tests_end");
 130         /* Restore previous cwd and remove scratch dir. */
 131         int ok = ok_unix(fchdir(current_dir), "fchdir");
 132         if (ok)
 133                 ok = ok_unix(close(current_dir), "close");
 134         if (ok) {
 135                 if (!keep_scratch_dir) {
 136                         ok = ok_unix(rmdir_recursive(scratch_dir), "rmdir_recursive");
 137                 }
 138         }
 139
 140         return ok;
 141 #else
 142     return 0;
 143 #endif
 144 }
 145
 146 static void usage(const char *progname)
 147 {
 148     fprintf(stderr, "usage: %s [-k][-w][testname [testargs] ...]\n", progname);
 149     exit(1);
 150 }
 151
 152 /* run one test, described by test, return info in test struct */
 153 static int tests_run_test(struct one_test_s *test, int argc, char * const *argv)
 154 {
 155     int ch;
 156
 157     while ((ch = getopt(argc, argv, "v")) != -1)
 158     {
 159         switch  (ch)
 160         {
 161             case 'v':
 162                 test_verbose++;
 163                 break;
 164             default:
 165                 usage(argv[0]);
 166         }
 167     }
 168
 169     if (test_onebatstest)
 170         fprintf(stdout, "[TEST] %s\n", test->name);
 171     else
 172         fprintf(stdout, "[BEGIN] %s\n", test->name);
 173
 174     const char *token = "PASS";
 175     if (test->entry == NULL) {
 176         fprintf(stdout, "%s:%d: error, no entry\n", __FILE__, __LINE__);
 177         test->failed_tests = 1;
 178     } else {
 179         struct timeval start, stop;
 180         gettimeofday(&start, NULL);
 181         @autoreleasepool {
 182             test->entry(argc, argv);
 183         }
 184         gettimeofday(&stop, NULL);
 185         /* this may overflow... */
 186         test->duration = (stop.tv_sec-start.tv_sec) * 1000 + (stop.tv_usec / 1000) - (start.tv_usec / 1000);
 187         if (test_plan_ok()) {
 188             token = "WARN";
 189         }
 190     }
 191
 192     /*
 193      * Check if we have any leaks, allow skipping test
 194      */
 195
 196     if (test_check_leaks) {
 197         char *cmd = NULL;
 198         int ret = 0;
 199
 200         asprintf(&cmd, "leaks %d >/dev/null", getpid());
 201         if (cmd) {
 202             ret = system(cmd);
 203             free(cmd);
 204         }
 205         if (ret != 0) {
 206             unsigned n = 0;
 207             fprintf(stdout, "leaks found in test %s\n", test->name);
 208
 209             if (test_skip_leaks_test) {
 210                 while (test_skip_leaks_test[n]) {
 211                     if (strcmp(test_skip_leaks_test[n], test->name) == 0) {
 212                         fprintf(stdout, "test %s known to be leaky, skipping\n", test->name);
 213                         ret = 0;
 214                         break;
 215                     }
 216                 }
 217             }
 218             if (ret) {
 219                 token = "FAIL";
 220             }
 221         } else {
 222             if (test_skip_leaks_test) {
 223                 unsigned n = 0;
 224
 225                 while (test_skip_leaks_test[n]) {
 226                     if (strcmp(test_skip_leaks_test[n], test->name) == 0) {
 227                         fprintf(stdout, "leaks didn't find leak in test %s, yet it was ignore\n", test->name);
 228                         token = "FAIL";
 229                         break;
 230                     }
 231                 }
 232             }
 233
 234         }
 235     }
 236
 237     test_plan_final(&test->failed_tests, &test->todo_pass_tests, &test->todo_tests,
 238                     &test->actual_tests, &test->planned_tests,
 239                     &test->plan_file, &test->plan_line);
 240     if (test_verbose) {
 241         // TODO Use ccperf timing and  printing routines that use proper si scaling
 242         fprintf(stdout, "%s  took %lu ms\n", test->name, test->duration);
 243     }
 244     if (test->failed_tests) {
 245         token = "FAIL";
 246     }
 247     fprintf(stdout, "[%s] %s\n", token, test->name);
 248
 249     return test->failed_tests;
 250 }
 251
 252 static int strcmp_under_is_dash(const char *s, const char *t) {
 253     for (;;) {
 254         char a = *s++, b = *t++;
 255         if (a != b) {
 256             if (a != '_' || b != '-')
 257                 return a - b;
 258         } else if (a == 0) {
 259             return 0;
 260         }
 261     }
 262 }
 263
 264 static int tests_named_index(const char *testcase)
 265 {
 266     int i;
 267
 268     for (i = 0; testlist[i].name; ++i) {
 269         if (strcmp_under_is_dash(testlist[i].name, testcase) == 0) {
 270             return i;
 271         }
 272     }
 273
 274     return -1;
 275 }
 276
 277 static int tests_run_all(int argc, char * const *argv)
 278 {
 279     int curroptind = optind;
 280     int i;
 281     int failcount=0;
 282
 283     for (i = 0; testlist[i].name; ++i) {
 284         if(!testlist[i].off) {
 285             @autoreleasepool {
 286                 failcount+=tests_run_test(&testlist[i], argc, argv);
 287             }
 288             optind = curroptind;
 289         }
 290     }
 291
 292     return failcount;
 293 }
 294
 295 static void
 296 tests_summary(const char *progname) {
 297     int failed_tests = 0;
 298     int todo_pass_tests = 0;
 299     int todo_tests = 0;
 300     int actual_tests = 0;
 301     int planned_tests = 0;
 302
 303     // First compute the totals to help decide if we need to print headers or not.
 304     for (int i = 0; testlist[i].name; ++i) {
 305         if (!testlist[i].off) {
 306             failed_tests += testlist[i].failed_tests;
 307             todo_pass_tests += testlist[i].todo_pass_tests;
 308             todo_tests += testlist[i].todo_tests;
 309             actual_tests += testlist[i].actual_tests;
 310             planned_tests += testlist[i].planned_tests;
 311         }
 312     }
 313
 314     if (!test_onebatstest) {
 315         fprintf(stdout, "[%s] %s\n", failed_tests ? "FAIL" : (actual_tests == planned_tests ? "PASS" : "WARN"), progname);
 316     }
 317
 318     fprintf(stdout, "[SUMMARY]\n");
 319
 320     // -v -v makes the summary verbose as well.
 321     fprintf(stdout, "Test name                                                 failed !failed  todo  total   plan\n");
 322     if (test_verbose > 1 || failed_tests || todo_pass_tests || actual_tests != planned_tests || (test_verbose && todo_tests)) {
 323         fprintf(stdout, "============================================================================================\n");
 324     }
 325     for (int i = 0; testlist[i].name; ++i) {
 326         if (!testlist[i].off) {
 327             const char *token = NULL;
 328             if (testlist[i].failed_tests) {
 329                 token = "FAIL";
 330             } else if (testlist[i].actual_tests != testlist[i].planned_tests) {
 331                 token = "WARN";
 332             } else if (testlist[i].todo_pass_tests) {
 333                 token = "WARN";
 334             } else if (test_verbose > 1 || (test_verbose && testlist[i].todo_tests)) {
 335                 token = "PASS";
 336             }
 337             if (token) {
 338                 fprintf(stdout, "[%s] %-50s %6d %6d %6d %6d %6d\n", token, testlist[i].name, testlist[i].failed_tests, testlist[i].todo_pass_tests, testlist[i].todo_tests, testlist[i].actual_tests, testlist[i].planned_tests);
 339             }
 340         }
 341     }
 342     if (test_verbose > 1 || failed_tests || todo_pass_tests || actual_tests != planned_tests || (test_verbose && todo_tests)) {
 343         fprintf(stdout, "============================================================================================\n");
 344     }
 345     fprintf(stdout, "Totals                                                    %6d %6d %6d %6d %6d\n", failed_tests, todo_pass_tests, todo_tests, actual_tests, planned_tests);
 346
 347 }
 348
 349 #define ASYNC_LOGGING  0
 350 #define DATE_LOGGING   0
 351
 352 int
 353 tests_begin(int argc, char * const *argv) {
 354     const char *testcase = NULL;
 355     bool initialized = false;
 356     __block bool print_security_logs = false;
 357     int testix = -1;
 358     int failcount = 0;
 359         int ch;
 360     int loop = 0;
 361     int list = 0;
 362
 363 #if ASYNC_LOGGING
 364     dispatch_queue_t show_queue = dispatch_queue_create("sec log queue", DISPATCH_QUEUE_SERIAL);
 365 #endif
 366
 367
 368 #if USEOLDLOGGING
 369     security_log_handler handle_logs = ^(int level, CFStringRef scope, const char *function, const char *file, int line, CFStringRef message) {
 370         time_t now = time(NULL);
 371
 372 #if DATE_LOGGING
 373         char *date = ctime(&now);
 374         date[19] = '\0';
 375         CFStringRef logStr = CFStringCreateWithFormat(kCFAllocatorDefault, NULL,
 376                                                       CFSTR("%s %@{} %s %@\n"), date + 4,
 377                                                       scope ? scope : CFSTR(""), function, message);
 378 #else
 379         CFStringRef logStr = CFStringCreateWithFormat(kCFAllocatorDefault, NULL,
 380                                                       CFSTR("%lu %@{} %s %@\n"), now,
 381                                                       scope ? scope : CFSTR(""), function, message);
 382 #endif
 383 #if ASYNC_LOGGING
 384         dispatch_async(show_queue, ^{ CFShow(logStr); CFReleaseSafe(logStr); });
 385 #else
 386         CFShow(logStr);
 387         CFReleaseSafe(logStr);
 388 #endif
 389     };
 390 #endif
 391
 392
 393     for (;;) {
 394         while (!testcase && (ch = getopt(argc, argv, "bklL1vwqs")) != -1)
 395         {
 396             switch  (ch)
 397             {
 398 #ifdef NO_SERVER
 399             case 'k':
 400                 keep_scratch_dir = true;
 401                 break;
 402 #endif
 403             case 's':
 404                 if (!print_security_logs) {
 405                     //add_security_log_handler(handle_logs);
 406                     print_security_logs = true;
 407                 }
 408                 break;
 409
 410             case 'b':
 411                 test_strict_bats = 0;
 412                 break;
 413
 414             case 'v':
 415                 test_verbose++;
 416                 break;
 417
 418             case 'w':
 419                 sleep(100);
 420                 break;
 421             case 'l':
 422                 loop=1;
 423                 break;
 424             case 'L':
 425                 list=1;
 426                 break;
 427             case '1':
 428                 test_onebatstest=1;
 429                 break;
 430             case '?':
 431             default:
 432                 printf("invalid option %c\n",ch);
 433                 usage(argv[0]);
 434             }
 435         }
 436
 437         if (optind < argc) {
 438             testix = tests_named_index(argv[optind]);
 439             if(testix<0) {
 440                 printf("invalid test %s\n",argv[optind]);
 441                 usage(argv[0]);
 442             }
 443         }
 444
 445         if(!print_security_logs) secLogDisable();
 446
 447
 448         if (!list && !initialized && !test_onebatstest)
 449             fprintf(stdout, "[TEST] %s\n", getprogname());
 450         if (testix < 0) {
 451             if (!initialized) {
 452                 tests_init();
 453                 initialized = true;
 454                 (void)initialized;
 455                 if (!list)
 456                     failcount+=tests_run_all(argc, argv);
 457             }
 458             break;
 459         } else {
 460             if (!initialized) {
 461                 tests_init();
 462                 initialized = true;
 463                 for (int i = 0; testlist[i].name; ++i) {
 464                     testlist[i].off = 1;
 465                 }
 466             }
 467             optind++;
 468             testlist[testix].off = 0;
 469             if (!list) {
 470                 @autoreleasepool {
 471                     failcount+=tests_run_test(&testlist[testix], argc, argv);
 472                 }
 473             }
 474             testix = -1;
 475         }
 476     }
 477     if (list) {
 478         for (int i = 0; testlist[i].name; ++i) {
 479             if (!testlist[i].off) {
 480                     fprintf(stdout, "%s\n", testlist[i].name);
 481             }
 482         }
 483     } else {
 484         tests_summary(getprogname());
 485     }
 486
 487     // remove_security_log_handler(handle_logs);
 488     fflush(stdout);
 489
 490
 491     /* Cleanups */
 492     tests_end();
 493
 494     if (loop) {
 495         printf("Looping until key press 'q'. You can run leaks now.\n");
 496         while(getchar()!='q');
 497     }
 498
 499     return failcount;
 500 }
 501