OSX/regressions/test/testmore.c

   1 /*
   2  * Copyright (c) 2005-2007,2012-2014 Apple Inc. All Rights Reserved.
   3  *
   4  * @APPLE_LICENSE_HEADER_START@
   5  *
   6  * This file contains Original Code and/or Modifications of Original Code
   7  * as defined in and that are subject to the Apple Public Source License
   8  * Version 2.0 (the 'License'). You may not use this file except in
   9  * compliance with the License. Please obtain a copy of the License at
  10  * http://www.opensource.apple.com/apsl/ and read it before using this
  11  * file.
  12  *
  13  * The Original Code and all software distributed under the License are
  14  * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  15  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  16  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
  17  * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
  18  * Please see the License for the specific language governing rights and
  19  * limitations under the License.
  20  *
  21  * @APPLE_LICENSE_HEADER_END@
  22  *
  23  * testmore.c
  24  */
  25
  26 #include <fcntl.h>
  27 #include <dispatch/dispatch.h>
  28 #include <pthread.h>
  29 #include <stdarg.h>
  30 #include <stdlib.h>
  31 #include <string.h>
  32 #include <sys/stat.h>
  33 #include <sys/types.h>
  34 #include <unistd.h>
  35 #include <AvailabilityMacros.h>
  36
  37 #include "testmore.h"
  38 #include "testenv.h"
  39
  40 pthread_mutex_t test_mutex; // protects the test number variables
  41 static int test_fails = 0;
  42 static int test_todo_pass = 0;
  43 static int test_todo = 0;
  44 static int test_num = 0;
  45 static int test_cases = 0;
  46 static int test_plan_line = 0;
  47 static const char *test_plan_file = NULL;
  48
  49 const char *test_directive = NULL;
  50 const char *test_reason = NULL;
  51
  52 static void fprint_string(FILE *file, CFStringRef string) {
  53     UInt8 buf[256];
  54     CFRange range = { .location = 0 };
  55     range.length = CFStringGetLength(string);
  56     while (range.length > 0) {
  57         CFIndex bytesUsed = 0;
  58         CFIndex converted = CFStringGetBytes(string, range, kCFStringEncodingUTF8, 0, false, buf, sizeof(buf), &bytesUsed);
  59         fwrite(buf, 1, bytesUsed, file);
  60         range.length -= converted;
  61         range.location += converted;
  62     }
  63 }
  64
  65 static void cffprint(FILE *file, CFStringRef fmt, ...) CF_FORMAT_FUNCTION(2,0);
  66
  67 static void cffprint_v(FILE *file, CFStringRef fmt, va_list args);
  68 static void cffprint_c_v(FILE *file, const char *fmt, va_list args);
  69
  70 static void cffprint_v(FILE *file, CFStringRef fmt, va_list args) {
  71     CFStringRef line = CFStringCreateWithFormatAndArguments(NULL, NULL, fmt, args);
  72     fprint_string(file, line);
  73     CFRelease(line);
  74 }
  75
  76 static void cffprint_c_v(FILE *file, const char *fmt, va_list args) {
  77     CFStringRef cffmt = CFStringCreateWithCString(kCFAllocatorDefault, fmt, kCFStringEncodingUTF8);
  78     cffprint_v(file, cffmt, args);
  79     CFRelease(cffmt);
  80 }
  81
  82 static void cffprint(FILE *file, CFStringRef fmt, ...) {
  83     va_list args;
  84     va_start(args, fmt);
  85     cffprint_v(file, fmt, args);
  86     va_end(args);
  87 }
  88
  89 void test_skip(const char *reason, int how_many, int unless)
  90 {
  91     if (unless)
  92         return;
  93
  94     int done;
  95     for (done = 0; done < how_many; ++done)
  96         test_ok(1, NULL, "skip", reason, __FILE__, __LINE__, NULL);
  97 }
  98
  99 void test_bail_out(const char *reason, const char *file, unsigned line)
 100 {
 101     fprintf(stdout, "[FAIL] BAIL OUT! (%s at line %u) %s\n", file, line, reason);
 102     fflush(stdout);
 103     exit(255);
 104 }
 105
 106 void test_plan_skip_all(const char *reason)
 107 {
 108     // Not super thread-safe. Don't test_plan_skip_all from multiple threads simultaneously.
 109     pthread_mutex_lock(&test_mutex);
 110     int skipN = test_cases - test_num;
 111     pthread_mutex_unlock(&test_mutex);
 112
 113     if (skipN > 0)
 114     {
 115         test_skip(reason, skipN, 0);
 116     }
 117 }
 118
 119 static const char *test_plan_name(void) {
 120     const char *plan_name = strrchr(test_plan_file, '/');
 121     plan_name = plan_name ? plan_name + 1 : test_plan_file;
 122     return plan_name;
 123 }
 124
 125 static int test_plan_pass(void) {
 126     if (test_verbose) {
 127         const char *name = test_plan_name();
 128         fprintf(stdout, "[BEGIN] %s plan\n[PASS] %s plan\n", name, name);
 129         // Update counts for summary
 130         //test_num++;
 131         //test_cases++;
 132     }
 133     return 0;
 134 }
 135
 136 static int test_plan_fail(CFStringRef reason, ...) {
 137     const char *name = test_plan_name();
 138     va_list ap;
 139     va_start(ap, reason);
 140     CFStringRef desc = CFStringCreateWithFormatAndArguments(kCFAllocatorDefault, NULL, reason, ap);
 141     cffprint(stdout, CFSTR("[BEGIN] %s plan\n%@[WARN] %s plan\n"), name, desc, name);
 142     CFRelease(desc);
 143     // Update counts for summary.  We consider test_plan_ok itself an unscheduled testcase for counts.
 144     //test_num++;
 145     //test_fails++;
 146     //test_cases++;
 147     return 1;
 148 }
 149
 150 int test_plan_ok(void) {
 151     int status = 0;
 152     fflush(stderr);
 153     const char *name = test_plan_name();
 154
 155     pthread_mutex_lock(&test_mutex);
 156     if (!test_num)
 157     {
 158         if (test_cases)
 159         {
 160             status = test_plan_fail(CFSTR("No tests run!\n"));
 161         }
 162         else
 163         {
 164             status = test_plan_fail(CFSTR("Looks like your test died before it could output anything.\n"));
 165         }
 166     }
 167     else if (test_num < test_cases)
 168     {
 169         status = test_plan_fail(CFSTR("Looks like you planned %d tests but only ran %d.\n"), test_cases, test_num);
 170     }
 171     else if (test_num > test_cases)
 172     {
 173         status = test_plan_fail(CFSTR("Looks like you planned %d tests but ran %d.\n"), test_cases, test_num);
 174     } else if (!test_fails) {
 175         status = test_plan_pass();
 176     }
 177     if (test_fails)
 178     {
 179         fprintf(stdout, "%s failed %d tests of %d.\n", name, test_fails, test_num);
 180         status = 1;
 181     }
 182     pthread_mutex_unlock(&test_mutex);
 183     fflush(stdout);
 184
 185     return status;
 186 }
 187
 188 // You should hold the test_mutex when you call this.
 189 static void test_plan_reset(void) {
 190     test_fails = 0;
 191     test_todo_pass = 0;
 192     test_todo = 0;
 193     test_num = 0;
 194     test_cases = 0;
 195     test_plan_file = NULL;
 196     test_plan_line = 0;
 197 }
 198
 199 void test_plan_final(int *failed, int *todo_pass, int *todo, int *actual, int *planned, const char **file, int *line) {
 200     pthread_mutex_lock(&test_mutex);
 201     if (failed)
 202         *failed = test_fails;
 203     if (todo_pass)
 204         *todo_pass = test_todo_pass;
 205     if (todo)
 206         *todo = test_todo;
 207     if (actual)
 208         *actual = test_num;
 209     if (planned)
 210         *planned = test_cases;
 211     if (file)
 212         *file = test_plan_file;
 213     if (line)
 214         *line = test_plan_line;
 215
 216     test_plan_reset();
 217     pthread_mutex_unlock(&test_mutex);
 218 }
 219
 220 void test_plan_tests(int count, const char *file, unsigned line) {
 221         if (test_cases)
 222     {
 223         fprintf(stdout,
 224                 "[FAIL] You tried to plan twice!\n");
 225
 226         fflush(stdout);
 227         exit(255);
 228     }
 229     else
 230         {
 231         if (!count)
 232         {
 233             fprintf(stdout, "[BEGIN] plan_tests\nYou said to run 0 tests!  "
 234                 "You've got to run something.\n[WARN] plan_tests\n");
 235             fflush(stdout);
 236         }
 237
 238         test_plan_file=file;
 239         test_plan_line=line;
 240
 241         test_cases = count;
 242
 243         static dispatch_once_t onceToken;
 244         dispatch_once(&onceToken, ^{
 245             if(pthread_mutex_init(&test_mutex, NULL) != 0) {
 246                 fprintf(stdout, "Failed to initialize mutex: %d\n", errno);
 247             }
 248         });
 249         }
 250 }
 251
 252 int
 253 test_diag(const char *directive, const char *reason,
 254         const char *file, unsigned line, const char *fmt, ...)
 255 {
 256         int is_todo = directive && !strcmp(directive, "TODO");
 257         va_list args;
 258
 259         va_start(args, fmt);
 260
 261         if (is_todo)
 262         {
 263                 fputs("# ", stdout);
 264                 if (fmt)
 265                         vprintf(fmt, args);
 266                 fputs("\n", stdout);
 267                 fflush(stdout);
 268         }
 269         else
 270         {
 271                 fputs("# ", stdout);
 272                 if (fmt)
 273                         vfprintf(stdout, fmt, args);
 274                 fputs("\n", stdout);
 275                 fflush(stdout);
 276         }
 277
 278         va_end(args);
 279
 280         return 1;
 281 }
 282
 283 int
 284 test_ok(int passed, __attribute((cf_consumed)) CFStringRef CF_CONSUMED description, const char *directive,
 285         const char *reason, const char *file, unsigned line,
 286         const char *fmt, ...)
 287 {
 288         int is_todo = directive && !strcmp(directive, "TODO");
 289         int is_setup = directive && !is_todo && !strcmp(directive, "SETUP");
 290
 291         if (is_setup)
 292         {
 293                 if (!passed)
 294                 {
 295                         fflush(stderr);
 296             fprintf(stdout, "[BEGIN] SETUP\n");
 297             if (fmt) {
 298                 va_list args;
 299                 va_start(args, fmt);
 300                 cffprint_c_v(stdout, fmt, args);
 301                 va_end(args);
 302             }
 303             cffprint(stdout, CFSTR("[WARN] SETUP%s%@%s%s\n"),
 304                                    description ? " - " : "",
 305                                    description ? description : CFSTR(""),
 306                                    reason ? " - " : "",
 307                                    reason ? reason : "");
 308             fflush(stdout);
 309                 }
 310         }
 311         else
 312         {
 313                 if (!test_cases)
 314                 {
 315             // Make having a plan optional? Commenting out the next 3 lines does - mb
 316                         //fprintf(stdout, "[FAIL] You tried to run a test without a plan!  "
 317                         //              "Gotta have a plan. at %s line %u\n", file, line);
 318                         //fflush(stdout);
 319                 }
 320
 321         pthread_mutex_lock(&test_mutex);
 322                 ++test_num;
 323         if (passed) {
 324             if (is_todo) {
 325                 test_todo_pass++;
 326             }
 327         } else if (is_todo) {
 328             test_todo++;
 329         } else {
 330             ++test_fails;
 331         }
 332
 333         /* We only print this when a test fails, unless verbose is enabled */
 334         if ((!passed && !is_todo) || test_verbose > 0) {
 335             fflush(stderr);
 336             if (test_strict_bats) {
 337                 cffprint(stdout, CFSTR("[BEGIN] %d%s%@\n"),
 338                          test_num,
 339                          description ? " - " : "",
 340                          description ? description : CFSTR(""));
 341             }
 342             if (is_todo && passed) {
 343                 fprintf(stdout, "%s:%d: warning: Unexpectedly passed (TODO) test\n", file, line);
 344             } else if (is_todo && !passed) {
 345                 /* Enable this to output TODO as warning */
 346                 fprintf(stdout, "%s:%d: ok: Failed (TODO) test\n", file, line);
 347             } else if (!passed) {
 348                 fprintf(stdout, "%s:%d: error: Failed test\n", file, line);
 349             }
 350             if (fmt) {
 351                 va_list args;
 352                 va_start(args, fmt);
 353                 cffprint_c_v(stdout, fmt, args);
 354                 va_end(args);
 355             }
 356             cffprint(stdout, CFSTR("[%s] %d%s%@%s%s%s%s\n"), passed ? (is_todo ? "PASS" : "PASS") : (is_todo ? "PASS" : "FAIL"),
 357                      test_num,
 358                      description ? " - " : "",
 359                      description ? description : CFSTR(""),
 360                      directive ? " # " : "",
 361                      directive ? directive : "",
 362                      reason ? " " : "",
 363                      reason ? reason : "");
 364             fflush(stdout);
 365         }
 366         pthread_mutex_unlock(&test_mutex);
 367     }
 368
 369     if (description)
 370         CFRelease(description);
 371
 372     return passed;
 373 }
 374
 375
 376 // TODO: Move this to testsec.h so that testmore and testenv can be shared
 377 static void buf_kill(void* p) {
 378     free(p);
 379 }
 380
 381 const char *
 382 sec_errstr(int err)
 383 {
 384     static pthread_key_t buffer0key;
 385     static pthread_key_t buffer1key;
 386     static pthread_key_t switchkey;
 387     static dispatch_once_t onceToken;
 388     dispatch_once(&onceToken, ^{
 389         pthread_key_create(&buffer0key, buf_kill);
 390         pthread_key_create(&buffer1key, buf_kill);
 391         pthread_key_create(&switchkey, buf_kill);
 392     });
 393
 394     uint32_t * switchp = (uint32_t*) pthread_getspecific(switchkey);
 395     if(switchp == NULL) {
 396         switchp = (uint32_t*) malloc(sizeof(uint32_t));
 397         *switchp = 0;
 398         pthread_setspecific(switchkey, switchp);
 399     }
 400
 401     char* buf = NULL;
 402
 403     pthread_key_t current = (*switchp) ? buffer0key : buffer1key;
 404     *switchp = !(*switchp);
 405
 406     buf = pthread_getspecific(current);
 407     if(buf == NULL) {
 408         buf = (char*) malloc(20);
 409         pthread_setspecific(current, buf);
 410     }
 411
 412     snprintf(buf, 20, "0x%X", err);
 413     return buf;
 414 }