OSX/regressions/test/testmore.c

   1 /*
   2  * Copyright (c) 2005-2007,2012-2014 Apple Inc. All Rights Reserved.
   3  *
   4  * @APPLE_LICENSE_HEADER_START@
   5  *
   6  * This file contains Original Code and/or Modifications of Original Code
   7  * as defined in and that are subject to the Apple Public Source License
   8  * Version 2.0 (the 'License'). You may not use this file except in
   9  * compliance with the License. Please obtain a copy of the License at
  10  * http://www.opensource.apple.com/apsl/ and read it before using this
  11  * file.
  12  *
  13  * The Original Code and all software distributed under the License are
  14  * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  15  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  16  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
  17  * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
  18  * Please see the License for the specific language governing rights and
  19  * limitations under the License.
  20  *
  21  * @APPLE_LICENSE_HEADER_END@
  22  *
  23  * testmore.c
  24  */
  25
  26 #include <fcntl.h>
  27 #include <dispatch/dispatch.h>
  28 #include <pthread.h>
  29 #include <stdarg.h>
  30 #include <stdlib.h>
  31 #include <string.h>
  32 #include <sys/stat.h>
  33 #include <sys/types.h>
  34 #include <unistd.h>
  35 #include <AvailabilityMacros.h>
  36
  37 #include "testmore.h"
  38 #include "testenv.h"
  39
  40 pthread_mutex_t test_mutex; // protects the test number variables
  41 static int test_fails = 0;
  42 static int test_todo_pass = 0;
  43 static int test_todo = 0;
  44 static int test_num = 0;
  45 static int test_cases = 0;
  46 static int test_plan_line = 0;
  47 static const char *test_plan_file = NULL;
  48
  49 const char *test_directive = NULL;
  50 const char *test_reason = NULL;
  51
  52 static void fprint_string(FILE *file, CFStringRef string) {
  53     UInt8 buf[256];
  54     CFRange range = { .location = 0 };
  55     range.length = CFStringGetLength(string);
  56     while (range.length > 0) {
  57         CFIndex bytesUsed = 0;
  58         CFIndex converted = CFStringGetBytes(string, range, kCFStringEncodingUTF8, 0, false, buf, sizeof(buf), &bytesUsed);
  59         fwrite(buf, 1, bytesUsed, file);
  60         range.length -= converted;
  61         range.location += converted;
  62     }
  63 }
  64
  65 static void cffprint(FILE *file, CFStringRef fmt, ...) CF_FORMAT_FUNCTION(2,3);
  66
  67 static void cffprint_v(FILE *file, CFStringRef fmt, va_list args) CF_FORMAT_FUNCTION(2,0);
  68 static void cffprint_c_v(FILE *file, const char *fmt, va_list args);
  69
  70 static void cffprint_v(FILE *file, CFStringRef fmt, va_list args) {
  71     CFStringRef line = CFStringCreateWithFormatAndArguments(NULL, NULL, fmt, args);
  72     fprint_string(file, line);
  73     CFRelease(line);
  74 }
  75
  76 #pragma clang diagnostic push
  77 #pragma clang diagnostic ignored "-Wformat-nonliteral"
  78
  79 static void cffprint_c_v(FILE *file, const char *fmt, va_list args) {
  80     CFStringRef cffmt = CFStringCreateWithCString(kCFAllocatorDefault, fmt, kCFStringEncodingUTF8);
  81     cffprint_v(file, cffmt, args);
  82     CFRelease(cffmt);
  83 }
  84
  85 #pragma clang diagnostic pop
  86
  87
  88 static void cffprint(FILE *file, CFStringRef fmt, ...) {
  89     va_list args;
  90     va_start(args, fmt);
  91     cffprint_v(file, fmt, args);
  92     va_end(args);
  93 }
  94
  95 void test_skip(const char *reason, int how_many, int unless)
  96 {
  97     if (unless)
  98         return;
  99
 100     int done;
 101     for (done = 0; done < how_many; ++done)
 102         test_ok(1, NULL, "skip", reason, __FILE__, __LINE__, NULL);
 103 }
 104
 105 void test_bail_out(const char *reason, const char *file, unsigned line)
 106 {
 107     fprintf(stdout, "[FAIL] BAIL OUT! (%s at line %u) %s\n", file, line, reason);
 108     fflush(stdout);
 109     exit(255);
 110 }
 111
 112 void test_plan_skip_all(const char *reason)
 113 {
 114     // Not super thread-safe. Don't test_plan_skip_all from multiple threads simultaneously.
 115     pthread_mutex_lock(&test_mutex);
 116     int skipN = test_cases - test_num;
 117     pthread_mutex_unlock(&test_mutex);
 118
 119     if (skipN > 0)
 120     {
 121         test_skip(reason, skipN, 0);
 122     }
 123 }
 124
 125 static const char *test_plan_name(void) {
 126     const char *plan_name = strrchr(test_plan_file, '/');
 127     plan_name = plan_name ? plan_name + 1 : test_plan_file;
 128     return plan_name;
 129 }
 130
 131 static int test_plan_pass(void) {
 132     if (test_verbose) {
 133         const char *name = test_plan_name();
 134         fprintf(stdout, "[BEGIN] %s plan\n[PASS] %s plan\n", name, name);
 135         // Update counts for summary
 136         //test_num++;
 137         //test_cases++;
 138     }
 139     return 0;
 140 }
 141
 142 static int test_plan_fail(CFStringRef reason, ...) CF_FORMAT_FUNCTION(1, 2);
 143
 144 static int test_plan_fail(CFStringRef reason, ...) {
 145     const char *name = test_plan_name();
 146     va_list ap;
 147     va_start(ap, reason);
 148     CFStringRef desc = CFStringCreateWithFormatAndArguments(kCFAllocatorDefault, NULL, reason, ap);
 149     cffprint(stdout, CFSTR("[BEGIN] %s plan\n%@[WARN] %s plan\n"), name, desc, name);
 150     CFRelease(desc);
 151     // Update counts for summary.  We consider test_plan_ok itself an unscheduled testcase for counts.
 152     //test_num++;
 153     //test_fails++;
 154     //test_cases++;
 155     return 1;
 156 }
 157
 158 int test_plan_ok(void) {
 159     int status = 0;
 160     fflush(stderr);
 161     const char *name = test_plan_name();
 162
 163     pthread_mutex_lock(&test_mutex);
 164     if (!test_num)
 165     {
 166         if (test_cases)
 167         {
 168             status = test_plan_fail(CFSTR("No tests run!\n"));
 169         }
 170         else
 171         {
 172             status = test_plan_fail(CFSTR("Looks like your test died before it could output anything.\n"));
 173         }
 174     }
 175     else if (test_num < test_cases)
 176     {
 177         status = test_plan_fail(CFSTR("Looks like you planned %d tests but only ran %d.\n"), test_cases, test_num);
 178     }
 179     else if (test_num > test_cases)
 180     {
 181         status = test_plan_fail(CFSTR("Looks like you planned %d tests but ran %d.\n"), test_cases, test_num);
 182     } else if (!test_fails) {
 183         status = test_plan_pass();
 184     }
 185     if (test_fails)
 186     {
 187         fprintf(stdout, "%s failed %d tests of %d.\n", name, test_fails, test_num);
 188         status = 1;
 189     }
 190     pthread_mutex_unlock(&test_mutex);
 191     fflush(stdout);
 192
 193     return status;
 194 }
 195
 196 // You should hold the test_mutex when you call this.
 197 static void test_plan_reset(void) {
 198     test_fails = 0;
 199     test_todo_pass = 0;
 200     test_todo = 0;
 201     test_num = 0;
 202     test_cases = 0;
 203     test_plan_file = NULL;
 204     test_plan_line = 0;
 205 }
 206
 207 void test_plan_final(int *failed, int *todo_pass, int *todo, int *actual, int *planned, const char **file, int *line) {
 208     pthread_mutex_lock(&test_mutex);
 209     if (failed)
 210         *failed = test_fails;
 211     if (todo_pass)
 212         *todo_pass = test_todo_pass;
 213     if (todo)
 214         *todo = test_todo;
 215     if (actual)
 216         *actual = test_num;
 217     if (planned)
 218         *planned = test_cases;
 219     if (file)
 220         *file = test_plan_file;
 221     if (line)
 222         *line = test_plan_line;
 223
 224     test_plan_reset();
 225     pthread_mutex_unlock(&test_mutex);
 226 }
 227
 228 void test_plan_tests(int count, const char *file, unsigned line) {
 229         if (test_cases)
 230     {
 231         fprintf(stdout,
 232                 "[FAIL] You tried to plan twice!\n");
 233
 234         fflush(stdout);
 235         exit(255);
 236     }
 237     else
 238         {
 239         if (!count)
 240         {
 241             fprintf(stdout, "[BEGIN] plan_tests\nYou said to run 0 tests!  "
 242                 "You've got to run something.\n[WARN] plan_tests\n");
 243             fflush(stdout);
 244         }
 245
 246         test_plan_file=file;
 247         test_plan_line=line;
 248
 249         test_cases = count;
 250
 251         static dispatch_once_t onceToken;
 252         dispatch_once(&onceToken, ^{
 253             if(pthread_mutex_init(&test_mutex, NULL) != 0) {
 254                 fprintf(stdout, "Failed to initialize mutex: %d\n", errno);
 255             }
 256         });
 257         }
 258 }
 259
 260 int
 261 test_diag(const char *directive, const char *reason,
 262         const char *file, unsigned line, const char *fmt, ...)
 263 {
 264         int is_todo = directive && !strcmp(directive, "TODO");
 265         va_list args;
 266
 267         va_start(args, fmt);
 268
 269         if (is_todo)
 270         {
 271                 fputs("# ", stdout);
 272                 if (fmt)
 273                         vprintf(fmt, args);
 274                 fputs("\n", stdout);
 275                 fflush(stdout);
 276         }
 277         else
 278         {
 279                 fputs("# ", stdout);
 280                 if (fmt)
 281                         vfprintf(stdout, fmt, args);
 282                 fputs("\n", stdout);
 283                 fflush(stdout);
 284         }
 285
 286         va_end(args);
 287
 288         return 1;
 289 }
 290
 291 int
 292 test_ok(int passed, __attribute((cf_consumed)) CFStringRef CF_CONSUMED description, const char *directive,
 293         const char *reason, const char *file, unsigned line,
 294         const char *fmt, ...)
 295 {
 296         int is_todo = directive && !strcmp(directive, "TODO");
 297         int is_setup = directive && !is_todo && !strcmp(directive, "SETUP");
 298
 299         if (is_setup)
 300         {
 301                 if (!passed)
 302                 {
 303                         fflush(stderr);
 304             fprintf(stdout, "[BEGIN] SETUP\n");
 305             if (fmt) {
 306                 va_list args;
 307                 va_start(args, fmt);
 308                 cffprint_c_v(stdout, fmt, args);
 309                 va_end(args);
 310             }
 311             cffprint(stdout, CFSTR("[WARN] SETUP%s%@%s%s\n"),
 312                                    description ? " - " : "",
 313                                    description ? description : CFSTR(""),
 314                                    reason ? " - " : "",
 315                                    reason ? reason : "");
 316             fflush(stdout);
 317                 }
 318         }
 319         else
 320         {
 321                 if (!test_cases)
 322                 {
 323             // Make having a plan optional? Commenting out the next 3 lines does - mb
 324                         //fprintf(stdout, "[FAIL] You tried to run a test without a plan!  "
 325                         //              "Gotta have a plan. at %s line %u\n", file, line);
 326                         //fflush(stdout);
 327                 }
 328
 329         pthread_mutex_lock(&test_mutex);
 330                 ++test_num;
 331         if (passed) {
 332             if (is_todo) {
 333                 test_todo_pass++;
 334             }
 335         } else if (is_todo) {
 336             test_todo++;
 337         } else {
 338             ++test_fails;
 339         }
 340
 341         /* We only print this when a test fails, unless verbose is enabled */
 342         if ((!passed && !is_todo) || test_verbose > 0) {
 343             fflush(stderr);
 344             if (test_strict_bats) {
 345                 cffprint(stdout, CFSTR("[BEGIN] %d%s%@\n"),
 346                          test_num,
 347                          description ? " - " : "",
 348                          description ? description : CFSTR(""));
 349             }
 350             if (is_todo && passed) {
 351                 fprintf(stdout, "%s:%d: warning: Unexpectedly passed (TODO) test\n", file, line);
 352             } else if (is_todo && !passed) {
 353                 /* Enable this to output TODO as warning */
 354                 fprintf(stdout, "%s:%d: ok: Failed (TODO) test\n", file, line);
 355             } else if (!passed) {
 356                 fprintf(stdout, "%s:%d: error: Failed test\n", file, line);
 357             }
 358             if (fmt) {
 359                 va_list args;
 360                 va_start(args, fmt);
 361                 cffprint_c_v(stdout, fmt, args);
 362                 va_end(args);
 363             }
 364             cffprint(stdout, CFSTR("[%s] %d%s%@%s%s%s%s\n"), passed ? (is_todo ? "PASS" : "PASS") : (is_todo ? "PASS" : "FAIL"),
 365                      test_num,
 366                      description ? " - " : "",
 367                      description ? description : CFSTR(""),
 368                      directive ? " # " : "",
 369                      directive ? directive : "",
 370                      reason ? " " : "",
 371                      reason ? reason : "");
 372             fflush(stdout);
 373         }
 374         pthread_mutex_unlock(&test_mutex);
 375     }
 376
 377     if (description)
 378         CFRelease(description);
 379
 380     return passed;
 381 }
 382
 383
 384 // TODO: Move this to testsec.h so that testmore and testenv can be shared
 385 static void buf_kill(void* p) {
 386     free(p);
 387 }
 388
 389 const char *
 390 sec_errstr(int err)
 391 {
 392     static pthread_key_t buffer0key;
 393     static pthread_key_t buffer1key;
 394     static pthread_key_t switchkey;
 395     static dispatch_once_t onceToken;
 396     dispatch_once(&onceToken, ^{
 397         pthread_key_create(&buffer0key, buf_kill);
 398         pthread_key_create(&buffer1key, buf_kill);
 399         pthread_key_create(&switchkey, buf_kill);
 400     });
 401
 402     uint32_t * switchp = (uint32_t*) pthread_getspecific(switchkey);
 403     if(switchp == NULL) {
 404         switchp = (uint32_t*) malloc(sizeof(uint32_t));
 405         *switchp = 0;
 406         pthread_setspecific(switchkey, switchp);
 407     }
 408
 409     char* buf = NULL;
 410
 411     pthread_key_t current = (*switchp) ? buffer0key : buffer1key;
 412     *switchp = !(*switchp);
 413
 414     buf = pthread_getspecific(current);
 415     if(buf == NULL) {
 416         buf = (char*) malloc(20);
 417         pthread_setspecific(current, buf);
 418     }
 419
 420     snprintf(buf, 20, "0x%X", err);
 421     return buf;
 422 }