OSX/regressions/test/testmore.c

   1 /*
   2  * Copyright (c) 2005-2007,2012-2014 Apple Inc. All Rights Reserved.
   3  *
   4  * @APPLE_LICENSE_HEADER_START@
   5  *
   6  * This file contains Original Code and/or Modifications of Original Code
   7  * as defined in and that are subject to the Apple Public Source License
   8  * Version 2.0 (the 'License'). You may not use this file except in
   9  * compliance with the License. Please obtain a copy of the License at
  10  * http://www.opensource.apple.com/apsl/ and read it before using this
  11  * file.
  12  *
  13  * The Original Code and all software distributed under the License are
  14  * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  15  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  16  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
  17  * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
  18  * Please see the License for the specific language governing rights and
  19  * limitations under the License.
  20  *
  21  * @APPLE_LICENSE_HEADER_END@
  22  *
  23  * testmore.c
  24  */
  25
  26 #include <fcntl.h>
  27 #include <stdarg.h>
  28 #include <stdlib.h>
  29 #include <string.h>
  30 #include <sys/stat.h>
  31 #include <sys/types.h>
  32 #include <unistd.h>
  33 #include <AvailabilityMacros.h>
  34
  35 #include "testmore.h"
  36 #include "testenv.h"
  37
  38 static int test_fails = 0;
  39 static int test_todo_pass = 0;
  40 static int test_todo = 0;
  41 static int test_num = 0;
  42 static int test_cases = 0;
  43 static int test_plan_line = 0;
  44 static const char *test_plan_file = NULL;
  45
  46 const char *test_directive = NULL;
  47 const char *test_reason = NULL;
  48
  49 static void fprint_string(FILE *file, CFStringRef string) {
  50     UInt8 buf[256];
  51     CFRange range = { .location = 0 };
  52     range.length = CFStringGetLength(string);
  53     while (range.length > 0) {
  54         CFIndex bytesUsed = 0;
  55         CFIndex converted = CFStringGetBytes(string, range, kCFStringEncodingUTF8, 0, false, buf, sizeof(buf), &bytesUsed);
  56         fwrite(buf, 1, bytesUsed, file);
  57         range.length -= converted;
  58         range.location += converted;
  59     }
  60 }
  61
  62 static void cffprint(FILE *file, CFStringRef fmt, ...) CF_FORMAT_FUNCTION(2,0);
  63
  64 static void cffprint_v(FILE *file, CFStringRef fmt, va_list args);
  65 static void cffprint_c_v(FILE *file, const char *fmt, va_list args);
  66
  67 static void cffprint_v(FILE *file, CFStringRef fmt, va_list args) {
  68     CFStringRef line = CFStringCreateWithFormatAndArguments(NULL, NULL, fmt, args);
  69     fprint_string(file, line);
  70     CFRelease(line);
  71 }
  72
  73 static void cffprint_c_v(FILE *file, const char *fmt, va_list args) {
  74     CFStringRef cffmt = CFStringCreateWithCString(kCFAllocatorDefault, fmt, kCFStringEncodingUTF8);
  75     cffprint_v(file, cffmt, args);
  76     CFRelease(cffmt);
  77 }
  78
  79 static void cffprint(FILE *file, CFStringRef fmt, ...) {
  80     va_list args;
  81     va_start(args, fmt);
  82     cffprint_v(file, fmt, args);
  83     va_end(args);
  84 }
  85
  86 void test_skip(const char *reason, int how_many, int unless)
  87 {
  88     if (unless)
  89         return;
  90
  91     int done;
  92     for (done = 0; done < how_many; ++done)
  93         test_ok(1, NULL, "skip", reason, __FILE__, __LINE__, NULL);
  94 }
  95
  96 void test_bail_out(const char *reason, const char *file, unsigned line)
  97 {
  98     fprintf(stdout, "[FAIL] BAIL OUT! (%s at line %u) %s\n", file, line, reason);
  99     fflush(stdout);
 100     exit(255);
 101 }
 102
 103 void test_plan_skip_all(const char *reason)
 104 {
 105     if (test_num < test_cases)
 106     {
 107         test_skip(reason, test_cases - test_num, 0);
 108     }
 109 }
 110
 111 static const char *test_plan_name(void) {
 112     const char *plan_name = strrchr(test_plan_file, '/');
 113     plan_name = plan_name ? plan_name + 1 : test_plan_file;
 114     return plan_name;
 115 }
 116
 117 static int test_plan_pass(void) {
 118     if (test_verbose) {
 119         const char *name = test_plan_name();
 120         fprintf(stdout, "[BEGIN] %s plan\n[PASS] %s plan\n", name, name);
 121         // Update counts for summary
 122         //test_num++;
 123         //test_cases++;
 124     }
 125     return 0;
 126 }
 127
 128 static int test_plan_fail(CFStringRef reason, ...) {
 129     const char *name = test_plan_name();
 130     va_list ap;
 131     va_start(ap, reason);
 132     CFStringRef desc = CFStringCreateWithFormatAndArguments(kCFAllocatorDefault, NULL, reason, ap);
 133     cffprint(stdout, CFSTR("[BEGIN] %s plan\n%@[WARN] %s plan\n"), name, desc, name);
 134     CFRelease(desc);
 135     // Update counts for summary.  We consider test_plan_ok itself an unscheduled testcase for counts.
 136     //test_num++;
 137     //test_fails++;
 138     //test_cases++;
 139     return 1;
 140 }
 141
 142 int test_plan_ok(void) {
 143     int status = 0;
 144     fflush(stderr);
 145     const char *name = test_plan_name();
 146
 147     if (!test_num)
 148     {
 149         if (test_cases)
 150         {
 151             status = test_plan_fail(CFSTR("No tests run!\n"));
 152         }
 153         else
 154         {
 155             status = test_plan_fail(CFSTR("Looks like your test died before it could output anything.\n"));
 156         }
 157     }
 158     else if (test_num < test_cases)
 159     {
 160         status = test_plan_fail(CFSTR("Looks like you planned %d tests but only ran %d.\n"), test_cases, test_num);
 161     }
 162     else if (test_num > test_cases)
 163     {
 164         status = test_plan_fail(CFSTR("Looks like you planned %d tests but ran %d.\n"), test_cases, test_num);
 165     } else if (!test_fails) {
 166         status = test_plan_pass();
 167     }
 168     if (test_fails)
 169     {
 170         fprintf(stdout, "%s failed %d tests of %d.\n", name, test_fails, test_num);
 171         status = 1;
 172     }
 173     fflush(stdout);
 174
 175     return status;
 176 }
 177
 178 static void test_plan_reset(void) {
 179     test_fails = 0;
 180     test_todo_pass = 0;
 181     test_todo = 0;
 182     test_num = 0;
 183     test_cases = 0;
 184     test_plan_file = NULL;
 185     test_plan_line = 0;
 186 }
 187
 188 void test_plan_final(int *failed, int *todo_pass, int *todo, int *actual, int *planned, const char **file, int *line) {
 189     if (failed)
 190         *failed = test_fails;
 191     if (todo_pass)
 192         *todo_pass = test_todo_pass;
 193     if (todo)
 194         *todo = test_todo;
 195     if (actual)
 196         *actual = test_num;
 197     if (planned)
 198         *planned = test_cases;
 199     if (file)
 200         *file = test_plan_file;
 201     if (line)
 202         *line = test_plan_line;
 203
 204     test_plan_reset();
 205 }
 206
 207 void test_plan_tests(int count, const char *file, unsigned line) {
 208         if (test_cases)
 209     {
 210         fprintf(stdout,
 211                 "[FAIL] You tried to plan twice!\n");
 212
 213         fflush(stdout);
 214         exit(255);
 215     }
 216     else
 217         {
 218         if (!count)
 219         {
 220             fprintf(stdout, "[BEGIN] plan_tests\nYou said to run 0 tests!  "
 221                 "You've got to run something.\n[WARN] plan_tests\n");
 222             fflush(stdout);
 223         }
 224
 225         test_plan_file=file;
 226         test_plan_line=line;
 227
 228         test_cases = count;
 229         }
 230 }
 231
 232 int
 233 test_diag(const char *directive, const char *reason,
 234         const char *file, unsigned line, const char *fmt, ...)
 235 {
 236         int is_todo = directive && !strcmp(directive, "TODO");
 237         va_list args;
 238
 239         va_start(args, fmt);
 240
 241         if (is_todo)
 242         {
 243                 fputs("# ", stdout);
 244                 if (fmt)
 245                         vprintf(fmt, args);
 246                 fputs("\n", stdout);
 247                 fflush(stdout);
 248         }
 249         else
 250         {
 251                 fputs("# ", stdout);
 252                 if (fmt)
 253                         vfprintf(stdout, fmt, args);
 254                 fputs("\n", stdout);
 255                 fflush(stdout);
 256         }
 257
 258         va_end(args);
 259
 260         return 1;
 261 }
 262
 263 int
 264 test_ok(int passed, __attribute((cf_consumed)) CFStringRef description, const char *directive,
 265         const char *reason, const char *file, unsigned line,
 266         const char *fmt, ...)
 267 {
 268         int is_todo = directive && !strcmp(directive, "TODO");
 269         int is_setup = directive && !is_todo && !strcmp(directive, "SETUP");
 270
 271         if (is_setup)
 272         {
 273                 if (!passed)
 274                 {
 275                         fflush(stderr);
 276             fprintf(stdout, "[BEGIN] SETUP\n");
 277             if (fmt) {
 278                 va_list args;
 279                 va_start(args, fmt);
 280                 cffprint_c_v(stdout, fmt, args);
 281                 va_end(args);
 282             }
 283             cffprint(stdout, CFSTR("[WARN] SETUP%s%@%s%s\n"),
 284                                    description ? " - " : "",
 285                                    description ? description : CFSTR(""),
 286                                    reason ? " - " : "",
 287                                    reason ? reason : "");
 288             fflush(stdout);
 289                 }
 290         }
 291         else
 292         {
 293                 if (!test_cases)
 294                 {
 295             // Make having a plan optional? Commenting out the next 3 lines does - mb
 296                         //fprintf(stdout, "[FAIL] You tried to run a test without a plan!  "
 297                         //              "Gotta have a plan. at %s line %u\n", file, line);
 298                         //fflush(stdout);
 299                 }
 300
 301                 ++test_num;
 302         if (passed) {
 303             if (is_todo) {
 304                 test_todo_pass++;
 305             }
 306         } else if (is_todo) {
 307             test_todo++;
 308         } else {
 309             ++test_fails;
 310         }
 311
 312         /* We only print this when a test fails, unless verbose is enabled */
 313         if ((!passed && !is_todo) || test_verbose > 0) {
 314             fflush(stderr);
 315             if (test_strict_bats) {
 316                 cffprint(stdout, CFSTR("[BEGIN] %d%s%@\n"),
 317                          test_num,
 318                          description ? " - " : "",
 319                          description ? description : CFSTR(""));
 320             }
 321             if (is_todo && passed) {
 322                 fprintf(stdout, "%s:%d: warning: Unexpectedly passed (TODO) test\n", file, line);
 323             } else if (is_todo && !passed) {
 324                 /* Enable this to output TODO as warning */
 325                 fprintf(stdout, "%s:%d: ok: Failed (TODO) test\n", file, line);
 326             } else if (!passed) {
 327                 fprintf(stdout, "%s:%d: error: Failed test\n", file, line);
 328             }
 329             if (fmt) {
 330                 va_list args;
 331                 va_start(args, fmt);
 332                 cffprint_c_v(stdout, fmt, args);
 333                 va_end(args);
 334             }
 335             cffprint(stdout, CFSTR("[%s] %d%s%@%s%s%s%s\n"), passed ? (is_todo ? "PASS" : "PASS") : (is_todo ? "PASS" : "FAIL"),
 336                      test_num,
 337                      description ? " - " : "",
 338                      description ? description : CFSTR(""),
 339                      directive ? " # " : "",
 340                      directive ? directive : "",
 341                      reason ? " " : "",
 342                      reason ? reason : "");
 343             fflush(stdout);
 344         }
 345     }
 346
 347     if (description)
 348         CFRelease(description);
 349
 350     return passed;
 351 }
 352
 353 // TODO: Move this to testsec.h so that testmore and testenv can be shared
 354 const char *
 355 sec_errstr(int err)
 356 {
 357 #if 1
 358         static int bufnum = 0;
 359     static char buf[2][20];
 360         bufnum = bufnum ? 0 : 1;
 361     sprintf(buf[bufnum], "0x%X", err);
 362     return buf[bufnum];
 363 #else /* !1 */
 364     if (err >= errSecErrnoBase && err <= errSecErrnoLimit)
 365         return strerror(err - 100000);
 366
 367 #ifdef MAC_OS_X_VERSION_10_4
 368     /* AvailabilityMacros.h would only define this if we are on a
 369        Tiger or later machine. */
 370     extern const char *cssmErrorString(long);
 371     return cssmErrorString(err);
 372 #else /* !defined(MAC_OS_X_VERSION_10_4) */
 373     extern const char *_ZN8Security15cssmErrorStringEl(long);
 374     return _ZN8Security15cssmErrorStringEl(err);
 375 #endif /* MAC_OS_X_VERSION_10_4 */
 376 #endif /* !1 */
 377 }