regressions/test/testmore.c

   1 /*
   2  * Copyright (c) 2005-2007 Apple Inc. All Rights Reserved.
   3  *
   4  * @APPLE_LICENSE_HEADER_START@
   5  *
   6  * This file contains Original Code and/or Modifications of Original Code
   7  * as defined in and that are subject to the Apple Public Source License
   8  * Version 2.0 (the 'License'). You may not use this file except in
   9  * compliance with the License. Please obtain a copy of the License at
  10  * http://www.opensource.apple.com/apsl/ and read it before using this
  11  * file.
  12  *
  13  * The Original Code and all software distributed under the License are
  14  * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  15  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  16  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
  17  * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
  18  * Please see the License for the specific language governing rights and
  19  * limitations under the License.
  20  *
  21  * @APPLE_LICENSE_HEADER_END@
  22  *
  23  * testmore.c
  24  */
  25
  26 #include <fcntl.h>
  27 #include <stdarg.h>
  28 #include <stdlib.h>
  29 #include <string.h>
  30 #include <sys/stat.h>
  31 #include <sys/types.h>
  32 #include <sys/time.h>
  33 #include <unistd.h>
  34 #include <AvailabilityMacros.h>
  35
  36 #include "testmore.h"
  37 #include "testenv.h"
  38
  39 static int test_num = 0;
  40 static int test_fails = 0;
  41 static int test_cases = 0;
  42 static const char *test_plan_file;
  43 static int test_plan_line=0;
  44
  45 const char *test_directive = NULL;
  46 const char *test_reason = NULL;
  47
  48 static void fprint_string(FILE *file, CFStringRef string) {
  49     UInt8 buf[256];
  50     CFRange range = { .location = 0 };
  51     range.length = CFStringGetLength(string);
  52     while (range.length > 0) {
  53         CFIndex bytesUsed = 0;
  54         CFIndex converted = CFStringGetBytes(string, range, kCFStringEncodingUTF8, 0, false, buf, sizeof(buf), &bytesUsed);
  55         fwrite(buf, 1, bytesUsed, file);
  56         range.length -= converted;
  57         range.location += converted;
  58     }
  59 }
  60
  61 static void cffprint(FILE *file, CFStringRef fmt, ...) CF_FORMAT_FUNCTION(2,0);
  62
  63 static void cffprint(FILE *file, CFStringRef fmt, ...) {
  64     va_list args;
  65     va_start(args, fmt);
  66     CFStringRef line = CFStringCreateWithFormatAndArguments(NULL, NULL, fmt, args);
  67     va_end(args);
  68     fprint_string(file, line);
  69     CFRelease(line);
  70 }
  71
  72 void test_skip(const char *reason, int how_many, int unless)
  73 {
  74     if (unless)
  75         return;
  76
  77     int done;
  78     for (done = 0; done < how_many; ++done)
  79         test_ok(1, NULL, "skip", reason, __FILE__, __LINE__, NULL);
  80 }
  81
  82 void test_bail_out(const char *reason, const char *file, unsigned line)
  83 {
  84     printf("BAIL OUT! (%s at line %u) %s\n", file, line, reason);
  85     fflush(stdout);
  86     exit(255);
  87 }
  88
  89 void test_plan_skip_all(const char *reason)
  90 {
  91     if (test_num < test_cases)
  92     {
  93         test_skip(reason, test_cases - test_num, 0);
  94     }
  95 }
  96
  97 static int test_plan_exit(void)
  98 {
  99     int status = 0;
 100     fflush(stdout);
 101
 102     if (!test_num)
 103     {
 104         if (test_cases)
 105         {
 106             fprintf(stderr, "%s:%u: warning: No tests run!\n", test_plan_file, test_plan_line);
 107             status = 255;
 108         }
 109         else
 110         {
 111             fprintf(stderr, "%s:%u: error: Looks like your test died before it could "
 112                     "output anything.\n", test_plan_file, test_plan_line);
 113             status = 255;
 114         }
 115     }
 116     else if (test_num < test_cases)
 117     {
 118         fprintf(stderr, "%s:%u: warning: Looks like you planned %d tests but only ran %d.\n",
 119                test_plan_file, test_plan_line, test_cases, test_num);
 120         status = test_fails + test_cases - test_num;
 121     }
 122     else if (test_num > test_cases)
 123     {
 124         fprintf(stderr, "%s:%u: warning: Looks like you planned %d tests but ran %d extra.\n",
 125                test_plan_file, test_plan_line, test_cases, test_num - test_cases);
 126         status = test_fails;
 127     }
 128     else if (test_fails)
 129     {
 130         fprintf(stderr, "%s:%u: error: Looks like you failed %d tests of %d.\n",
 131                test_plan_file, test_plan_line, test_fails, test_cases);
 132         status = test_fails;
 133     }
 134
 135     fflush(stderr);
 136
 137     /* reset the test plan */
 138     test_num = 0;
 139     test_fails = 0;
 140     test_cases = 0;
 141
 142     return status;
 143 }
 144
 145 void test_plan_tests(int count, const char *file, unsigned line)
 146 {
 147 #if 0
 148     if (atexit(test_plan_exit) < 0)
 149     {
 150         fprintf(stderr, "failed to setup atexit handler: %s\n",
 151                 strerror(errno));
 152         fflush(stderr);
 153         exit(255);
 154     }
 155 #endif
 156
 157         if (test_cases)
 158     {
 159         fprintf(stderr,
 160                 "%s:%u: error: You tried to plan twice!\n",
 161                 file, line);
 162
 163         fflush(stderr);
 164         exit(255);
 165     }
 166     else
 167         {
 168         if (!count)
 169         {
 170             fprintf(stderr, "%s:%u: warning: You said to run 0 tests!  You've got to run "
 171                     "something.\n", file, line);
 172             fflush(stderr);
 173             exit(255);
 174         }
 175
 176         test_plan_file=file;
 177         test_plan_line=line;
 178
 179         test_cases = count;
 180                 fprintf(stderr, "%s:%u: note: 1..%d\n", file, line, test_cases);
 181                 fflush(stdout);
 182         }
 183 }
 184
 185 int
 186 test_diag(const char *directive, const char *reason,
 187         const char *file, unsigned line, const char *fmt, ...)
 188 {
 189         int is_todo = directive && !strcmp(directive, "TODO");
 190         va_list args;
 191
 192         va_start(args, fmt);
 193
 194         if (is_todo)
 195         {
 196                 fputs("# ", stdout);
 197                 if (fmt)
 198                         vprintf(fmt, args);
 199                 fputs("\n", stdout);
 200                 fflush(stdout);
 201         }
 202         else
 203         {
 204                 fflush(stdout);
 205                 fputs("# ", stderr);
 206                 if (fmt)
 207                         vfprintf(stderr, fmt, args);
 208                 fputs("\n", stderr);
 209                 fflush(stderr);
 210         }
 211
 212         va_end(args);
 213
 214         return 1;
 215 }
 216
 217 int
 218 test_ok(int passed, __attribute((cf_consumed)) CFStringRef description, const char *directive,
 219         const char *reason, const char *file, unsigned line,
 220         const char *fmt, ...)
 221 {
 222         int is_todo = !passed && directive && !strcmp(directive, "TODO");
 223         int is_setup = directive && !is_todo && !strcmp(directive, "SETUP");
 224
 225         if (is_setup)
 226         {
 227                 if (!passed)
 228                 {
 229                         fflush(stdout);
 230             cffprint(stderr, CFSTR("# SETUP not ok%s%@%s%s\n"),
 231                                    description ? " - " : "",
 232                                    description ? description : CFSTR(""),
 233                                    reason ? " - " : "",
 234                                    reason ? reason : "");
 235                 }
 236         }
 237         else
 238         {
 239                 if (!test_cases)
 240                 {
 241                         atexit((void(*)(void))test_plan_exit);
 242                         fprintf(stderr, "You tried to run a test without a plan!  "
 243                                         "Gotta have a plan. at %s line %u\n", file, line);
 244                         fflush(stderr);
 245                         exit(255);
 246                 }
 247
 248                 ++test_num;
 249                 if (test_num > test_cases || (!passed && !is_todo))
 250                         ++test_fails;
 251
 252         /* We only print this when a test fail, unless verbose is enabled */
 253         if ((!passed) || (test_verbose > 0)) {
 254             cffprint(stderr, CFSTR("%s:%u: note: %sok %d%s%@%s%s%s%s\n"),
 255                       file, line, passed ? "" : "not ", test_num,
 256                       description ? " - " : "",
 257                       description ? description : CFSTR(""),
 258                       directive ? " # " : "",
 259                       directive ? directive : "",
 260                       reason ? " " : "",
 261                       reason ? reason : "");
 262         }
 263     }
 264
 265     if (passed)
 266                 fflush(stdout);
 267         else
 268     {
 269                 va_list args;
 270
 271                 va_start(args, fmt);
 272
 273                 if (is_todo)
 274                 {
 275 /* Enable this to output TODO as warning */
 276 #if 0
 277                         printf("%s:%d: warning: Failed (TODO) test\n", file, line);
 278                         if (fmt)
 279                                 vprintf(fmt, args);
 280 #endif
 281                         fflush(stdout);
 282                 }
 283         else
 284                 {
 285                         fflush(stdout);
 286                         fprintf(stderr, "%s:%d: error: Failed test\n", file, line);
 287                         if (fmt)
 288                                 vfprintf(stderr, fmt, args);
 289                         fflush(stderr);
 290                 }
 291
 292                 va_end(args);
 293     }
 294
 295     if (description)
 296         CFRelease(description);
 297
 298     return passed;
 299 }
 300
 301
 302 const char *
 303 sec_errstr(int err)
 304 {
 305 #if 1
 306         static int bufnum = 0;
 307     static char buf[2][20];
 308         bufnum = bufnum ? 0 : 1;
 309     sprintf(buf[bufnum], "0x%X", err);
 310     return buf[bufnum];
 311 #else /* !1 */
 312     if (err >= errSecErrnoBase && err <= errSecErrnoLimit)
 313         return strerror(err - 100000);
 314
 315 #ifdef MAC_OS_X_VERSION_10_4
 316     /* AvailabilityMacros.h would only define this if we are on a
 317        Tiger or later machine. */
 318     extern const char *cssmErrorString(long);
 319     return cssmErrorString(err);
 320 #else /* !defined(MAC_OS_X_VERSION_10_4) */
 321     extern const char *_ZN8Security15cssmErrorStringEl(long);
 322     return _ZN8Security15cssmErrorStringEl(err);
 323 #endif /* MAC_OS_X_VERSION_10_4 */
 324 #endif /* !1 */
 325 }
 326
 327 /* run one test, described by test, return info in test struct */
 328 int run_one_test(struct one_test_s *test, int argc, char * const *argv)
 329 {
 330     struct timeval start, stop;
 331
 332     if(test->entry==NULL) {
 333         printf("%s:%d: error: wtf?\n", __FILE__, __LINE__);
 334         return -1;
 335     }
 336
 337     gettimeofday(&start, NULL);
 338     test->entry(argc, argv);
 339     gettimeofday(&stop, NULL);
 340
 341
 342     /* this may overflow... */
 343     test->duration=(stop.tv_sec-start.tv_sec)*1000+(stop.tv_usec/1000)-(start.tv_usec/1000);
 344     test->failed_tests=test_fails;
 345
 346     return test_plan_exit();
 347  };