tests/TrustTests/TestRunners/main.m

   1 //
   2 //  TrustTests
   3 //
   4 //  Stolen from Mark Pauley / CDEntitledTestRunner who stole it from Drew Terry / MobileContainerManager
   5 //  Copyright 2016-2018 Apple. All rights reserved.
   6 //
   7
   8 #import <Foundation/Foundation.h>
   9 #import <unistd.h>
  10 #import <XCTest/XCTest.h>
  11
  12 @interface TestRunner : NSObject <XCTestObservation> {
  13     NSBundle *_bundle;
  14     XCTestSuite *_testSuite;
  15 }
  16
  17 - (instancetype)initWithBundlePath:(NSString *)path andTestNames:(NSArray *)names;
  18 - (NSUInteger)runUnitTestSuite;
  19 - (void)testLogWithFormat:(NSString *)format, ... NS_FORMAT_FUNCTION(1, 2);
  20 - (void)testLogWithFormat:(NSString *)format arguments:(va_list)arguments NS_FORMAT_FUNCTION(1,0);
  21 @end
  22
  23 @implementation TestRunner
  24 - (instancetype)initWithBundlePath:(NSString *)path andTestNames:(NSArray *)names
  25 {
  26     self = [super init];
  27     if (self) {
  28         NSError *error = nil;
  29
  30         _bundle = [NSBundle bundleWithPath:path];
  31         if (!_bundle) {
  32             [self testLogWithFormat:@"No bundle at location %@ (%s)\n", path, strerror(errno)];
  33             return nil;
  34         }
  35         if (![_bundle loadAndReturnError:&error]) {
  36             [self testLogWithFormat:@"Test Bundle at %@ didn't load: %@\n", path, error];
  37             return nil;
  38         }
  39
  40         if(names) {
  41             XCTestSuite* testSuite = [[XCTestSuite alloc] initWithName:[[path lastPathComponent] stringByDeletingPathExtension]];
  42             XCTestSuite* loadedSuite = [XCTestSuite testSuiteForBundlePath:path];
  43             // Filter out only the tests that were named.
  44             [loadedSuite.tests enumerateObjectsUsingBlock:^(__kindof XCTest * _Nonnull test, NSUInteger __unused idx, BOOL * __unused _Nonnull stop) {
  45                 [self testLogWithFormat:@"Checking test %@\n", test.name];
  46                 if([names containsObject:test.name]) {
  47                     [testSuite addTest:test];
  48                 }
  49             }];
  50             _testSuite = testSuite;
  51         }
  52         else {
  53             _testSuite = [XCTestSuite testSuiteForBundlePath:path];
  54         }
  55     }
  56     return self;
  57 }
  58
  59 - (NSUInteger)runUnitTestSuite
  60 {
  61     [[XCTestObservationCenter sharedTestObservationCenter] addTestObserver:self];
  62
  63     [_testSuite runTest];
  64
  65     XCTestRun *testRun = [_testSuite testRun];
  66
  67     return testRun.totalFailureCount;
  68 }
  69
  70 - (NSFileHandle *)logFileHandle
  71 {
  72     return [NSFileHandle fileHandleWithStandardOutput];
  73 }
  74
  75 - (void)testLogWithFormat:(NSString *)format, ...
  76 {
  77     va_list ap;
  78     va_start(ap, format);
  79     [self testLogWithFormat:format arguments:ap];
  80     va_end(ap);
  81 }
  82
  83 - (void)testLogWithFormat:(NSString *)format arguments:(va_list)arguments
  84 {
  85     NSString *message = [[NSString alloc] initWithFormat:format arguments:arguments];
  86     [self.logFileHandle writeData:[message dataUsingEncoding:NSUTF8StringEncoding]];
  87 }
  88
  89 - (NSDateFormatter *)dateFormatter
  90 {
  91     static NSDateFormatter *dateFormatter;
  92     static dispatch_once_t onceToken;
  93     dispatch_once(&onceToken, ^{
  94         dateFormatter = [NSDateFormatter new];
  95         dateFormatter.dateFormat = @"yyyy-MM-dd HH:mm:ss.SSS";
  96     });
  97     return dateFormatter;
  98 }
  99
 100 /* -testBundleWillStart:                            // exactly once per test bundle
 101  *      -testSuiteWillStart:                        // exactly once per test suite
 102  *          -testCaseWillStart:                     // exactly once per test case
 103  *          -testCase:didFailWithDescription:...    // zero or more times per test case, any time between test case start and finish
 104  *          -testCaseDidFinish:                     // exactly once per test case
 105  *      -testSuite:didFailWithDescription:...       // zero or more times per test suite, any time between test suite start and finish
 106  *      -testSuiteDidFinish:                        // exactly once per test suite
 107  * -testBundleDidFinish:                            // exactly once per test bundle
 108  */
 109
 110 - (void)testSuiteWillStart:(XCTestSuite *)testSuite
 111 {
 112     [self testLogWithFormat:@"Test Suite '%@' started at %@\n", testSuite.name, [self.dateFormatter stringFromDate:testSuite.testRun.startDate]];
 113 }
 114
 115 - (void)testSuite:(XCTestSuite *)testSuite didFailWithDescription:(NSString *)description inFile:(nullable NSString *)filePath atLine:(NSUInteger)lineNumber
 116 {
 117     [self testLogWithFormat:@"%@:%lu: error: %@ : %@\n", ((nil != filePath) ? filePath : @"<unknown>"), ((unsigned long)((nil != filePath) ? lineNumber : 0)), testSuite.name, description];
 118 }
 119
 120 - (void)testSuiteDidFinish:(XCTestSuite *)testSuite
 121 {
 122     XCTestRun *testRun = testSuite.testRun;
 123     [self testLogWithFormat:@"Test Suite '%@' %s at %@.\n\t Executed %lu test%s, with %lu failure%s (%lu unexpected) in %.3f (%.3f) seconds\n",
 124      testSuite.name,
 125      (testRun.hasSucceeded ? "passed" : "failed"),
 126      [self.dateFormatter stringFromDate:testRun.stopDate],
 127      ((unsigned long)testRun.executionCount), (testRun.executionCount != 1 ? "s" : ""),
 128      ((unsigned long)testRun.totalFailureCount), (testRun.totalFailureCount != 1 ? "s" : ""),
 129      ((unsigned long)testRun.unexpectedExceptionCount),
 130      testRun.testDuration,
 131      testRun.totalDuration];
 132 }
 133
 134 - (void)testCaseWillStart:(XCTestCase *)testCase
 135 {
 136     [self testLogWithFormat:@"Test Case '%@' started.\n", testCase.name];
 137 }
 138
 139 - (void)testCase:(XCTestCase *)testCase didFailWithDescription:(NSString *)description inFile:(nullable NSString *)filePath atLine:(NSUInteger)lineNumber
 140 {
 141     [self testLogWithFormat:@"%@:%lu: error: %@ : %@\n", ((nil != filePath) ? filePath : @"<unknown>"), ((unsigned long)((nil != filePath) ? lineNumber : 0)), testCase.name, description];
 142 }
 143
 144 - (void)testCaseDidFinish:(XCTestCase *)testCase
 145 {
 146     [self testLogWithFormat:@"Test Case '%@' %s (%.3f seconds).\n", testCase.name, (testCase.testRun.hasSucceeded ? "passed" : "failed"), testCase.testRun.totalDuration];
 147
 148 }
 149 @end
 150
 151
 152
 153
 154 static char* gTestBundleDir = "/AppleInternal/XCTests/com.apple.security/";
 155 static char* gTestBundleName = "TrustTests";
 156
 157 static NSMutableArray* gTestCaseNames = nil;
 158
 159 static const char* opt_str = "d:t:c:h";
 160
 161 static void usage(char*const binName, bool longUsage) {
 162     fprintf(stderr, "Usage: %s [-d <test_dir>] -t test_bundle_name [(-c test_case_name)*]\n", binName);
 163     if (longUsage) {
 164         fprintf(stderr, "-d: argument = path to directory where test bundles live\n");
 165         fprintf(stderr, "-t: argument = name of test bundle to be run (without extension)\n");
 166         fprintf(stderr, "-c: argument = name of test case to be run (multiple)\n");
 167     }
 168 }
 169
 170 static void getOptions(int argc, char *const *argv) {
 171     int ch;
 172     while ( (ch = getopt(argc, argv, opt_str)) != -1 ) {
 173         switch(ch)
 174         {
 175             case 'd':
 176                 gTestBundleDir = optarg;
 177                 break;
 178             case 't':
 179                 gTestBundleName = optarg;
 180                 break;
 181             case 'c':
 182                 if(!gTestCaseNames) {
 183                     gTestCaseNames = [NSMutableArray new];
 184                 }
 185                 [gTestCaseNames addObject:@(optarg)];
 186                 break;
 187             case 'h':
 188             case '?':
 189             default:
 190                 usage(argv[0], true);
 191                 exit(0);
 192                 break;
 193         }
 194     }
 195 }
 196
 197 int main (int argc, const char * argv[])
 198 {
 199     @autoreleasepool {
 200         getOptions(argc, (char*const*)argv);
 201         NSString *testBundleDir = [NSString stringWithCString:gTestBundleDir encoding:NSUTF8StringEncoding];
 202         NSString *testBundleName = [NSString stringWithCString:gTestBundleName encoding:NSUTF8StringEncoding];
 203         NSString *testBundlePath = [[testBundleDir stringByAppendingPathComponent:testBundleName] stringByAppendingPathExtension:@"xctest"];
 204
 205         printf("Running unit tests %s at: %s\n", gTestCaseNames?gTestCaseNames.description.UTF8String:"[All]", testBundlePath.UTF8String);
 206
 207         TestRunner *unitTest = [[TestRunner alloc] initWithBundlePath:testBundlePath andTestNames:gTestCaseNames];
 208         if (!unitTest) {
 209             fprintf(stderr, "Failed to load unit test runner at: %s\n", testBundlePath.UTF8String);
 210             return 1;
 211         }
 212
 213         //runUnitTestSuite returns the number of failures. 0 = success, non-zero means failure. This complies with BATS testing standards.
 214         return (int)[unitTest runUnitTestSuite];
 215
 216         return 0;
 217     }
 218 }
 219