tests/TrustTests/TestRunners/main.m

   1 //
   2 //  TrustTests
   3 //
   4 //  Stolen from Mark Pauley / CDEntitledTestRunner who stole it from Drew Terry / MobileContainerManager
   5 //  Copyright 2016-2018 Apple. All rights reserved.
   6 //
   7
   8 #import <Foundation/Foundation.h>
   9 #import <unistd.h>
  10 #import <XCTest/XCTest.h>
  11
  12 @interface TestRunner : NSObject <XCTestObservation> {
  13     NSBundle *_bundle;
  14     XCTestSuite *_testSuite;
  15 }
  16
  17 - (instancetype)initWithBundlePath:(NSString *)path andTestNames:(NSArray *)names;
  18 - (NSUInteger)runUnitTestSuite;
  19 - (void)testLogWithFormat:(NSString *)format, ... NS_FORMAT_FUNCTION(1, 2);
  20 - (void)testLogWithFormat:(NSString *)format arguments:(va_list)arguments NS_FORMAT_FUNCTION(1,0);
  21 @end
  22
  23 @implementation TestRunner
  24 - (instancetype)initWithBundlePath:(NSString *)path andTestNames:(NSArray *)names
  25 {
  26     if ((self = [super init])) {
  27         NSError *error = nil;
  28
  29         _bundle = [NSBundle bundleWithPath:path];
  30         if (!_bundle) {
  31             [self testLogWithFormat:@"No bundle at location %@ (%s)\n", path, strerror(errno)];
  32             return nil;
  33         }
  34         if (![_bundle loadAndReturnError:&error]) {
  35             [self testLogWithFormat:@"Test Bundle at %@ didn't load: %@\n", path, error];
  36             return nil;
  37         }
  38
  39         if(names) {
  40             XCTestSuite* testSuite = [[XCTestSuite alloc] initWithName:[[path lastPathComponent] stringByDeletingPathExtension]];
  41             XCTestSuite* loadedSuite = [XCTestSuite testSuiteForBundlePath:path];
  42             // Filter out only the tests that were named.
  43             [loadedSuite.tests enumerateObjectsUsingBlock:^(__kindof XCTest * _Nonnull test, NSUInteger __unused idx, BOOL * __unused _Nonnull stop) {
  44                 [self testLogWithFormat:@"Checking test %@\n", test.name];
  45                 if([names containsObject:test.name]) {
  46                     [testSuite addTest:test];
  47                 }
  48             }];
  49             _testSuite = testSuite;
  50         }
  51         else {
  52             _testSuite = [XCTestSuite testSuiteForBundlePath:path];
  53         }
  54     }
  55     return self;
  56 }
  57
  58 - (NSUInteger)runUnitTestSuite
  59 {
  60     [[XCTestObservationCenter sharedTestObservationCenter] addTestObserver:self];
  61
  62     [_testSuite runTest];
  63
  64     XCTestRun *testRun = [_testSuite testRun];
  65
  66     return testRun.totalFailureCount;
  67 }
  68
  69 - (NSFileHandle *)logFileHandle
  70 {
  71     return [NSFileHandle fileHandleWithStandardOutput];
  72 }
  73
  74 - (void)testLogWithFormat:(NSString *)format, ...
  75 {
  76     va_list ap;
  77     va_start(ap, format);
  78     [self testLogWithFormat:format arguments:ap];
  79     va_end(ap);
  80 }
  81
  82 - (void)testLogWithFormat:(NSString *)format arguments:(va_list)arguments
  83 {
  84     NSString *message = [[NSString alloc] initWithFormat:format arguments:arguments];
  85     [self.logFileHandle writeData:[message dataUsingEncoding:NSUTF8StringEncoding]];
  86 }
  87
  88 - (NSDateFormatter *)dateFormatter
  89 {
  90     static NSDateFormatter *dateFormatter;
  91     static dispatch_once_t onceToken;
  92     dispatch_once(&onceToken, ^{
  93         dateFormatter = [NSDateFormatter new];
  94         dateFormatter.dateFormat = @"yyyy-MM-dd HH:mm:ss.SSS";
  95     });
  96     return dateFormatter;
  97 }
  98
  99 /* -testBundleWillStart:                            // exactly once per test bundle
 100  *      -testSuiteWillStart:                        // exactly once per test suite
 101  *          -testCaseWillStart:                     // exactly once per test case
 102  *          -testCase:didFailWithDescription:...    // zero or more times per test case, any time between test case start and finish
 103  *          -testCaseDidFinish:                     // exactly once per test case
 104  *      -testSuite:didFailWithDescription:...       // zero or more times per test suite, any time between test suite start and finish
 105  *      -testSuiteDidFinish:                        // exactly once per test suite
 106  * -testBundleDidFinish:                            // exactly once per test bundle
 107  */
 108
 109 - (void)testSuiteWillStart:(XCTestSuite *)testSuite
 110 {
 111     [self testLogWithFormat:@"Test Suite '%@' started at %@\n", testSuite.name, [self.dateFormatter stringFromDate:testSuite.testRun.startDate]];
 112 }
 113
 114 - (void)testSuite:(XCTestSuite *)testSuite didRecordIssue:(XCTIssue *)issue
 115 {
 116     NSString *filePath = [issue.sourceCodeContext.location.fileURL absoluteString];
 117     NSInteger lineNumber = issue.sourceCodeContext.location.lineNumber;
 118     NSString *description = issue.description;
 119     [self testLogWithFormat:@"%@:%lu: error: %@ : %@\n", ((nil != filePath) ? filePath : @"<unknown>"), ((unsigned long)((nil != filePath) ? lineNumber : 0)), testSuite.name, description];
 120 }
 121
 122 - (void)testSuiteDidFinish:(XCTestSuite *)testSuite
 123 {
 124     XCTestRun *testRun = testSuite.testRun;
 125     [self testLogWithFormat:@"Test Suite '%@' %s at %@.\n\t Executed %lu test%s, with %lu failure%s (%lu unexpected) in %.3f (%.3f) seconds\n",
 126      testSuite.name,
 127      (testRun.hasSucceeded ? "passed" : "failed"),
 128      [self.dateFormatter stringFromDate:testRun.stopDate],
 129      ((unsigned long)testRun.executionCount), (testRun.executionCount != 1 ? "s" : ""),
 130      ((unsigned long)testRun.totalFailureCount), (testRun.totalFailureCount != 1 ? "s" : ""),
 131      ((unsigned long)testRun.unexpectedExceptionCount),
 132      testRun.testDuration,
 133      testRun.totalDuration];
 134 }
 135
 136 - (void)testCaseWillStart:(XCTestCase *)testCase
 137 {
 138     [self testLogWithFormat:@"Test Case '%@' started.\n", testCase.name];
 139 }
 140
 141 - (void)testCase:(XCTestCase *)testCase didRecordIssue:(XCTIssue *)issue
 142 {
 143     NSString *filePath = [issue.sourceCodeContext.location.fileURL absoluteString];
 144     NSInteger lineNumber = issue.sourceCodeContext.location.lineNumber;
 145     NSString *description = issue.description;
 146     [self testLogWithFormat:@"%@:%lu: error: %@ : %@\n", ((nil != filePath) ? filePath : @"<unknown>"), ((unsigned long)((nil != filePath) ? lineNumber : 0)), testCase.name, description];
 147 }
 148
 149 - (void)testCaseDidFinish:(XCTestCase *)testCase
 150 {
 151     [self testLogWithFormat:@"Test Case '%@' %s (%.3f seconds).\n", testCase.name, (testCase.testRun.hasSucceeded ? "passed" : "failed"), testCase.testRun.totalDuration];
 152
 153 }
 154 @end
 155
 156
 157
 158
 159 static char* gTestBundleDir = "/AppleInternal/XCTests/com.apple.security/";
 160 static char* gTestBundleName = "TrustTests";
 161
 162 static NSMutableArray* gTestCaseNames = nil;
 163
 164 static const char* opt_str = "d:t:c:h";
 165
 166 static void usage(char*const binName, bool longUsage) {
 167     fprintf(stderr, "Usage: %s [-d <test_dir>] -t test_bundle_name [(-c test_case_name)*]\n", binName);
 168     if (longUsage) {
 169         fprintf(stderr, "-d: argument = path to directory where test bundles live\n");
 170         fprintf(stderr, "-t: argument = name of test bundle to be run (without extension)\n");
 171         fprintf(stderr, "-c: argument = name of test case to be run (multiple)\n");
 172     }
 173 }
 174
 175 static void getOptions(int argc, char *const *argv) {
 176     int ch;
 177     while ( (ch = getopt(argc, argv, opt_str)) != -1 ) {
 178         switch(ch)
 179         {
 180             case 'd':
 181                 gTestBundleDir = optarg;
 182                 break;
 183             case 't':
 184                 gTestBundleName = optarg;
 185                 break;
 186             case 'c':
 187                 if(!gTestCaseNames) {
 188                     gTestCaseNames = [NSMutableArray new];
 189                 }
 190                 [gTestCaseNames addObject:@(optarg)];
 191                 break;
 192             case 'h':
 193             case '?':
 194             default:
 195                 usage(argv[0], true);
 196                 exit(0);
 197                 break;
 198         }
 199     }
 200 }
 201
 202 int main (int argc, const char * argv[])
 203 {
 204     @autoreleasepool {
 205         getOptions(argc, (char*const*)argv);
 206         NSString *testBundleDir = [NSString stringWithCString:gTestBundleDir encoding:NSUTF8StringEncoding];
 207         NSString *testBundleName = [NSString stringWithCString:gTestBundleName encoding:NSUTF8StringEncoding];
 208         NSString *testBundlePath = [[testBundleDir stringByAppendingPathComponent:testBundleName] stringByAppendingPathExtension:@"xctest"];
 209
 210         printf("Running unit tests %s at: %s\n", gTestCaseNames?gTestCaseNames.description.UTF8String:"[All]", testBundlePath.UTF8String);
 211
 212         TestRunner *unitTest = [[TestRunner alloc] initWithBundlePath:testBundlePath andTestNames:gTestCaseNames];
 213         if (!unitTest) {
 214             fprintf(stderr, "Failed to load unit test runner at: %s\n", testBundlePath.UTF8String);
 215             return 1;
 216         }
 217
 218         //runUnitTestSuite returns the number of failures. 0 = success, non-zero means failure. This complies with BATS testing standards.
 219         return (int)[unitTest runUnitTestSuite];
 220
 221         return 0;
 222     }
 223 }
 224