keychain/ckks/tests/testrunner/KeychainEntitledTestRunner.m

   1 //
   2 //  KeychainEntitledTestRunner.m
   3 //  KeychainEntitledTestRunner
   4 //
   5 //  Stolen from Mark Pauley / CDEntitledTestRunner who stole it from Drew Terry / MobileContainerManager
   6 //  Copyright 2016-2017 Apple. All rights reserved.
   7 //
   8
   9 #import <Foundation/Foundation.h>
  10 #import <unistd.h>
  11 #import <XCTest/XCTest.h>
  12
  13 @interface TestRunner : NSObject <XCTestObservation> {
  14     NSBundle *_bundle;
  15     XCTestSuite *_testSuite;
  16 }
  17
  18 - (instancetype)initWithBundlePath:(NSString *)path andTestNames:(NSArray *)names;
  19 - (NSUInteger)runUnitTestSuite;
  20
  21 - (void)testLogWithFormat:(NSString *)format, ... NS_FORMAT_FUNCTION(1,2);
  22 - (void)testLogWithFormat:(NSString *)format arguments:(va_list)arguments NS_FORMAT_FUNCTION(1,0);
  23
  24 @end
  25
  26 @implementation TestRunner
  27 - (instancetype)initWithBundlePath:(NSString *)path andTestNames:(NSArray *)names
  28 {
  29     self = [super init];
  30     if (self) {
  31         NSError *error = nil;
  32
  33         _bundle = [NSBundle bundleWithPath:path];
  34         if (!_bundle) {
  35             [self testLogWithFormat:@"No bundle at location %@ (%s)\n", path, strerror(errno)];
  36             return nil;
  37         }
  38         if (![_bundle loadAndReturnError:&error]) {
  39             [self testLogWithFormat:@"Test Bundle at %@ didn't load: %@\n", path, error];
  40             return nil;
  41         }
  42
  43         if(names) {
  44             XCTestSuite* testSuite = [[XCTestSuite alloc] initWithName:[[path lastPathComponent] stringByDeletingPathExtension]];
  45             XCTestSuite* loadedSuite = [XCTestSuite testSuiteForBundlePath:path];
  46             // Filter out only the tests that were named.
  47             [loadedSuite.tests enumerateObjectsUsingBlock:^(__kindof XCTest * _Nonnull test, NSUInteger __unused idx, BOOL * __unused _Nonnull stop) {
  48                 [self testLogWithFormat:@"Checking test %@\n", test.name];
  49                 if([names containsObject:test.name]) {
  50                     [testSuite addTest:test];
  51                 }
  52             }];
  53             _testSuite = testSuite;
  54         }
  55         else {
  56             _testSuite = [XCTestSuite testSuiteForBundlePath:path];
  57         }
  58     }
  59     return self;
  60 }
  61
  62 - (NSUInteger)runUnitTestSuite
  63 {
  64     [[XCTestObservationCenter sharedTestObservationCenter] addTestObserver:self];
  65
  66     [_testSuite runTest];
  67
  68     XCTestRun *testRun = [_testSuite testRun];
  69
  70     return testRun.totalFailureCount;
  71 }
  72
  73 - (NSFileHandle *)logFileHandle
  74 {
  75     return [NSFileHandle fileHandleWithStandardOutput];
  76 }
  77
  78 - (void)testLogWithFormat:(NSString *)format, ...
  79 {
  80     va_list ap;
  81     va_start(ap, format);
  82     [self testLogWithFormat:format arguments:ap];
  83     va_end(ap);
  84 }
  85
  86 - (void)testLogWithFormat:(NSString *)format arguments:(va_list)arguments
  87 {
  88     NSString *message = [[NSString alloc] initWithFormat:format arguments:arguments];
  89     [self.logFileHandle writeData:[message dataUsingEncoding:NSUTF8StringEncoding]];
  90 }
  91
  92 - (NSDateFormatter *)dateFormatter
  93 {
  94     static NSDateFormatter *dateFormatter;
  95     static dispatch_once_t onceToken;
  96     dispatch_once(&onceToken, ^{
  97         dateFormatter = [NSDateFormatter new];
  98         dateFormatter.dateFormat = @"yyyy-MM-dd HH:mm:ss.SSS";
  99     });
 100     return dateFormatter;
 101 }
 102
 103 /* -testBundleWillStart:                            // exactly once per test bundle
 104  *      -testSuiteWillStart:                        // exactly once per test suite
 105  *          -testCaseWillStart:                     // exactly once per test case
 106  *          -testCase:didFailWithDescription:...    // zero or more times per test case, any time between test case start and finish
 107  *          -testCaseDidFinish:                     // exactly once per test case
 108  *      -testSuite:didFailWithDescription:...       // zero or more times per test suite, any time between test suite start and finish
 109  *      -testSuiteDidFinish:                        // exactly once per test suite
 110  * -testBundleDidFinish:                            // exactly once per test bundle
 111  */
 112
 113 - (void)testSuiteWillStart:(XCTestSuite *)testSuite
 114 {
 115     [self testLogWithFormat:@"Test Suite '%@' started at %@\n", testSuite.name, [self.dateFormatter stringFromDate:testSuite.testRun.startDate]];
 116 }
 117
 118 - (void)testSuite:(XCTestSuite *)testSuite didFailWithDescription:(NSString *)description inFile:(nullable NSString *)filePath atLine:(NSUInteger)lineNumber
 119 {
 120     [self testLogWithFormat:@"%@:%lu: error: %@ : %@\n", ((nil != filePath) ? filePath : @"<unknown>"), ((unsigned long)((nil != filePath) ? lineNumber : 0)), testSuite.name, description];
 121 }
 122
 123 - (void)testSuiteDidFinish:(XCTestSuite *)testSuite
 124 {
 125     XCTestRun *testRun = testSuite.testRun;
 126     [self testLogWithFormat:@"Test Suite '%@' %s at %@.\n\t Executed %lu test%s, with %lu failure%s (%lu unexpected) in %.3f (%.3f) seconds\n",
 127      testSuite.name,
 128      (testRun.hasSucceeded ? "passed" : "failed"),
 129      [self.dateFormatter stringFromDate:testRun.stopDate],
 130      ((unsigned long)testRun.executionCount), (testRun.executionCount != 1 ? "s" : ""),
 131      ((unsigned long)testRun.totalFailureCount), (testRun.totalFailureCount != 1 ? "s" : ""),
 132      ((unsigned long)testRun.unexpectedExceptionCount),
 133      testRun.testDuration,
 134      testRun.totalDuration];
 135 }
 136
 137 - (void)testCaseWillStart:(XCTestCase *)testCase
 138 {
 139     [self testLogWithFormat:@"Test Case '%@' started.\n", testCase.name];
 140 }
 141
 142 - (void)testCase:(XCTestCase *)testCase didFailWithDescription:(NSString *)description inFile:(nullable NSString *)filePath atLine:(NSUInteger)lineNumber
 143 {
 144     [self testLogWithFormat:@"%@:%lu: error: %@ : %@\n", ((nil != filePath) ? filePath : @"<unknown>"), ((unsigned long)((nil != filePath) ? lineNumber : 0)), testCase.name, description];
 145 }
 146
 147 - (void)testCaseDidFinish:(XCTestCase *)testCase
 148 {
 149     [self testLogWithFormat:@"Test Case '%@' %s (%.3f seconds).\n", testCase.name, (testCase.testRun.hasSucceeded ? "passed" : "failed"), testCase.testRun.totalDuration];
 150
 151 }
 152 @end
 153
 154
 155
 156
 157 static char* gTestBundleDir = "/AppleInternal/XCTests/com.apple.security";
 158 static char* gTestBundleName = "CKKSCloudKitTests";
 159
 160 static NSMutableArray* gTestCaseNames = nil;
 161
 162 static const char* opt_str = "d:t:c:h";
 163
 164 static void usage(char*const binName, bool longUsage) {
 165     fprintf(stderr, "Usage: %s [-d <test_dir>] -t test_bundle_name [(-c test_case_name)*]\n", binName);
 166     if (longUsage) {
 167         fprintf(stderr, "-d: argument = path to directory where test bundles live\n");
 168         fprintf(stderr, "-t: argument = name of test bundle to be run (without extension)\n");
 169         fprintf(stderr, "-c: argument = name of test case to be run (multiple)\n");
 170     }
 171 }
 172
 173 static void getOptions(int argc, char *const *argv) {
 174     int ch;
 175     while ( (ch = getopt(argc, argv, opt_str)) != -1 ) {
 176         switch(ch)
 177         {
 178             case 'd':
 179                 gTestBundleDir = optarg;
 180                 break;
 181             case 't':
 182                 gTestBundleName = optarg;
 183                 break;
 184             case 'c':
 185                 if(!gTestCaseNames) {
 186                     gTestCaseNames = [NSMutableArray new];
 187                 }
 188                 [gTestCaseNames addObject:@(optarg)];
 189                 break;
 190             case 'h':
 191             case '?':
 192             default:
 193                 usage(argv[0], true);
 194                 exit(0);
 195                 break;
 196         }
 197     }
 198 }
 199
 200 int main (int argc, const char * argv[])
 201 {
 202     @autoreleasepool {
 203         getOptions(argc, (char*const*)argv);
 204         NSString *testBundleDir = [NSString stringWithCString:gTestBundleDir encoding:NSUTF8StringEncoding];
 205         NSString *testBundleName = [NSString stringWithCString:gTestBundleName encoding:NSUTF8StringEncoding];
 206         NSString *testBundlePath = [[testBundleDir stringByAppendingPathComponent:testBundleName] stringByAppendingPathExtension:@"xctest"];
 207
 208         printf("Running unit tests %s at: %s\n", gTestCaseNames?gTestCaseNames.description.UTF8String:"[All]", testBundlePath.UTF8String);
 209
 210         TestRunner *unitTest = [[TestRunner alloc] initWithBundlePath:testBundlePath andTestNames:gTestCaseNames];
 211         if (!unitTest) {
 212             fprintf(stderr, "Failed to load unit test runner at: %s\n", testBundlePath.UTF8String);
 213             return 1;
 214         }
 215
 216         //runUnitTestSuite returns the number of failures. 0 = success, non-zero means failure. This complies with BATS testing standards.
 217         return (int)[unitTest runUnitTestSuite];
 218
 219         return 0;
 220     }
 221 }