tests/TestHostBinaries/KeychainEntitledTestRunner/KeychainEntitledTestRunner.m

   1 //
   2 //  KeychainEntitledTestRunner.m
   3 //  KeychainEntitledTestRunner
   4 //
   5 //  Stolen from Mark Pauley / CDEntitledTestRunner who stole it from Drew Terry / MobileContainerManager
   6 //  Copyright 2016-2017 Apple. All rights reserved.
   7 //
   8
   9 #import <Foundation/Foundation.h>
  10 #import <unistd.h>
  11 #import <XCTest/XCTest.h>
  12
  13 @interface TestRunner : NSObject <XCTestObservation> {
  14     NSBundle *_bundle;
  15     XCTestSuite *_testSuite;
  16 }
  17
  18 - (instancetype)initWithBundlePath:(NSString *)path andTestNames:(NSArray *)names;
  19 - (NSUInteger)runUnitTestSuite;
  20
  21 - (void)testLogWithFormat:(NSString *)format, ... NS_FORMAT_FUNCTION(1,2);
  22 - (void)testLogWithFormat:(NSString *)format arguments:(va_list)arguments NS_FORMAT_FUNCTION(1,0);
  23
  24 @end
  25
  26 @implementation TestRunner
  27 - (instancetype)initWithBundlePath:(NSString *)path andTestNames:(NSArray *)names
  28 {
  29     if ((self = [super init])) {
  30         NSError *error = nil;
  31
  32         _bundle = [NSBundle bundleWithPath:path];
  33         if (!_bundle) {
  34             [self testLogWithFormat:@"No bundle at location %@ (%s)\n", path, strerror(errno)];
  35             return nil;
  36         }
  37         if (![_bundle loadAndReturnError:&error]) {
  38             [self testLogWithFormat:@"Test Bundle at %@ didn't load: %@\n", path, error];
  39             return nil;
  40         }
  41
  42         if(names) {
  43             XCTestSuite* testSuite = [[XCTestSuite alloc] initWithName:[[path lastPathComponent] stringByDeletingPathExtension]];
  44             XCTestSuite* loadedSuite = [XCTestSuite testSuiteForBundlePath:path];
  45             // Filter out only the tests that were named.
  46             [loadedSuite.tests enumerateObjectsUsingBlock:^(__kindof XCTest * _Nonnull test, NSUInteger __unused idx, BOOL * __unused _Nonnull stop) {
  47                 [self testLogWithFormat:@"Checking test %@\n", test.name];
  48                 if([names containsObject:test.name]) {
  49                     [testSuite addTest:test];
  50                 }
  51             }];
  52             _testSuite = testSuite;
  53         }
  54         else {
  55             _testSuite = [XCTestSuite testSuiteForBundlePath:path];
  56         }
  57     }
  58     return self;
  59 }
  60
  61 - (NSUInteger)runUnitTestSuite
  62 {
  63     [[XCTestObservationCenter sharedTestObservationCenter] addTestObserver:self];
  64
  65     [_testSuite runTest];
  66
  67     XCTestRun *testRun = [_testSuite testRun];
  68
  69     return testRun.totalFailureCount;
  70 }
  71
  72 - (NSFileHandle *)logFileHandle
  73 {
  74     return [NSFileHandle fileHandleWithStandardOutput];
  75 }
  76
  77 - (void)testLogWithFormat:(NSString *)format, ...
  78 {
  79     va_list ap;
  80     va_start(ap, format);
  81     [self testLogWithFormat:format arguments:ap];
  82     va_end(ap);
  83 }
  84
  85 - (void)testLogWithFormat:(NSString *)format arguments:(va_list)arguments
  86 {
  87     NSString *message = [[NSString alloc] initWithFormat:format arguments:arguments];
  88     [self.logFileHandle writeData:[message dataUsingEncoding:NSUTF8StringEncoding]];
  89 }
  90
  91 - (NSDateFormatter *)dateFormatter
  92 {
  93     static NSDateFormatter *dateFormatter;
  94     static dispatch_once_t onceToken;
  95     dispatch_once(&onceToken, ^{
  96         dateFormatter = [NSDateFormatter new];
  97         dateFormatter.dateFormat = @"yyyy-MM-dd HH:mm:ss.SSS";
  98     });
  99     return dateFormatter;
 100 }
 101
 102 /* -testBundleWillStart:                            // exactly once per test bundle
 103  *      -testSuiteWillStart:                        // exactly once per test suite
 104  *          -testCaseWillStart:                     // exactly once per test case
 105  *          -testCase:didFailWithDescription:...    // zero or more times per test case, any time between test case start and finish
 106  *          -testCaseDidFinish:                     // exactly once per test case
 107  *      -testSuite:didFailWithDescription:...       // zero or more times per test suite, any time between test suite start and finish
 108  *      -testSuiteDidFinish:                        // exactly once per test suite
 109  * -testBundleDidFinish:                            // exactly once per test bundle
 110  */
 111
 112 - (void)testSuiteWillStart:(XCTestSuite *)testSuite
 113 {
 114     [self testLogWithFormat:@"Test Suite '%@' started at %@\n", testSuite.name, [self.dateFormatter stringFromDate:testSuite.testRun.startDate]];
 115 }
 116
 117 - (void)testSuite:(XCTestSuite *)testSuite didRecordIssue:(XCTIssue *)issue {
 118         [self testLogWithFormat:@"(%@)%@:%lu: error: %@", testSuite.name, issue.sourceCodeContext.location.fileURL,
 119                                                                                                            issue.sourceCodeContext.location.lineNumber, issue.compactDescription];
 120 }
 121
 122 - (void)testSuiteDidFinish:(XCTestSuite *)testSuite
 123 {
 124     XCTestRun *testRun = testSuite.testRun;
 125     [self testLogWithFormat:@"Test Suite '%@' %s at %@.\n\t Executed %lu test%s, with %lu failure%s (%lu unexpected) in %.3f (%.3f) seconds\n",
 126      testSuite.name,
 127      (testRun.hasSucceeded ? "passed" : "failed"),
 128      [self.dateFormatter stringFromDate:testRun.stopDate],
 129      ((unsigned long)testRun.executionCount), (testRun.executionCount != 1 ? "s" : ""),
 130      ((unsigned long)testRun.totalFailureCount), (testRun.totalFailureCount != 1 ? "s" : ""),
 131      ((unsigned long)testRun.unexpectedExceptionCount),
 132      testRun.testDuration,
 133      testRun.totalDuration];
 134 }
 135
 136 - (void)testCaseWillStart:(XCTestCase *)testCase
 137 {
 138     [self testLogWithFormat:@"Test Case '%@' started.\n", testCase.name];
 139 }
 140
 141 - (void)testCase:(XCTestCase *)testCase didRecordIssue:(XCTIssue *)issue {
 142         [self testLogWithFormat:@"(%@)%@:%lu error: %@\n%@", testCase.name, issue.sourceCodeContext.location.fileURL, issue.sourceCodeContext.location.lineNumber, issue.detailedDescription, issue.sourceCodeContext.callStack];
 143 }
 144
 145 - (void)testCaseDidFinish:(XCTestCase *)testCase
 146 {
 147     [self testLogWithFormat:@"Test Case '%@' %s (%.3f seconds).\n", testCase.name, (testCase.testRun.hasSucceeded ? "passed" : "failed"), testCase.testRun.totalDuration];
 148
 149 }
 150 @end
 151
 152
 153
 154
 155 static char* gTestBundleDir = "/AppleInternal/XCTests/com.apple.security";
 156 static char* gTestBundleName = "CKKSCloudKitTests";
 157
 158 static NSMutableArray* gTestCaseNames = nil;
 159
 160 static const char* opt_str = "d:t:c:h";
 161
 162 static void usage(char*const binName, bool longUsage) {
 163     fprintf(stderr, "Usage: %s [-d <test_dir>] -t test_bundle_name [(-c test_case_name)*]\n", binName);
 164     if (longUsage) {
 165         fprintf(stderr, "-d: argument = path to directory where test bundles live\n");
 166         fprintf(stderr, "-t: argument = name of test bundle to be run (without extension)\n");
 167         fprintf(stderr, "-c: argument = name of test case to be run (multiple)\n");
 168     }
 169 }
 170
 171 static void getOptions(int argc, char *const *argv) {
 172     int ch;
 173     while ( (ch = getopt(argc, argv, opt_str)) != -1 ) {
 174         switch(ch)
 175         {
 176             case 'd':
 177                 gTestBundleDir = optarg;
 178                 break;
 179             case 't':
 180                 gTestBundleName = optarg;
 181                 break;
 182             case 'c':
 183                 if(!gTestCaseNames) {
 184                     gTestCaseNames = [NSMutableArray new];
 185                 }
 186                 [gTestCaseNames addObject:@(optarg)];
 187                 break;
 188             case 'h':
 189             case '?':
 190             default:
 191                 usage(argv[0], true);
 192                 exit(0);
 193                 break;
 194         }
 195     }
 196 }
 197
 198 int main (int argc, const char * argv[])
 199 {
 200     @autoreleasepool {
 201         getOptions(argc, (char*const*)argv);
 202         NSString *testBundleDir = [NSString stringWithCString:gTestBundleDir encoding:NSUTF8StringEncoding];
 203         NSString *testBundleName = [NSString stringWithCString:gTestBundleName encoding:NSUTF8StringEncoding];
 204         NSString *testBundlePath = [[testBundleDir stringByAppendingPathComponent:testBundleName] stringByAppendingPathExtension:@"xctest"];
 205
 206         printf("Running unit tests %s at: %s\n", gTestCaseNames?gTestCaseNames.description.UTF8String:"[All]", testBundlePath.UTF8String);
 207
 208         TestRunner *unitTest = [[TestRunner alloc] initWithBundlePath:testBundlePath andTestNames:gTestCaseNames];
 209         if (!unitTest) {
 210             fprintf(stderr, "Failed to load unit test runner at: %s\n", testBundlePath.UTF8String);
 211             return 1;
 212         }
 213
 214         //runUnitTestSuite returns the number of failures. 0 = success, non-zero means failure. This complies with BATS testing standards.
 215         return (int)[unitTest runUnitTestSuite];
 216
 217         return 0;
 218     }
 219 }