local_test_runner/ContainerizedTestRunner.mm

   1 /*
   2 * Copyright (c) 2019 Apple Inc. All rights reserved.
   3 *
   4 * @APPLE_LICENSE_HEADER_START@
   5 *
   6 * "Portions Copyright (c) 1999 Apple Computer, Inc.  All Rights
   7 * Reserved.  This file contains Original Code and/or Modifications of
   8 * Original Code as defined in and that are subject to the Apple Public
   9 * Source License Version 1.0 (the 'License').  You may not use this file
  10 * except in compliance with the License.  Please obtain a copy of the
  11 * License at http://www.apple.com/publicsource and read it before using
  12 * this file.
  13 *
  14 * The Original Code and all software distributed under the License are
  15 * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  16 * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  17 * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
  18 * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
  19 * License for the specific language governing rights and limitations
  20 * under the License."
  21 *
  22 * @APPLE_LICENSE_HEADER_END@
  23 */
  24
  25 #import <XCTest/XCTest.h>
  26 #include <CommonCrypto/CommonDigest.h>
  27 #include <objc/runtime.h>
  28
  29 #include "test_support.h"
  30
  31 struct TestInfo
  32 {
  33     const char*         testName;
  34     const char*         runLine;
  35 };
  36
  37 #include "XCTestGenerated.h"
  38
  39 @interface ContainerizedTestRunner : XCTestCase
  40 - (void)launchTest:(const char *)test withRunLine:(const char *)runLine;
  41 @end
  42
  43 @implementation ContainerizedTestRunner
  44 #if 1
  45 + (void)registerTest:(const TestInfo &)info withEnv:(const char *)env andExtensions:(const char *)extension {
  46     char *fullRunline = NULL;
  47     asprintf(&fullRunline, "TMPDIR=/tmp/ TEST_OUTPUT=XCTest  %s %s", env, info.runLine);
  48     unsigned char hash[CC_SHA1_DIGEST_LENGTH];
  49     char hashStr[CC_SHA1_DIGEST_LENGTH*2+1];
  50     CC_SHA1(fullRunline, (CC_LONG)strlen(fullRunline), &hash[0]);
  51     snprintf(&hashStr[0], CC_SHA1_DIGEST_LENGTH*2+1, "%x%x%x%x", hash[0], hash[1], hash[2], hash[3]);
  52     char buffer[4096];
  53     snprintf(&buffer[0], 4096, "test_%s_%s_%s", info.testName, extension, hashStr);
  54     SEL newSel = sel_registerName(buffer);
  55     IMP newIMP = imp_implementationWithBlock(^(id self) {
  56         [self launchTest:info.testName withRunLine:fullRunline];
  57     });
  58     class_addMethod([self class], newSel, newIMP, "v@:");
  59 }
  60
  61 + (void)load {
  62     for (const TestInfo& info : sTests) {
  63         if ( strstr(info.runLine, "run-static") != nullptr ) {
  64             [self registerTest:info withEnv:"" andExtensions:"static"];
  65         } else {
  66             [self registerTest:info withEnv:"DYLD_USE_CLOSURES=0" andExtensions:"dyld2"];
  67             [self registerTest:info withEnv:"DYLD_USE_CLOSURES=1" andExtensions:"dyld3"];
  68         }
  69 #if 0
  70         [self registerTest:info withEnv:"DYLD_USE_CLOSURES=0 MallocStackLogging=1 MallocDebugReport=none" andExtensions:"dyld2_leaks"];
  71         [self registerTest:info withEnv:"DYLD_USE_CLOSURES=1 MallocStackLogging=1 MallocDebugReport=none" andExtensions:"dyld3_leaks"];
  72 #endif
  73     };
  74 }
  75
  76 #else
  77 // This would be the way to do it if XCTest did not insist on using the selector name for differentiating results in the Xcode UI
  78 + (NSArray<NSInvocation *> *)testInvocations {
  79     static NSArray<NSInvocation *> *invocations = nil;
  80     static dispatch_once_t onceToken;
  81     dispatch_once(&onceToken, ^{
  82         NSMutableArray<NSInvocation *> *invocationArray = [[NSMutableArray alloc] init];
  83         for (const TestInfo& info : sTests) {
  84             NSInvocation *invocation = [NSInvocation invocationWithMethodSignature:[[ContainerizedTestRunner class]
  85                                                 instanceMethodSignatureForSelector:@selector(launchTest:withRunLine:)]];
  86             invocation.selector = @selector(launchTest:withRunLine:);
  87             [invocation setArgument:(void*)&info.testName atIndex:2];
  88             [invocation setArgument:(void*)&info.runLine atIndex:3];
  89             [invocationArray addObject:invocation];
  90         }
  91         invocations = [NSArray arrayWithArray:invocationArray];
  92     });
  93     return [invocations copy];
  94 }
  95
  96 - (NSString *) name {
  97     const char *testName = NULL;
  98     const char *runLine = NULL;
  99     [self.invocation getArgument:(void*)&testName atIndex:2];
 100     [self.invocation getArgument:(void*)&runLine atIndex:3];
 101     return [NSString stringWithFormat:@"%s: %s", testName, runLine];
 102 }
 103
 104 - (NSString *)nameForLegacyLogging
 105 {
 106     return self.name;
 107 }
 108 #endif
 109
 110 - (void)setUp {
 111     static dispatch_once_t onceToken;
 112     dispatch_once(&onceToken, ^{
 113         _process process;
 114         fprintf(stderr, "CHROOT: %s\n", CHROOT_PATH);
 115         process.set_executable_path("/usr/sbin/chroot");
 116         const char *args[] = {CHROOT_PATH, "/bin/sh", "-c", "/sbin/mount -t devfs devfs /dev", NULL};
 117         process.set_args(args);
 118         process.launch();
 119     });
 120 }
 121
 122 - (void)tearDown {
 123     // Put teardown code here. This method is called after the invocation of each test method in the class.
 124 }
 125
 126 - (void) executeCommandInContainer:(const char *)command {
 127     _process process;
 128     process.set_executable_path("/usr/sbin/chroot");
 129     const char *args[] = {CHROOT_PATH, "/bin/sh", "-c", command, NULL};
 130     process.set_args(args);
 131     __block dispatch_data_t output = NULL;
 132     process.set_stdout_handler(^(int fd) {
 133         ssize_t size = 0;
 134         do {
 135             char buffer[16384];
 136             size = read(fd, &buffer[0], 16384);
 137             if (size == -1) { break; }
 138             dispatch_data_t data = dispatch_data_create(&buffer[0], size, NULL, DISPATCH_DATA_DESTRUCTOR_DEFAULT);
 139             if (!output) {
 140                 output = data;
 141             } else {
 142                 output = dispatch_data_create_concat(output, data);
 143             }
 144         } while (size > 0);
 145     });
 146     process.set_exit_handler(^(pid_t pid) {
 147         int status = 0;
 148         (void)waitpid(pid, &status, 0);
 149
 150         int exitStatus = WEXITSTATUS(status);
 151         if (exitStatus != 0) {
 152             NSString *failure = [NSString stringWithFormat:@"Test exited with return code %d\n%s", exitStatus, command];
 153             XCTSourceCodeContext *context = [[XCTSourceCodeContext alloc] initWithLocation:[[XCTSourceCodeLocation alloc] initWithFilePath:@__FILE__ lineNumber:__LINE__]];
 154             XCTIssue *issue = [[XCTIssue alloc] initWithType:XCTIssueTypeUncaughtException compactDescription:failure detailedDescription:NULL sourceCodeContext:context associatedError:NULL attachments:[[NSArray alloc] init]];
 155             [self recordIssue: issue];
 156             return;
 157         }
 158         if (!output) {
 159             NSString *failure = [NSString stringWithFormat:@"Test did not write any data to stdout\n%s", command];
 160             XCTSourceCodeContext *context = [[XCTSourceCodeContext alloc] initWithLocation:[[XCTSourceCodeLocation alloc] initWithFilePath:@__FILE__ lineNumber:__LINE__]];
 161             XCTIssue *issue = [[XCTIssue alloc] initWithType:XCTIssueTypeUncaughtException compactDescription:failure detailedDescription:NULL sourceCodeContext:context associatedError:NULL attachments:[[NSArray alloc] init]];
 162             [self recordIssue: issue];
 163             return;
 164         }
 165         NSError *error = nil;
 166         NSDictionary *dict = [NSPropertyListSerialization propertyListWithData:(NSData *)output options:NSPropertyListImmutable format:nil error:&error];
 167         if (!dict) {
 168             NSString *failure = [NSString stringWithFormat:@"Could not convert stdout \"%@\" to property list. Got Error %@\n%s", output, error, command];
 169             XCTSourceCodeContext *context = [[XCTSourceCodeContext alloc] initWithLocation:[[XCTSourceCodeLocation alloc] initWithFilePath:@__FILE__ lineNumber:__LINE__]];
 170             XCTIssue *issue = [[XCTIssue alloc] initWithType:XCTIssueTypeUncaughtException compactDescription:failure detailedDescription:NULL sourceCodeContext:context associatedError:NULL attachments:[[NSArray alloc] init]];
 171             [self recordIssue: issue];
 172             return;
 173         }
 174
 175         if (dict[@"LOGS"]) {
 176             NSLog(@"LOGS:\n%@",[NSString stringWithFormat:@"%@\n", dict[@"LOGS"]]);
 177         }
 178
 179         if (![dict[@"PASS"] boolValue]) {
 180             NSString *failure = [NSString stringWithFormat:@"%@\n%s", dict[@"INFO"], command];
 181             XCTSourceCodeContext *context = [[XCTSourceCodeContext alloc] initWithLocation:[[XCTSourceCodeLocation alloc] initWithFilePath:dict[@"FILE"] lineNumber:(NSInteger)dict[@"LINE"]]];
 182             XCTIssue *issue = [[XCTIssue alloc] initWithType:XCTIssueTypeUncaughtException compactDescription:failure detailedDescription:NULL sourceCodeContext:context associatedError:NULL attachments:[[NSArray alloc] init]];
 183             [self recordIssue: issue];
 184             return;
 185         }
 186     });
 187     process.launch();
 188 }
 189
 190
 191 - (void) launchTest:(const char *)test withRunLine:(const char *)runLine {
 192     char command[4096];
 193     snprintf(&command[0], 4096, "cd /AppleInternal/CoreOS/tests/dyld/%s; %s", test, runLine);
 194     [self executeCommandInContainer:command];
 195 // sudo chroot . /bin/sh -c 'TEST_OUTPUT=BATS /AppleInternal/CoreOS/tests/dyld/dyld_get_sdk_version/sdk-check.exe'
 196
 197 }
 198
 199 //
 200 //- (void)testExample {
 201 //    // This is an example of a functional test case.
 202 //    // Use XCTAssert and related functions to verify your tests produce the correct results.
 203 //}
 204 //
 205 //- (void)testPerformanceExample {
 206 //    // This is an example of a performance test case.
 207 //    [self measureBlock:^{
 208 //        // Put the code you want to measure the time of here.
 209 //    }];
 210 //}
 211
 212 @end