experiment/test/SecExperimentTests.m

   1 //
   2 //  SecExperimentTests.m
   3 //
   4 //
   5
   6 #import <XCTest/XCTest.h>
   7 #import <OCMock/OCMock.h>
   8 #import <CoreFoundation/CFXPCBridge.h>
   9 #import <nw/private.h>
  10 #import <Security/SecProtocolPriv.h>
  11 #import "SecExperimentInternal.h"
  12 #import "SecExperimentPriv.h"
  13
  14 @interface SecExperimentTests : XCTestCase
  15 @end
  16
  17 @implementation SecExperimentTests
  18
  19 - (NSDictionary *)copyRandomConfiguration
  20 {
  21     const char *testKey = "test_defaults_experiment_key";
  22     const char *testValue = "test_value";
  23     NSString *testKeyString = [NSString stringWithUTF8String:testKey];
  24     NSString *testValueString = [NSString stringWithUTF8String:testValue];
  25     NSDictionary *testConfigData = @{testKeyString: testValueString};
  26     NSDictionary *testConfig = @{SecExperimentConfigurationKeyFleetSampleRate : @(100),
  27                                  SecExperimentConfigurationKeyDeviceSampleRate : @(100),
  28                                  SecExperimentConfigurationKeyExperimentIdentifier : @"identifier",
  29                                  SecExperimentConfigurationKeyConfigurationData : testConfigData};
  30     return testConfig;
  31 }
  32
  33 - (void)testCopyFromDefaults {
  34     const char *testKey = "test_defaults_experiment_key";
  35     const char *testValue = "test_value";
  36     NSString *testKeyString = [NSString stringWithUTF8String:testKey];
  37     NSString *testValueString = [NSString stringWithUTF8String:testValue];
  38     NSDictionary *testConfigData = @{testKeyString: testValueString};
  39     NSDictionary *testConfig = @{SecExperimentConfigurationKeyFleetSampleRate : @(100),
  40                                  SecExperimentConfigurationKeyDeviceSampleRate : @(100),
  41                                  SecExperimentConfigurationKeyExperimentIdentifier : @"identifier",
  42                                  SecExperimentConfigurationKeyConfigurationData : testConfigData};
  43
  44     SecExperiment *mockExperiment = OCMPartialMock([[SecExperiment alloc] initWithName:kSecExperimentTLSProbe]);
  45     OCMStub([mockExperiment copyExperimentConfigurationFromUserDefaults]).andReturn(testConfig);
  46     sec_experiment_t experiment = sec_experiment_create_with_inner_experiment(mockExperiment);
  47
  48     sec_experiment_set_sampling_disabled(experiment, true);
  49     xpc_object_t actualConfig = sec_experiment_copy_configuration(experiment);
  50     XCTAssertNotNil(actualConfig);
  51
  52     xpc_type_t configType = xpc_get_type(actualConfig);
  53     XCTAssertTrue(configType == XPC_TYPE_DICTIONARY);
  54
  55     const char *actualValue = xpc_dictionary_get_string(actualConfig, testKey);
  56     XCTAssertTrue(actualValue != NULL && strncmp(actualValue, testValue, strlen(testValue)) == 0);
  57 }
  58
  59 - (void)testInitializeWithIdentifier {
  60     sec_experiment_t experiment = sec_experiment_create(kSecExperimentTLSProbe);
  61     const char *identifier = sec_experiment_get_identifier(experiment);
  62     XCTAssertTrue(identifier == NULL);
  63 }
  64
  65 - (void)testExperimentRunAsynchronously_SkipSampling {
  66     dispatch_queue_t test_queue = dispatch_queue_create("test_queue", NULL);
  67
  68     __block dispatch_semaphore_t blocker = dispatch_semaphore_create(0);
  69     __block bool experiment_invoked = false;
  70     sec_experiment_run_block_t run_block = ^bool(const char *identifier, xpc_object_t config) {
  71         experiment_invoked = identifier != NULL && config != NULL;
  72         dispatch_semaphore_signal(blocker);
  73         return experiment_invoked;
  74     };
  75
  76     NSDictionary *testConfig = [self copyRandomConfiguration];
  77
  78     SecExperiment *mockExperiment = OCMPartialMock([[SecExperiment alloc] initWithName:kSecExperimentTLSProbe]);
  79     OCMStub([mockExperiment copyExperimentConfigurationFromUserDefaults]).andReturn(testConfig);
  80     sec_experiment_t experiment = sec_experiment_create_with_inner_experiment(mockExperiment);
  81
  82     sec_experiment_run_internal(experiment, true, test_queue, run_block, nil, false);
  83     XCTAssertTrue(dispatch_semaphore_wait(blocker, dispatch_time(DISPATCH_TIME_NOW, (uint64_t)(5 * NSEC_PER_SEC))) == 0L);
  84     XCTAssertTrue(experiment_invoked);
  85 }
  86
  87 - (void)testExperimentRun_SkipWithoutConfig {
  88     SecExperiment *mockExperiment = OCMPartialMock([[SecExperiment alloc] initWithName:kSecExperimentTLSProbe]);
  89     OCMStub([mockExperiment experimentIsAllowedForProcess]).andReturn(YES);
  90     OCMStub([mockExperiment copyRemoteExperimentAsset]).andReturn(nil);
  91     OCMStub([mockExperiment copyExperimentConfigurationFromUserDefaults]).andReturn(nil);
  92     sec_experiment_t experiment = sec_experiment_create_with_inner_experiment(mockExperiment);
  93
  94     dispatch_queue_t test_queue = dispatch_queue_create("test_queue", NULL);
  95     __block bool run = false;
  96     sec_experiment_run_block_t run_block = ^bool(__unused const char *identifier, __unused xpc_object_t config) {
  97         run = true;
  98         return true;
  99     };
 100     __block BOOL skipped = false;
 101     sec_experiment_skip_block_t skip_block = ^(__unused const char *identifier) {
 102         skipped = true;
 103     };
 104
 105     bool result = sec_experiment_run_internal(experiment, true, test_queue, run_block, skip_block, true);
 106     XCTAssertTrue(result);
 107     XCTAssertTrue(skipped);
 108     XCTAssertFalse(run);
 109 }
 110
 111 - (void)testExperimentRunSuccess {
 112     NSDictionary *testConfig = [self copyRandomConfiguration];
 113
 114     SecExperiment *mockExperiment = OCMPartialMock([[SecExperiment alloc] initWithName:kSecExperimentTLSProbe]);
 115     OCMStub([mockExperiment isSamplingDisabled]).andReturn(YES);
 116     OCMStub([mockExperiment experimentIsAllowedForProcess]).andReturn(YES);
 117     OCMStub([mockExperiment copyRemoteExperimentAsset]).andReturn(testConfig);
 118     OCMStub([mockExperiment copyExperimentConfigurationFromUserDefaults]).andReturn(nil);
 119     OCMStub([mockExperiment copyRandomExperimentConfigurationFromAsset:[OCMArg any]]).andReturn(testConfig);
 120     sec_experiment_t experiment = sec_experiment_create_with_inner_experiment(mockExperiment);
 121
 122     dispatch_queue_t test_queue = dispatch_queue_create("test_queue", NULL);
 123     __block bool run = false;
 124     sec_experiment_run_block_t run_block = ^bool(__unused const char *identifier, __unused xpc_object_t config) {
 125         run = true;
 126         return true; // Signal that the experiment run was successful
 127     };
 128     __block BOOL skipped = false;
 129     sec_experiment_skip_block_t skip_block = ^(__unused const char *identifier) {
 130         skipped = true;
 131     };
 132
 133     bool result = sec_experiment_run_internal(experiment, true, test_queue, run_block, skip_block, true);
 134     XCTAssertTrue(result);
 135     XCTAssertFalse(skipped);
 136     XCTAssertTrue(run);
 137     XCTAssertTrue(sec_experiment_get_run_count(experiment) == 1);
 138     XCTAssertTrue(sec_experiment_get_successful_run_count(experiment) == 1);
 139 }
 140
 141 - (void)testExperimentRunFailure {
 142     NSDictionary *testConfig = [self copyRandomConfiguration];
 143
 144     SecExperiment *mockExperiment = OCMPartialMock([[SecExperiment alloc] initWithName:kSecExperimentTLSProbe]);
 145     OCMStub([mockExperiment experimentIsAllowedForProcess]).andReturn(YES);
 146     OCMStub([mockExperiment isSamplingDisabled]).andReturn(YES);
 147     OCMStub([mockExperiment copyRemoteExperimentAsset]).andReturn(testConfig);
 148     OCMStub([mockExperiment copyExperimentConfigurationFromUserDefaults]).andReturn(nil);
 149     OCMStub([mockExperiment copyRandomExperimentConfigurationFromAsset:[OCMArg any]]).andReturn(testConfig);
 150     sec_experiment_t experiment = sec_experiment_create_with_inner_experiment(mockExperiment);
 151
 152     dispatch_queue_t test_queue = dispatch_queue_create("test_queue", NULL);
 153     __block bool run = false;
 154     sec_experiment_run_block_t run_block = ^bool(__unused const char *identifier, __unused xpc_object_t config) {
 155         run = true;
 156         return false; // Signal that the experiment run failed
 157     };
 158     __block BOOL skipped = false;
 159     sec_experiment_skip_block_t skip_block = ^(__unused const char *identifier) {
 160         skipped = true;
 161     };
 162
 163     bool result = sec_experiment_run_internal(experiment, true, test_queue, run_block, skip_block, true);
 164     XCTAssertTrue(result);
 165     XCTAssertFalse(skipped);
 166     XCTAssertTrue(run);
 167     XCTAssertTrue(sec_experiment_get_run_count(experiment) == 1);
 168     XCTAssertTrue(sec_experiment_get_successful_run_count(experiment) == 0);
 169 }
 170
 171 - (void)testExperimentRun_SamplingEnabled_NotInFleet {
 172     size_t fleetNumber = 10;
 173     NSDictionary *testConfig = @{SecExperimentConfigurationKeyFleetSampleRate: @(fleetNumber), SecExperimentConfigurationKeyDeviceSampleRate : @(2)};
 174
 175     SecExperimentConfig *mockConfig = OCMPartialMock([[SecExperimentConfig alloc] initWithConfiguration:testConfig]);
 176     OCMStub([mockConfig hostHash]).andReturn(fleetNumber + 1); // Ensure that fleetNumber < hostHash
 177
 178     SecExperiment *mockExperiment = OCMPartialMock([[SecExperiment alloc] initWithName:kSecExperimentTLSProbe]);
 179     OCMStub([mockExperiment experimentIsAllowedForProcess]).andReturn(YES);
 180     OCMStub([mockExperiment isSamplingDisabled]).andReturn(NO);
 181     OCMStub([mockExperiment copyExperimentConfiguration]).andReturn(mockConfig);
 182
 183     sec_experiment_t experiment = sec_experiment_create_with_inner_experiment(mockExperiment);
 184
 185     dispatch_queue_t test_queue = dispatch_queue_create("test_queue", NULL);
 186     __block bool run = false;
 187     sec_experiment_run_block_t run_block = ^bool(__unused const char *identifier, __unused xpc_object_t config) {
 188         run = true;
 189         return false; // Signal that the experiment run failed
 190     };
 191     __block BOOL skipped = false;
 192     sec_experiment_skip_block_t skip_block = ^(__unused const char *identifier) {
 193         skipped = true;
 194     };
 195
 196     bool result = sec_experiment_run_internal(experiment, true, test_queue, run_block, skip_block, true);
 197     XCTAssertTrue(result);
 198     XCTAssertTrue(skipped);
 199     XCTAssertFalse(run);
 200     XCTAssertTrue(sec_experiment_get_run_count(experiment) == 0);
 201     XCTAssertTrue(sec_experiment_get_successful_run_count(experiment) == 0);
 202 }
 203
 204 - (void)testExperimentRun_SamplingEnabled_InFleetNotInSample {
 205     size_t fleetNumber = 10;
 206     NSDictionary *testConfig = @{SecExperimentConfigurationKeyFleetSampleRate: @(fleetNumber), SecExperimentConfigurationKeyDeviceSampleRate : @(2)};
 207
 208     SecExperimentConfig *mockConfig = OCMPartialMock([[SecExperimentConfig alloc] initWithConfiguration:testConfig]);
 209     OCMStub([mockConfig hostHash]).andReturn(fleetNumber - 1); // Ensure that fleetNumber > hostHash
 210     OCMStub([mockConfig shouldRunWithSamplingRate:[OCMArg any]]).andReturn(NO); // Determine that we're not in the fleet
 211
 212     SecExperiment *mockExperiment = OCMPartialMock([[SecExperiment alloc] initWithName:kSecExperimentTLSProbe]);
 213     OCMStub([mockExperiment experimentIsAllowedForProcess]).andReturn(YES);
 214     OCMStub([mockExperiment isSamplingDisabled]).andReturn(NO);
 215     OCMStub([mockExperiment copyExperimentConfiguration]).andReturn(mockConfig);
 216
 217     sec_experiment_t experiment = sec_experiment_create_with_inner_experiment(mockExperiment);
 218
 219     dispatch_queue_t test_queue = dispatch_queue_create("test_queue", NULL);
 220     __block bool run = false;
 221     sec_experiment_run_block_t run_block = ^bool(__unused const char *identifier, __unused xpc_object_t config) {
 222         run = true;
 223         return false; // Signal that the experiment run failed
 224     };
 225     __block BOOL skipped = false;
 226     sec_experiment_skip_block_t skip_block = ^(__unused const char *identifier) {
 227         skipped = true;
 228     };
 229
 230     bool result = sec_experiment_run_internal(experiment, true, test_queue, run_block, skip_block, true);
 231     XCTAssertTrue(result);
 232     XCTAssertTrue(skipped);
 233     XCTAssertFalse(run);
 234     XCTAssertTrue(sec_experiment_get_run_count(experiment) == 0);
 235     XCTAssertTrue(sec_experiment_get_successful_run_count(experiment) == 0);
 236 }
 237
 238 - (void)testExperimentRun_DisallowedProcess {
 239     SecExperiment *mockExperiment = OCMPartialMock([[SecExperiment alloc] initWithName:kSecExperimentTLSProbe]);
 240     OCMStub([mockExperiment experimentIsAllowedForProcess]).andReturn(NO);
 241     sec_experiment_t experiment = sec_experiment_create_with_inner_experiment(mockExperiment);
 242
 243     dispatch_queue_t test_queue = dispatch_queue_create("test_queue", NULL);
 244     sec_experiment_run_block_t run_block = ^bool(__unused const char *identifier, __unused xpc_object_t config) {
 245         return true;
 246     };
 247
 248     bool result = sec_experiment_run_internal(experiment, true, test_queue, run_block, nil, false);
 249     XCTAssertFalse(result);
 250 }
 251
 252 - (void)testExperimentRunSynchronously_SkipSampling {
 253     dispatch_queue_t test_queue = dispatch_queue_create("test_queue", NULL);
 254
 255     __block bool experiment_invoked = false;
 256     sec_experiment_run_block_t run_block = ^bool(const char *identifier, xpc_object_t config) {
 257         experiment_invoked = identifier != NULL && config != NULL;
 258         return experiment_invoked;
 259     };
 260
 261     NSDictionary *testConfig = [self copyRandomConfiguration];
 262
 263     SecExperiment *mockExperiment = OCMPartialMock([[SecExperiment alloc] initWithName:kSecExperimentTLSProbe]);
 264     OCMStub([mockExperiment copyExperimentConfigurationFromUserDefaults]).andReturn(testConfig);
 265     sec_experiment_t experiment = sec_experiment_create_with_inner_experiment(mockExperiment);
 266
 267     sec_experiment_run_internal(experiment, true, test_queue, run_block, nil, true);
 268     XCTAssertTrue(experiment_invoked);
 269 }
 270
 271 - (void)testDefaultsConfigCopy {
 272     const char *testKey = "test_defaults_experiment_key";
 273     const char *testValue = "test_value";
 274     NSString *testKeyString = [NSString stringWithUTF8String:testKey];
 275     NSString *testValueString = [NSString stringWithUTF8String:testValue];
 276     NSDictionary *testConfigData = @{testKeyString: testValueString};
 277     NSString *experimentName = @"TestExperiment";
 278     NSDictionary *testConfig = @{experimentName: @{SecExperimentConfigurationKeyFleetSampleRate : @(100),
 279                                                    SecExperimentConfigurationKeyDeviceSampleRate : @(100),
 280                                                    SecExperimentConfigurationKeyExperimentIdentifier : @"identifier",
 281                                                    SecExperimentConfigurationKeyConfigurationData : testConfigData}};
 282
 283     sec_experiment_t experiment = sec_experiment_create([experimentName UTF8String]);
 284     XCTAssert(experiment, @"sec_experiment_create");
 285
 286     NSUserDefaults *defaults = [NSUserDefaults standardUserDefaults];
 287     [defaults setPersistentDomain:testConfig forName:[NSString stringWithUTF8String:kSecExperimentDefaultsDomain]];
 288
 289     sec_experiment_set_sampling_disabled(experiment, true);
 290     xpc_object_t tlsconfig = sec_experiment_copy_configuration(experiment);
 291     XCTAssertNotNil(tlsconfig);
 292     if (tlsconfig) {
 293         XCTAssertTrue(xpc_get_type(tlsconfig) == XPC_TYPE_DICTIONARY);
 294         if (xpc_get_type(tlsconfig) == XPC_TYPE_DICTIONARY) {
 295             const char *actualValue = xpc_dictionary_get_string(tlsconfig, testKey);
 296             XCTAssertTrue(actualValue != NULL);
 297             if (actualValue != NULL) {
 298                 XCTAssertTrue(strcmp(actualValue, testValue) == 0);
 299             }
 300         }
 301     }
 302
 303     // Clear the persistent domain
 304     [defaults removePersistentDomainForName:[NSString stringWithUTF8String:kSecExperimentDefaultsDomain]];
 305 }
 306
 307 - (void)testRunWithIdentifier {
 308     const char *testKey = "test_defaults_experiment_key";
 309     const char *testValue = "test_value";
 310     NSString *testKeyString = [NSString stringWithUTF8String:testKey];
 311     NSString *testValueString = [NSString stringWithUTF8String:testValue];
 312     NSDictionary *testConfigData = @{testKeyString: testValueString};
 313     NSString *experimentName = @"TestExperiment";
 314     NSString *identifierName = @"ExperimentIdentifier";
 315     NSDictionary *testConfig = @{experimentName: @{SecExperimentConfigurationKeyFleetSampleRate : @(100),
 316                                                    SecExperimentConfigurationKeyDeviceSampleRate : @(100),
 317                                                    SecExperimentConfigurationKeyExperimentIdentifier : identifierName,
 318                                                    SecExperimentConfigurationKeyConfigurationData : testConfigData}};
 319
 320     SecExperiment *mockExperiment = OCMPartialMock([[SecExperiment alloc] initWithName:[experimentName UTF8String]]);
 321     OCMStub([mockExperiment copyExperimentConfigurationFromUserDefaults]).andReturn(testConfig);
 322     sec_experiment_t experiment = sec_experiment_create_with_inner_experiment(mockExperiment);
 323
 324     sec_experiment_run_internal(experiment, true, nil, ^bool(const char * _Nonnull identifier, xpc_object_t  _Nonnull experiment_config) {
 325         XCTAssertTrue(identifier != NULL);
 326         if (identifier != NULL) {
 327             XCTAssertTrue(strcmp([identifierName UTF8String], identifier) == 0);
 328         }
 329     }, ^(const char * _Nonnull identifier) {
 330         XCTAssertFalse(true);
 331     }, true);
 332 }
 333
 334 - (void)testGeneration {
 335     nw_protocol_options_t tls_options = nw_tls_create_options();
 336     sec_protocol_options_t sec_options = nw_tls_copy_sec_protocol_options(tls_options);
 337
 338     sec_protocol_options_set_tls_grease_enabled(sec_options, true);
 339     xpc_object_t config = sec_protocol_options_create_config(sec_options);
 340
 341     xpc_dictionary_apply(config, ^bool(const char * _Nonnull key, xpc_object_t  _Nonnull value) {
 342         if (xpc_get_type(value) == XPC_TYPE_BOOL) {
 343             NSLog(@"%s -> %d", key, xpc_bool_get_value(value));
 344         } else if (xpc_get_type(value) == XPC_TYPE_UINT64) {
 345             NSLog(@"%s -> %zu", key, (size_t)xpc_uint64_get_value(value));
 346         } else {
 347             NSLog(@"%s -> %d", key, (int)xpc_get_type(value));
 348         }
 349
 350         return true;
 351     });
 352
 353     CFDictionaryRef config_dictionary = _CFXPCCreateCFObjectFromXPCObject(config);
 354     XCTAssertTrue(config_dictionary != NULL);
 355     NSLog(@"%@", (__bridge NSDictionary *)config_dictionary);
 356 }
 357
 358 @end