supd/Tests/SupdTests.m

   1 /*
   2  * Copyright (c) 2017-2018 Apple Inc. All Rights Reserved.
   3  *
   4  * @APPLE_LICENSE_HEADER_START@
   5  *
   6  * This file contains Original Code and/or Modifications of Original Code
   7  * as defined in and that are subject to the Apple Public Source License
   8  * Version 2.0 (the 'License'). You may not use this file except in
   9  * compliance with the License. Please obtain a copy of the License at
  10  * http://www.opensource.apple.com/apsl/ and read it before using this
  11  * file.
  12  *
  13  * The Original Code and all software distributed under the License are
  14  * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  15  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  16  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
  17  * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
  18  * Please see the License for the specific language governing rights and
  19  * limitations under the License.
  20  *
  21  * @APPLE_LICENSE_HEADER_END@
  22  */
  23
  24 #import <XCTest/XCTest.h>
  25
  26 // securityuploadd does not do anything or build meaningful code on simulator, so no tests either.
  27 #if TARGET_OS_SIMULATOR
  28
  29 @interface SupdTests : XCTestCase
  30 @end
  31
  32 @implementation SupdTests
  33 @end
  34
  35 #else
  36
  37 #import <OCMock/OCMock.h>
  38 #import "supd.h"
  39 #import <Security/SFAnalytics.h>
  40 #import "SFAnalyticsDefines.h"
  41 #import <CoreFoundation/CFPriv.h>
  42
  43 static NSString* _path;
  44 static NSInteger _testnum;
  45 static NSString* build = NULL;
  46 static NSString* product = NULL;
  47 static NSInteger _reporterWrites;
  48
  49 // MARK: Stub FakeCKKSAnalytics
  50
  51 @interface FakeCKKSAnalytics : SFAnalytics
  52
  53 @end
  54
  55 @implementation FakeCKKSAnalytics
  56
  57 + (NSString*)databasePath
  58 {
  59     return [_path stringByAppendingFormat:@"/ckks_%ld.db", (long)_testnum];
  60 }
  61
  62 @end
  63
  64
  65 // MARK: Stub FakeSOSAnalytics
  66
  67 @interface FakeSOSAnalytics : SFAnalytics
  68
  69 @end
  70
  71 @implementation FakeSOSAnalytics
  72
  73 + (NSString*)databasePath
  74 {
  75     return [_path stringByAppendingFormat:@"/sos_%ld.db", (long)_testnum];
  76 }
  77
  78 @end
  79
  80
  81 // MARK: Stub FakePCSAnalytics
  82
  83 @interface FakePCSAnalytics : SFAnalytics
  84
  85 @end
  86
  87 @implementation FakePCSAnalytics
  88
  89 + (NSString*)databasePath
  90 {
  91     return [_path stringByAppendingFormat:@"/pcs_%ld.db", (long)_testnum];
  92 }
  93
  94 @end
  95
  96 // MARK: Stub FakeTLSAnalytics
  97
  98 @interface FakeTLSAnalytics : SFAnalytics
  99
 100 @end
 101
 102 @implementation FakeTLSAnalytics
 103
 104 + (NSString*)databasePath
 105 {
 106     return [_path stringByAppendingFormat:@"/tls_%ld.db", (long)_testnum];
 107 }
 108
 109 @end
 110
 111 // MARK: Start SupdTests
 112
 113 @interface SupdTests : XCTestCase
 114
 115 @end
 116
 117 @implementation SupdTests {
 118     supd* _supd;
 119     id mockReporter;
 120     FakeCKKSAnalytics* _ckksAnalytics;
 121     FakeSOSAnalytics* _sosAnalytics;
 122     FakePCSAnalytics* _pcsAnalytics;
 123     FakeTLSAnalytics* _tlsAnalytics;
 124 }
 125
 126 // MARK: Test helper methods
 127 - (SFAnalyticsTopic *)keySyncTopic {
 128     for (SFAnalyticsTopic *topic in _supd.analyticsTopics) {
 129         if ([topic.internalTopicName isEqualToString:SFAnalyticsTopicKeySync]) {
 130             return topic;
 131         }
 132     }
 133     return nil;
 134 }
 135
 136 - (SFAnalyticsTopic *)TrustTopic {
 137     for (SFAnalyticsTopic *topic in _supd.analyticsTopics) {
 138         if ([topic.internalTopicName isEqualToString:SFAnalyticsTopicTrust]) {
 139             return topic;
 140         }
 141     }
 142     return nil;
 143 }
 144
 145 - (void)inspectDataBlobStructure:(NSDictionary*)data
 146 {
 147     [self inspectDataBlobStructure:data forTopic:[[self keySyncTopic] splunkTopicName]];
 148 }
 149
 150 - (void)inspectDataBlobStructure:(NSDictionary*)data forTopic:(NSString*)topic
 151 {
 152     if (!data || ![data isKindOfClass:[NSDictionary class]]) {
 153         XCTFail(@"data is an NSDictionary");
 154     }
 155
 156     XCTAssert(_supd.analyticsTopics, @"supd has nonnull topics list");
 157     XCTAssert([[self keySyncTopic] splunkTopicName], @"keysync topic has a splunk name");
 158     XCTAssert([[self TrustTopic] splunkTopicName], @"trust topic has a splunk name");
 159     XCTAssertEqual([data count], 2ul, @"dictionary event and posttime objects");
 160     XCTAssertTrue(data[@"events"] && [data[@"events"] isKindOfClass:[NSArray class]], @"data blob contains an NSArray 'events'");
 161     XCTAssertTrue(data[@"postTime"] && [data[@"postTime"] isKindOfClass:[NSNumber class]], @"data blob contains an NSNumber 'postTime");
 162     NSDate* postTime = [NSDate dateWithTimeIntervalSince1970:[data[@"postTime"] doubleValue]];
 163     XCTAssertTrue([[NSDate date] timeIntervalSinceDate:postTime] < 3, @"postTime is sane");
 164
 165     for (NSDictionary* event in data[@"events"]) {
 166         if ([event isKindOfClass:[NSDictionary class]]) {
 167             NSLog(@"build: \"%@\", eventbuild: \"%@\"", build, event[@"build"]);
 168             XCTAssertEqualObjects(event[@"build"], build, @"event contains correct build string");
 169             XCTAssertEqualObjects(event[@"product"], product, @"event contains correct product string");
 170             XCTAssertTrue([event[@"eventTime"] isKindOfClass:[NSNumber class]], @"event contains an NSNumber 'eventTime");
 171             NSDate* eventTime = [NSDate dateWithTimeIntervalSince1970:[event[@"eventTime"] doubleValue]];
 172             XCTAssertTrue([[NSDate date] timeIntervalSinceDate:eventTime] < 3, @"eventTime is sane");
 173             XCTAssertTrue([event[@"eventType"] isKindOfClass:[NSString class]], @"all events have a type");
 174             XCTAssertEqualObjects(event[@"topic"], topic, @"all events have a topic name");
 175         } else {
 176             XCTFail(@"event %@ is an NSDictionary", event);
 177         }
 178     }
 179 }
 180
 181 - (BOOL)event:(NSDictionary*)event containsAttributes:(NSDictionary*)attrs {
 182     if (!attrs) {
 183         return YES;
 184     }
 185     __block BOOL equal = YES;
 186     [attrs enumerateKeysAndObjectsUsingBlock:^(id  _Nonnull key, id  _Nonnull obj, BOOL * _Nonnull stop) {
 187         equal &= [event[key] isEqualToString:obj];
 188     }];
 189     return equal;
 190 }
 191
 192 - (int)failures:(NSDictionary*)data eventType:(NSString*)type attributes:(NSDictionary*)attrs class:(SFAnalyticsEventClass)class
 193 {
 194     int encountered = 0;
 195     for (NSDictionary* event in data[@"events"]) {
 196         if ([event[@"eventType"] isEqualToString:type] &&
 197             [event[@"eventClass"] isKindOfClass:[NSNumber class]] &&
 198             [event[@"eventClass"] intValue] == class && [self event:event containsAttributes:attrs]) {
 199             ++encountered;
 200         }
 201     }
 202     return encountered;
 203 }
 204
 205 - (void)checkTotalEventCount:(NSDictionary*)data hard:(int)hard soft:(int)soft accuracy:(int)accuracy summaries:(int)summ
 206 {
 207     int hardfound = 0, softfound = 0, summfound = 0;
 208     for (NSDictionary* event in data[@"events"]) {
 209         if ([event[SFAnalyticsEventType] hasSuffix:@"HealthSummary"]) {
 210             ++summfound;
 211         } else if ([event[SFAnalyticsEventClassKey] integerValue] == SFAnalyticsEventClassHardFailure) {
 212             ++hardfound;
 213         } else if ([event[SFAnalyticsEventClassKey] integerValue] == SFAnalyticsEventClassSoftFailure) {
 214             ++softfound;
 215         }
 216     }
 217
 218     XCTAssertLessThanOrEqual(((NSArray*)data[@"events"]).count, 1000ul, @"Total event count fits in alloted data");
 219     XCTAssertEqual(summfound, summ);
 220
 221     // Add customizable fuzziness
 222     XCTAssertEqualWithAccuracy(hardfound, hard, accuracy);
 223     XCTAssertEqualWithAccuracy(softfound, soft, accuracy);
 224 }
 225
 226 - (void)checkTotalEventCount:(NSDictionary*)data hard:(int)hard soft:(int)soft
 227 {
 228     [self checkTotalEventCount:data hard:hard soft:soft accuracy:10 summaries:(int)[[[self keySyncTopic] topicClients] count]];
 229 }
 230
 231 - (void)checkTotalEventCount:(NSDictionary*)data hard:(int)hard soft:(int)soft accuracy:(int)accuracy
 232 {
 233     [self checkTotalEventCount:data hard:hard soft:soft accuracy:accuracy summaries:(int)[[[self keySyncTopic] topicClients] count]];
 234 }
 235
 236 // This is a dumb hack, but inlining stringWithFormat causes the compiler to growl for unknown reasons
 237 - (NSString*)string:(NSString*)name item:(NSString*)item
 238 {
 239     return [NSString stringWithFormat:@"%@-%@", name, item];
 240 }
 241
 242 - (void)sampleStatisticsInEvents:(NSArray*)events name:(NSString*)name values:(NSArray*)values
 243 {
 244     [self sampleStatisticsInEvents:events name:name values:values amount:1];
 245 }
 246
 247 // Usually amount == 1 but for testing sampler with same name in different subclasses this is higher
 248 - (void)sampleStatisticsInEvents:(NSArray*)events name:(NSString*)name values:(NSArray*)values amount:(int)num
 249 {
 250     int found = 0;
 251     for (NSDictionary* event in events) {
 252         if (([values count] == 1 && ![event objectForKey:[NSString stringWithFormat:@"%@", name]]) ||
 253             ([values count] > 1 && ![event objectForKey:[NSString stringWithFormat:@"%@-min", name]])) {
 254             continue;
 255         }
 256
 257         ++found;
 258         if (values.count == 1) {
 259             XCTAssertEqual([event[name] doubleValue], [values[0] doubleValue]);
 260             XCTAssertNil(event[[self string:name item:@"min"]]);
 261             XCTAssertNil(event[[self string:name item:@"max"]]);
 262             XCTAssertNil(event[[self string:name item:@"avg"]]);
 263             XCTAssertNil(event[[self string:name item:@"med"]]);
 264         } else {
 265             XCTAssertEqualWithAccuracy([event[[self string:name item:@"min"]] doubleValue], [values[0] doubleValue], 0.01f);
 266             XCTAssertEqualWithAccuracy([event[[self string:name item:@"max"]] doubleValue], [values[1] doubleValue], 0.01f);
 267             XCTAssertEqualWithAccuracy([event[[self string:name item:@"avg"]] doubleValue], [values[2] doubleValue], 0.01f);
 268             XCTAssertEqualWithAccuracy([event[[self string:name item:@"med"]] doubleValue], [values[3] doubleValue], 0.01f);
 269         }
 270
 271         if (values.count > 4) {
 272             XCTAssertEqualWithAccuracy([event[[self string:name item:@"dev"]] doubleValue], [values[4] doubleValue], 0.01f);
 273         } else {
 274             XCTAssertNil(event[[self string:name item:@"dev"]]);
 275         }
 276
 277         if (values.count > 5) {
 278             XCTAssertEqualWithAccuracy([event[[self string:name item:@"1q"]] doubleValue], [values[5] doubleValue], 0.01f);
 279             XCTAssertEqualWithAccuracy([event[[self string:name item:@"3q"]] doubleValue], [values[6] doubleValue], 0.01f);
 280         } else {
 281             XCTAssertNil(event[[self string:name item:@"1q"]]);
 282             XCTAssertNil(event[[self string:name item:@"3q"]]);
 283         }
 284     }
 285     XCTAssertEqual(found, num);
 286 }
 287
 288 - (NSDictionary*)getJSONDataFromSupd
 289 {
 290     return [self getJSONDataFromSupdWithTopic:SFAnalyticsTopicKeySync];
 291 }
 292
 293 - (NSDictionary*)getJSONDataFromSupdWithTopic:(NSString*)topic
 294 {
 295     dispatch_semaphore_t sema = dispatch_semaphore_create(0);
 296     __block NSDictionary* data;
 297     [_supd getLoggingJSON:YES topic:topic reply:^(NSData *json, NSError *error) {
 298         XCTAssertNil(error);
 299         XCTAssertNotNil(json);
 300         if (!error) {
 301             data = [NSJSONSerialization JSONObjectWithData:json options:0 error:&error];
 302         }
 303         XCTAssertNil(error, @"no error deserializing json: %@", error);
 304         dispatch_semaphore_signal(sema);
 305     }];
 306     if (dispatch_semaphore_wait(sema, dispatch_time(DISPATCH_TIME_NOW, NSEC_PER_SEC * 5)) != 0) {
 307         XCTFail(@"supd returns JSON data in a timely fashion");
 308     }
 309     return data;
 310 }
 311
 312 // MARK: Test administration
 313
 314 + (void)setUp
 315 {
 316     NSError* error;
 317     _path = [NSTemporaryDirectory() stringByAppendingPathComponent:[NSString stringWithFormat:@"%@/", [[NSUUID UUID] UUIDString]]];
 318     [[NSFileManager defaultManager] createDirectoryAtPath:_path
 319                               withIntermediateDirectories:YES
 320                                                attributes:nil
 321                                                     error:&error];
 322     if (error) {
 323         NSLog(@"sad trombone, couldn't create path");
 324     }
 325
 326     NSDictionary *version = CFBridgingRelease(_CFCopySystemVersionDictionary());
 327     if (version) {
 328         build = version[(__bridge NSString *)_kCFSystemVersionBuildVersionKey];
 329         product = version[(__bridge NSString *)_kCFSystemVersionProductNameKey];
 330     } else {
 331         NSLog(@"could not get build version/product, tests should fail");
 332     }
 333 }
 334
 335 - (void)setUp
 336 {
 337     [super setUp];
 338     self.continueAfterFailure = NO;
 339     ++_testnum;
 340
 341     id mockTopic = OCMStrictClassMock([SFAnalyticsTopic class]);
 342     NSString *ckksPath = [_path stringByAppendingFormat:@"/ckks_%ld.db", (long)_testnum];
 343     NSString *sosPath = [_path stringByAppendingFormat:@"/sos_%ld.db", (long)_testnum];
 344     NSString *pcsPath = [_path stringByAppendingFormat:@"/pcs_%ld.db", (long)_testnum];
 345     NSString *tlsPath = [_path stringByAppendingFormat:@"/tls_%ld.db", (long)_testnum];
 346     NSString *signInPath = [_path stringByAppendingFormat:@"/signin_%ld.db", (long)_testnum];
 347     NSString *cloudServicesPath = [_path stringByAppendingFormat:@"/cloudServices_%ld.db", (long)_testnum];
 348     OCMStub([mockTopic databasePathForCKKS]).andReturn(ckksPath);
 349     OCMStub([mockTopic databasePathForSOS]).andReturn(sosPath);
 350     OCMStub([mockTopic databasePathForPCS]).andReturn(pcsPath);
 351     OCMStub([mockTopic databasePathForTLS]).andReturn(tlsPath);
 352     OCMStub([mockTopic databasePathForSignIn]).andReturn(signInPath);
 353     OCMStub([mockTopic databasePathForCloudServices]).andReturn(cloudServicesPath);
 354
 355     // These are not used for testing, but real data can pollute tests so point to empty DBs
 356     NSString *localpath = [_path stringByAppendingFormat:@"/local_empty_%ld.db", (long)_testnum];
 357     NSString *trustPath = [_path stringByAppendingFormat:@"/trust_empty_%ld.db", (long)_testnum];
 358     NSString *trustdhealthPath = [_path stringByAppendingFormat:@"/trustdhealth_empty_%ld.db", (long)_testnum];
 359     OCMStub([mockTopic databasePathForLocal]).andReturn(localpath);
 360     OCMStub([mockTopic databasePathForTrust]).andReturn(trustPath);
 361     OCMStub([mockTopic databasePathForTrustdHealth]).andReturn(trustdhealthPath);
 362
 363     _reporterWrites = 0;
 364     mockReporter = OCMClassMock([SFAnalyticsReporter class]);
 365     OCMStub([mockReporter saveReport:[OCMArg isNotNil] fileName:[OCMArg isNotNil]]).andDo(^(NSInvocation *invocation) {
 366         _reporterWrites++;
 367     }).andReturn(YES);
 368
 369     [supd removeInstance];
 370     _supd = [[supd alloc] initWithReporter:mockReporter];
 371     _ckksAnalytics = [FakeCKKSAnalytics new];
 372     _sosAnalytics = [FakeSOSAnalytics new];
 373     _pcsAnalytics = [FakePCSAnalytics new];
 374     _tlsAnalytics = [FakeTLSAnalytics new];
 375
 376     // Forcibly override analytics flags and enable them by default
 377     deviceAnalyticsOverride = YES;
 378     deviceAnalyticsEnabled = YES;
 379     iCloudAnalyticsOverride = YES;
 380     iCloudAnalyticsEnabled = YES;
 381     runningTests = YES;
 382 }
 383
 384 - (void)tearDown
 385 {
 386
 387     [super tearDown];
 388 }
 389
 390 // MARK: Actual tests
 391
 392 // Note! This test relies on Security being installed because supd reads from a plist in Security.framework
 393 - (void)testSplunkDefaultTopicNameExists
 394 {
 395     XCTAssertNotNil([[self keySyncTopic] splunkTopicName]);
 396 }
 397
 398 // Note! This test relies on Security being installed because supd reads from a plist in Security.framework
 399 - (void)testSplunkDefaultBagURLExists
 400 {
 401     XCTAssertNotNil([[self keySyncTopic] splunkBagURL]);
 402 }
 403
 404 - (void)testHaveEligibleClientsKeySync
 405 {
 406     // KeySyncTopic has no clients requiring deviceAnalytics currently
 407     SFAnalyticsTopic* keytopic = [[SFAnalyticsTopic alloc] initWithDictionary:@{} name:@"KeySyncTopic" samplingRates:@{}];
 408
 409     XCTAssertTrue([keytopic haveEligibleClients], @"Both analytics enabled -> we have keysync clients");
 410
 411     deviceAnalyticsEnabled = NO;
 412     XCTAssertTrue([keytopic haveEligibleClients], @"Only iCloud analytics enabled -> we have keysync clients");
 413
 414     iCloudAnalyticsEnabled = NO;
 415     XCTAssertFalse([keytopic haveEligibleClients], @"Both analytics disabled -> no keysync clients");
 416
 417     deviceAnalyticsEnabled = YES;
 418     XCTAssertTrue([keytopic haveEligibleClients], @"Only device analytics enabled -> we have keysync clients (localkeychain for now)");
 419 }
 420
 421 - (void)testHaveEligibleClientsTrust
 422 {
 423     // TrustTopic has no clients requiring iCloudAnalytics currently
 424     SFAnalyticsTopic* trusttopic = [[SFAnalyticsTopic alloc] initWithDictionary:@{} name:@"TrustTopic" samplingRates:@{}];
 425
 426     XCTAssertTrue([trusttopic haveEligibleClients], @"Both analytics enabled -> we have trust clients");
 427
 428     deviceAnalyticsEnabled = NO;
 429     XCTAssertFalse([trusttopic haveEligibleClients], @"Only iCloud analytics enabled -> no trust clients");
 430
 431     iCloudAnalyticsEnabled = NO;
 432     XCTAssertFalse([trusttopic haveEligibleClients], @"Both analytics disabled -> no trust clients");
 433
 434     deviceAnalyticsEnabled = YES;
 435     XCTAssertTrue([trusttopic haveEligibleClients], @"Only device analytics enabled -> we have trust clients");
 436 }
 437
 438 - (void)testLoggingJSONSimple:(BOOL)analyticsEnabled
 439 {
 440     iCloudAnalyticsEnabled = analyticsEnabled;
 441
 442     [_ckksAnalytics logSuccessForEventNamed:@"ckksunittestevent"];
 443     NSDictionary* ckksAttrs = @{@"cattr" : @"cvalue"};
 444     [_ckksAnalytics logHardFailureForEventNamed:@"ckksunittestevent" withAttributes:ckksAttrs];
 445     [_ckksAnalytics logSoftFailureForEventNamed:@"ckksunittestevent" withAttributes:ckksAttrs];
 446     [_sosAnalytics logSuccessForEventNamed:@"unittestevent"];
 447     NSDictionary* utAttrs = @{@"uattr" : @"uvalue"};
 448     [_sosAnalytics logHardFailureForEventNamed:@"unittestevent" withAttributes:utAttrs];
 449     [_sosAnalytics logSoftFailureForEventNamed:@"unittestevent" withAttributes:utAttrs];
 450
 451     NSDictionary* data = [self getJSONDataFromSupd];
 452     [self inspectDataBlobStructure:data];
 453
 454     // TODO: inspect health summaries
 455
 456     if (analyticsEnabled) {
 457         XCTAssertEqual([self failures:data eventType:@"ckksunittestevent" attributes:ckksAttrs class:SFAnalyticsEventClassHardFailure], 1);
 458         XCTAssertEqual([self failures:data eventType:@"ckksunittestevent" attributes:ckksAttrs class:SFAnalyticsEventClassSoftFailure], 1);
 459         XCTAssertEqual([self failures:data eventType:@"unittestevent" attributes:utAttrs class:SFAnalyticsEventClassHardFailure], 1);
 460         XCTAssertEqual([self failures:data eventType:@"unittestevent" attributes:utAttrs class:SFAnalyticsEventClassSoftFailure], 1);
 461
 462         [self checkTotalEventCount:data hard:2 soft:2 accuracy:0];
 463     } else {
 464         // localkeychain requires device analytics only so we still get it
 465         [self checkTotalEventCount:data hard:0 soft:0 accuracy:0 summaries:1];
 466     }
 467 }
 468
 469 - (void)testLoggingJSONSimpleWithiCloudAnalyticsEnabled
 470 {
 471     [self testLoggingJSONSimple:YES];
 472 }
 473
 474 - (void)testLoggingJSONSimpleWithiCloudAnalyticsDisabled
 475 {
 476     [self testLoggingJSONSimple:NO];
 477 }
 478
 479 - (void)testTLSLoggingJSONSimple:(BOOL)analyticsEnabled
 480 {
 481     deviceAnalyticsEnabled = analyticsEnabled;
 482
 483     [_tlsAnalytics logSuccessForEventNamed:@"tlsunittestevent"];
 484     NSDictionary* tlsAttrs = @{@"cattr" : @"cvalue"};
 485     [_tlsAnalytics logHardFailureForEventNamed:@"tlsunittestevent" withAttributes:tlsAttrs];
 486     [_tlsAnalytics logSoftFailureForEventNamed:@"tlsunittestevent" withAttributes:tlsAttrs];
 487
 488     NSDictionary* data = [self getJSONDataFromSupdWithTopic:SFAnalyticsTopicTrust];
 489     [self inspectDataBlobStructure:data forTopic:[[self TrustTopic] splunkTopicName]];
 490
 491     if (analyticsEnabled) {
 492         [self checkTotalEventCount:data hard:1 soft:1 accuracy:0 summaries:(int)[[[self TrustTopic] topicClients] count]];
 493     } else {
 494         [self checkTotalEventCount:data hard:0 soft:0 accuracy:0 summaries:0];
 495     }
 496 }
 497
 498 - (void)testTLSLoggingJSONSimpleWithDeviceAnalyticsEnabled
 499 {
 500     [self testTLSLoggingJSONSimple:YES];
 501 }
 502
 503 - (void)testTLSLoggingJSONSimpleWithDeviceAnalyticsDisabled
 504 {
 505     [self testTLSLoggingJSONSimple:NO];
 506 }
 507
 508 - (void)testMockDiagnosticReportGeneration
 509 {
 510     SFAnalyticsReporter *reporter = mockReporter;
 511
 512     uint8_t report_data[] = {0x00, 0x01, 0x02, 0x03};
 513     NSData *reportData = [[NSData alloc] initWithBytes:report_data length:sizeof(report_data)];
 514     BOOL writtenToLog = YES;
 515     size_t numWrites = 5;
 516     for (size_t i = 0; i < numWrites; i++) {
 517         writtenToLog &= [reporter saveReport:reportData fileName:@"log.txt"];
 518     }
 519
 520     XCTAssertTrue(writtenToLog, "Failed to write to log");
 521     XCTAssertTrue((int)_reporterWrites == (int)numWrites, "Expected %zu report, got %d", numWrites, (int)_reporterWrites);
 522 }
 523
 524 - (void)testSuccessCounts
 525 {
 526     NSString* eventName1 = @"successCountsEvent1";
 527     NSString* eventName2 = @"successCountsEvent2";
 528
 529     for (int idx = 0; idx < 3; ++idx) {
 530         [_ckksAnalytics logSuccessForEventNamed:eventName1];
 531         [_ckksAnalytics logSuccessForEventNamed:eventName2];
 532         [_ckksAnalytics logHardFailureForEventNamed:eventName1 withAttributes:nil];
 533         [_ckksAnalytics logSoftFailureForEventNamed:eventName2 withAttributes:nil];
 534     }
 535     [_ckksAnalytics logSuccessForEventNamed:eventName2];
 536
 537     NSDictionary* data = [self getJSONDataFromSupd];
 538     [self inspectDataBlobStructure:data];
 539
 540     NSDictionary* hs;
 541     for (NSDictionary* event in data[@"events"]) {
 542         if ([event[SFAnalyticsEventType] isEqual:@"ckksHealthSummary"]) {
 543             hs = event;
 544             break;
 545         }
 546     }
 547     XCTAssert(hs);
 548
 549     XCTAssertEqual([hs[SFAnalyticsColumnSuccessCount] integerValue], 7);
 550     XCTAssertEqual([hs[SFAnalyticsColumnHardFailureCount] integerValue], 3);
 551     XCTAssertEqual([hs[SFAnalyticsColumnSoftFailureCount] integerValue], 3);
 552     XCTAssertEqual([hs[[self string:eventName1 item:@"success"]] integerValue], 3);
 553     XCTAssertEqual([hs[[self string:eventName1 item:@"hardfail"]] integerValue], 3);
 554     XCTAssertEqual([hs[[self string:eventName1 item:@"softfail"]] integerValue], 0);
 555     XCTAssertEqual([hs[[self string:eventName2 item:@"success"]] integerValue], 4);
 556     XCTAssertEqual([hs[[self string:eventName2 item:@"hardfail"]] integerValue], 0);
 557     XCTAssertEqual([hs[[self string:eventName2 item:@"softfail"]] integerValue], 3);
 558 }
 559
 560 // There was a failure with thresholds if some, but not all clients exceeded their 'threshold' number of failures,
 561 // causing the addFailures:toUploadRecords:threshold method to crash with out of bounds.
 562 // This is also implicitly tested in testTooManyHardFailures and testTooManyCombinedFailures but I wanted an explicit case.
 563 - (void)testExceedThresholdForOneClientOnly
 564 {
 565     int testAmount = ((int)SFAnalyticsMaxEventsToReport / 4);
 566     for (int idx = 0; idx < testAmount; ++idx) {
 567         [_ckksAnalytics logHardFailureForEventNamed:@"ckkshardfail" withAttributes:nil];
 568         [_ckksAnalytics logSoftFailureForEventNamed:@"ckkssoftfail" withAttributes:nil];
 569     }
 570
 571     [_sosAnalytics logHardFailureForEventNamed:@"soshardfail" withAttributes:nil];
 572     [_sosAnalytics logSoftFailureForEventNamed:@"sossoftfail" withAttributes:nil];
 573
 574     NSDictionary* data = [self getJSONDataFromSupd];
 575     [self inspectDataBlobStructure:data];
 576
 577     [self checkTotalEventCount:data hard:testAmount + 1 soft:testAmount + 1 accuracy:0];
 578
 579     XCTAssertEqual([self failures:data eventType:@"ckkshardfail" attributes:nil class:SFAnalyticsEventClassHardFailure], testAmount);
 580     XCTAssertEqual([self failures:data eventType:@"ckkssoftfail" attributes:nil class:SFAnalyticsEventClassSoftFailure], testAmount);
 581     XCTAssertEqual([self failures:data eventType:@"soshardfail" attributes:nil class:SFAnalyticsEventClassHardFailure], 1);
 582     XCTAssertEqual([self failures:data eventType:@"sossoftfail" attributes:nil class:SFAnalyticsEventClassSoftFailure], 1);
 583 }
 584
 585
 586 // We have so many hard failures they won't fit in the upload buffer
 587 - (void)testTooManyHardFailures
 588 {
 589     NSDictionary* ckksAttrs = @{@"cattr" : @"cvalue"};
 590     NSDictionary* utAttrs = @{@"uattr" : @"uvalue"};
 591     for (int idx = 0; idx < 400; ++idx) {
 592         [_ckksAnalytics logHardFailureForEventNamed:@"ckksunittestfailure" withAttributes:ckksAttrs];
 593         [_ckksAnalytics logHardFailureForEventNamed:@"ckksunittestfailure" withAttributes:ckksAttrs];
 594         [_sosAnalytics logHardFailureForEventNamed:@"utunittestfailure" withAttributes:utAttrs];
 595     }
 596
 597     NSDictionary* data = [self getJSONDataFromSupd];
 598     [self inspectDataBlobStructure:data];
 599
 600     [self checkTotalEventCount:data hard:998 soft:0];
 601     // Based on threshold = records_to_upload/10 with a nice margin
 602     XCTAssertEqualWithAccuracy([self failures:data eventType:@"ckksunittestfailure" attributes:ckksAttrs class:SFAnalyticsEventClassHardFailure], 658, 50);
 603     XCTAssertEqualWithAccuracy([self failures:data eventType:@"utunittestfailure" attributes:utAttrs class:SFAnalyticsEventClassHardFailure], 339, 50);
 604 }
 605
 606 // So many soft failures they won't fit in the buffer
 607 - (void)testTooManySoftFailures
 608 {
 609     NSDictionary* ckksAttrs = @{@"cattr" : @"cvalue"};
 610     NSDictionary* utAttrs = @{@"uattr" : @"uvalue"};
 611     for (int idx = 0; idx < 400; ++idx) {
 612         [_ckksAnalytics logSoftFailureForEventNamed:@"ckksunittestfailure" withAttributes:ckksAttrs];
 613         [_ckksAnalytics logSoftFailureForEventNamed:@"ckksunittestfailure" withAttributes:ckksAttrs];
 614         [_sosAnalytics logSoftFailureForEventNamed:@"utunittestfailure" withAttributes:utAttrs];
 615     }
 616
 617     NSDictionary* data = [self getJSONDataFromSupd];
 618     [self inspectDataBlobStructure:data];
 619
 620     [self checkTotalEventCount:data hard:0 soft:998];
 621     // Based on threshold = records_to_upload/10 with a nice margin
 622     XCTAssertEqualWithAccuracy([self failures:data eventType:@"ckksunittestfailure" attributes:ckksAttrs class:SFAnalyticsEventClassSoftFailure], 665, 50);
 623     XCTAssertEqualWithAccuracy([self failures:data eventType:@"utunittestfailure" attributes:utAttrs class:SFAnalyticsEventClassSoftFailure], 332, 50);
 624 }
 625
 626 - (void)testTooManyCombinedFailures
 627 {
 628     NSDictionary* ckksAttrs = @{@"cattr1" : @"cvalue1", @"cattrthatisalotlongerthanthepreviousone" : @"cvaluethatisalsoalotlongerthantheother"};
 629     NSDictionary* utAttrs = @{@"uattr" : @"uvalue", @"uattrthatisalotlongerthanthepreviousone" : @"uvaluethatisalsoalotlongerthantheother"};
 630     for (int idx = 0; idx < 400; ++idx) {
 631         [_ckksAnalytics logHardFailureForEventNamed:@"ckksunittestfailure" withAttributes:ckksAttrs];
 632         [_ckksAnalytics logSoftFailureForEventNamed:@"ckksunittestfailure" withAttributes:ckksAttrs];
 633         [_sosAnalytics logHardFailureForEventNamed:@"utunittestfailure" withAttributes:utAttrs];
 634         [_sosAnalytics logSoftFailureForEventNamed:@"utunittestfailure" withAttributes:utAttrs];
 635     }
 636
 637     NSDictionary* data = [self getJSONDataFromSupd];
 638     [self inspectDataBlobStructure:data];
 639
 640     [self checkTotalEventCount:data hard:800 soft:198];
 641     // Based on threshold = records_to_upload/10 with a nice margin
 642     XCTAssertEqualWithAccuracy([self failures:data eventType:@"ckksunittestfailure" attributes:ckksAttrs class:SFAnalyticsEventClassHardFailure], 400, 50);
 643     XCTAssertEqualWithAccuracy([self failures:data eventType:@"utunittestfailure" attributes:utAttrs class:SFAnalyticsEventClassHardFailure], 400, 50);
 644     XCTAssertEqualWithAccuracy([self failures:data eventType:@"ckksunittestfailure" attributes:ckksAttrs class:SFAnalyticsEventClassSoftFailure], 100, 50);
 645     XCTAssertEqualWithAccuracy([self failures:data eventType:@"utunittestfailure" attributes:utAttrs class:SFAnalyticsEventClassSoftFailure], 100, 50);
 646 }
 647
 648 // There's an even number of samples
 649 - (void)testSamplesEvenSampleCount
 650 {
 651     NSString* sampleNameEven = @"evenSample";
 652
 653     for (NSNumber* value in @[@36.831855250339714, @90.78721762172914, @49.24392301762506,
 654                               @42.806362283260036, @16.76725375576855, @34.50969130579674,
 655                               @25.956509180834637, @36.8268555935645, @35.54069258036879,
 656                               @7.26364884595062, @45.414180770615395, @5.223213570809022]) {
 657         [_ckksAnalytics logMetric:value withName:sampleNameEven];
 658     }
 659
 660     NSDictionary* data = [self getJSONDataFromSupd];
 661     [self inspectDataBlobStructure:data];
 662
 663     // min, max, avg, med, dev, 1q, 3q
 664     [self checkTotalEventCount:data hard:0 soft:0 accuracy:0];
 665     [self sampleStatisticsInEvents:data[@"events"] name:sampleNameEven values:@[@5.22, @90.78, @35.60, @36.18, @21.52, @21.36, @44.11]];
 666 }
 667
 668 // There are 4*n + 1 samples
 669 - (void)testSamples4n1SampleCount
 670 {
 671     NSString* sampleName4n1 = @"4n1Sample";
 672     for (NSNumber* value in @[@37.76544251068022, @27.36378948426223, @45.10503077614114,
 673                               @43.90635413191473, @54.78709742040113, @52.34879597889124,
 674                               @70.95760312196856, @23.23648158872921, @75.34678687445064,
 675                               @10.723238854026203, @41.98468801166455, @17.074404554908476,
 676                               @94.24252031232739]) {
 677         [_ckksAnalytics logMetric:value withName:sampleName4n1];
 678     }
 679
 680     NSDictionary* data = [self getJSONDataFromSupd];
 681     [self inspectDataBlobStructure:data];
 682
 683     [self checkTotalEventCount:data hard:0 soft:0 accuracy:0];
 684     [self sampleStatisticsInEvents:data[@"events"] name:sampleName4n1 values:@[@10.72, @94.24, @45.76, @43.90, @23.14, @26.33, @58.83]];
 685 }
 686
 687 // There are 4*n + 3 samples
 688 - (void)testSamples4n3SampleCount
 689 {
 690     NSString* sampleName4n3 = @"4n3Sample";
 691
 692     for (NSNumber* value in @[@42.012971885655496, @87.85629592375282, @5.748491212287082,
 693                               @38.451850063872975, @81.96900109690873, @99.83098790545392,
 694                               @80.89400981437815, @5.719237885152143, @1.6740622555032196,
 695                               @14.437000556079038, @29.046050177512395]) {
 696         [_sosAnalytics logMetric:value withName:sampleName4n3];
 697     }
 698
 699     NSDictionary* data = [self getJSONDataFromSupd];
 700     [self inspectDataBlobStructure:data];
 701     [self checkTotalEventCount:data hard:0 soft:0 accuracy:0];
 702
 703     [self sampleStatisticsInEvents:data[@"events"] name:sampleName4n3 values:@[@1.67, @99.83, @44.33, @38.45, @35.28, @7.92, @81.70]];
 704 }
 705
 706 // stddev and quartiles undefined for single sample
 707 - (void)testSamplesSingleSample
 708 {
 709     NSString* sampleName = @"singleSample";
 710
 711     [_ckksAnalytics logMetric:@3.14159 withName:sampleName];
 712
 713     NSDictionary* data = [self getJSONDataFromSupd];
 714     [self inspectDataBlobStructure:data];
 715     [self checkTotalEventCount:data hard:0 soft:0 accuracy:0];
 716
 717     [self sampleStatisticsInEvents:data[@"events"] name:sampleName values:@[@3.14159]];
 718 }
 719
 720 // quartiles meaningless for fewer than 4 samples (but stddev exists)
 721 - (void)testSamplesFewerThanFour
 722 {
 723     NSString* sampleName = @"fewSamples";
 724
 725     [_ckksAnalytics logMetric:@3.14159 withName:sampleName];
 726     [_ckksAnalytics logMetric:@6.28318 withName:sampleName];
 727
 728     NSDictionary* data = [self getJSONDataFromSupd];
 729     [self inspectDataBlobStructure:data];
 730     [self checkTotalEventCount:data hard:0 soft:0 accuracy:0];
 731
 732     [self sampleStatisticsInEvents:data[@"events"] name:sampleName values:@[@3.14, @6.28, @4.71, @4.71, @1.57]];
 733 }
 734
 735 - (void)testSamplesSameNameDifferentSubclass
 736 {
 737     NSString* sampleName = @"differentSubclassSamples";
 738
 739     [_sosAnalytics logMetric:@313.37 withName:sampleName];
 740     [_ckksAnalytics logMetric:@313.37 withName:sampleName];
 741
 742     NSDictionary* data = [self getJSONDataFromSupd];
 743     [self inspectDataBlobStructure:data];
 744     [self checkTotalEventCount:data hard:0 soft:0 accuracy:0];
 745
 746     [self sampleStatisticsInEvents:data[@"events"] name:sampleName values:@[@313.37] amount:2];
 747 }
 748
 749 - (void)testInvalidJSON
 750 {
 751     NSData* bad = [@"let's break supd!" dataUsingEncoding:NSUTF8StringEncoding];
 752     [_ckksAnalytics logHardFailureForEventNamed:@"testEvent" withAttributes:@{ @"dataAttribute" : bad}];
 753
 754     NSDictionary* data = [self getJSONDataFromSupd];
 755     XCTAssertNotNil(data);
 756     XCTAssertNotNil(data[@"events"]);
 757     NSUInteger foundErrorEvents = 0;
 758     for (NSDictionary* event in data[@"events"]) {
 759         if ([event[SFAnalyticsEventType] isEqualToString:SFAnalyticsEventTypeErrorEvent] && [event[SFAnalyticsEventErrorDestription] isEqualToString:@"JSON:testEvent"]) {
 760             ++foundErrorEvents;
 761         }
 762     }
 763     XCTAssertEqual(foundErrorEvents, 1);
 764 }
 765
 766
 767 // TODO
 768 - (void)testGetSysdiagnoseDump
 769 {
 770
 771 }
 772
 773 // TODO (need mock server)
 774 - (void)testSplunkUpload
 775 {
 776
 777 }
 778
 779 // TODO (need mock server)
 780 - (void)testDBIsEmptiedAfterUpload
 781 {
 782
 783 }
 784
 785 @end
 786
 787 #endif  // !TARGET_OS_SIMULATOR