supd/Tests/SupdTests.m

   1 /*
   2  * Copyright (c) 2017-2018 Apple Inc. All Rights Reserved.
   3  *
   4  * @APPLE_LICENSE_HEADER_START@
   5  *
   6  * This file contains Original Code and/or Modifications of Original Code
   7  * as defined in and that are subject to the Apple Public Source License
   8  * Version 2.0 (the 'License'). You may not use this file except in
   9  * compliance with the License. Please obtain a copy of the License at
  10  * http://www.opensource.apple.com/apsl/ and read it before using this
  11  * file.
  12  *
  13  * The Original Code and all software distributed under the License are
  14  * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  15  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  16  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
  17  * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
  18  * Please see the License for the specific language governing rights and
  19  * limitations under the License.
  20  *
  21  * @APPLE_LICENSE_HEADER_END@
  22  */
  23
  24 #import <XCTest/XCTest.h>
  25
  26 // securityuploadd does not do anything or build meaningful code on simulator, so no tests either.
  27 #if TARGET_OS_SIMULATOR
  28
  29 @interface SupdTests : XCTestCase
  30 @end
  31
  32 @implementation SupdTests
  33 @end
  34
  35 #else
  36
  37 #import <OCMock/OCMock.h>
  38 #import "supd.h"
  39 #import <Security/SFAnalytics.h>
  40 #import "SFAnalyticsDefines.h"
  41 #import <CoreFoundation/CFPriv.h>
  42
  43 static NSString* _path;
  44 static NSInteger _testnum;
  45 static NSString* build = NULL;
  46 static NSString* product = NULL;
  47 static NSInteger _reporterWrites;
  48
  49 // MARK: Stub FakeCKKSAnalytics
  50
  51 @interface FakeCKKSAnalytics : SFAnalytics
  52
  53 @end
  54
  55 @implementation FakeCKKSAnalytics
  56
  57 + (NSString*)databasePath
  58 {
  59     return [_path stringByAppendingFormat:@"/ckks_%ld.db", (long)_testnum];
  60 }
  61
  62 @end
  63
  64
  65 // MARK: Stub FakeSOSAnalytics
  66
  67 @interface FakeSOSAnalytics : SFAnalytics
  68
  69 @end
  70
  71 @implementation FakeSOSAnalytics
  72
  73 + (NSString*)databasePath
  74 {
  75     return [_path stringByAppendingFormat:@"/sos_%ld.db", (long)_testnum];
  76 }
  77
  78 @end
  79
  80
  81 // MARK: Stub FakePCSAnalytics
  82
  83 @interface FakePCSAnalytics : SFAnalytics
  84
  85 @end
  86
  87 @implementation FakePCSAnalytics
  88
  89 + (NSString*)databasePath
  90 {
  91     return [_path stringByAppendingFormat:@"/pcs_%ld.db", (long)_testnum];
  92 }
  93
  94 @end
  95
  96 // MARK: Stub FakeTLSAnalytics
  97
  98 @interface FakeTLSAnalytics : SFAnalytics
  99
 100 @end
 101
 102 @implementation FakeTLSAnalytics
 103
 104 + (NSString*)databasePath
 105 {
 106     return [_path stringByAppendingFormat:@"/tls_%ld.db", (long)_testnum];
 107 }
 108
 109 @end
 110
 111 // MARK: Start SupdTests
 112
 113 @interface SupdTests : XCTestCase
 114
 115 @end
 116
 117 @implementation SupdTests {
 118     supd* _supd;
 119     id mockReporter;
 120     FakeCKKSAnalytics* _ckksAnalytics;
 121     FakeSOSAnalytics* _sosAnalytics;
 122     FakePCSAnalytics* _pcsAnalytics;
 123     FakeTLSAnalytics* _tlsAnalytics;
 124 }
 125
 126 // MARK: Test helper methods
 127 - (SFAnalyticsTopic *)keySyncTopic {
 128     for (SFAnalyticsTopic *topic in _supd.analyticsTopics) {
 129         if ([topic.internalTopicName isEqualToString:SFAnalyticsTopicKeySync]) {
 130             return topic;
 131         }
 132     }
 133     return nil;
 134 }
 135
 136 - (SFAnalyticsTopic *)TrustTopic {
 137     for (SFAnalyticsTopic *topic in _supd.analyticsTopics) {
 138         if ([topic.internalTopicName isEqualToString:SFAnalyticsTopicTrust]) {
 139             return topic;
 140         }
 141     }
 142     return nil;
 143 }
 144
 145 - (void)inspectDataBlobStructure:(NSDictionary*)data
 146 {
 147     [self inspectDataBlobStructure:data forTopic:[[self keySyncTopic] splunkTopicName]];
 148 }
 149
 150 - (void)inspectDataBlobStructure:(NSDictionary*)data forTopic:(NSString*)topic
 151 {
 152     if (!data || ![data isKindOfClass:[NSDictionary class]]) {
 153         XCTFail(@"data is an NSDictionary");
 154     }
 155
 156     XCTAssert(_supd.analyticsTopics, @"supd has nonnull topics list");
 157     XCTAssert([[self keySyncTopic] splunkTopicName], @"keysync topic has a splunk name");
 158     XCTAssert([[self TrustTopic] splunkTopicName], @"trust topic has a splunk name");
 159     XCTAssertEqual([data count], 2ul, @"dictionary event and posttime objects");
 160     XCTAssertTrue(data[@"events"] && [data[@"events"] isKindOfClass:[NSArray class]], @"data blob contains an NSArray 'events'");
 161     XCTAssertTrue(data[@"postTime"] && [data[@"postTime"] isKindOfClass:[NSNumber class]], @"data blob contains an NSNumber 'postTime");
 162     NSDate* postTime = [NSDate dateWithTimeIntervalSince1970:[data[@"postTime"] doubleValue]];
 163     XCTAssertTrue([[NSDate date] timeIntervalSinceDate:postTime] < 3, @"postTime is sane");
 164
 165     for (NSDictionary* event in data[@"events"]) {
 166         if ([event isKindOfClass:[NSDictionary class]]) {
 167             NSLog(@"build: \"%@\", eventbuild: \"%@\"", build, event[@"build"]);
 168             XCTAssertEqualObjects(event[@"build"], build, @"event contains correct build string");
 169             XCTAssertEqualObjects(event[@"product"], product, @"event contains correct product string");
 170             XCTAssertTrue([event[@"eventTime"] isKindOfClass:[NSNumber class]], @"event contains an NSNumber 'eventTime");
 171             NSDate* eventTime = [NSDate dateWithTimeIntervalSince1970:[event[@"eventTime"] doubleValue]];
 172             XCTAssertTrue([[NSDate date] timeIntervalSinceDate:eventTime] < 3, @"eventTime is sane");
 173             XCTAssertTrue([event[@"eventType"] isKindOfClass:[NSString class]], @"all events have a type");
 174             XCTAssertEqualObjects(event[@"topic"], topic, @"all events have a topic name");
 175         } else {
 176             XCTFail(@"event %@ is an NSDictionary", event);
 177         }
 178     }
 179 }
 180
 181 - (BOOL)event:(NSDictionary*)event containsAttributes:(NSDictionary*)attrs {
 182     if (!attrs) {
 183         return YES;
 184     }
 185     __block BOOL equal = YES;
 186     [attrs enumerateKeysAndObjectsUsingBlock:^(id  _Nonnull key, id  _Nonnull obj, BOOL * _Nonnull stop) {
 187         equal &= [event[key] isEqualToString:obj];
 188     }];
 189     return equal;
 190 }
 191
 192 - (int)failures:(NSDictionary*)data eventType:(NSString*)type attributes:(NSDictionary*)attrs class:(SFAnalyticsEventClass)class
 193 {
 194     int encountered = 0;
 195     for (NSDictionary* event in data[@"events"]) {
 196         if ([event[@"eventType"] isEqualToString:type] &&
 197             [event[@"eventClass"] isKindOfClass:[NSNumber class]] &&
 198             [event[@"eventClass"] intValue] == class && [self event:event containsAttributes:attrs]) {
 199             ++encountered;
 200         }
 201     }
 202     return encountered;
 203 }
 204
 205 - (void)checkTotalEventCount:(NSDictionary*)data hard:(int)hard soft:(int)soft accuracy:(int)accuracy summaries:(int)summ
 206 {
 207     int hardfound = 0, softfound = 0, summfound = 0;
 208     for (NSDictionary* event in data[@"events"]) {
 209         if ([event[SFAnalyticsEventType] hasSuffix:@"HealthSummary"]) {
 210             ++summfound;
 211         } else if ([event[SFAnalyticsEventClassKey] integerValue] == SFAnalyticsEventClassHardFailure) {
 212             ++hardfound;
 213         } else if ([event[SFAnalyticsEventClassKey] integerValue] == SFAnalyticsEventClassSoftFailure) {
 214             ++softfound;
 215         }
 216     }
 217
 218     XCTAssertLessThanOrEqual(((NSArray*)data[@"events"]).count, 1000ul, @"Total event count fits in alloted data");
 219     XCTAssertEqual(summfound, summ);
 220
 221     // Add customizable fuzziness
 222     XCTAssertEqualWithAccuracy(hardfound, hard, accuracy);
 223     XCTAssertEqualWithAccuracy(softfound, soft, accuracy);
 224 }
 225
 226 - (void)checkTotalEventCount:(NSDictionary*)data hard:(int)hard soft:(int)soft
 227 {
 228     [self checkTotalEventCount:data hard:hard soft:soft accuracy:10 summaries:(int)[[[self keySyncTopic] topicClients] count]];
 229 }
 230
 231 - (void)checkTotalEventCount:(NSDictionary*)data hard:(int)hard soft:(int)soft accuracy:(int)accuracy
 232 {
 233     [self checkTotalEventCount:data hard:hard soft:soft accuracy:accuracy summaries:(int)[[[self keySyncTopic] topicClients] count]];
 234 }
 235
 236 // This is a dumb hack, but inlining stringWithFormat causes the compiler to growl for unknown reasons
 237 - (NSString*)string:(NSString*)name item:(NSString*)item
 238 {
 239     return [NSString stringWithFormat:@"%@-%@", name, item];
 240 }
 241
 242 - (void)sampleStatisticsInEvents:(NSArray*)events name:(NSString*)name values:(NSArray*)values
 243 {
 244     [self sampleStatisticsInEvents:events name:name values:values amount:1];
 245 }
 246
 247 // Usually amount == 1 but for testing sampler with same name in different subclasses this is higher
 248 - (void)sampleStatisticsInEvents:(NSArray*)events name:(NSString*)name values:(NSArray*)values amount:(int)num
 249 {
 250     int found = 0;
 251     for (NSDictionary* event in events) {
 252         if (([values count] == 1 && ![event objectForKey:[NSString stringWithFormat:@"%@", name]]) ||
 253             ([values count] > 1 && ![event objectForKey:[NSString stringWithFormat:@"%@-min", name]])) {
 254             continue;
 255         }
 256
 257         ++found;
 258         if (values.count == 1) {
 259             XCTAssertEqual([event[name] doubleValue], [values[0] doubleValue]);
 260             XCTAssertNil(event[[self string:name item:@"min"]]);
 261             XCTAssertNil(event[[self string:name item:@"max"]]);
 262             XCTAssertNil(event[[self string:name item:@"avg"]]);
 263             XCTAssertNil(event[[self string:name item:@"med"]]);
 264         } else {
 265             XCTAssertEqualWithAccuracy([event[[self string:name item:@"min"]] doubleValue], [values[0] doubleValue], 0.01f);
 266             XCTAssertEqualWithAccuracy([event[[self string:name item:@"max"]] doubleValue], [values[1] doubleValue], 0.01f);
 267             XCTAssertEqualWithAccuracy([event[[self string:name item:@"avg"]] doubleValue], [values[2] doubleValue], 0.01f);
 268             XCTAssertEqualWithAccuracy([event[[self string:name item:@"med"]] doubleValue], [values[3] doubleValue], 0.01f);
 269         }
 270
 271         if (values.count > 4) {
 272             XCTAssertEqualWithAccuracy([event[[self string:name item:@"dev"]] doubleValue], [values[4] doubleValue], 0.01f);
 273         } else {
 274             XCTAssertNil(event[[self string:name item:@"dev"]]);
 275         }
 276
 277         if (values.count > 5) {
 278             XCTAssertEqualWithAccuracy([event[[self string:name item:@"1q"]] doubleValue], [values[5] doubleValue], 0.01f);
 279             XCTAssertEqualWithAccuracy([event[[self string:name item:@"3q"]] doubleValue], [values[6] doubleValue], 0.01f);
 280         } else {
 281             XCTAssertNil(event[[self string:name item:@"1q"]]);
 282             XCTAssertNil(event[[self string:name item:@"3q"]]);
 283         }
 284     }
 285     XCTAssertEqual(found, num);
 286 }
 287
 288 - (NSDictionary*)getJSONDataFromSupd
 289 {
 290     return [self getJSONDataFromSupdWithTopic:SFAnalyticsTopicKeySync];
 291 }
 292
 293 - (NSDictionary*)getJSONDataFromSupdWithTopic:(NSString*)topic
 294 {
 295     dispatch_semaphore_t sema = dispatch_semaphore_create(0);
 296     __block NSDictionary* data;
 297     [_supd createLoggingJSON:YES topic:topic reply:^(NSData *json, NSError *error) {
 298         XCTAssertNil(error);
 299         XCTAssertNotNil(json);
 300         if (!error) {
 301             data = [NSJSONSerialization JSONObjectWithData:json options:0 error:&error];
 302         }
 303         XCTAssertNil(error, @"no error deserializing json: %@", error);
 304         dispatch_semaphore_signal(sema);
 305     }];
 306     if (dispatch_semaphore_wait(sema, dispatch_time(DISPATCH_TIME_NOW, NSEC_PER_SEC * 5)) != 0) {
 307         XCTFail(@"supd returns JSON data in a timely fashion");
 308     }
 309     return data;
 310 }
 311
 312 // MARK: Test administration
 313
 314 + (void)setUp
 315 {
 316     NSError* error;
 317     _path = [NSTemporaryDirectory() stringByAppendingPathComponent:[NSString stringWithFormat:@"%@/", [[NSUUID UUID] UUIDString]]];
 318     [[NSFileManager defaultManager] createDirectoryAtPath:_path
 319                               withIntermediateDirectories:YES
 320                                                attributes:nil
 321                                                     error:&error];
 322     if (error) {
 323         NSLog(@"sad trombone, couldn't create path");
 324     }
 325
 326     NSDictionary *version = CFBridgingRelease(_CFCopySystemVersionDictionary());
 327     if (version) {
 328         build = version[(__bridge NSString *)_kCFSystemVersionBuildVersionKey];
 329         product = version[(__bridge NSString *)_kCFSystemVersionProductNameKey];
 330     } else {
 331         NSLog(@"could not get build version/product, tests should fail");
 332     }
 333 }
 334
 335 - (void)setUp
 336 {
 337     [super setUp];
 338     self.continueAfterFailure = NO;
 339     ++_testnum;
 340
 341     id mockTopic = OCMStrictClassMock([SFAnalyticsTopic class]);
 342     NSString *ckksPath = [_path stringByAppendingFormat:@"/ckks_%ld.db", (long)_testnum];
 343     NSString *sosPath = [_path stringByAppendingFormat:@"/sos_%ld.db", (long)_testnum];
 344     NSString *pcsPath = [_path stringByAppendingFormat:@"/pcs_%ld.db", (long)_testnum];
 345     NSString *tlsPath = [_path stringByAppendingFormat:@"/tls_%ld.db", (long)_testnum];
 346     NSString *signInPath = [_path stringByAppendingFormat:@"/signin_%ld.db", (long)_testnum];
 347     NSString *cloudServicesPath = [_path stringByAppendingFormat:@"/cloudServices_%ld.db", (long)_testnum];
 348     OCMStub([mockTopic databasePathForCKKS]).andReturn(ckksPath);
 349     OCMStub([mockTopic databasePathForSOS]).andReturn(sosPath);
 350     OCMStub([mockTopic databasePathForPCS]).andReturn(pcsPath);
 351     OCMStub([mockTopic databasePathForTLS]).andReturn(tlsPath);
 352     OCMStub([mockTopic databasePathForSignIn]).andReturn(signInPath);
 353     OCMStub([mockTopic databasePathForCloudServices]).andReturn(cloudServicesPath);
 354
 355     // These are not used for testing, but real data can pollute tests so point to empty DBs
 356     NSString *localpath = [_path stringByAppendingFormat:@"/local_empty_%ld.db", (long)_testnum];
 357     NSString *trustPath = [_path stringByAppendingFormat:@"/trust_empty_%ld.db", (long)_testnum];
 358     NSString *trustdhealthPath = [_path stringByAppendingFormat:@"/trustdhealth_empty_%ld.db", (long)_testnum];
 359     OCMStub([mockTopic databasePathForLocal]).andReturn(localpath);
 360     OCMStub([mockTopic databasePathForTrust]).andReturn(trustPath);
 361     OCMStub([mockTopic databasePathForTrustdHealth]).andReturn(trustdhealthPath);
 362
 363     _reporterWrites = 0;
 364     mockReporter = OCMClassMock([SFAnalyticsReporter class]);
 365     OCMStub([mockReporter saveReport:[OCMArg isNotNil] fileName:[OCMArg isNotNil]]).andDo(^(NSInvocation *invocation) {
 366         _reporterWrites++;
 367     }).andReturn(YES);
 368
 369     [supd removeInstance];
 370     _supd = [[supd alloc] initWithReporter:mockReporter];
 371     _ckksAnalytics = [FakeCKKSAnalytics new];
 372     _sosAnalytics = [FakeSOSAnalytics new];
 373     _pcsAnalytics = [FakePCSAnalytics new];
 374     _tlsAnalytics = [FakeTLSAnalytics new];
 375
 376     // Forcibly override analytics flags and enable them by default
 377     deviceAnalyticsOverride = YES;
 378     deviceAnalyticsEnabled = YES;
 379     iCloudAnalyticsOverride = YES;
 380     iCloudAnalyticsEnabled = YES;
 381     runningTests = YES;
 382 }
 383
 384 - (void)tearDown
 385 {
 386
 387     [super tearDown];
 388 }
 389
 390 // MARK: Actual tests
 391
 392 // Note! This test relies on Security being installed because supd reads from a plist in Security.framework
 393 - (void)testSplunkDefaultTopicNameExists
 394 {
 395     XCTAssertNotNil([[self keySyncTopic] splunkTopicName]);
 396 }
 397
 398 // Note! This test relies on Security being installed because supd reads from a plist in Security.framework
 399 - (void)testSplunkDefaultBagURLExists
 400 {
 401     XCTAssertNotNil([[self keySyncTopic] splunkBagURL]);
 402 }
 403
 404 - (void)testHaveEligibleClientsKeySync
 405 {
 406     // KeySyncTopic has no clients requiring deviceAnalytics currently
 407     SFAnalyticsTopic* keytopic = [[SFAnalyticsTopic alloc] initWithDictionary:@{} name:@"KeySyncTopic" samplingRates:@{}];
 408
 409     XCTAssertTrue([keytopic haveEligibleClients], @"Both analytics enabled -> we have keysync clients");
 410
 411     deviceAnalyticsEnabled = NO;
 412     XCTAssertTrue([keytopic haveEligibleClients], @"Only iCloud analytics enabled -> we have keysync clients");
 413
 414     iCloudAnalyticsEnabled = NO;
 415     XCTAssertFalse([keytopic haveEligibleClients], @"Both analytics disabled -> no keysync clients");
 416
 417     deviceAnalyticsEnabled = YES;
 418     XCTAssertTrue([keytopic haveEligibleClients], @"Only device analytics enabled -> we have keysync clients (localkeychain for now)");
 419 }
 420
 421 - (void)testHaveEligibleClientsTrust
 422 {
 423     // TrustTopic has no clients requiring iCloudAnalytics currently
 424     SFAnalyticsTopic* trusttopic = [[SFAnalyticsTopic alloc] initWithDictionary:@{} name:@"TrustTopic" samplingRates:@{}];
 425
 426     XCTAssertTrue([trusttopic haveEligibleClients], @"Both analytics enabled -> we have trust clients");
 427
 428     deviceAnalyticsEnabled = NO;
 429     XCTAssertFalse([trusttopic haveEligibleClients], @"Only iCloud analytics enabled -> no trust clients");
 430
 431     iCloudAnalyticsEnabled = NO;
 432     XCTAssertFalse([trusttopic haveEligibleClients], @"Both analytics disabled -> no trust clients");
 433
 434     deviceAnalyticsEnabled = YES;
 435     XCTAssertTrue([trusttopic haveEligibleClients], @"Only device analytics enabled -> we have trust clients");
 436 }
 437
 438 - (void)testLoggingJSONSimple:(BOOL)analyticsEnabled
 439 {
 440     iCloudAnalyticsEnabled = analyticsEnabled;
 441
 442     [_ckksAnalytics logSuccessForEventNamed:@"ckksunittestevent"];
 443     NSDictionary* ckksAttrs = @{@"cattr" : @"cvalue"};
 444     [_ckksAnalytics logHardFailureForEventNamed:@"ckksunittestevent" withAttributes:ckksAttrs];
 445     [_ckksAnalytics logSoftFailureForEventNamed:@"ckksunittestevent" withAttributes:ckksAttrs];
 446     [_sosAnalytics logSuccessForEventNamed:@"unittestevent"];
 447     NSDictionary* utAttrs = @{@"uattr" : @"uvalue"};
 448     [_sosAnalytics logHardFailureForEventNamed:@"unittestevent" withAttributes:utAttrs];
 449     [_sosAnalytics logSoftFailureForEventNamed:@"unittestevent" withAttributes:utAttrs];
 450
 451     NSDictionary *data =  [self getJSONDataFromSupd];
 452
 453     [self inspectDataBlobStructure:data];
 454
 455     // TODO: inspect health summaries
 456
 457     if (analyticsEnabled) {
 458         XCTAssertEqual([self failures:data eventType:@"ckksunittestevent" attributes:ckksAttrs class:SFAnalyticsEventClassHardFailure], 1);
 459         XCTAssertEqual([self failures:data eventType:@"ckksunittestevent" attributes:ckksAttrs class:SFAnalyticsEventClassSoftFailure], 1);
 460         XCTAssertEqual([self failures:data eventType:@"unittestevent" attributes:utAttrs class:SFAnalyticsEventClassHardFailure], 1);
 461         XCTAssertEqual([self failures:data eventType:@"unittestevent" attributes:utAttrs class:SFAnalyticsEventClassSoftFailure], 1);
 462
 463         [self checkTotalEventCount:data hard:2 soft:2 accuracy:0];
 464     } else {
 465         // localkeychain requires device analytics only so we still get it
 466         [self checkTotalEventCount:data hard:0 soft:0 accuracy:0 summaries:1];
 467     }
 468 }
 469
 470 - (void)testLoggingJSONSimpleWithiCloudAnalyticsEnabled
 471 {
 472     [self testLoggingJSONSimple:YES];
 473 }
 474
 475 - (void)testLoggingJSONSimpleWithiCloudAnalyticsDisabled
 476 {
 477     [self testLoggingJSONSimple:NO];
 478 }
 479
 480 - (void)testTLSLoggingJSONSimple:(BOOL)analyticsEnabled
 481 {
 482     deviceAnalyticsEnabled = analyticsEnabled;
 483
 484     [_tlsAnalytics logSuccessForEventNamed:@"tlsunittestevent"];
 485     NSDictionary* tlsAttrs = @{@"cattr" : @"cvalue"};
 486     [_tlsAnalytics logHardFailureForEventNamed:@"tlsunittestevent" withAttributes:tlsAttrs];
 487     [_tlsAnalytics logSoftFailureForEventNamed:@"tlsunittestevent" withAttributes:tlsAttrs];
 488
 489     NSDictionary* data = [self getJSONDataFromSupdWithTopic:SFAnalyticsTopicTrust];
 490     [self inspectDataBlobStructure:data forTopic:[[self TrustTopic] splunkTopicName]];
 491
 492     if (analyticsEnabled) {
 493         [self checkTotalEventCount:data hard:1 soft:1 accuracy:0 summaries:(int)[[[self TrustTopic] topicClients] count]];
 494     } else {
 495         [self checkTotalEventCount:data hard:0 soft:0 accuracy:0 summaries:0];
 496     }
 497 }
 498
 499 - (void)testTLSLoggingJSONSimpleWithDeviceAnalyticsEnabled
 500 {
 501     [self testTLSLoggingJSONSimple:YES];
 502 }
 503
 504 - (void)testTLSLoggingJSONSimpleWithDeviceAnalyticsDisabled
 505 {
 506     [self testTLSLoggingJSONSimple:NO];
 507 }
 508
 509 - (void)testMockDiagnosticReportGeneration
 510 {
 511     SFAnalyticsReporter *reporter = mockReporter;
 512
 513     uint8_t report_data[] = {0x00, 0x01, 0x02, 0x03};
 514     NSData *reportData = [[NSData alloc] initWithBytes:report_data length:sizeof(report_data)];
 515     BOOL writtenToLog = YES;
 516     size_t numWrites = 5;
 517     for (size_t i = 0; i < numWrites; i++) {
 518         writtenToLog &= [reporter saveReport:reportData fileName:@"log.txt"];
 519     }
 520
 521     XCTAssertTrue(writtenToLog, "Failed to write to log");
 522     XCTAssertTrue((int)_reporterWrites == (int)numWrites, "Expected %zu report, got %d", numWrites, (int)_reporterWrites);
 523 }
 524
 525 - (void)testSuccessCounts
 526 {
 527     NSString* eventName1 = @"successCountsEvent1";
 528     NSString* eventName2 = @"successCountsEvent2";
 529
 530     for (int idx = 0; idx < 3; ++idx) {
 531         [_ckksAnalytics logSuccessForEventNamed:eventName1];
 532         [_ckksAnalytics logSuccessForEventNamed:eventName2];
 533         [_ckksAnalytics logHardFailureForEventNamed:eventName1 withAttributes:nil];
 534         [_ckksAnalytics logSoftFailureForEventNamed:eventName2 withAttributes:nil];
 535     }
 536     [_ckksAnalytics logSuccessForEventNamed:eventName2];
 537
 538     NSDictionary* data = [self getJSONDataFromSupd];
 539     [self inspectDataBlobStructure:data];
 540
 541     NSDictionary* hs;
 542     for (NSDictionary* event in data[@"events"]) {
 543         if ([event[SFAnalyticsEventType] isEqual:@"ckksHealthSummary"]) {
 544             hs = event;
 545             break;
 546         }
 547     }
 548     XCTAssert(hs);
 549
 550     XCTAssertEqual([hs[SFAnalyticsColumnSuccessCount] integerValue], 7);
 551     XCTAssertEqual([hs[SFAnalyticsColumnHardFailureCount] integerValue], 3);
 552     XCTAssertEqual([hs[SFAnalyticsColumnSoftFailureCount] integerValue], 3);
 553     XCTAssertEqual([hs[[self string:eventName1 item:@"success"]] integerValue], 3);
 554     XCTAssertEqual([hs[[self string:eventName1 item:@"hardfail"]] integerValue], 3);
 555     XCTAssertEqual([hs[[self string:eventName1 item:@"softfail"]] integerValue], 0);
 556     XCTAssertEqual([hs[[self string:eventName2 item:@"success"]] integerValue], 4);
 557     XCTAssertEqual([hs[[self string:eventName2 item:@"hardfail"]] integerValue], 0);
 558     XCTAssertEqual([hs[[self string:eventName2 item:@"softfail"]] integerValue], 3);
 559 }
 560
 561 // There was a failure with thresholds if some, but not all clients exceeded their 'threshold' number of failures,
 562 // causing the addFailures:toUploadRecords:threshold method to crash with out of bounds.
 563 // This is also implicitly tested in testTooManyHardFailures and testTooManyCombinedFailures but I wanted an explicit case.
 564 - (void)testExceedThresholdForOneClientOnly
 565 {
 566     int testAmount = ((int)SFAnalyticsMaxEventsToReport / 4);
 567     for (int idx = 0; idx < testAmount; ++idx) {
 568         [_ckksAnalytics logHardFailureForEventNamed:@"ckkshardfail" withAttributes:nil];
 569         [_ckksAnalytics logSoftFailureForEventNamed:@"ckkssoftfail" withAttributes:nil];
 570     }
 571
 572     [_sosAnalytics logHardFailureForEventNamed:@"soshardfail" withAttributes:nil];
 573     [_sosAnalytics logSoftFailureForEventNamed:@"sossoftfail" withAttributes:nil];
 574
 575     NSDictionary* data = [self getJSONDataFromSupd];
 576     [self inspectDataBlobStructure:data];
 577
 578     [self checkTotalEventCount:data hard:testAmount + 1 soft:testAmount + 1 accuracy:0];
 579
 580     XCTAssertEqual([self failures:data eventType:@"ckkshardfail" attributes:nil class:SFAnalyticsEventClassHardFailure], testAmount);
 581     XCTAssertEqual([self failures:data eventType:@"ckkssoftfail" attributes:nil class:SFAnalyticsEventClassSoftFailure], testAmount);
 582     XCTAssertEqual([self failures:data eventType:@"soshardfail" attributes:nil class:SFAnalyticsEventClassHardFailure], 1);
 583     XCTAssertEqual([self failures:data eventType:@"sossoftfail" attributes:nil class:SFAnalyticsEventClassSoftFailure], 1);
 584 }
 585
 586
 587 // We have so many hard failures they won't fit in the upload buffer
 588 - (void)testTooManyHardFailures
 589 {
 590     NSDictionary* ckksAttrs = @{@"cattr" : @"cvalue"};
 591     NSDictionary* utAttrs = @{@"uattr" : @"uvalue"};
 592     for (int idx = 0; idx < 400; ++idx) {
 593         [_ckksAnalytics logHardFailureForEventNamed:@"ckksunittestfailure" withAttributes:ckksAttrs];
 594         [_ckksAnalytics logHardFailureForEventNamed:@"ckksunittestfailure" withAttributes:ckksAttrs];
 595         [_sosAnalytics logHardFailureForEventNamed:@"utunittestfailure" withAttributes:utAttrs];
 596     }
 597
 598     NSDictionary* data = [self getJSONDataFromSupd];
 599     [self inspectDataBlobStructure:data];
 600
 601     [self checkTotalEventCount:data hard:998 soft:0];
 602     // Based on threshold = records_to_upload/10 with a nice margin
 603     XCTAssertEqualWithAccuracy([self failures:data eventType:@"ckksunittestfailure" attributes:ckksAttrs class:SFAnalyticsEventClassHardFailure], 658, 50);
 604     XCTAssertEqualWithAccuracy([self failures:data eventType:@"utunittestfailure" attributes:utAttrs class:SFAnalyticsEventClassHardFailure], 339, 50);
 605 }
 606
 607 // So many soft failures they won't fit in the buffer
 608 - (void)testTooManySoftFailures
 609 {
 610     NSDictionary* ckksAttrs = @{@"cattr" : @"cvalue"};
 611     NSDictionary* utAttrs = @{@"uattr" : @"uvalue"};
 612     for (int idx = 0; idx < 400; ++idx) {
 613         [_ckksAnalytics logSoftFailureForEventNamed:@"ckksunittestfailure" withAttributes:ckksAttrs];
 614         [_ckksAnalytics logSoftFailureForEventNamed:@"ckksunittestfailure" withAttributes:ckksAttrs];
 615         [_sosAnalytics logSoftFailureForEventNamed:@"utunittestfailure" withAttributes:utAttrs];
 616     }
 617
 618     NSDictionary* data = [self getJSONDataFromSupd];
 619     [self inspectDataBlobStructure:data];
 620
 621     [self checkTotalEventCount:data hard:0 soft:998];
 622     // Based on threshold = records_to_upload/10 with a nice margin
 623     XCTAssertEqualWithAccuracy([self failures:data eventType:@"ckksunittestfailure" attributes:ckksAttrs class:SFAnalyticsEventClassSoftFailure], 665, 50);
 624     XCTAssertEqualWithAccuracy([self failures:data eventType:@"utunittestfailure" attributes:utAttrs class:SFAnalyticsEventClassSoftFailure], 332, 50);
 625 }
 626
 627 - (void)testTooManyCombinedFailures
 628 {
 629     NSDictionary* ckksAttrs = @{@"cattr1" : @"cvalue1", @"cattrthatisalotlongerthanthepreviousone" : @"cvaluethatisalsoalotlongerthantheother"};
 630     NSDictionary* utAttrs = @{@"uattr" : @"uvalue", @"uattrthatisalotlongerthanthepreviousone" : @"uvaluethatisalsoalotlongerthantheother"};
 631     for (int idx = 0; idx < 400; ++idx) {
 632         [_ckksAnalytics logHardFailureForEventNamed:@"ckksunittestfailure" withAttributes:ckksAttrs];
 633         [_ckksAnalytics logSoftFailureForEventNamed:@"ckksunittestfailure" withAttributes:ckksAttrs];
 634         [_sosAnalytics logHardFailureForEventNamed:@"utunittestfailure" withAttributes:utAttrs];
 635         [_sosAnalytics logSoftFailureForEventNamed:@"utunittestfailure" withAttributes:utAttrs];
 636     }
 637
 638     NSDictionary* data = [self getJSONDataFromSupd];
 639     [self inspectDataBlobStructure:data];
 640
 641     [self checkTotalEventCount:data hard:800 soft:198];
 642     // Based on threshold = records_to_upload/10 with a nice margin
 643     XCTAssertEqualWithAccuracy([self failures:data eventType:@"ckksunittestfailure" attributes:ckksAttrs class:SFAnalyticsEventClassHardFailure], 400, 50);
 644     XCTAssertEqualWithAccuracy([self failures:data eventType:@"utunittestfailure" attributes:utAttrs class:SFAnalyticsEventClassHardFailure], 400, 50);
 645     XCTAssertEqualWithAccuracy([self failures:data eventType:@"ckksunittestfailure" attributes:ckksAttrs class:SFAnalyticsEventClassSoftFailure], 100, 50);
 646     XCTAssertEqualWithAccuracy([self failures:data eventType:@"utunittestfailure" attributes:utAttrs class:SFAnalyticsEventClassSoftFailure], 100, 50);
 647 }
 648
 649 // There's an even number of samples
 650 - (void)testSamplesEvenSampleCount
 651 {
 652     NSString* sampleNameEven = @"evenSample";
 653
 654     for (NSNumber* value in @[@36.831855250339714, @90.78721762172914, @49.24392301762506,
 655                               @42.806362283260036, @16.76725375576855, @34.50969130579674,
 656                               @25.956509180834637, @36.8268555935645, @35.54069258036879,
 657                               @7.26364884595062, @45.414180770615395, @5.223213570809022]) {
 658         [_ckksAnalytics logMetric:value withName:sampleNameEven];
 659     }
 660
 661     NSDictionary* data = [self getJSONDataFromSupd];
 662     [self inspectDataBlobStructure:data];
 663
 664     // min, max, avg, med, dev, 1q, 3q
 665     [self checkTotalEventCount:data hard:0 soft:0 accuracy:0];
 666     [self sampleStatisticsInEvents:data[@"events"] name:sampleNameEven values:@[@5.22, @90.78, @35.60, @36.18, @21.52, @21.36, @44.11]];
 667 }
 668
 669 // There are 4*n + 1 samples
 670 - (void)testSamples4n1SampleCount
 671 {
 672     NSString* sampleName4n1 = @"4n1Sample";
 673     for (NSNumber* value in @[@37.76544251068022, @27.36378948426223, @45.10503077614114,
 674                               @43.90635413191473, @54.78709742040113, @52.34879597889124,
 675                               @70.95760312196856, @23.23648158872921, @75.34678687445064,
 676                               @10.723238854026203, @41.98468801166455, @17.074404554908476,
 677                               @94.24252031232739]) {
 678         [_ckksAnalytics logMetric:value withName:sampleName4n1];
 679     }
 680
 681     NSDictionary* data = [self getJSONDataFromSupd];
 682     [self inspectDataBlobStructure:data];
 683
 684     [self checkTotalEventCount:data hard:0 soft:0 accuracy:0];
 685     [self sampleStatisticsInEvents:data[@"events"] name:sampleName4n1 values:@[@10.72, @94.24, @45.76, @43.90, @23.14, @26.33, @58.83]];
 686 }
 687
 688 // There are 4*n + 3 samples
 689 - (void)testSamples4n3SampleCount
 690 {
 691     NSString* sampleName4n3 = @"4n3Sample";
 692
 693     for (NSNumber* value in @[@42.012971885655496, @87.85629592375282, @5.748491212287082,
 694                               @38.451850063872975, @81.96900109690873, @99.83098790545392,
 695                               @80.89400981437815, @5.719237885152143, @1.6740622555032196,
 696                               @14.437000556079038, @29.046050177512395]) {
 697         [_sosAnalytics logMetric:value withName:sampleName4n3];
 698     }
 699
 700     NSDictionary* data = [self getJSONDataFromSupd];
 701     [self inspectDataBlobStructure:data];
 702     [self checkTotalEventCount:data hard:0 soft:0 accuracy:0];
 703
 704     [self sampleStatisticsInEvents:data[@"events"] name:sampleName4n3 values:@[@1.67, @99.83, @44.33, @38.45, @35.28, @7.92, @81.70]];
 705 }
 706
 707 // stddev and quartiles undefined for single sample
 708 - (void)testSamplesSingleSample
 709 {
 710     NSString* sampleName = @"singleSample";
 711
 712     [_ckksAnalytics logMetric:@3.14159 withName:sampleName];
 713
 714     NSDictionary* data = [self getJSONDataFromSupd];
 715     [self inspectDataBlobStructure:data];
 716     [self checkTotalEventCount:data hard:0 soft:0 accuracy:0];
 717
 718     [self sampleStatisticsInEvents:data[@"events"] name:sampleName values:@[@3.14159]];
 719 }
 720
 721 // quartiles meaningless for fewer than 4 samples (but stddev exists)
 722 - (void)testSamplesFewerThanFour
 723 {
 724     NSString* sampleName = @"fewSamples";
 725
 726     [_ckksAnalytics logMetric:@3.14159 withName:sampleName];
 727     [_ckksAnalytics logMetric:@6.28318 withName:sampleName];
 728
 729     NSDictionary* data = [self getJSONDataFromSupd];
 730     [self inspectDataBlobStructure:data];
 731     [self checkTotalEventCount:data hard:0 soft:0 accuracy:0];
 732
 733     [self sampleStatisticsInEvents:data[@"events"] name:sampleName values:@[@3.14, @6.28, @4.71, @4.71, @1.57]];
 734 }
 735
 736 - (void)testSamplesSameNameDifferentSubclass
 737 {
 738     NSString* sampleName = @"differentSubclassSamples";
 739
 740     [_sosAnalytics logMetric:@313.37 withName:sampleName];
 741     [_ckksAnalytics logMetric:@313.37 withName:sampleName];
 742
 743     NSDictionary* data = [self getJSONDataFromSupd];
 744     [self inspectDataBlobStructure:data];
 745     [self checkTotalEventCount:data hard:0 soft:0 accuracy:0];
 746
 747     [self sampleStatisticsInEvents:data[@"events"] name:sampleName values:@[@313.37] amount:2];
 748 }
 749
 750 - (void)testInvalidJSON
 751 {
 752     NSData* bad = [@"let's break supd!" dataUsingEncoding:NSUTF8StringEncoding];
 753     [_ckksAnalytics logHardFailureForEventNamed:@"testEvent" withAttributes:@{ @"dataAttribute" : bad}];
 754
 755     NSDictionary* data = [self getJSONDataFromSupd];
 756     XCTAssertNotNil(data);
 757     XCTAssertNotNil(data[@"events"]);
 758     NSUInteger foundErrorEvents = 0;
 759     for (NSDictionary* event in data[@"events"]) {
 760         if ([event[SFAnalyticsEventType] isEqualToString:SFAnalyticsEventTypeErrorEvent] && [event[SFAnalyticsEventErrorDestription] isEqualToString:@"JSON:testEvent"]) {
 761             ++foundErrorEvents;
 762         }
 763     }
 764     XCTAssertEqual(foundErrorEvents, 1);
 765 }
 766
 767 - (void)testUploadSizeLimits
 768 {
 769     SFAnalyticsTopic *trustTopic = [self TrustTopic];
 770     XCTAssertEqual(1000000, trustTopic.uploadSizeLimit);
 771
 772     SFAnalyticsTopic *keySyncTopic = [self keySyncTopic];
 773     XCTAssertEqual(1000000, keySyncTopic.uploadSizeLimit);
 774 }
 775
 776 - (NSArray<NSDictionary *> *)createRandomEventList:(size_t)count
 777 {
 778     NSMutableArray<NSDictionary *> *eventSet = [[NSMutableArray<NSDictionary *> alloc] init];
 779
 780     const size_t dataSize = 100;
 781     uint8_t backingBuffer[dataSize] = {};
 782     for (size_t i = 0; i < count; i++) {
 783         NSData *data = [[NSData alloc] initWithBytes:backingBuffer length:dataSize];
 784         NSDictionary *entry = @{@"key" : [data base64EncodedStringWithOptions:0]};
 785         [eventSet addObject:entry];
 786     }
 787
 788     return eventSet;
 789 }
 790
 791 - (void)testCreateLoggingJSON
 792 {
 793     NSArray<NSDictionary *> *summaries = [self createRandomEventList:5];
 794     NSArray<NSDictionary *> *failures = [self createRandomEventList:100];
 795     NSMutableArray<NSDictionary *> *visitedEvents = [[NSMutableArray<NSDictionary *> alloc] init];
 796
 797     SFAnalyticsTopic *topic = [self TrustTopic];
 798     const size_t sizeLimit = 10000; // total size of the encoded data
 799     topic.uploadSizeLimit = sizeLimit;
 800
 801     NSError *error = nil;
 802     NSArray<NSDictionary *> *eventSet = [topic createChunkedLoggingJSON:summaries failures:failures error:&error];
 803     XCTAssertNil(error);
 804
 805     for (NSDictionary *event in eventSet) {
 806         XCTAssertNotNil([event objectForKey:@"events"]);
 807         XCTAssertNotNil([event objectForKey:SFAnalyticsPostTime]);
 808         NSArray *events = [event objectForKey:@"events"];
 809         for (NSDictionary *summary in summaries) {
 810             BOOL foundSummary = NO;
 811             for (NSDictionary *innerEvent in events) {
 812                 if ([summary isEqualToDictionary:innerEvent]) {
 813                     foundSummary = YES;
 814                     break;
 815                 }
 816             }
 817             XCTAssertTrue(foundSummary);
 818         }
 819
 820         // Record the events we've seen so far
 821         for (NSDictionary *innerEvent in events) {
 822             [visitedEvents addObject:innerEvent];
 823         }
 824     }
 825
 826     // Check that each summary and failure is in the visitedEvents
 827     for (NSDictionary *summary in summaries) {
 828         BOOL foundSummary = NO;
 829         for (NSDictionary *innerEvent in visitedEvents) {
 830             if ([summary isEqualToDictionary:innerEvent]) {
 831                 foundSummary = YES;
 832                 break;
 833             }
 834         }
 835         XCTAssertTrue(foundSummary);
 836     }
 837     for (NSDictionary *failure in failures) {
 838         BOOL foundFailure = NO;
 839         for (NSDictionary *innerEvent in visitedEvents) {
 840             if ([failure isEqualToDictionary:innerEvent]) {
 841                 foundFailure = YES;
 842                 break;
 843             }
 844         }
 845         XCTAssertTrue(foundFailure);
 846     }
 847 }
 848
 849 - (void)testEventSetChunking
 850 {
 851     NSArray<NSDictionary *> *eventSet = [self createRandomEventList:100];
 852     SFAnalyticsTopic *topic = [self TrustTopic];
 853
 854     const size_t sizeLimit = 10000; // total size of the encoded data
 855     size_t encodedEventSize = [topic serializedEventSize:eventSet[0] error:nil];
 856     topic.uploadSizeLimit = sizeLimit; // fix the upload limit
 857
 858     // Chunk up the set, assuming that each chunk already has one event in it.
 859     // In practice, this is the health summary.
 860     NSError *error = nil;
 861     NSArray<NSArray *> *chunkedEvents = [topic chunkFailureSet:(sizeLimit - encodedEventSize) events:eventSet error:nil];
 862     XCTAssertNil(error);
 863
 864     // There should be two resulting chunks, since the set of chunks overflows.
 865     XCTAssertEqual(2, [chunkedEvents count]);
 866 }
 867
 868 // TODO
 869 - (void)testGetSysdiagnoseDump
 870 {
 871
 872 }
 873
 874 // TODO (need mock server)
 875 - (void)testSplunkUpload
 876 {
 877
 878 }
 879
 880 // TODO (need mock server)
 881 - (void)testDBIsEmptiedAfterUpload
 882 {
 883
 884 }
 885
 886 @end
 887
 888 #endif  // !TARGET_OS_SIMULATOR