supd/Tests/SupdTests.m

   1 /*
   2  * Copyright (c) 2017-2018 Apple Inc. All Rights Reserved.
   3  *
   4  * @APPLE_LICENSE_HEADER_START@
   5  *
   6  * This file contains Original Code and/or Modifications of Original Code
   7  * as defined in and that are subject to the Apple Public Source License
   8  * Version 2.0 (the 'License'). You may not use this file except in
   9  * compliance with the License. Please obtain a copy of the License at
  10  * http://www.opensource.apple.com/apsl/ and read it before using this
  11  * file.
  12  *
  13  * The Original Code and all software distributed under the License are
  14  * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  15  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  16  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
  17  * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
  18  * Please see the License for the specific language governing rights and
  19  * limitations under the License.
  20  *
  21  * @APPLE_LICENSE_HEADER_END@
  22  */
  23
  24 #import <XCTest/XCTest.h>
  25
  26 // securityuploadd does not do anything or build meaningful code on simulator, so no tests either.
  27 #if TARGET_OS_SIMULATOR
  28
  29 @interface SupdTests : XCTestCase
  30 @end
  31
  32 @implementation SupdTests
  33 @end
  34
  35 #else
  36
  37 #import <OCMock/OCMock.h>
  38 #import "supd.h"
  39 #import <Security/SFAnalytics.h>
  40 #import "SFAnalyticsDefines.h"
  41 #import <CoreFoundation/CFPriv.h>
  42
  43 static NSString* _path;
  44 static NSInteger _testnum;
  45 static NSString* build = NULL;
  46 static NSString* product = NULL;
  47 static NSInteger _reporterWrites;
  48
  49 // MARK: Stub FakeCKKSAnalytics
  50
  51 @interface FakeCKKSAnalytics : SFAnalytics
  52
  53 @end
  54
  55 @implementation FakeCKKSAnalytics
  56
  57 + (NSString*)databasePath
  58 {
  59     return [_path stringByAppendingFormat:@"/ckks_%ld.db", (long)_testnum];
  60 }
  61
  62 @end
  63
  64
  65 // MARK: Stub FakeSOSAnalytics
  66
  67 @interface FakeSOSAnalytics : SFAnalytics
  68
  69 @end
  70
  71 @implementation FakeSOSAnalytics
  72
  73 + (NSString*)databasePath
  74 {
  75     return [_path stringByAppendingFormat:@"/sos_%ld.db", (long)_testnum];
  76 }
  77
  78 @end
  79
  80
  81 // MARK: Stub FakePCSAnalytics
  82
  83 @interface FakePCSAnalytics : SFAnalytics
  84
  85 @end
  86
  87 @implementation FakePCSAnalytics
  88
  89 + (NSString*)databasePath
  90 {
  91     return [_path stringByAppendingFormat:@"/pcs_%ld.db", (long)_testnum];
  92 }
  93
  94 @end
  95
  96 // MARK: Stub FakeTLSAnalytics
  97
  98 @interface FakeTLSAnalytics : SFAnalytics
  99
 100 @end
 101
 102 @implementation FakeTLSAnalytics
 103
 104 + (NSString*)databasePath
 105 {
 106     return [_path stringByAppendingFormat:@"/tls_%ld.db", (long)_testnum];
 107 }
 108
 109 @end
 110
 111 // MARK: Start SupdTests
 112
 113 @interface SupdTests : XCTestCase
 114
 115 @end
 116
 117 @implementation SupdTests {
 118     supd* _supd;
 119     id mockReporter;
 120     FakeCKKSAnalytics* _ckksAnalytics;
 121     FakeSOSAnalytics* _sosAnalytics;
 122     FakePCSAnalytics* _pcsAnalytics;
 123     FakeTLSAnalytics* _tlsAnalytics;
 124 }
 125
 126 // MARK: Test helper methods
 127 - (SFAnalyticsTopic *)keySyncTopic {
 128     for (SFAnalyticsTopic *topic in _supd.analyticsTopics) {
 129         if ([topic.internalTopicName isEqualToString:SFAnalyticsTopicKeySync]) {
 130             return topic;
 131         }
 132     }
 133     return nil;
 134 }
 135
 136 - (SFAnalyticsTopic *)TrustTopic {
 137     for (SFAnalyticsTopic *topic in _supd.analyticsTopics) {
 138         if ([topic.internalTopicName isEqualToString:SFAnalyticsTopicTrust]) {
 139             return topic;
 140         }
 141     }
 142     return nil;
 143 }
 144
 145 - (void)inspectDataBlobStructure:(NSDictionary*)data
 146 {
 147     [self inspectDataBlobStructure:data forTopic:[[self keySyncTopic] splunkTopicName]];
 148 }
 149
 150 - (void)inspectDataBlobStructure:(NSDictionary*)data forTopic:(NSString*)topic
 151 {
 152     if (!data || ![data isKindOfClass:[NSDictionary class]]) {
 153         XCTFail(@"data is an NSDictionary");
 154     }
 155
 156     XCTAssert(_supd.analyticsTopics, @"supd has nonnull topics list");
 157     XCTAssert([[self keySyncTopic] splunkTopicName], @"keysync topic has a splunk name");
 158     XCTAssert([[self TrustTopic] splunkTopicName], @"trust topic has a splunk name");
 159     XCTAssertEqual([data count], 2ul, @"dictionary event and posttime objects");
 160     XCTAssertTrue(data[@"events"] && [data[@"events"] isKindOfClass:[NSArray class]], @"data blob contains an NSArray 'events'");
 161     XCTAssertTrue(data[@"postTime"] && [data[@"postTime"] isKindOfClass:[NSNumber class]], @"data blob contains an NSNumber 'postTime");
 162     NSDate* postTime = [NSDate dateWithTimeIntervalSince1970:[data[@"postTime"] doubleValue]];
 163     XCTAssertTrue([[NSDate date] timeIntervalSinceDate:postTime] < 3, @"postTime is sane");
 164
 165     for (NSDictionary* event in data[@"events"]) {
 166         if ([event isKindOfClass:[NSDictionary class]]) {
 167             NSLog(@"build: \"%@\", eventbuild: \"%@\"", build, event[@"build"]);
 168             XCTAssertEqualObjects(event[@"build"], build, @"event contains correct build string");
 169             XCTAssertEqualObjects(event[@"product"], product, @"event contains correct product string");
 170             XCTAssertTrue([event[@"eventTime"] isKindOfClass:[NSNumber class]], @"event contains an NSNumber 'eventTime");
 171             NSDate* eventTime = [NSDate dateWithTimeIntervalSince1970:[event[@"eventTime"] doubleValue]];
 172             XCTAssertTrue([[NSDate date] timeIntervalSinceDate:eventTime] < 3, @"eventTime is sane");
 173             XCTAssertTrue([event[@"eventType"] isKindOfClass:[NSString class]], @"all events have a type");
 174             XCTAssertEqualObjects(event[@"topic"], topic, @"all events have a topic name");
 175         } else {
 176             XCTFail(@"event %@ is an NSDictionary", event);
 177         }
 178     }
 179 }
 180
 181 - (BOOL)event:(NSDictionary*)event containsAttributes:(NSDictionary*)attrs {
 182     if (!attrs) {
 183         return YES;
 184     }
 185     __block BOOL equal = YES;
 186     [attrs enumerateKeysAndObjectsUsingBlock:^(id  _Nonnull key, id  _Nonnull obj, BOOL * _Nonnull stop) {
 187         equal &= [event[key] isEqualToString:obj];
 188     }];
 189     return equal;
 190 }
 191
 192 - (int)failures:(NSDictionary*)data eventType:(NSString*)type attributes:(NSDictionary*)attrs class:(SFAnalyticsEventClass)class
 193 {
 194     int encountered = 0;
 195     for (NSDictionary* event in data[@"events"]) {
 196         if ([event[@"eventType"] isEqualToString:type] &&
 197             [event[@"eventClass"] isKindOfClass:[NSNumber class]] &&
 198             [event[@"eventClass"] intValue] == class && [self event:event containsAttributes:attrs]) {
 199             ++encountered;
 200         }
 201     }
 202     return encountered;
 203 }
 204
 205 - (void)checkTotalEventCount:(NSDictionary*)data hard:(int)hard soft:(int)soft accuracy:(int)accuracy summaries:(int)summ
 206 {
 207     int hardfound = 0, softfound = 0, summfound = 0;
 208     for (NSDictionary* event in data[@"events"]) {
 209         if ([event[SFAnalyticsEventType] hasSuffix:@"HealthSummary"]) {
 210             ++summfound;
 211         } else if ([event[SFAnalyticsEventClassKey] integerValue] == SFAnalyticsEventClassHardFailure) {
 212             ++hardfound;
 213         } else if ([event[SFAnalyticsEventClassKey] integerValue] == SFAnalyticsEventClassSoftFailure) {
 214             ++softfound;
 215         }
 216     }
 217
 218     XCTAssertLessThanOrEqual(((NSArray*)data[@"events"]).count, 1000ul, @"Total event count fits in alloted data");
 219     XCTAssertEqual(summfound, summ);
 220
 221     // Add customizable fuzziness
 222     XCTAssertEqualWithAccuracy(hardfound, hard, accuracy);
 223     XCTAssertEqualWithAccuracy(softfound, soft, accuracy);
 224 }
 225
 226 - (void)checkTotalEventCount:(NSDictionary*)data hard:(int)hard soft:(int)soft
 227 {
 228     [self checkTotalEventCount:data hard:hard soft:soft accuracy:10 summaries:(int)[[[self keySyncTopic] topicClients] count]];
 229 }
 230
 231 - (void)checkTotalEventCount:(NSDictionary*)data hard:(int)hard soft:(int)soft accuracy:(int)accuracy
 232 {
 233     [self checkTotalEventCount:data hard:hard soft:soft accuracy:accuracy summaries:(int)[[[self keySyncTopic] topicClients] count]];
 234 }
 235
 236 // This is a dumb hack, but inlining stringWithFormat causes the compiler to growl for unknown reasons
 237 - (NSString*)string:(NSString*)name item:(NSString*)item
 238 {
 239     return [NSString stringWithFormat:@"%@-%@", name, item];
 240 }
 241
 242 - (void)sampleStatisticsInEvents:(NSArray*)events name:(NSString*)name values:(NSArray*)values
 243 {
 244     [self sampleStatisticsInEvents:events name:name values:values amount:1];
 245 }
 246
 247 // Usually amount == 1 but for testing sampler with same name in different subclasses this is higher
 248 - (void)sampleStatisticsInEvents:(NSArray*)events name:(NSString*)name values:(NSArray*)values amount:(int)num
 249 {
 250     int found = 0;
 251     for (NSDictionary* event in events) {
 252         if (([values count] == 1 && ![event objectForKey:[NSString stringWithFormat:@"%@", name]]) ||
 253             ([values count] > 1 && ![event objectForKey:[NSString stringWithFormat:@"%@-min", name]])) {
 254             continue;
 255         }
 256
 257         ++found;
 258         if (values.count == 1) {
 259             XCTAssertEqual([event[name] doubleValue], [values[0] doubleValue]);
 260             XCTAssertNil(event[[self string:name item:@"min"]]);
 261             XCTAssertNil(event[[self string:name item:@"max"]]);
 262             XCTAssertNil(event[[self string:name item:@"avg"]]);
 263             XCTAssertNil(event[[self string:name item:@"med"]]);
 264         } else {
 265             XCTAssertEqualWithAccuracy([event[[self string:name item:@"min"]] doubleValue], [values[0] doubleValue], 0.01f);
 266             XCTAssertEqualWithAccuracy([event[[self string:name item:@"max"]] doubleValue], [values[1] doubleValue], 0.01f);
 267             XCTAssertEqualWithAccuracy([event[[self string:name item:@"avg"]] doubleValue], [values[2] doubleValue], 0.01f);
 268             XCTAssertEqualWithAccuracy([event[[self string:name item:@"med"]] doubleValue], [values[3] doubleValue], 0.01f);
 269         }
 270
 271         if (values.count > 4) {
 272             XCTAssertEqualWithAccuracy([event[[self string:name item:@"dev"]] doubleValue], [values[4] doubleValue], 0.01f);
 273         } else {
 274             XCTAssertNil(event[[self string:name item:@"dev"]]);
 275         }
 276
 277         if (values.count > 5) {
 278             XCTAssertEqualWithAccuracy([event[[self string:name item:@"1q"]] doubleValue], [values[5] doubleValue], 0.01f);
 279             XCTAssertEqualWithAccuracy([event[[self string:name item:@"3q"]] doubleValue], [values[6] doubleValue], 0.01f);
 280         } else {
 281             XCTAssertNil(event[[self string:name item:@"1q"]]);
 282             XCTAssertNil(event[[self string:name item:@"3q"]]);
 283         }
 284     }
 285     XCTAssertEqual(found, num);
 286 }
 287
 288 - (NSDictionary*)getJSONDataFromSupd
 289 {
 290     return [self getJSONDataFromSupdWithTopic:SFAnalyticsTopicKeySync];
 291 }
 292
 293 - (NSDictionary*)getJSONDataFromSupdWithTopic:(NSString*)topic
 294 {
 295     dispatch_semaphore_t sema = dispatch_semaphore_create(0);
 296     __block NSDictionary* data;
 297     [_supd createLoggingJSON:YES topic:topic reply:^(NSData *json, NSError *error) {
 298         XCTAssertNil(error);
 299         XCTAssertNotNil(json);
 300         if (!error) {
 301             data = [NSJSONSerialization JSONObjectWithData:json options:0 error:&error];
 302         }
 303         XCTAssertNil(error, @"no error deserializing json: %@", error);
 304         dispatch_semaphore_signal(sema);
 305     }];
 306     if (dispatch_semaphore_wait(sema, dispatch_time(DISPATCH_TIME_NOW, NSEC_PER_SEC * 5)) != 0) {
 307         XCTFail(@"supd returns JSON data in a timely fashion");
 308     }
 309     return data;
 310 }
 311
 312 // MARK: Test administration
 313
 314 + (void)setUp
 315 {
 316     NSError* error;
 317     _path = [NSTemporaryDirectory() stringByAppendingPathComponent:[NSString stringWithFormat:@"%@/", [[NSUUID UUID] UUIDString]]];
 318     [[NSFileManager defaultManager] createDirectoryAtPath:_path
 319                               withIntermediateDirectories:YES
 320                                                attributes:nil
 321                                                     error:&error];
 322     if (error) {
 323         NSLog(@"sad trombone, couldn't create path");
 324     }
 325
 326     NSDictionary *version = CFBridgingRelease(_CFCopySystemVersionDictionary());
 327     if (version) {
 328         build = version[(__bridge NSString *)_kCFSystemVersionBuildVersionKey];
 329         product = version[(__bridge NSString *)_kCFSystemVersionProductNameKey];
 330     } else {
 331         NSLog(@"could not get build version/product, tests should fail");
 332     }
 333 }
 334
 335 - (void)setUp
 336 {
 337     [super setUp];
 338     self.continueAfterFailure = NO;
 339     ++_testnum;
 340
 341     id mockTopic = OCMStrictClassMock([SFAnalyticsTopic class]);
 342     NSString *ckksPath = [_path stringByAppendingFormat:@"/ckks_%ld.db", (long)_testnum];
 343     NSString *sosPath = [_path stringByAppendingFormat:@"/sos_%ld.db", (long)_testnum];
 344     NSString *pcsPath = [_path stringByAppendingFormat:@"/pcs_%ld.db", (long)_testnum];
 345     NSString *tlsPath = [_path stringByAppendingFormat:@"/tls_%ld.db", (long)_testnum];
 346     NSString *signInPath = [_path stringByAppendingFormat:@"/signin_%ld.db", (long)_testnum];
 347     NSString *cloudServicesPath = [_path stringByAppendingFormat:@"/cloudServices_%ld.db", (long)_testnum];
 348     OCMStub([mockTopic databasePathForCKKS]).andReturn(ckksPath);
 349     OCMStub([mockTopic databasePathForSOS]).andReturn(sosPath);
 350     OCMStub([mockTopic databasePathForPCS]).andReturn(pcsPath);
 351     OCMStub([mockTopic databasePathForTrust]).andReturn(tlsPath);
 352     OCMStub([mockTopic databasePathForSignIn]).andReturn(signInPath);
 353     OCMStub([mockTopic databasePathForCloudServices]).andReturn(cloudServicesPath);
 354
 355     // These are not used for testing, but real data can pollute tests so point to empty DBs
 356     NSString *localpath = [_path stringByAppendingFormat:@"/local_empty_%ld.db", (long)_testnum];
 357     NSString *networkingPath = [_path stringByAppendingFormat:@"/networking_empty_%ld.db", (long)_testnum];
 358     OCMStub([mockTopic databasePathForLocal]).andReturn(localpath);
 359     OCMStub([mockTopic databasePathForNetworking]).andReturn(networkingPath);
 360
 361 #if TARGET_OS_OSX
 362     NSString *rootTrustPath = [_path stringByAppendingFormat:@"/root_trust_empty_%ld.db", (long)_testnum];
 363     NSString *rootNetworkingPath = [_path stringByAppendingFormat:@"/root_networking_empty_%ld.db", (long)_testnum];
 364     OCMStub([mockTopic databasePathForRootTrust]).andReturn(rootTrustPath);
 365     OCMStub([mockTopic databasePathForRootNetworking]).andReturn(rootNetworkingPath);
 366 #endif
 367
 368     _reporterWrites = 0;
 369     mockReporter = OCMClassMock([SFAnalyticsReporter class]);
 370     OCMStub([mockReporter saveReport:[OCMArg isNotNil] fileName:[OCMArg isNotNil]]).andDo(^(NSInvocation *invocation) {
 371         _reporterWrites++;
 372     }).andReturn(YES);
 373
 374     [supd removeInstance];
 375     _supd = [[supd alloc] initWithReporter:mockReporter];
 376     _ckksAnalytics = [FakeCKKSAnalytics new];
 377     _sosAnalytics = [FakeSOSAnalytics new];
 378     _pcsAnalytics = [FakePCSAnalytics new];
 379     _tlsAnalytics = [FakeTLSAnalytics new];
 380
 381     // Forcibly override analytics flags and enable them by default
 382     deviceAnalyticsOverride = YES;
 383     deviceAnalyticsEnabled = YES;
 384     iCloudAnalyticsOverride = YES;
 385     iCloudAnalyticsEnabled = YES;
 386     runningTests = YES;
 387 }
 388
 389 - (void)tearDown
 390 {
 391
 392     [super tearDown];
 393 }
 394
 395 // MARK: Actual tests
 396
 397 // Note! This test relies on Security being installed because supd reads from a plist in Security.framework
 398 - (void)testSplunkDefaultTopicNameExists
 399 {
 400     XCTAssertNotNil([[self keySyncTopic] splunkTopicName]);
 401 }
 402
 403 // Note! This test relies on Security being installed because supd reads from a plist in Security.framework
 404 - (void)testSplunkDefaultBagURLExists
 405 {
 406     XCTAssertNotNil([[self keySyncTopic] splunkBagURL]);
 407 }
 408
 409 - (void)testHaveEligibleClientsKeySync
 410 {
 411     // KeySyncTopic has no clients requiring deviceAnalytics currently
 412     SFAnalyticsTopic* keytopic = [[SFAnalyticsTopic alloc] initWithDictionary:@{} name:@"KeySyncTopic" samplingRates:@{}];
 413
 414     XCTAssertTrue([keytopic haveEligibleClients], @"Both analytics enabled -> we have keysync clients");
 415
 416     deviceAnalyticsEnabled = NO;
 417     XCTAssertTrue([keytopic haveEligibleClients], @"Only iCloud analytics enabled -> we have keysync clients");
 418
 419     iCloudAnalyticsEnabled = NO;
 420     XCTAssertFalse([keytopic haveEligibleClients], @"Both analytics disabled -> no keysync clients");
 421
 422     deviceAnalyticsEnabled = YES;
 423     XCTAssertTrue([keytopic haveEligibleClients], @"Only device analytics enabled -> we have keysync clients (localkeychain for now)");
 424 }
 425
 426 - (void)testHaveEligibleClientsTrust
 427 {
 428     // TrustTopic has no clients requiring iCloudAnalytics currently
 429     SFAnalyticsTopic* trusttopic = [[SFAnalyticsTopic alloc] initWithDictionary:@{} name:@"TrustTopic" samplingRates:@{}];
 430
 431     XCTAssertTrue([trusttopic haveEligibleClients], @"Both analytics enabled -> we have trust clients");
 432
 433     deviceAnalyticsEnabled = NO;
 434     XCTAssertFalse([trusttopic haveEligibleClients], @"Only iCloud analytics enabled -> no trust clients");
 435
 436     iCloudAnalyticsEnabled = NO;
 437     XCTAssertFalse([trusttopic haveEligibleClients], @"Both analytics disabled -> no trust clients");
 438
 439     deviceAnalyticsEnabled = YES;
 440     XCTAssertTrue([trusttopic haveEligibleClients], @"Only device analytics enabled -> we have trust clients");
 441 }
 442
 443 - (void)testLoggingJSONSimple:(BOOL)analyticsEnabled
 444 {
 445     iCloudAnalyticsEnabled = analyticsEnabled;
 446
 447     [_ckksAnalytics logSuccessForEventNamed:@"ckksunittestevent"];
 448     NSDictionary* ckksAttrs = @{@"cattr" : @"cvalue"};
 449     [_ckksAnalytics logHardFailureForEventNamed:@"ckksunittestevent" withAttributes:ckksAttrs];
 450     [_ckksAnalytics logSoftFailureForEventNamed:@"ckksunittestevent" withAttributes:ckksAttrs];
 451     [_sosAnalytics logSuccessForEventNamed:@"unittestevent"];
 452     NSDictionary* utAttrs = @{@"uattr" : @"uvalue"};
 453     [_sosAnalytics logHardFailureForEventNamed:@"unittestevent" withAttributes:utAttrs];
 454     [_sosAnalytics logSoftFailureForEventNamed:@"unittestevent" withAttributes:utAttrs];
 455
 456     NSDictionary *data =  [self getJSONDataFromSupd];
 457
 458     [self inspectDataBlobStructure:data];
 459
 460     // TODO: inspect health summaries
 461
 462     if (analyticsEnabled) {
 463         XCTAssertEqual([self failures:data eventType:@"ckksunittestevent" attributes:ckksAttrs class:SFAnalyticsEventClassHardFailure], 1);
 464         XCTAssertEqual([self failures:data eventType:@"ckksunittestevent" attributes:ckksAttrs class:SFAnalyticsEventClassSoftFailure], 1);
 465         XCTAssertEqual([self failures:data eventType:@"unittestevent" attributes:utAttrs class:SFAnalyticsEventClassHardFailure], 1);
 466         XCTAssertEqual([self failures:data eventType:@"unittestevent" attributes:utAttrs class:SFAnalyticsEventClassSoftFailure], 1);
 467
 468         [self checkTotalEventCount:data hard:2 soft:2 accuracy:0];
 469     } else {
 470         // localkeychain requires device analytics only so we still get it
 471         [self checkTotalEventCount:data hard:0 soft:0 accuracy:0 summaries:1];
 472     }
 473 }
 474
 475 - (void)testLoggingJSONSimpleWithiCloudAnalyticsEnabled
 476 {
 477     [self testLoggingJSONSimple:YES];
 478 }
 479
 480 - (void)testLoggingJSONSimpleWithiCloudAnalyticsDisabled
 481 {
 482     [self testLoggingJSONSimple:NO];
 483 }
 484
 485 - (void)testTLSLoggingJSONSimple:(BOOL)analyticsEnabled
 486 {
 487     deviceAnalyticsEnabled = analyticsEnabled;
 488
 489     [_tlsAnalytics logSuccessForEventNamed:@"tlsunittestevent"];
 490     NSDictionary* tlsAttrs = @{@"cattr" : @"cvalue"};
 491     [_tlsAnalytics logHardFailureForEventNamed:@"tlsunittestevent" withAttributes:tlsAttrs];
 492     [_tlsAnalytics logSoftFailureForEventNamed:@"tlsunittestevent" withAttributes:tlsAttrs];
 493
 494     NSDictionary* data = [self getJSONDataFromSupdWithTopic:SFAnalyticsTopicTrust];
 495     [self inspectDataBlobStructure:data forTopic:[[self TrustTopic] splunkTopicName]];
 496
 497     if (analyticsEnabled) {
 498         [self checkTotalEventCount:data hard:1 soft:1 accuracy:0 summaries:(int)[[[self TrustTopic] topicClients] count]];
 499     } else {
 500         [self checkTotalEventCount:data hard:0 soft:0 accuracy:0 summaries:0];
 501     }
 502 }
 503
 504 - (void)testTLSLoggingJSONSimpleWithDeviceAnalyticsEnabled
 505 {
 506     [self testTLSLoggingJSONSimple:YES];
 507 }
 508
 509 - (void)testTLSLoggingJSONSimpleWithDeviceAnalyticsDisabled
 510 {
 511     [self testTLSLoggingJSONSimple:NO];
 512 }
 513
 514 - (void)testMockDiagnosticReportGeneration
 515 {
 516     SFAnalyticsReporter *reporter = mockReporter;
 517
 518     uint8_t report_data[] = {0x00, 0x01, 0x02, 0x03};
 519     NSData *reportData = [[NSData alloc] initWithBytes:report_data length:sizeof(report_data)];
 520     BOOL writtenToLog = YES;
 521     size_t numWrites = 5;
 522     for (size_t i = 0; i < numWrites; i++) {
 523         writtenToLog &= [reporter saveReport:reportData fileName:@"log.txt"];
 524     }
 525
 526     XCTAssertTrue(writtenToLog, "Failed to write to log");
 527     XCTAssertTrue((int)_reporterWrites == (int)numWrites, "Expected %zu report, got %d", numWrites, (int)_reporterWrites);
 528 }
 529
 530 - (void)testSuccessCounts
 531 {
 532     NSString* eventName1 = @"successCountsEvent1";
 533     NSString* eventName2 = @"successCountsEvent2";
 534
 535     for (int idx = 0; idx < 3; ++idx) {
 536         [_ckksAnalytics logSuccessForEventNamed:eventName1];
 537         [_ckksAnalytics logSuccessForEventNamed:eventName2];
 538         [_ckksAnalytics logHardFailureForEventNamed:eventName1 withAttributes:nil];
 539         [_ckksAnalytics logSoftFailureForEventNamed:eventName2 withAttributes:nil];
 540     }
 541     [_ckksAnalytics logSuccessForEventNamed:eventName2];
 542
 543     NSDictionary* data = [self getJSONDataFromSupd];
 544     [self inspectDataBlobStructure:data];
 545
 546     NSDictionary* hs;
 547     for (NSDictionary* event in data[@"events"]) {
 548         if ([event[SFAnalyticsEventType] isEqual:@"ckksHealthSummary"]) {
 549             hs = event;
 550             break;
 551         }
 552     }
 553     XCTAssert(hs);
 554
 555     XCTAssertEqual([hs[SFAnalyticsColumnSuccessCount] integerValue], 7);
 556     XCTAssertEqual([hs[SFAnalyticsColumnHardFailureCount] integerValue], 3);
 557     XCTAssertEqual([hs[SFAnalyticsColumnSoftFailureCount] integerValue], 3);
 558     XCTAssertEqual([hs[[self string:eventName1 item:@"success"]] integerValue], 3);
 559     XCTAssertEqual([hs[[self string:eventName1 item:@"hardfail"]] integerValue], 3);
 560     XCTAssertEqual([hs[[self string:eventName1 item:@"softfail"]] integerValue], 0);
 561     XCTAssertEqual([hs[[self string:eventName2 item:@"success"]] integerValue], 4);
 562     XCTAssertEqual([hs[[self string:eventName2 item:@"hardfail"]] integerValue], 0);
 563     XCTAssertEqual([hs[[self string:eventName2 item:@"softfail"]] integerValue], 3);
 564 }
 565
 566 // There was a failure with thresholds if some, but not all clients exceeded their 'threshold' number of failures,
 567 // causing the addFailures:toUploadRecords:threshold method to crash with out of bounds.
 568 // This is also implicitly tested in testTooManyHardFailures and testTooManyCombinedFailures but I wanted an explicit case.
 569 - (void)testExceedThresholdForOneClientOnly
 570 {
 571     int testAmount = ((int)SFAnalyticsMaxEventsToReport / 4);
 572     for (int idx = 0; idx < testAmount; ++idx) {
 573         [_ckksAnalytics logHardFailureForEventNamed:@"ckkshardfail" withAttributes:nil];
 574         [_ckksAnalytics logSoftFailureForEventNamed:@"ckkssoftfail" withAttributes:nil];
 575     }
 576
 577     [_sosAnalytics logHardFailureForEventNamed:@"soshardfail" withAttributes:nil];
 578     [_sosAnalytics logSoftFailureForEventNamed:@"sossoftfail" withAttributes:nil];
 579
 580     NSDictionary* data = [self getJSONDataFromSupd];
 581     [self inspectDataBlobStructure:data];
 582
 583     [self checkTotalEventCount:data hard:testAmount + 1 soft:testAmount + 1 accuracy:0];
 584
 585     XCTAssertEqual([self failures:data eventType:@"ckkshardfail" attributes:nil class:SFAnalyticsEventClassHardFailure], testAmount);
 586     XCTAssertEqual([self failures:data eventType:@"ckkssoftfail" attributes:nil class:SFAnalyticsEventClassSoftFailure], testAmount);
 587     XCTAssertEqual([self failures:data eventType:@"soshardfail" attributes:nil class:SFAnalyticsEventClassHardFailure], 1);
 588     XCTAssertEqual([self failures:data eventType:@"sossoftfail" attributes:nil class:SFAnalyticsEventClassSoftFailure], 1);
 589 }
 590
 591
 592 // We have so many hard failures they won't fit in the upload buffer
 593 - (void)testTooManyHardFailures
 594 {
 595     NSDictionary* ckksAttrs = @{@"cattr" : @"cvalue"};
 596     NSDictionary* utAttrs = @{@"uattr" : @"uvalue"};
 597     for (int idx = 0; idx < 400; ++idx) {
 598         [_ckksAnalytics logHardFailureForEventNamed:@"ckksunittestfailure" withAttributes:ckksAttrs];
 599         [_ckksAnalytics logHardFailureForEventNamed:@"ckksunittestfailure" withAttributes:ckksAttrs];
 600         [_sosAnalytics logHardFailureForEventNamed:@"utunittestfailure" withAttributes:utAttrs];
 601     }
 602
 603     NSDictionary* data = [self getJSONDataFromSupd];
 604     [self inspectDataBlobStructure:data];
 605
 606     [self checkTotalEventCount:data hard:998 soft:0];
 607     // Based on threshold = records_to_upload/10 with a nice margin
 608     XCTAssertEqualWithAccuracy([self failures:data eventType:@"ckksunittestfailure" attributes:ckksAttrs class:SFAnalyticsEventClassHardFailure], 658, 50);
 609     XCTAssertEqualWithAccuracy([self failures:data eventType:@"utunittestfailure" attributes:utAttrs class:SFAnalyticsEventClassHardFailure], 339, 50);
 610 }
 611
 612 // So many soft failures they won't fit in the buffer
 613 - (void)testTooManySoftFailures
 614 {
 615     NSDictionary* ckksAttrs = @{@"cattr" : @"cvalue"};
 616     NSDictionary* utAttrs = @{@"uattr" : @"uvalue"};
 617     for (int idx = 0; idx < 400; ++idx) {
 618         [_ckksAnalytics logSoftFailureForEventNamed:@"ckksunittestfailure" withAttributes:ckksAttrs];
 619         [_ckksAnalytics logSoftFailureForEventNamed:@"ckksunittestfailure" withAttributes:ckksAttrs];
 620         [_sosAnalytics logSoftFailureForEventNamed:@"utunittestfailure" withAttributes:utAttrs];
 621     }
 622
 623     NSDictionary* data = [self getJSONDataFromSupd];
 624     [self inspectDataBlobStructure:data];
 625
 626     [self checkTotalEventCount:data hard:0 soft:998];
 627     // Based on threshold = records_to_upload/10 with a nice margin
 628     XCTAssertEqualWithAccuracy([self failures:data eventType:@"ckksunittestfailure" attributes:ckksAttrs class:SFAnalyticsEventClassSoftFailure], 665, 50);
 629     XCTAssertEqualWithAccuracy([self failures:data eventType:@"utunittestfailure" attributes:utAttrs class:SFAnalyticsEventClassSoftFailure], 332, 50);
 630 }
 631
 632 - (void)testTooManyCombinedFailures
 633 {
 634     NSDictionary* ckksAttrs = @{@"cattr1" : @"cvalue1", @"cattrthatisalotlongerthanthepreviousone" : @"cvaluethatisalsoalotlongerthantheother"};
 635     NSDictionary* utAttrs = @{@"uattr" : @"uvalue", @"uattrthatisalotlongerthanthepreviousone" : @"uvaluethatisalsoalotlongerthantheother"};
 636     for (int idx = 0; idx < 400; ++idx) {
 637         [_ckksAnalytics logHardFailureForEventNamed:@"ckksunittestfailure" withAttributes:ckksAttrs];
 638         [_ckksAnalytics logSoftFailureForEventNamed:@"ckksunittestfailure" withAttributes:ckksAttrs];
 639         [_sosAnalytics logHardFailureForEventNamed:@"utunittestfailure" withAttributes:utAttrs];
 640         [_sosAnalytics logSoftFailureForEventNamed:@"utunittestfailure" withAttributes:utAttrs];
 641     }
 642
 643     NSDictionary* data = [self getJSONDataFromSupd];
 644     [self inspectDataBlobStructure:data];
 645
 646     [self checkTotalEventCount:data hard:800 soft:198];
 647     // Based on threshold = records_to_upload/10 with a nice margin
 648     XCTAssertEqualWithAccuracy([self failures:data eventType:@"ckksunittestfailure" attributes:ckksAttrs class:SFAnalyticsEventClassHardFailure], 400, 50);
 649     XCTAssertEqualWithAccuracy([self failures:data eventType:@"utunittestfailure" attributes:utAttrs class:SFAnalyticsEventClassHardFailure], 400, 50);
 650     XCTAssertEqualWithAccuracy([self failures:data eventType:@"ckksunittestfailure" attributes:ckksAttrs class:SFAnalyticsEventClassSoftFailure], 100, 50);
 651     XCTAssertEqualWithAccuracy([self failures:data eventType:@"utunittestfailure" attributes:utAttrs class:SFAnalyticsEventClassSoftFailure], 100, 50);
 652 }
 653
 654 // There's an even number of samples
 655 - (void)testSamplesEvenSampleCount
 656 {
 657     NSString* sampleNameEven = @"evenSample";
 658
 659     for (NSNumber* value in @[@36.831855250339714, @90.78721762172914, @49.24392301762506,
 660                               @42.806362283260036, @16.76725375576855, @34.50969130579674,
 661                               @25.956509180834637, @36.8268555935645, @35.54069258036879,
 662                               @7.26364884595062, @45.414180770615395, @5.223213570809022]) {
 663         [_ckksAnalytics logMetric:value withName:sampleNameEven];
 664     }
 665
 666     NSDictionary* data = [self getJSONDataFromSupd];
 667     [self inspectDataBlobStructure:data];
 668
 669     // min, max, avg, med, dev, 1q, 3q
 670     [self checkTotalEventCount:data hard:0 soft:0 accuracy:0];
 671     [self sampleStatisticsInEvents:data[@"events"] name:sampleNameEven values:@[@5.22, @90.78, @35.60, @36.18, @21.52, @21.36, @44.11]];
 672 }
 673
 674 // There are 4*n + 1 samples
 675 - (void)testSamples4n1SampleCount
 676 {
 677     NSString* sampleName4n1 = @"4n1Sample";
 678     for (NSNumber* value in @[@37.76544251068022, @27.36378948426223, @45.10503077614114,
 679                               @43.90635413191473, @54.78709742040113, @52.34879597889124,
 680                               @70.95760312196856, @23.23648158872921, @75.34678687445064,
 681                               @10.723238854026203, @41.98468801166455, @17.074404554908476,
 682                               @94.24252031232739]) {
 683         [_ckksAnalytics logMetric:value withName:sampleName4n1];
 684     }
 685
 686     NSDictionary* data = [self getJSONDataFromSupd];
 687     [self inspectDataBlobStructure:data];
 688
 689     [self checkTotalEventCount:data hard:0 soft:0 accuracy:0];
 690     [self sampleStatisticsInEvents:data[@"events"] name:sampleName4n1 values:@[@10.72, @94.24, @45.76, @43.90, @23.14, @26.33, @58.83]];
 691 }
 692
 693 // There are 4*n + 3 samples
 694 - (void)testSamples4n3SampleCount
 695 {
 696     NSString* sampleName4n3 = @"4n3Sample";
 697
 698     for (NSNumber* value in @[@42.012971885655496, @87.85629592375282, @5.748491212287082,
 699                               @38.451850063872975, @81.96900109690873, @99.83098790545392,
 700                               @80.89400981437815, @5.719237885152143, @1.6740622555032196,
 701                               @14.437000556079038, @29.046050177512395]) {
 702         [_sosAnalytics logMetric:value withName:sampleName4n3];
 703     }
 704
 705     NSDictionary* data = [self getJSONDataFromSupd];
 706     [self inspectDataBlobStructure:data];
 707     [self checkTotalEventCount:data hard:0 soft:0 accuracy:0];
 708
 709     [self sampleStatisticsInEvents:data[@"events"] name:sampleName4n3 values:@[@1.67, @99.83, @44.33, @38.45, @35.28, @7.92, @81.70]];
 710 }
 711
 712 // stddev and quartiles undefined for single sample
 713 - (void)testSamplesSingleSample
 714 {
 715     NSString* sampleName = @"singleSample";
 716
 717     [_ckksAnalytics logMetric:@3.14159 withName:sampleName];
 718
 719     NSDictionary* data = [self getJSONDataFromSupd];
 720     [self inspectDataBlobStructure:data];
 721     [self checkTotalEventCount:data hard:0 soft:0 accuracy:0];
 722
 723     [self sampleStatisticsInEvents:data[@"events"] name:sampleName values:@[@3.14159]];
 724 }
 725
 726 // quartiles meaningless for fewer than 4 samples (but stddev exists)
 727 - (void)testSamplesFewerThanFour
 728 {
 729     NSString* sampleName = @"fewSamples";
 730
 731     [_ckksAnalytics logMetric:@3.14159 withName:sampleName];
 732     [_ckksAnalytics logMetric:@6.28318 withName:sampleName];
 733
 734     NSDictionary* data = [self getJSONDataFromSupd];
 735     [self inspectDataBlobStructure:data];
 736     [self checkTotalEventCount:data hard:0 soft:0 accuracy:0];
 737
 738     [self sampleStatisticsInEvents:data[@"events"] name:sampleName values:@[@3.14, @6.28, @4.71, @4.71, @1.57]];
 739 }
 740
 741 - (void)testSamplesSameNameDifferentSubclass
 742 {
 743     NSString* sampleName = @"differentSubclassSamples";
 744
 745     [_sosAnalytics logMetric:@313.37 withName:sampleName];
 746     [_ckksAnalytics logMetric:@313.37 withName:sampleName];
 747
 748     NSDictionary* data = [self getJSONDataFromSupd];
 749     [self inspectDataBlobStructure:data];
 750     [self checkTotalEventCount:data hard:0 soft:0 accuracy:0];
 751
 752     [self sampleStatisticsInEvents:data[@"events"] name:sampleName values:@[@313.37] amount:2];
 753 }
 754
 755 - (void)testInvalidJSON
 756 {
 757     NSData* bad = [@"let's break supd!" dataUsingEncoding:NSUTF8StringEncoding];
 758     [_ckksAnalytics logHardFailureForEventNamed:@"testEvent" withAttributes:@{ @"dataAttribute" : bad}];
 759
 760     NSDictionary* data = [self getJSONDataFromSupd];
 761     XCTAssertNotNil(data);
 762     XCTAssertNotNil(data[@"events"]);
 763     NSUInteger foundErrorEvents = 0;
 764     for (NSDictionary* event in data[@"events"]) {
 765         if ([event[SFAnalyticsEventType] isEqualToString:SFAnalyticsEventTypeErrorEvent] && [event[SFAnalyticsEventErrorDestription] isEqualToString:@"JSON:testEvent"]) {
 766             ++foundErrorEvents;
 767         }
 768     }
 769     XCTAssertEqual(foundErrorEvents, 1);
 770 }
 771
 772 - (void)testUploadSizeLimits
 773 {
 774     SFAnalyticsTopic *trustTopic = [self TrustTopic];
 775     XCTAssertEqual(1000000, trustTopic.uploadSizeLimit);
 776
 777     SFAnalyticsTopic *keySyncTopic = [self keySyncTopic];
 778     XCTAssertEqual(1000000, keySyncTopic.uploadSizeLimit);
 779 }
 780
 781 - (NSArray<NSDictionary *> *)createRandomEventList:(size_t)count
 782 {
 783     NSMutableArray<NSDictionary *> *eventSet = [[NSMutableArray<NSDictionary *> alloc] init];
 784
 785     const size_t dataSize = 100;
 786     uint8_t backingBuffer[dataSize] = {};
 787     for (size_t i = 0; i < count; i++) {
 788         NSData *data = [[NSData alloc] initWithBytes:backingBuffer length:dataSize];
 789         NSDictionary *entry = @{@"key" : [data base64EncodedStringWithOptions:0]};
 790         [eventSet addObject:entry];
 791     }
 792
 793     return eventSet;
 794 }
 795
 796 - (void)testCreateLoggingJSON
 797 {
 798     NSArray<NSDictionary *> *summaries = [self createRandomEventList:5];
 799     NSArray<NSDictionary *> *failures = [self createRandomEventList:100];
 800     NSMutableArray<NSDictionary *> *visitedEvents = [[NSMutableArray<NSDictionary *> alloc] init];
 801
 802     SFAnalyticsTopic *topic = [self TrustTopic];
 803     const size_t sizeLimit = 10000; // total size of the encoded data
 804     topic.uploadSizeLimit = sizeLimit;
 805
 806     NSError *error = nil;
 807     NSArray<NSDictionary *> *eventSet = [topic createChunkedLoggingJSON:summaries failures:failures error:&error];
 808     XCTAssertNil(error);
 809
 810     for (NSDictionary *event in eventSet) {
 811         XCTAssertNotNil([event objectForKey:@"events"]);
 812         XCTAssertNotNil([event objectForKey:SFAnalyticsPostTime]);
 813         NSArray *events = [event objectForKey:@"events"];
 814         for (NSDictionary *summary in summaries) {
 815             BOOL foundSummary = NO;
 816             for (NSDictionary *innerEvent in events) {
 817                 if ([summary isEqualToDictionary:innerEvent]) {
 818                     foundSummary = YES;
 819                     break;
 820                 }
 821             }
 822             XCTAssertTrue(foundSummary);
 823         }
 824
 825         // Record the events we've seen so far
 826         for (NSDictionary *innerEvent in events) {
 827             [visitedEvents addObject:innerEvent];
 828         }
 829     }
 830
 831     // Check that each summary and failure is in the visitedEvents
 832     for (NSDictionary *summary in summaries) {
 833         BOOL foundSummary = NO;
 834         for (NSDictionary *innerEvent in visitedEvents) {
 835             if ([summary isEqualToDictionary:innerEvent]) {
 836                 foundSummary = YES;
 837                 break;
 838             }
 839         }
 840         XCTAssertTrue(foundSummary);
 841     }
 842     for (NSDictionary *failure in failures) {
 843         BOOL foundFailure = NO;
 844         for (NSDictionary *innerEvent in visitedEvents) {
 845             if ([failure isEqualToDictionary:innerEvent]) {
 846                 foundFailure = YES;
 847                 break;
 848             }
 849         }
 850         XCTAssertTrue(foundFailure);
 851     }
 852 }
 853
 854 - (void)testEventSetChunking
 855 {
 856     NSArray<NSDictionary *> *eventSet = [self createRandomEventList:100];
 857     SFAnalyticsTopic *topic = [self TrustTopic];
 858
 859     const size_t sizeLimit = 10000; // total size of the encoded data
 860     size_t encodedEventSize = [topic serializedEventSize:eventSet[0] error:nil];
 861     topic.uploadSizeLimit = sizeLimit; // fix the upload limit
 862
 863     // Chunk up the set, assuming that each chunk already has one event in it.
 864     // In practice, this is the health summary.
 865     NSError *error = nil;
 866     NSArray<NSArray *> *chunkedEvents = [topic chunkFailureSet:(sizeLimit - encodedEventSize) events:eventSet error:nil];
 867     XCTAssertNil(error);
 868
 869     // There should be two resulting chunks, since the set of chunks overflows.
 870     XCTAssertEqual(2, [chunkedEvents count]);
 871 }
 872
 873 // TODO
 874 - (void)testGetSysdiagnoseDump
 875 {
 876
 877 }
 878
 879 // TODO (need mock server)
 880 - (void)testSplunkUpload
 881 {
 882
 883 }
 884
 885 // TODO (need mock server)
 886 - (void)testDBIsEmptiedAfterUpload
 887 {
 888
 889 }
 890
 891 @end
 892
 893 #endif  // !TARGET_OS_SIMULATOR