supd/Tests/SupdTests.m

   1 /*
   2  * Copyright (c) 2017-2018 Apple Inc. All Rights Reserved.
   3  *
   4  * @APPLE_LICENSE_HEADER_START@
   5  *
   6  * This file contains Original Code and/or Modifications of Original Code
   7  * as defined in and that are subject to the Apple Public Source License
   8  * Version 2.0 (the 'License'). You may not use this file except in
   9  * compliance with the License. Please obtain a copy of the License at
  10  * http://www.opensource.apple.com/apsl/ and read it before using this
  11  * file.
  12  *
  13  * The Original Code and all software distributed under the License are
  14  * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  15  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  16  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
  17  * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
  18  * Please see the License for the specific language governing rights and
  19  * limitations under the License.
  20  *
  21  * @APPLE_LICENSE_HEADER_END@
  22  */
  23
  24 #import <OCMock/OCMock.h>
  25 #import <XCTest/XCTest.h>
  26 #import "supd.h"
  27 #import "SFAnalytics.h"
  28 #import "SFAnalyticsDefines.h"
  29 #import <CoreFoundation/CFPriv.h>
  30
  31 static NSString* _path;
  32 static NSInteger _testnum;
  33 static NSString* build = NULL;
  34 static NSString* product = NULL;
  35 static NSInteger _reporterWrites;
  36 static NSInteger _reporterCleanups;
  37
  38 // MARK: Stub FakeCKKSAnalytics
  39
  40 @interface FakeCKKSAnalytics : SFAnalytics
  41
  42 @end
  43
  44 @implementation FakeCKKSAnalytics
  45
  46 + (NSString*)databasePath
  47 {
  48     return [_path stringByAppendingFormat:@"/ckks_%ld.db", _testnum];
  49 }
  50
  51 @end
  52
  53
  54 // MARK: Stub FakeSOSAnalytics
  55
  56 @interface FakeSOSAnalytics : SFAnalytics
  57
  58 @end
  59
  60 @implementation FakeSOSAnalytics
  61
  62 + (NSString*)databasePath
  63 {
  64     return [_path stringByAppendingFormat:@"/sos_%ld.db", _testnum];
  65 }
  66
  67 @end
  68
  69
  70 // MARK: Stub FakePCSAnalytics
  71
  72 @interface FakePCSAnalytics : SFAnalytics
  73
  74 @end
  75
  76 @implementation FakePCSAnalytics
  77
  78 + (NSString*)databasePath
  79 {
  80     return [_path stringByAppendingFormat:@"/pcs_%ld.db", _testnum];
  81 }
  82
  83 @end
  84
  85 // MARK: Stub FakeTLSAnalytics
  86
  87 @interface FakeTLSAnalytics : SFAnalytics
  88
  89 @end
  90
  91 @implementation FakeTLSAnalytics
  92
  93 + (NSString*)databasePath
  94 {
  95     return [_path stringByAppendingFormat:@"/tls_%ld.db", _testnum];
  96 }
  97
  98 @end
  99
 100 // MARK: Start SupdTests
 101
 102 @interface SupdTests : XCTestCase
 103
 104 @end
 105
 106 @implementation SupdTests {
 107     supd* _supd;
 108     id mockReporter;
 109     FakeCKKSAnalytics* _ckksAnalytics;
 110     FakeSOSAnalytics* _sosAnalytics;
 111     FakePCSAnalytics* _pcsAnalytics;
 112     FakeTLSAnalytics* _tlsAnalytics;
 113 }
 114
 115 // MARK: Test helper methods
 116 - (SFAnalyticsTopic *)keySyncTopic {
 117     for (SFAnalyticsTopic *topic in _supd.analyticsTopics) {
 118         if ([topic.internalTopicName isEqualToString:SFAnalyticsTopicKeySync]) {
 119             return topic;
 120         }
 121     }
 122     return nil;
 123 }
 124
 125 - (void)inspectDataBlobStructure:(NSDictionary*)data
 126 {
 127     if (!data || ![data isKindOfClass:[NSDictionary class]]) {
 128         XCTFail(@"data is an NSDictionary");
 129     }
 130
 131     XCTAssert(_supd.analyticsTopics, @"supd has nonnull topics list");
 132     SFAnalyticsTopic *keySyncTopic = [self keySyncTopic];
 133     XCTAssert([keySyncTopic splunkTopicName], @"supd has a nonnull topic name");
 134     XCTAssertEqual([data count], 2ul, @"dictionary event and posttime objects");
 135     XCTAssertTrue(data[@"events"] && [data[@"events"] isKindOfClass:[NSArray class]], @"data blob contains an NSArray 'events'");
 136     XCTAssertTrue(data[@"postTime"] && [data[@"postTime"] isKindOfClass:[NSNumber class]], @"data blob contains an NSNumber 'postTime");
 137     NSDate* postTime = [NSDate dateWithTimeIntervalSince1970:[data[@"postTime"] doubleValue]];
 138     XCTAssertTrue([[NSDate date] timeIntervalSinceDate:postTime] < 3, @"postTime is sane");
 139
 140     for (NSDictionary* event in data[@"events"]) {
 141         if ([event isKindOfClass:[NSDictionary class]]) {
 142             XCTAssertTrue([event[@"build"] isEqual:build], @"event contains correct build string");
 143             XCTAssertTrue([event[@"product"] isEqual:product], @"event contains correct product string");
 144             XCTAssertTrue([event[@"eventTime"] isKindOfClass:[NSNumber class]], @"event contains an NSNumber 'eventTime");
 145             NSDate* eventTime = [NSDate dateWithTimeIntervalSince1970:[event[@"eventTime"] doubleValue]];
 146             XCTAssertTrue([[NSDate date] timeIntervalSinceDate:eventTime] < 3, @"eventTime is sane");
 147             XCTAssertTrue([event[@"eventType"] isKindOfClass:[NSString class]], @"all events have a type");
 148             XCTAssertTrue([event[@"topic"] isEqual:[keySyncTopic splunkTopicName]], @"all events have a topic name");
 149         } else {
 150             XCTFail(@"event %@ is an NSDictionary", event);
 151         }
 152     }
 153 }
 154
 155 - (BOOL)event:(NSDictionary*)event containsAttributes:(NSDictionary*)attrs {
 156     if (!attrs) {
 157         return YES;
 158     }
 159     __block BOOL equal = YES;
 160     [attrs enumerateKeysAndObjectsUsingBlock:^(id  _Nonnull key, id  _Nonnull obj, BOOL * _Nonnull stop) {
 161         equal &= [event[key] isEqualToString:obj];
 162     }];
 163     return equal;
 164 }
 165
 166 - (int)failures:(NSDictionary*)data eventType:(NSString*)type attributes:(NSDictionary*)attrs class:(SFAnalyticsEventClass)class
 167 {
 168     int encountered = 0;
 169     for (NSDictionary* event in data[@"events"]) {
 170         if ([event[@"eventType"] isEqualToString:type] &&
 171             [event[@"eventClass"] isKindOfClass:[NSNumber class]] &&
 172             [event[@"eventClass"] intValue] == class && [self event:event containsAttributes:attrs]) {
 173             ++encountered;
 174         }
 175     }
 176     return encountered;
 177 }
 178
 179 - (void)checkTotalEventCount:(NSDictionary*)data hard:(int)hard soft:(int)soft forcedFail:(BOOL)forcedFail
 180 {
 181     int hardfound = 0, softfound = 0;
 182     NSUInteger summfound = 0;
 183     for (NSDictionary* event in data[@"events"]) {
 184         if ([event[SFAnalyticsEventType] hasSuffix:@"HealthSummary"]) {
 185             ++summfound;
 186         } else if ([event[SFAnalyticsEventClassKey] integerValue] == SFAnalyticsEventClassHardFailure) {
 187             ++hardfound;
 188         } else if ([event[SFAnalyticsEventClassKey] integerValue] == SFAnalyticsEventClassSoftFailure) {
 189             ++softfound;
 190         }
 191     }
 192
 193     XCTAssertLessThanOrEqual(((NSArray*)data[@"events"]).count, 1000ul, @"Total event count fits in alloted data");
 194     if (!forcedFail) {
 195         XCTAssertEqual(summfound, [[[self keySyncTopic] topicClients] count]);
 196     }
 197     // Add fuzziness, we're not testing exact implementation details
 198     XCTAssertEqualWithAccuracy(hardfound, hard, 10);
 199     XCTAssertEqualWithAccuracy(softfound, soft, 10);
 200 }
 201
 202 - (void)checkTotalEventCount:(NSDictionary*)data hard:(int)hard soft:(int)soft
 203 {
 204     [self checkTotalEventCount:data hard:hard soft:soft forcedFail:NO];
 205 }
 206
 207 // This is a dumb hack, but inlining stringWithFormat causes the compiler to growl for unknown reasons
 208 - (NSString*)string:(NSString*)name item:(NSString*)item
 209 {
 210     return [NSString stringWithFormat:@"%@-%@", name, item];
 211 }
 212
 213 - (void)sampleStatisticsInEvents:(NSArray*)events name:(NSString*)name values:(NSArray*)values
 214 {
 215     [self sampleStatisticsInEvents:events name:name values:values amount:1];
 216 }
 217
 218 // Usually amount == 1 but for testing sampler with same name in different subclasses this is higher
 219 - (void)sampleStatisticsInEvents:(NSArray*)events name:(NSString*)name values:(NSArray*)values amount:(int)num
 220 {
 221     int found = 0;
 222     for (NSDictionary* event in events) {
 223         if (([values count] == 1 && ![event objectForKey:[NSString stringWithFormat:@"%@", name]]) ||
 224             ([values count] > 1 && ![event objectForKey:[NSString stringWithFormat:@"%@-min", name]])) {
 225             continue;
 226         }
 227
 228         ++found;
 229         if (values.count == 1) {
 230             XCTAssertEqual([event[name] doubleValue], [values[0] doubleValue]);
 231             XCTAssertNil(event[[self string:name item:@"min"]]);
 232             XCTAssertNil(event[[self string:name item:@"max"]]);
 233             XCTAssertNil(event[[self string:name item:@"avg"]]);
 234             XCTAssertNil(event[[self string:name item:@"med"]]);
 235         } else {
 236             XCTAssertEqualWithAccuracy([event[[self string:name item:@"min"]] doubleValue], [values[0] doubleValue], 0.01f);
 237             XCTAssertEqualWithAccuracy([event[[self string:name item:@"max"]] doubleValue], [values[1] doubleValue], 0.01f);
 238             XCTAssertEqualWithAccuracy([event[[self string:name item:@"avg"]] doubleValue], [values[2] doubleValue], 0.01f);
 239             XCTAssertEqualWithAccuracy([event[[self string:name item:@"med"]] doubleValue], [values[3] doubleValue], 0.01f);
 240         }
 241
 242         if (values.count > 4) {
 243             XCTAssertEqualWithAccuracy([event[[self string:name item:@"dev"]] doubleValue], [values[4] doubleValue], 0.01f);
 244         } else {
 245             XCTAssertNil(event[[self string:name item:@"dev"]]);
 246         }
 247
 248         if (values.count > 5) {
 249             XCTAssertEqualWithAccuracy([event[[self string:name item:@"1q"]] doubleValue], [values[5] doubleValue], 0.01f);
 250             XCTAssertEqualWithAccuracy([event[[self string:name item:@"3q"]] doubleValue], [values[6] doubleValue], 0.01f);
 251         } else {
 252             XCTAssertNil(event[[self string:name item:@"1q"]]);
 253             XCTAssertNil(event[[self string:name item:@"3q"]]);
 254         }
 255     }
 256     XCTAssertEqual(found, num);
 257 }
 258
 259 - (NSDictionary*)getJSONDataFromSupd
 260 {
 261     dispatch_semaphore_t sema = dispatch_semaphore_create(0);
 262     __block NSDictionary* data;
 263     [_supd getLoggingJSON:YES topic:SFAnalyticsTopicKeySync reply:^(NSData *json, NSError *error) {
 264         XCTAssertNil(error);
 265         XCTAssertNotNil(json);
 266         if (!error) {
 267             data = [NSJSONSerialization JSONObjectWithData:json options:0 error:&error];
 268         }
 269         XCTAssertNil(error, @"no error deserializing json: %@", error);
 270         dispatch_semaphore_signal(sema);
 271     }];
 272     if (dispatch_semaphore_wait(sema, dispatch_time(DISPATCH_TIME_NOW, NSEC_PER_SEC * 5)) != 0) {
 273         XCTFail(@"supd returns JSON data in a timely fashion");
 274     }
 275     return data;
 276 }
 277
 278 // MARK: Test administration
 279
 280 + (void)setUp
 281 {
 282     NSError* error;
 283     _path = [NSTemporaryDirectory() stringByAppendingPathComponent:[NSString stringWithFormat:@"%@/", [[NSUUID UUID] UUIDString]]];
 284     [[NSFileManager defaultManager] createDirectoryAtPath:_path
 285                               withIntermediateDirectories:YES
 286                                                attributes:nil
 287                                                     error:&error];
 288     if (error) {
 289         NSLog(@"sad trombone, couldn't create path");
 290     }
 291
 292     NSDictionary *version = CFBridgingRelease(_CFCopySystemVersionDictionary());
 293     if (version) {
 294         build = version[(__bridge NSString *)_kCFSystemVersionBuildVersionKey];
 295         product = version[(__bridge NSString *)_kCFSystemVersionProductNameKey];
 296     } else {
 297         NSLog(@"could not get build version/product, tests should fail");
 298     }
 299 }
 300
 301 - (void)setUp
 302 {
 303     [super setUp];
 304     self.continueAfterFailure = NO;
 305     ++_testnum;
 306
 307     id mockTopic = OCMStrictClassMock([SFAnalyticsTopic class]);
 308     NSString *ckksPath = [_path stringByAppendingFormat:@"/ckks_%ld.db", _testnum];
 309     NSString *sosPath = [_path stringByAppendingFormat:@"/sos_%ld.db", _testnum];
 310     NSString *pcsPath = [_path stringByAppendingFormat:@"/pcs_%ld.db", _testnum];
 311     NSString *tlsPath = [_path stringByAppendingFormat:@"/tls_%ld.db", _testnum];
 312     OCMStub([mockTopic databasePathForCKKS]).andReturn(ckksPath);
 313     OCMStub([mockTopic databasePathForSOS]).andReturn(sosPath);
 314     OCMStub([mockTopic databasePathForPCS]).andReturn(pcsPath);
 315     OCMStub([mockTopic databasePathForTLS]).andReturn(tlsPath);
 316
 317     _reporterWrites = 0;
 318     mockReporter = OCMClassMock([SFAnalyticsReporter class]);
 319     OCMStub([mockReporter saveReport:[OCMArg isNotNil] fileName:[OCMArg isNotNil]]).andDo(^(NSInvocation *invocation) {
 320         _reporterWrites++;
 321     }).andReturn(YES);
 322
 323     [supd removeInstance];
 324     _supd = [[supd alloc] initWithReporter:mockReporter];
 325     _ckksAnalytics = [FakeCKKSAnalytics new];
 326     _sosAnalytics = [FakeSOSAnalytics new];
 327     _pcsAnalytics = [FakePCSAnalytics new];
 328     _tlsAnalytics = [FakeTLSAnalytics new];
 329     NSLog(@"ckks sqlite3 %@", [FakeCKKSAnalytics databasePath]);
 330     NSLog(@"sos  sqlite3 %@", [FakeSOSAnalytics databasePath]);
 331     NSLog(@"pcs  sqlite3 %@", [FakePCSAnalytics databasePath]);
 332     NSLog(@"tls  sqlite3 %@", [FakeTLSAnalytics databasePath]);
 333
 334     // Forcibly override analytics flags and enable them by default
 335     deviceAnalyticsOverride = YES;
 336     deviceAnalyticsEnabled = YES;
 337     iCloudAnalyticsOverride = YES;
 338     iCloudAnalyticsEnabled = YES;
 339 }
 340
 341 - (void)tearDown
 342 {
 343
 344     [super tearDown];
 345 }
 346
 347 // MARK: Actual tests
 348
 349 // Note! This test relies on Security being installed because supd reads from a plist in Security.framework
 350 - (void)testSplunkDefaultTopicNameExists
 351 {
 352     XCTAssertNotNil([[self keySyncTopic] splunkTopicName]);
 353 }
 354
 355 // Note! This test relies on Security being installed because supd reads from a plist in Security.framework
 356 - (void)testSplunkDefaultBagURLExists
 357 {
 358     XCTAssertNotNil([[self keySyncTopic] splunkBagURL]);
 359 }
 360
 361 - (void)testHaveEligibleClientsKeySync
 362 {
 363     // KeySyncTopic has no clients requiring deviceAnalytics currently
 364     SFAnalyticsTopic* keytopic = [[SFAnalyticsTopic alloc] initWithDictionary:@{} name:@"KeySyncTopic" samplingRates:@{}];
 365
 366     XCTAssertTrue([keytopic haveEligibleClients], @"Both analytics enabled -> we have keysync clients");
 367
 368     deviceAnalyticsEnabled = NO;
 369     XCTAssertTrue([keytopic haveEligibleClients], @"Only iCloud analytics enabled -> we have keysync clients");
 370
 371     iCloudAnalyticsEnabled = NO;
 372     XCTAssertFalse([keytopic haveEligibleClients], @"Both analytics disabled -> no keysync clients");
 373
 374     deviceAnalyticsEnabled = YES;
 375     XCTAssertFalse([keytopic haveEligibleClients], @"Only device analytics enabled -> no keysync clients");
 376 }
 377
 378 - (void)testHaveEligibleClientsTrust
 379 {
 380     // TrustTopic has no clients requiring iCloudAnalytics currently
 381     SFAnalyticsTopic* trusttopic = [[SFAnalyticsTopic alloc] initWithDictionary:@{} name:@"TrustTopic" samplingRates:@{}];
 382
 383     XCTAssertTrue([trusttopic haveEligibleClients], @"Both analytics enabled -> we have trust clients");
 384
 385     deviceAnalyticsEnabled = NO;
 386     XCTAssertFalse([trusttopic haveEligibleClients], @"Only iCloud analytics enabled -> no trust clients");
 387
 388     iCloudAnalyticsEnabled = NO;
 389     XCTAssertFalse([trusttopic haveEligibleClients], @"Both analytics disabled -> no trust clients");
 390
 391     deviceAnalyticsEnabled = YES;
 392     XCTAssertTrue([trusttopic haveEligibleClients], @"Only device analytics enabled -> we have trust clients");
 393 }
 394
 395 - (void)testLoggingJSONSimple:(BOOL)analyticsEnabled
 396 {
 397     iCloudAnalyticsEnabled = analyticsEnabled;
 398
 399     [_ckksAnalytics logSuccessForEventNamed:@"ckksunittestevent"];
 400     NSDictionary* ckksAttrs = @{@"cattr" : @"cvalue"};
 401     [_ckksAnalytics logHardFailureForEventNamed:@"ckksunittestevent" withAttributes:ckksAttrs];
 402     [_ckksAnalytics logSoftFailureForEventNamed:@"ckksunittestevent" withAttributes:ckksAttrs];
 403     [_sosAnalytics logSuccessForEventNamed:@"unittestevent"];
 404     NSDictionary* utAttrs = @{@"uattr" : @"uvalue"};
 405     [_sosAnalytics logHardFailureForEventNamed:@"unittestevent" withAttributes:utAttrs];
 406     [_sosAnalytics logSoftFailureForEventNamed:@"unittestevent" withAttributes:utAttrs];
 407
 408     NSDictionary* data = [self getJSONDataFromSupd];
 409     [self inspectDataBlobStructure:data];
 410
 411     // TODO: inspect health summaries
 412
 413     if (analyticsEnabled) {
 414         XCTAssertEqual([self failures:data eventType:@"ckksunittestevent" attributes:ckksAttrs class:SFAnalyticsEventClassHardFailure], 1);
 415         XCTAssertEqual([self failures:data eventType:@"ckksunittestevent" attributes:ckksAttrs class:SFAnalyticsEventClassSoftFailure], 1);
 416         XCTAssertEqual([self failures:data eventType:@"unittestevent" attributes:utAttrs class:SFAnalyticsEventClassHardFailure], 1);
 417         XCTAssertEqual([self failures:data eventType:@"unittestevent" attributes:utAttrs class:SFAnalyticsEventClassSoftFailure], 1);
 418
 419         [self checkTotalEventCount:data hard:2 soft:2];
 420     } else {
 421         [self checkTotalEventCount:data hard:0 soft:0 forcedFail:YES];
 422     }
 423 }
 424
 425 - (void)testLoggingJSONSimpleWithiCloudAnalyticsEnabled
 426 {
 427     [self testLoggingJSONSimple:YES];
 428 }
 429
 430 - (void)testLoggingJSONSimpleWithiCloudAnalyticsDisabled
 431 {
 432     [self testLoggingJSONSimple:NO];
 433 }
 434
 435 - (void)testTLSLoggingJSONSimple:(BOOL)analyticsEnabled
 436 {
 437     deviceAnalyticsEnabled = analyticsEnabled;
 438
 439     [_tlsAnalytics logSuccessForEventNamed:@"tlsunittestevent"];
 440     NSDictionary* tlsAttrs = @{@"cattr" : @"cvalue"};
 441     [_tlsAnalytics logHardFailureForEventNamed:@"tlsunittestevent" withAttributes:tlsAttrs];
 442     [_tlsAnalytics logSoftFailureForEventNamed:@"tlsunittestevent" withAttributes:tlsAttrs];
 443
 444     NSDictionary* data = [self getJSONDataFromSupd];
 445     [self inspectDataBlobStructure:data];
 446
 447     if (analyticsEnabled) {
 448         [self checkTotalEventCount:data hard:1 soft:1];
 449     } else {
 450         [self checkTotalEventCount:data hard:0 soft:0 forcedFail:YES];
 451     }
 452 }
 453
 454 - (void)testTLSLoggingJSONSimpleWithDeviceAnalyticsEnabled
 455 {
 456     [self testTLSLoggingJSONSimple:YES];
 457 }
 458
 459 - (void)testTLSLoggingJSONSimpleWithDeviceAnalyticsDisabled
 460 {
 461     [self testTLSLoggingJSONSimple:NO];
 462 }
 463
 464 - (void)testMockDiagnosticReportGeneration
 465 {
 466     SFAnalyticsReporter *reporter = mockReporter;
 467
 468     uint8_t report_data[] = {0x00, 0x01, 0x02, 0x03};
 469     NSData *reportData = [[NSData alloc] initWithBytes:report_data length:sizeof(report_data)];
 470     BOOL writtenToLog = YES;
 471     size_t numWrites = 5;
 472     for (size_t i = 0; i < numWrites; i++) {
 473         writtenToLog &= [reporter saveReport:reportData fileName:@"log.txt"];
 474     }
 475
 476     XCTAssertTrue(writtenToLog, "Failed to write to log");
 477     XCTAssertTrue((int)_reporterWrites == (int)numWrites, "Expected %zu report, got %d", numWrites, (int)_reporterWrites);
 478 }
 479
 480 - (void)testSuccessCounts
 481 {
 482     NSString* eventName1 = @"successCountsEvent1";
 483     NSString* eventName2 = @"successCountsEvent2";
 484
 485     for (int idx = 0; idx < 3; ++idx) {
 486         [_ckksAnalytics logSuccessForEventNamed:eventName1];
 487         [_ckksAnalytics logSuccessForEventNamed:eventName2];
 488         [_ckksAnalytics logHardFailureForEventNamed:eventName1 withAttributes:nil];
 489         [_ckksAnalytics logSoftFailureForEventNamed:eventName2 withAttributes:nil];
 490     }
 491     [_ckksAnalytics logSuccessForEventNamed:eventName2];
 492
 493     NSDictionary* data = [self getJSONDataFromSupd];
 494     [self inspectDataBlobStructure:data];
 495
 496     NSDictionary* hs;
 497     for (NSDictionary* event in data[@"events"]) {
 498         if ([event[SFAnalyticsEventType] isEqual:@"ckksHealthSummary"]) {
 499             hs = event;
 500             break;
 501         }
 502     }
 503     XCTAssert(hs);
 504
 505     XCTAssertEqual([hs[SFAnalyticsColumnSuccessCount] integerValue], 7);
 506     XCTAssertEqual([hs[SFAnalyticsColumnHardFailureCount] integerValue], 3);
 507     XCTAssertEqual([hs[SFAnalyticsColumnSoftFailureCount] integerValue], 3);
 508     XCTAssertEqual([hs[[self string:eventName1 item:@"success"]] integerValue], 3);
 509     XCTAssertEqual([hs[[self string:eventName1 item:@"hardfail"]] integerValue], 3);
 510     XCTAssertEqual([hs[[self string:eventName1 item:@"softfail"]] integerValue], 0);
 511     XCTAssertEqual([hs[[self string:eventName2 item:@"success"]] integerValue], 4);
 512     XCTAssertEqual([hs[[self string:eventName2 item:@"hardfail"]] integerValue], 0);
 513     XCTAssertEqual([hs[[self string:eventName2 item:@"softfail"]] integerValue], 3);
 514 }
 515
 516 // There was a failure with thresholds if some, but not all clients exceeded their 'threshold' number of failures,
 517 // causing the addFailures:toUploadRecords:threshold method to crash with out of bounds.
 518 // This is also implicitly tested in testTooManyHardFailures and testTooManyCombinedFailures but I wanted an explicit case.
 519 - (void)testExceedThresholdForOneClientOnly
 520 {
 521     int testAmount = ((int)SFAnalyticsMaxEventsToReport / 4);
 522     for (int idx = 0; idx < testAmount; ++idx) {
 523         [_ckksAnalytics logHardFailureForEventNamed:@"ckkshardfail" withAttributes:nil];
 524         [_ckksAnalytics logSoftFailureForEventNamed:@"ckkssoftfail" withAttributes:nil];
 525     }
 526
 527     [_sosAnalytics logHardFailureForEventNamed:@"soshardfail" withAttributes:nil];
 528     [_sosAnalytics logSoftFailureForEventNamed:@"sossoftfail" withAttributes:nil];
 529
 530     NSDictionary* data = [self getJSONDataFromSupd];
 531     [self inspectDataBlobStructure:data];
 532
 533     [self checkTotalEventCount:data hard:testAmount + 1 soft:testAmount + 1];
 534
 535     XCTAssertEqual([self failures:data eventType:@"ckkshardfail" attributes:nil class:SFAnalyticsEventClassHardFailure], testAmount);
 536     XCTAssertEqual([self failures:data eventType:@"ckkssoftfail" attributes:nil class:SFAnalyticsEventClassSoftFailure], testAmount);
 537     XCTAssertEqual([self failures:data eventType:@"soshardfail" attributes:nil class:SFAnalyticsEventClassHardFailure], 1);
 538     XCTAssertEqual([self failures:data eventType:@"sossoftfail" attributes:nil class:SFAnalyticsEventClassSoftFailure], 1);
 539 }
 540
 541
 542 // We have so many hard failures they won't fit in the upload buffer
 543 - (void)testTooManyHardFailures
 544 {
 545     NSDictionary* ckksAttrs = @{@"cattr" : @"cvalue"};
 546     NSDictionary* utAttrs = @{@"uattr" : @"uvalue"};
 547     for (int idx = 0; idx < 400; ++idx) {
 548         [_ckksAnalytics logHardFailureForEventNamed:@"ckksunittestfailure" withAttributes:ckksAttrs];
 549         [_ckksAnalytics logHardFailureForEventNamed:@"ckksunittestfailure" withAttributes:ckksAttrs];
 550         [_sosAnalytics logHardFailureForEventNamed:@"utunittestfailure" withAttributes:utAttrs];
 551     }
 552
 553     NSDictionary* data = [self getJSONDataFromSupd];
 554     [self inspectDataBlobStructure:data];
 555
 556     [self checkTotalEventCount:data hard:998 soft:0];
 557     // Based on threshold = records_to_upload/10 with a nice margin
 558     XCTAssertEqualWithAccuracy([self failures:data eventType:@"ckksunittestfailure" attributes:ckksAttrs class:SFAnalyticsEventClassHardFailure], 658, 50);
 559     XCTAssertEqualWithAccuracy([self failures:data eventType:@"utunittestfailure" attributes:utAttrs class:SFAnalyticsEventClassHardFailure], 339, 50);
 560 }
 561
 562 // So many soft failures they won't fit in the buffer
 563 - (void)testTooManySoftFailures
 564 {
 565     NSDictionary* ckksAttrs = @{@"cattr" : @"cvalue"};
 566     NSDictionary* utAttrs = @{@"uattr" : @"uvalue"};
 567     for (int idx = 0; idx < 400; ++idx) {
 568         [_ckksAnalytics logSoftFailureForEventNamed:@"ckksunittestfailure" withAttributes:ckksAttrs];
 569         [_ckksAnalytics logSoftFailureForEventNamed:@"ckksunittestfailure" withAttributes:ckksAttrs];
 570         [_sosAnalytics logSoftFailureForEventNamed:@"utunittestfailure" withAttributes:utAttrs];
 571     }
 572
 573     NSDictionary* data = [self getJSONDataFromSupd];
 574     [self inspectDataBlobStructure:data];
 575
 576     [self checkTotalEventCount:data hard:0 soft:998];
 577     // Based on threshold = records_to_upload/10 with a nice margin
 578     XCTAssertEqualWithAccuracy([self failures:data eventType:@"ckksunittestfailure" attributes:ckksAttrs class:SFAnalyticsEventClassSoftFailure], 665, 50);
 579     XCTAssertEqualWithAccuracy([self failures:data eventType:@"utunittestfailure" attributes:utAttrs class:SFAnalyticsEventClassSoftFailure], 332, 50);
 580 }
 581
 582 - (void)testTooManyCombinedFailures
 583 {
 584     NSDictionary* ckksAttrs = @{@"cattr1" : @"cvalue1", @"cattrthatisalotlongerthanthepreviousone" : @"cvaluethatisalsoalotlongerthantheother"};
 585     NSDictionary* utAttrs = @{@"uattr" : @"uvalue", @"uattrthatisalotlongerthanthepreviousone" : @"uvaluethatisalsoalotlongerthantheother"};
 586     for (int idx = 0; idx < 400; ++idx) {
 587         [_ckksAnalytics logHardFailureForEventNamed:@"ckksunittestfailure" withAttributes:ckksAttrs];
 588         [_ckksAnalytics logSoftFailureForEventNamed:@"ckksunittestfailure" withAttributes:ckksAttrs];
 589         [_sosAnalytics logHardFailureForEventNamed:@"utunittestfailure" withAttributes:utAttrs];
 590         [_sosAnalytics logSoftFailureForEventNamed:@"utunittestfailure" withAttributes:utAttrs];
 591     }
 592
 593     NSDictionary* data = [self getJSONDataFromSupd];
 594     [self inspectDataBlobStructure:data];
 595
 596     [self checkTotalEventCount:data hard:800 soft:198];
 597     // Based on threshold = records_to_upload/10 with a nice margin
 598     XCTAssertEqualWithAccuracy([self failures:data eventType:@"ckksunittestfailure" attributes:ckksAttrs class:SFAnalyticsEventClassHardFailure], 400, 50);
 599     XCTAssertEqualWithAccuracy([self failures:data eventType:@"utunittestfailure" attributes:utAttrs class:SFAnalyticsEventClassHardFailure], 400, 50);
 600     XCTAssertEqualWithAccuracy([self failures:data eventType:@"ckksunittestfailure" attributes:ckksAttrs class:SFAnalyticsEventClassSoftFailure], 100, 50);
 601     XCTAssertEqualWithAccuracy([self failures:data eventType:@"utunittestfailure" attributes:utAttrs class:SFAnalyticsEventClassSoftFailure], 100, 50);
 602 }
 603
 604 // There's an even number of samples
 605 - (void)testSamplesEvenSampleCount
 606 {
 607     NSString* sampleNameEven = @"evenSample";
 608
 609     for (NSNumber* value in @[@36.831855250339714, @90.78721762172914, @49.24392301762506,
 610                               @42.806362283260036, @16.76725375576855, @34.50969130579674,
 611                               @25.956509180834637, @36.8268555935645, @35.54069258036879,
 612                               @7.26364884595062, @45.414180770615395, @5.223213570809022]) {
 613         [_ckksAnalytics logMetric:value withName:sampleNameEven];
 614     }
 615
 616     NSDictionary* data = [self getJSONDataFromSupd];
 617     [self inspectDataBlobStructure:data];
 618
 619     // min, max, avg, med, dev, 1q, 3q
 620     [self checkTotalEventCount:data hard:0 soft:0];
 621     [self sampleStatisticsInEvents:data[@"events"] name:sampleNameEven values:@[@5.22, @90.78, @35.60, @36.18, @21.52, @21.36, @44.11]];
 622 }
 623
 624 // There are 4*n + 1 samples
 625 - (void)testSamples4n1SampleCount
 626 {
 627     NSString* sampleName4n1 = @"4n1Sample";
 628     for (NSNumber* value in @[@37.76544251068022, @27.36378948426223, @45.10503077614114,
 629                               @43.90635413191473, @54.78709742040113, @52.34879597889124,
 630                               @70.95760312196856, @23.23648158872921, @75.34678687445064,
 631                               @10.723238854026203, @41.98468801166455, @17.074404554908476,
 632                               @94.24252031232739]) {
 633         [_ckksAnalytics logMetric:value withName:sampleName4n1];
 634     }
 635
 636     NSDictionary* data = [self getJSONDataFromSupd];
 637     [self inspectDataBlobStructure:data];
 638
 639     [self checkTotalEventCount:data hard:0 soft:0];
 640     [self sampleStatisticsInEvents:data[@"events"] name:sampleName4n1 values:@[@10.72, @94.24, @45.76, @43.90, @23.14, @26.33, @58.83]];
 641 }
 642
 643 // There are 4*n + 3 samples
 644 - (void)testSamples4n3SampleCount
 645 {
 646     NSString* sampleName4n3 = @"4n3Sample";
 647
 648     for (NSNumber* value in @[@42.012971885655496, @87.85629592375282, @5.748491212287082,
 649                               @38.451850063872975, @81.96900109690873, @99.83098790545392,
 650                               @80.89400981437815, @5.719237885152143, @1.6740622555032196,
 651                               @14.437000556079038, @29.046050177512395]) {
 652         [_sosAnalytics logMetric:value withName:sampleName4n3];
 653     }
 654
 655     NSDictionary* data = [self getJSONDataFromSupd];
 656     [self inspectDataBlobStructure:data];
 657     [self checkTotalEventCount:data hard:0 soft:0];
 658
 659     [self sampleStatisticsInEvents:data[@"events"] name:sampleName4n3 values:@[@1.67, @99.83, @44.33, @38.45, @35.28, @7.92, @81.70]];
 660 }
 661
 662 // stddev and quartiles undefined for single sample
 663 - (void)testSamplesSingleSample
 664 {
 665     NSString* sampleName = @"singleSample";
 666
 667     [_ckksAnalytics logMetric:@3.14159 withName:sampleName];
 668
 669     NSDictionary* data = [self getJSONDataFromSupd];
 670     [self inspectDataBlobStructure:data];
 671     [self checkTotalEventCount:data hard:0 soft:0];
 672
 673     [self sampleStatisticsInEvents:data[@"events"] name:sampleName values:@[@3.14159]];
 674 }
 675
 676 // quartiles meaningless for fewer than 4 samples (but stddev exists)
 677 - (void)testSamplesFewerThanFour
 678 {
 679     NSString* sampleName = @"fewSamples";
 680
 681     [_ckksAnalytics logMetric:@3.14159 withName:sampleName];
 682     [_ckksAnalytics logMetric:@6.28318 withName:sampleName];
 683
 684     NSDictionary* data = [self getJSONDataFromSupd];
 685     [self inspectDataBlobStructure:data];
 686     [self checkTotalEventCount:data hard:0 soft:0];
 687
 688     [self sampleStatisticsInEvents:data[@"events"] name:sampleName values:@[@3.14, @6.28, @4.71, @4.71, @1.57]];
 689 }
 690
 691 - (void)testSamplesSameNameDifferentSubclass
 692 {
 693     NSString* sampleName = @"differentSubclassSamples";
 694
 695     [_sosAnalytics logMetric:@313.37 withName:sampleName];
 696     [_ckksAnalytics logMetric:@313.37 withName:sampleName];
 697
 698     NSDictionary* data = [self getJSONDataFromSupd];
 699     [self inspectDataBlobStructure:data];
 700     [self checkTotalEventCount:data hard:0 soft:0];
 701
 702     [self sampleStatisticsInEvents:data[@"events"] name:sampleName values:@[@313.37] amount:2];
 703 }
 704
 705
 706
 707 // TODO
 708 - (void)testGetSysdiagnoseDump
 709 {
 710
 711 }
 712
 713 // TODO (need mock server)
 714 - (void)testSplunkUpload
 715 {
 716
 717 }
 718
 719 // TODO (need mock server)
 720 - (void)testDBIsEmptiedAfterUpload
 721 {
 722
 723 }
 724
 725 @end