supd/Tests/SupdTests.m

   1 /*
   2  * Copyright (c) 2017-2018 Apple Inc. All Rights Reserved.
   3  *
   4  * @APPLE_LICENSE_HEADER_START@
   5  *
   6  * This file contains Original Code and/or Modifications of Original Code
   7  * as defined in and that are subject to the Apple Public Source License
   8  * Version 2.0 (the 'License'). You may not use this file except in
   9  * compliance with the License. Please obtain a copy of the License at
  10  * http://www.opensource.apple.com/apsl/ and read it before using this
  11  * file.
  12  *
  13  * The Original Code and all software distributed under the License are
  14  * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  15  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  16  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
  17  * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
  18  * Please see the License for the specific language governing rights and
  19  * limitations under the License.
  20  *
  21  * @APPLE_LICENSE_HEADER_END@
  22  */
  23
  24 #import <OCMock/OCMock.h>
  25 #import <XCTest/XCTest.h>
  26 #import "supd.h"
  27 #import <Security/SFAnalytics.h>
  28 #import "SFAnalyticsDefines.h"
  29 #import <CoreFoundation/CFPriv.h>
  30
  31 static NSString* _path;
  32 static NSInteger _testnum;
  33 static NSString* build = NULL;
  34 static NSString* product = NULL;
  35 static NSInteger _reporterWrites;
  36
  37 // MARK: Stub FakeCKKSAnalytics
  38
  39 @interface FakeCKKSAnalytics : SFAnalytics
  40
  41 @end
  42
  43 @implementation FakeCKKSAnalytics
  44
  45 + (NSString*)databasePath
  46 {
  47     return [_path stringByAppendingFormat:@"/ckks_%ld.db", _testnum];
  48 }
  49
  50 @end
  51
  52
  53 // MARK: Stub FakeSOSAnalytics
  54
  55 @interface FakeSOSAnalytics : SFAnalytics
  56
  57 @end
  58
  59 @implementation FakeSOSAnalytics
  60
  61 + (NSString*)databasePath
  62 {
  63     return [_path stringByAppendingFormat:@"/sos_%ld.db", _testnum];
  64 }
  65
  66 @end
  67
  68
  69 // MARK: Stub FakePCSAnalytics
  70
  71 @interface FakePCSAnalytics : SFAnalytics
  72
  73 @end
  74
  75 @implementation FakePCSAnalytics
  76
  77 + (NSString*)databasePath
  78 {
  79     return [_path stringByAppendingFormat:@"/pcs_%ld.db", _testnum];
  80 }
  81
  82 @end
  83
  84 // MARK: Stub FakeTLSAnalytics
  85
  86 @interface FakeTLSAnalytics : SFAnalytics
  87
  88 @end
  89
  90 @implementation FakeTLSAnalytics
  91
  92 + (NSString*)databasePath
  93 {
  94     return [_path stringByAppendingFormat:@"/tls_%ld.db", _testnum];
  95 }
  96
  97 @end
  98
  99 // MARK: Start SupdTests
 100
 101 @interface SupdTests : XCTestCase
 102
 103 @end
 104
 105 @implementation SupdTests {
 106     supd* _supd;
 107     id mockReporter;
 108     FakeCKKSAnalytics* _ckksAnalytics;
 109     FakeSOSAnalytics* _sosAnalytics;
 110     FakePCSAnalytics* _pcsAnalytics;
 111     FakeTLSAnalytics* _tlsAnalytics;
 112 }
 113
 114 // MARK: Test helper methods
 115 - (SFAnalyticsTopic *)keySyncTopic {
 116     for (SFAnalyticsTopic *topic in _supd.analyticsTopics) {
 117         if ([topic.internalTopicName isEqualToString:SFAnalyticsTopicKeySync]) {
 118             return topic;
 119         }
 120     }
 121     return nil;
 122 }
 123
 124 - (SFAnalyticsTopic *)TrustTopic {
 125     for (SFAnalyticsTopic *topic in _supd.analyticsTopics) {
 126         if ([topic.internalTopicName isEqualToString:SFAnaltyicsTopicTrust]) {
 127             return topic;
 128         }
 129     }
 130     return nil;
 131 }
 132
 133 - (void)inspectDataBlobStructure:(NSDictionary*)data
 134 {
 135     [self inspectDataBlobStructure:data forTopic:[[self keySyncTopic] splunkTopicName]];
 136 }
 137
 138 - (void)inspectDataBlobStructure:(NSDictionary*)data forTopic:(NSString*)topic
 139 {
 140     if (!data || ![data isKindOfClass:[NSDictionary class]]) {
 141         XCTFail(@"data is an NSDictionary");
 142     }
 143
 144     XCTAssert(_supd.analyticsTopics, @"supd has nonnull topics list");
 145     XCTAssert([[self keySyncTopic] splunkTopicName], @"keysync topic has a splunk name");
 146     XCTAssert([[self TrustTopic] splunkTopicName], @"trust topic has a splunk name");
 147     XCTAssertEqual([data count], 2ul, @"dictionary event and posttime objects");
 148     XCTAssertTrue(data[@"events"] && [data[@"events"] isKindOfClass:[NSArray class]], @"data blob contains an NSArray 'events'");
 149     XCTAssertTrue(data[@"postTime"] && [data[@"postTime"] isKindOfClass:[NSNumber class]], @"data blob contains an NSNumber 'postTime");
 150     NSDate* postTime = [NSDate dateWithTimeIntervalSince1970:[data[@"postTime"] doubleValue]];
 151     XCTAssertTrue([[NSDate date] timeIntervalSinceDate:postTime] < 3, @"postTime is sane");
 152
 153     for (NSDictionary* event in data[@"events"]) {
 154         if ([event isKindOfClass:[NSDictionary class]]) {
 155             NSLog(@"build: \"%@\", eventbuild: \"%@\"", build, event[@"build"]);
 156             XCTAssertTrue([event[@"build"] isEqual:build], @"event contains correct build string");
 157             XCTAssertTrue([event[@"product"] isEqual:product], @"event contains correct product string");
 158             XCTAssertTrue([event[@"eventTime"] isKindOfClass:[NSNumber class]], @"event contains an NSNumber 'eventTime");
 159             NSDate* eventTime = [NSDate dateWithTimeIntervalSince1970:[event[@"eventTime"] doubleValue]];
 160             XCTAssertTrue([[NSDate date] timeIntervalSinceDate:eventTime] < 3, @"eventTime is sane");
 161             XCTAssertTrue([event[@"eventType"] isKindOfClass:[NSString class]], @"all events have a type");
 162             XCTAssertTrue([event[@"topic"] isEqual:topic], @"all events have a topic name");
 163         } else {
 164             XCTFail(@"event %@ is an NSDictionary", event);
 165         }
 166     }
 167 }
 168
 169 - (BOOL)event:(NSDictionary*)event containsAttributes:(NSDictionary*)attrs {
 170     if (!attrs) {
 171         return YES;
 172     }
 173     __block BOOL equal = YES;
 174     [attrs enumerateKeysAndObjectsUsingBlock:^(id  _Nonnull key, id  _Nonnull obj, BOOL * _Nonnull stop) {
 175         equal &= [event[key] isEqualToString:obj];
 176     }];
 177     return equal;
 178 }
 179
 180 - (int)failures:(NSDictionary*)data eventType:(NSString*)type attributes:(NSDictionary*)attrs class:(SFAnalyticsEventClass)class
 181 {
 182     int encountered = 0;
 183     for (NSDictionary* event in data[@"events"]) {
 184         if ([event[@"eventType"] isEqualToString:type] &&
 185             [event[@"eventClass"] isKindOfClass:[NSNumber class]] &&
 186             [event[@"eventClass"] intValue] == class && [self event:event containsAttributes:attrs]) {
 187             ++encountered;
 188         }
 189     }
 190     return encountered;
 191 }
 192
 193 - (void)checkTotalEventCount:(NSDictionary*)data hard:(int)hard soft:(int)soft accuracy:(int)accuracy summaries:(int)summ
 194 {
 195     int hardfound = 0, softfound = 0, summfound = 0;
 196     for (NSDictionary* event in data[@"events"]) {
 197         if ([event[SFAnalyticsEventType] hasSuffix:@"HealthSummary"]) {
 198             ++summfound;
 199         } else if ([event[SFAnalyticsEventClassKey] integerValue] == SFAnalyticsEventClassHardFailure) {
 200             ++hardfound;
 201         } else if ([event[SFAnalyticsEventClassKey] integerValue] == SFAnalyticsEventClassSoftFailure) {
 202             ++softfound;
 203         }
 204     }
 205
 206     XCTAssertLessThanOrEqual(((NSArray*)data[@"events"]).count, 1000ul, @"Total event count fits in alloted data");
 207     XCTAssertEqual(summfound, summ);
 208
 209     // Add customizable fuzziness
 210     XCTAssertEqualWithAccuracy(hardfound, hard, accuracy);
 211     XCTAssertEqualWithAccuracy(softfound, soft, accuracy);
 212 }
 213
 214 - (void)checkTotalEventCount:(NSDictionary*)data hard:(int)hard soft:(int)soft
 215 {
 216     [self checkTotalEventCount:data hard:hard soft:soft accuracy:10 summaries:(int)[[[self keySyncTopic] topicClients] count]];
 217 }
 218
 219 - (void)checkTotalEventCount:(NSDictionary*)data hard:(int)hard soft:(int)soft accuracy:(int)accuracy
 220 {
 221     [self checkTotalEventCount:data hard:hard soft:soft accuracy:accuracy summaries:(int)[[[self keySyncTopic] topicClients] count]];
 222 }
 223
 224 // This is a dumb hack, but inlining stringWithFormat causes the compiler to growl for unknown reasons
 225 - (NSString*)string:(NSString*)name item:(NSString*)item
 226 {
 227     return [NSString stringWithFormat:@"%@-%@", name, item];
 228 }
 229
 230 - (void)sampleStatisticsInEvents:(NSArray*)events name:(NSString*)name values:(NSArray*)values
 231 {
 232     [self sampleStatisticsInEvents:events name:name values:values amount:1];
 233 }
 234
 235 // Usually amount == 1 but for testing sampler with same name in different subclasses this is higher
 236 - (void)sampleStatisticsInEvents:(NSArray*)events name:(NSString*)name values:(NSArray*)values amount:(int)num
 237 {
 238     int found = 0;
 239     for (NSDictionary* event in events) {
 240         if (([values count] == 1 && ![event objectForKey:[NSString stringWithFormat:@"%@", name]]) ||
 241             ([values count] > 1 && ![event objectForKey:[NSString stringWithFormat:@"%@-min", name]])) {
 242             continue;
 243         }
 244
 245         ++found;
 246         if (values.count == 1) {
 247             XCTAssertEqual([event[name] doubleValue], [values[0] doubleValue]);
 248             XCTAssertNil(event[[self string:name item:@"min"]]);
 249             XCTAssertNil(event[[self string:name item:@"max"]]);
 250             XCTAssertNil(event[[self string:name item:@"avg"]]);
 251             XCTAssertNil(event[[self string:name item:@"med"]]);
 252         } else {
 253             XCTAssertEqualWithAccuracy([event[[self string:name item:@"min"]] doubleValue], [values[0] doubleValue], 0.01f);
 254             XCTAssertEqualWithAccuracy([event[[self string:name item:@"max"]] doubleValue], [values[1] doubleValue], 0.01f);
 255             XCTAssertEqualWithAccuracy([event[[self string:name item:@"avg"]] doubleValue], [values[2] doubleValue], 0.01f);
 256             XCTAssertEqualWithAccuracy([event[[self string:name item:@"med"]] doubleValue], [values[3] doubleValue], 0.01f);
 257         }
 258
 259         if (values.count > 4) {
 260             XCTAssertEqualWithAccuracy([event[[self string:name item:@"dev"]] doubleValue], [values[4] doubleValue], 0.01f);
 261         } else {
 262             XCTAssertNil(event[[self string:name item:@"dev"]]);
 263         }
 264
 265         if (values.count > 5) {
 266             XCTAssertEqualWithAccuracy([event[[self string:name item:@"1q"]] doubleValue], [values[5] doubleValue], 0.01f);
 267             XCTAssertEqualWithAccuracy([event[[self string:name item:@"3q"]] doubleValue], [values[6] doubleValue], 0.01f);
 268         } else {
 269             XCTAssertNil(event[[self string:name item:@"1q"]]);
 270             XCTAssertNil(event[[self string:name item:@"3q"]]);
 271         }
 272     }
 273     XCTAssertEqual(found, num);
 274 }
 275
 276 - (NSDictionary*)getJSONDataFromSupd
 277 {
 278     return [self getJSONDataFromSupdWithTopic:SFAnalyticsTopicKeySync];
 279 }
 280
 281 - (NSDictionary*)getJSONDataFromSupdWithTopic:(NSString*)topic
 282 {
 283     dispatch_semaphore_t sema = dispatch_semaphore_create(0);
 284     __block NSDictionary* data;
 285     [_supd getLoggingJSON:YES topic:topic reply:^(NSData *json, NSError *error) {
 286         XCTAssertNil(error);
 287         XCTAssertNotNil(json);
 288         if (!error) {
 289             data = [NSJSONSerialization JSONObjectWithData:json options:0 error:&error];
 290         }
 291         XCTAssertNil(error, @"no error deserializing json: %@", error);
 292         dispatch_semaphore_signal(sema);
 293     }];
 294     if (dispatch_semaphore_wait(sema, dispatch_time(DISPATCH_TIME_NOW, NSEC_PER_SEC * 5)) != 0) {
 295         XCTFail(@"supd returns JSON data in a timely fashion");
 296     }
 297     return data;
 298 }
 299
 300 // MARK: Test administration
 301
 302 + (void)setUp
 303 {
 304     NSError* error;
 305     _path = [NSTemporaryDirectory() stringByAppendingPathComponent:[NSString stringWithFormat:@"%@/", [[NSUUID UUID] UUIDString]]];
 306     [[NSFileManager defaultManager] createDirectoryAtPath:_path
 307                               withIntermediateDirectories:YES
 308                                                attributes:nil
 309                                                     error:&error];
 310     if (error) {
 311         NSLog(@"sad trombone, couldn't create path");
 312     }
 313
 314     NSDictionary *version = CFBridgingRelease(_CFCopySystemVersionDictionary());
 315     if (version) {
 316         build = version[(__bridge NSString *)_kCFSystemVersionBuildVersionKey];
 317         product = version[(__bridge NSString *)_kCFSystemVersionProductNameKey];
 318     } else {
 319         NSLog(@"could not get build version/product, tests should fail");
 320     }
 321 }
 322
 323 - (void)setUp
 324 {
 325     [super setUp];
 326     self.continueAfterFailure = NO;
 327     ++_testnum;
 328
 329     id mockTopic = OCMStrictClassMock([SFAnalyticsTopic class]);
 330     NSString *ckksPath = [_path stringByAppendingFormat:@"/ckks_%ld.db", _testnum];
 331     NSString *sosPath = [_path stringByAppendingFormat:@"/sos_%ld.db", _testnum];
 332     NSString *pcsPath = [_path stringByAppendingFormat:@"/pcs_%ld.db", _testnum];
 333     NSString *tlsPath = [_path stringByAppendingFormat:@"/tls_%ld.db", _testnum];
 334     OCMStub([mockTopic databasePathForCKKS]).andReturn(ckksPath);
 335     OCMStub([mockTopic databasePathForSOS]).andReturn(sosPath);
 336     OCMStub([mockTopic databasePathForPCS]).andReturn(pcsPath);
 337     OCMStub([mockTopic databasePathForTLS]).andReturn(tlsPath);
 338
 339     // These are not used for testing, but real data can pollute tests so point to empty DBs
 340     NSString *localpath = [_path stringByAppendingFormat:@"/local_empty_%ld.db", _testnum];
 341     NSString *trustPath = [_path stringByAppendingFormat:@"/trust_empty_%ld.db", _testnum];
 342     NSString *trustdhealthPath = [_path stringByAppendingFormat:@"/trustdhealth_empty_%ld.db", _testnum];
 343     OCMStub([mockTopic databasePathForLocal]).andReturn(localpath);
 344     OCMStub([mockTopic databasePathForTrust]).andReturn(trustPath);
 345     OCMStub([mockTopic databasePathForTrustdHealth]).andReturn(trustdhealthPath);
 346
 347     _reporterWrites = 0;
 348     mockReporter = OCMClassMock([SFAnalyticsReporter class]);
 349     OCMStub([mockReporter saveReport:[OCMArg isNotNil] fileName:[OCMArg isNotNil]]).andDo(^(NSInvocation *invocation) {
 350         _reporterWrites++;
 351     }).andReturn(YES);
 352
 353     [supd removeInstance];
 354     _supd = [[supd alloc] initWithReporter:mockReporter];
 355     _ckksAnalytics = [FakeCKKSAnalytics new];
 356     _sosAnalytics = [FakeSOSAnalytics new];
 357     _pcsAnalytics = [FakePCSAnalytics new];
 358     _tlsAnalytics = [FakeTLSAnalytics new];
 359
 360     // These are only useful for debugging
 361 //    NSLog(@"ckks sqlite3 %@", [FakeCKKSAnalytics databasePath]);
 362 //    NSLog(@"sos  sqlite3 %@", [FakeSOSAnalytics databasePath]);
 363 //    NSLog(@"pcs  sqlite3 %@", [FakePCSAnalytics databasePath]);
 364 //    NSLog(@"tls  sqlite3 %@", [FakeTLSAnalytics databasePath]);
 365
 366     // Forcibly override analytics flags and enable them by default
 367     deviceAnalyticsOverride = YES;
 368     deviceAnalyticsEnabled = YES;
 369     iCloudAnalyticsOverride = YES;
 370     iCloudAnalyticsEnabled = YES;
 371     runningTests = YES;
 372 }
 373
 374 - (void)tearDown
 375 {
 376
 377     [super tearDown];
 378 }
 379
 380 // MARK: Actual tests
 381
 382 // Note! This test relies on Security being installed because supd reads from a plist in Security.framework
 383 - (void)testSplunkDefaultTopicNameExists
 384 {
 385     XCTAssertNotNil([[self keySyncTopic] splunkTopicName]);
 386 }
 387
 388 // Note! This test relies on Security being installed because supd reads from a plist in Security.framework
 389 - (void)testSplunkDefaultBagURLExists
 390 {
 391     XCTAssertNotNil([[self keySyncTopic] splunkBagURL]);
 392 }
 393
 394 - (void)testHaveEligibleClientsKeySync
 395 {
 396     // KeySyncTopic has no clients requiring deviceAnalytics currently
 397     SFAnalyticsTopic* keytopic = [[SFAnalyticsTopic alloc] initWithDictionary:@{} name:@"KeySyncTopic" samplingRates:@{}];
 398
 399     XCTAssertTrue([keytopic haveEligibleClients], @"Both analytics enabled -> we have keysync clients");
 400
 401     deviceAnalyticsEnabled = NO;
 402     XCTAssertTrue([keytopic haveEligibleClients], @"Only iCloud analytics enabled -> we have keysync clients");
 403
 404     iCloudAnalyticsEnabled = NO;
 405     XCTAssertFalse([keytopic haveEligibleClients], @"Both analytics disabled -> no keysync clients");
 406
 407     deviceAnalyticsEnabled = YES;
 408     XCTAssertTrue([keytopic haveEligibleClients], @"Only device analytics enabled -> we have keysync clients (localkeychain for now)");
 409 }
 410
 411 - (void)testHaveEligibleClientsTrust
 412 {
 413     // TrustTopic has no clients requiring iCloudAnalytics currently
 414     SFAnalyticsTopic* trusttopic = [[SFAnalyticsTopic alloc] initWithDictionary:@{} name:@"TrustTopic" samplingRates:@{}];
 415
 416     XCTAssertTrue([trusttopic haveEligibleClients], @"Both analytics enabled -> we have trust clients");
 417
 418     deviceAnalyticsEnabled = NO;
 419     XCTAssertFalse([trusttopic haveEligibleClients], @"Only iCloud analytics enabled -> no trust clients");
 420
 421     iCloudAnalyticsEnabled = NO;
 422     XCTAssertFalse([trusttopic haveEligibleClients], @"Both analytics disabled -> no trust clients");
 423
 424     deviceAnalyticsEnabled = YES;
 425     XCTAssertTrue([trusttopic haveEligibleClients], @"Only device analytics enabled -> we have trust clients");
 426 }
 427
 428 - (void)testLoggingJSONSimple:(BOOL)analyticsEnabled
 429 {
 430     iCloudAnalyticsEnabled = analyticsEnabled;
 431
 432     [_ckksAnalytics logSuccessForEventNamed:@"ckksunittestevent"];
 433     NSDictionary* ckksAttrs = @{@"cattr" : @"cvalue"};
 434     [_ckksAnalytics logHardFailureForEventNamed:@"ckksunittestevent" withAttributes:ckksAttrs];
 435     [_ckksAnalytics logSoftFailureForEventNamed:@"ckksunittestevent" withAttributes:ckksAttrs];
 436     [_sosAnalytics logSuccessForEventNamed:@"unittestevent"];
 437     NSDictionary* utAttrs = @{@"uattr" : @"uvalue"};
 438     [_sosAnalytics logHardFailureForEventNamed:@"unittestevent" withAttributes:utAttrs];
 439     [_sosAnalytics logSoftFailureForEventNamed:@"unittestevent" withAttributes:utAttrs];
 440
 441     NSDictionary* data = [self getJSONDataFromSupd];
 442     [self inspectDataBlobStructure:data];
 443
 444     // TODO: inspect health summaries
 445
 446     if (analyticsEnabled) {
 447         XCTAssertEqual([self failures:data eventType:@"ckksunittestevent" attributes:ckksAttrs class:SFAnalyticsEventClassHardFailure], 1);
 448         XCTAssertEqual([self failures:data eventType:@"ckksunittestevent" attributes:ckksAttrs class:SFAnalyticsEventClassSoftFailure], 1);
 449         XCTAssertEqual([self failures:data eventType:@"unittestevent" attributes:utAttrs class:SFAnalyticsEventClassHardFailure], 1);
 450         XCTAssertEqual([self failures:data eventType:@"unittestevent" attributes:utAttrs class:SFAnalyticsEventClassSoftFailure], 1);
 451
 452         [self checkTotalEventCount:data hard:2 soft:2 accuracy:0];
 453     } else {
 454         // localkeychain requires device analytics only so we still get it
 455         [self checkTotalEventCount:data hard:0 soft:0 accuracy:0 summaries:1];
 456     }
 457 }
 458
 459 - (void)testLoggingJSONSimpleWithiCloudAnalyticsEnabled
 460 {
 461     [self testLoggingJSONSimple:YES];
 462 }
 463
 464 - (void)testLoggingJSONSimpleWithiCloudAnalyticsDisabled
 465 {
 466     [self testLoggingJSONSimple:NO];
 467 }
 468
 469 - (void)testTLSLoggingJSONSimple:(BOOL)analyticsEnabled
 470 {
 471     deviceAnalyticsEnabled = analyticsEnabled;
 472
 473     [_tlsAnalytics logSuccessForEventNamed:@"tlsunittestevent"];
 474     NSDictionary* tlsAttrs = @{@"cattr" : @"cvalue"};
 475     [_tlsAnalytics logHardFailureForEventNamed:@"tlsunittestevent" withAttributes:tlsAttrs];
 476     [_tlsAnalytics logSoftFailureForEventNamed:@"tlsunittestevent" withAttributes:tlsAttrs];
 477
 478     NSDictionary* data = [self getJSONDataFromSupdWithTopic:SFAnaltyicsTopicTrust];
 479     [self inspectDataBlobStructure:data forTopic:[[self TrustTopic] splunkTopicName]];
 480
 481     if (analyticsEnabled) {
 482         [self checkTotalEventCount:data hard:1 soft:1 accuracy:0 summaries:(int)[[[self TrustTopic] topicClients] count]];
 483     } else {
 484         [self checkTotalEventCount:data hard:0 soft:0 accuracy:0 summaries:0];
 485     }
 486 }
 487
 488 - (void)testTLSLoggingJSONSimpleWithDeviceAnalyticsEnabled
 489 {
 490     [self testTLSLoggingJSONSimple:YES];
 491 }
 492
 493 - (void)testTLSLoggingJSONSimpleWithDeviceAnalyticsDisabled
 494 {
 495     [self testTLSLoggingJSONSimple:NO];
 496 }
 497
 498 - (void)testMockDiagnosticReportGeneration
 499 {
 500     SFAnalyticsReporter *reporter = mockReporter;
 501
 502     uint8_t report_data[] = {0x00, 0x01, 0x02, 0x03};
 503     NSData *reportData = [[NSData alloc] initWithBytes:report_data length:sizeof(report_data)];
 504     BOOL writtenToLog = YES;
 505     size_t numWrites = 5;
 506     for (size_t i = 0; i < numWrites; i++) {
 507         writtenToLog &= [reporter saveReport:reportData fileName:@"log.txt"];
 508     }
 509
 510     XCTAssertTrue(writtenToLog, "Failed to write to log");
 511     XCTAssertTrue((int)_reporterWrites == (int)numWrites, "Expected %zu report, got %d", numWrites, (int)_reporterWrites);
 512 }
 513
 514 - (void)testSuccessCounts
 515 {
 516     NSString* eventName1 = @"successCountsEvent1";
 517     NSString* eventName2 = @"successCountsEvent2";
 518
 519     for (int idx = 0; idx < 3; ++idx) {
 520         [_ckksAnalytics logSuccessForEventNamed:eventName1];
 521         [_ckksAnalytics logSuccessForEventNamed:eventName2];
 522         [_ckksAnalytics logHardFailureForEventNamed:eventName1 withAttributes:nil];
 523         [_ckksAnalytics logSoftFailureForEventNamed:eventName2 withAttributes:nil];
 524     }
 525     [_ckksAnalytics logSuccessForEventNamed:eventName2];
 526
 527     NSDictionary* data = [self getJSONDataFromSupd];
 528     [self inspectDataBlobStructure:data];
 529
 530     NSDictionary* hs;
 531     for (NSDictionary* event in data[@"events"]) {
 532         if ([event[SFAnalyticsEventType] isEqual:@"ckksHealthSummary"]) {
 533             hs = event;
 534             break;
 535         }
 536     }
 537     XCTAssert(hs);
 538
 539     XCTAssertEqual([hs[SFAnalyticsColumnSuccessCount] integerValue], 7);
 540     XCTAssertEqual([hs[SFAnalyticsColumnHardFailureCount] integerValue], 3);
 541     XCTAssertEqual([hs[SFAnalyticsColumnSoftFailureCount] integerValue], 3);
 542     XCTAssertEqual([hs[[self string:eventName1 item:@"success"]] integerValue], 3);
 543     XCTAssertEqual([hs[[self string:eventName1 item:@"hardfail"]] integerValue], 3);
 544     XCTAssertEqual([hs[[self string:eventName1 item:@"softfail"]] integerValue], 0);
 545     XCTAssertEqual([hs[[self string:eventName2 item:@"success"]] integerValue], 4);
 546     XCTAssertEqual([hs[[self string:eventName2 item:@"hardfail"]] integerValue], 0);
 547     XCTAssertEqual([hs[[self string:eventName2 item:@"softfail"]] integerValue], 3);
 548 }
 549
 550 // There was a failure with thresholds if some, but not all clients exceeded their 'threshold' number of failures,
 551 // causing the addFailures:toUploadRecords:threshold method to crash with out of bounds.
 552 // This is also implicitly tested in testTooManyHardFailures and testTooManyCombinedFailures but I wanted an explicit case.
 553 - (void)testExceedThresholdForOneClientOnly
 554 {
 555     int testAmount = ((int)SFAnalyticsMaxEventsToReport / 4);
 556     for (int idx = 0; idx < testAmount; ++idx) {
 557         [_ckksAnalytics logHardFailureForEventNamed:@"ckkshardfail" withAttributes:nil];
 558         [_ckksAnalytics logSoftFailureForEventNamed:@"ckkssoftfail" withAttributes:nil];
 559     }
 560
 561     [_sosAnalytics logHardFailureForEventNamed:@"soshardfail" withAttributes:nil];
 562     [_sosAnalytics logSoftFailureForEventNamed:@"sossoftfail" withAttributes:nil];
 563
 564     NSDictionary* data = [self getJSONDataFromSupd];
 565     [self inspectDataBlobStructure:data];
 566
 567     [self checkTotalEventCount:data hard:testAmount + 1 soft:testAmount + 1 accuracy:0];
 568
 569     XCTAssertEqual([self failures:data eventType:@"ckkshardfail" attributes:nil class:SFAnalyticsEventClassHardFailure], testAmount);
 570     XCTAssertEqual([self failures:data eventType:@"ckkssoftfail" attributes:nil class:SFAnalyticsEventClassSoftFailure], testAmount);
 571     XCTAssertEqual([self failures:data eventType:@"soshardfail" attributes:nil class:SFAnalyticsEventClassHardFailure], 1);
 572     XCTAssertEqual([self failures:data eventType:@"sossoftfail" attributes:nil class:SFAnalyticsEventClassSoftFailure], 1);
 573 }
 574
 575
 576 // We have so many hard failures they won't fit in the upload buffer
 577 - (void)testTooManyHardFailures
 578 {
 579     NSDictionary* ckksAttrs = @{@"cattr" : @"cvalue"};
 580     NSDictionary* utAttrs = @{@"uattr" : @"uvalue"};
 581     for (int idx = 0; idx < 400; ++idx) {
 582         [_ckksAnalytics logHardFailureForEventNamed:@"ckksunittestfailure" withAttributes:ckksAttrs];
 583         [_ckksAnalytics logHardFailureForEventNamed:@"ckksunittestfailure" withAttributes:ckksAttrs];
 584         [_sosAnalytics logHardFailureForEventNamed:@"utunittestfailure" withAttributes:utAttrs];
 585     }
 586
 587     NSDictionary* data = [self getJSONDataFromSupd];
 588     [self inspectDataBlobStructure:data];
 589
 590     [self checkTotalEventCount:data hard:998 soft:0];
 591     // Based on threshold = records_to_upload/10 with a nice margin
 592     XCTAssertEqualWithAccuracy([self failures:data eventType:@"ckksunittestfailure" attributes:ckksAttrs class:SFAnalyticsEventClassHardFailure], 658, 50);
 593     XCTAssertEqualWithAccuracy([self failures:data eventType:@"utunittestfailure" attributes:utAttrs class:SFAnalyticsEventClassHardFailure], 339, 50);
 594 }
 595
 596 // So many soft failures they won't fit in the buffer
 597 - (void)testTooManySoftFailures
 598 {
 599     NSDictionary* ckksAttrs = @{@"cattr" : @"cvalue"};
 600     NSDictionary* utAttrs = @{@"uattr" : @"uvalue"};
 601     for (int idx = 0; idx < 400; ++idx) {
 602         [_ckksAnalytics logSoftFailureForEventNamed:@"ckksunittestfailure" withAttributes:ckksAttrs];
 603         [_ckksAnalytics logSoftFailureForEventNamed:@"ckksunittestfailure" withAttributes:ckksAttrs];
 604         [_sosAnalytics logSoftFailureForEventNamed:@"utunittestfailure" withAttributes:utAttrs];
 605     }
 606
 607     NSDictionary* data = [self getJSONDataFromSupd];
 608     [self inspectDataBlobStructure:data];
 609
 610     [self checkTotalEventCount:data hard:0 soft:998];
 611     // Based on threshold = records_to_upload/10 with a nice margin
 612     XCTAssertEqualWithAccuracy([self failures:data eventType:@"ckksunittestfailure" attributes:ckksAttrs class:SFAnalyticsEventClassSoftFailure], 665, 50);
 613     XCTAssertEqualWithAccuracy([self failures:data eventType:@"utunittestfailure" attributes:utAttrs class:SFAnalyticsEventClassSoftFailure], 332, 50);
 614 }
 615
 616 - (void)testTooManyCombinedFailures
 617 {
 618     NSDictionary* ckksAttrs = @{@"cattr1" : @"cvalue1", @"cattrthatisalotlongerthanthepreviousone" : @"cvaluethatisalsoalotlongerthantheother"};
 619     NSDictionary* utAttrs = @{@"uattr" : @"uvalue", @"uattrthatisalotlongerthanthepreviousone" : @"uvaluethatisalsoalotlongerthantheother"};
 620     for (int idx = 0; idx < 400; ++idx) {
 621         [_ckksAnalytics logHardFailureForEventNamed:@"ckksunittestfailure" withAttributes:ckksAttrs];
 622         [_ckksAnalytics logSoftFailureForEventNamed:@"ckksunittestfailure" withAttributes:ckksAttrs];
 623         [_sosAnalytics logHardFailureForEventNamed:@"utunittestfailure" withAttributes:utAttrs];
 624         [_sosAnalytics logSoftFailureForEventNamed:@"utunittestfailure" withAttributes:utAttrs];
 625     }
 626
 627     NSDictionary* data = [self getJSONDataFromSupd];
 628     [self inspectDataBlobStructure:data];
 629
 630     [self checkTotalEventCount:data hard:800 soft:198];
 631     // Based on threshold = records_to_upload/10 with a nice margin
 632     XCTAssertEqualWithAccuracy([self failures:data eventType:@"ckksunittestfailure" attributes:ckksAttrs class:SFAnalyticsEventClassHardFailure], 400, 50);
 633     XCTAssertEqualWithAccuracy([self failures:data eventType:@"utunittestfailure" attributes:utAttrs class:SFAnalyticsEventClassHardFailure], 400, 50);
 634     XCTAssertEqualWithAccuracy([self failures:data eventType:@"ckksunittestfailure" attributes:ckksAttrs class:SFAnalyticsEventClassSoftFailure], 100, 50);
 635     XCTAssertEqualWithAccuracy([self failures:data eventType:@"utunittestfailure" attributes:utAttrs class:SFAnalyticsEventClassSoftFailure], 100, 50);
 636 }
 637
 638 // There's an even number of samples
 639 - (void)testSamplesEvenSampleCount
 640 {
 641     NSString* sampleNameEven = @"evenSample";
 642
 643     for (NSNumber* value in @[@36.831855250339714, @90.78721762172914, @49.24392301762506,
 644                               @42.806362283260036, @16.76725375576855, @34.50969130579674,
 645                               @25.956509180834637, @36.8268555935645, @35.54069258036879,
 646                               @7.26364884595062, @45.414180770615395, @5.223213570809022]) {
 647         [_ckksAnalytics logMetric:value withName:sampleNameEven];
 648     }
 649
 650     NSDictionary* data = [self getJSONDataFromSupd];
 651     [self inspectDataBlobStructure:data];
 652
 653     // min, max, avg, med, dev, 1q, 3q
 654     [self checkTotalEventCount:data hard:0 soft:0 accuracy:0];
 655     [self sampleStatisticsInEvents:data[@"events"] name:sampleNameEven values:@[@5.22, @90.78, @35.60, @36.18, @21.52, @21.36, @44.11]];
 656 }
 657
 658 // There are 4*n + 1 samples
 659 - (void)testSamples4n1SampleCount
 660 {
 661     NSString* sampleName4n1 = @"4n1Sample";
 662     for (NSNumber* value in @[@37.76544251068022, @27.36378948426223, @45.10503077614114,
 663                               @43.90635413191473, @54.78709742040113, @52.34879597889124,
 664                               @70.95760312196856, @23.23648158872921, @75.34678687445064,
 665                               @10.723238854026203, @41.98468801166455, @17.074404554908476,
 666                               @94.24252031232739]) {
 667         [_ckksAnalytics logMetric:value withName:sampleName4n1];
 668     }
 669
 670     NSDictionary* data = [self getJSONDataFromSupd];
 671     [self inspectDataBlobStructure:data];
 672
 673     [self checkTotalEventCount:data hard:0 soft:0 accuracy:0];
 674     [self sampleStatisticsInEvents:data[@"events"] name:sampleName4n1 values:@[@10.72, @94.24, @45.76, @43.90, @23.14, @26.33, @58.83]];
 675 }
 676
 677 // There are 4*n + 3 samples
 678 - (void)testSamples4n3SampleCount
 679 {
 680     NSString* sampleName4n3 = @"4n3Sample";
 681
 682     for (NSNumber* value in @[@42.012971885655496, @87.85629592375282, @5.748491212287082,
 683                               @38.451850063872975, @81.96900109690873, @99.83098790545392,
 684                               @80.89400981437815, @5.719237885152143, @1.6740622555032196,
 685                               @14.437000556079038, @29.046050177512395]) {
 686         [_sosAnalytics logMetric:value withName:sampleName4n3];
 687     }
 688
 689     NSDictionary* data = [self getJSONDataFromSupd];
 690     [self inspectDataBlobStructure:data];
 691     [self checkTotalEventCount:data hard:0 soft:0 accuracy:0];
 692
 693     [self sampleStatisticsInEvents:data[@"events"] name:sampleName4n3 values:@[@1.67, @99.83, @44.33, @38.45, @35.28, @7.92, @81.70]];
 694 }
 695
 696 // stddev and quartiles undefined for single sample
 697 - (void)testSamplesSingleSample
 698 {
 699     NSString* sampleName = @"singleSample";
 700
 701     [_ckksAnalytics logMetric:@3.14159 withName:sampleName];
 702
 703     NSDictionary* data = [self getJSONDataFromSupd];
 704     [self inspectDataBlobStructure:data];
 705     [self checkTotalEventCount:data hard:0 soft:0 accuracy:0];
 706
 707     [self sampleStatisticsInEvents:data[@"events"] name:sampleName values:@[@3.14159]];
 708 }
 709
 710 // quartiles meaningless for fewer than 4 samples (but stddev exists)
 711 - (void)testSamplesFewerThanFour
 712 {
 713     NSString* sampleName = @"fewSamples";
 714
 715     [_ckksAnalytics logMetric:@3.14159 withName:sampleName];
 716     [_ckksAnalytics logMetric:@6.28318 withName:sampleName];
 717
 718     NSDictionary* data = [self getJSONDataFromSupd];
 719     [self inspectDataBlobStructure:data];
 720     [self checkTotalEventCount:data hard:0 soft:0 accuracy:0];
 721
 722     [self sampleStatisticsInEvents:data[@"events"] name:sampleName values:@[@3.14, @6.28, @4.71, @4.71, @1.57]];
 723 }
 724
 725 - (void)testSamplesSameNameDifferentSubclass
 726 {
 727     NSString* sampleName = @"differentSubclassSamples";
 728
 729     [_sosAnalytics logMetric:@313.37 withName:sampleName];
 730     [_ckksAnalytics logMetric:@313.37 withName:sampleName];
 731
 732     NSDictionary* data = [self getJSONDataFromSupd];
 733     [self inspectDataBlobStructure:data];
 734     [self checkTotalEventCount:data hard:0 soft:0 accuracy:0];
 735
 736     [self sampleStatisticsInEvents:data[@"events"] name:sampleName values:@[@313.37] amount:2];
 737 }
 738
 739
 740
 741 // TODO
 742 - (void)testGetSysdiagnoseDump
 743 {
 744
 745 }
 746
 747 // TODO (need mock server)
 748 - (void)testSplunkUpload
 749 {
 750
 751 }
 752
 753 // TODO (need mock server)
 754 - (void)testDBIsEmptiedAfterUpload
 755 {
 756
 757 }
 758
 759 @end