supd/supd.m

   1 /*
   2  * Copyright (c) 2017-2018 Apple Inc. All Rights Reserved.
   3  *
   4  * @APPLE_LICENSE_HEADER_START@
   5  *
   6  * This file contains Original Code and/or Modifications of Original Code
   7  * as defined in and that are subject to the Apple Public Source License
   8  * Version 2.0 (the 'License'). You may not use this file except in
   9  * compliance with the License. Please obtain a copy of the License at
  10  * http://www.opensource.apple.com/apsl/ and read it before using this
  11  * file.
  12  *
  13  * The Original Code and all software distributed under the License are
  14  * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  15  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  16  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
  17  * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
  18  * Please see the License for the specific language governing rights and
  19  * limitations under the License.
  20  *
  21  * @APPLE_LICENSE_HEADER_END@
  22  */
  23
  24 #import "supd.h"
  25
  26 #if !TARGET_OS_SIMULATOR
  27
  28 #import "SFAnalyticsDefines.h"
  29 #import "SFAnalyticsSQLiteStore.h"
  30 #import <Security/SFAnalytics.h>
  31
  32 #include <utilities/SecFileLocations.h>
  33 #import "utilities/debugging.h"
  34 #import <os/variant_private.h>
  35 #import <xpc/xpc.h>
  36 #include <notify.h>
  37 #import "keychain/ckks/CKKSControl.h"
  38 #import <zlib.h>
  39
  40 #import <AuthKit/AKAppleIDAuthenticationContext.h>
  41 #import <AuthKit/AKAppleIDAuthenticationController.h>
  42 #import <AuthKit/AKAppleIDAuthenticationController_Private.h>
  43
  44 #if TARGET_OS_OSX
  45 #include "dirhelper_priv.h"
  46 #endif
  47
  48 #if TARGET_OS_OSX
  49 #import <CrashReporterSupport/CrashReporterSupportPrivate.h>
  50 #else
  51 #import <CrashReporterSupport/CrashReporterSupport.h>
  52 #endif
  53
  54 #import <Accounts/Accounts.h>
  55 #import <Accounts/ACAccountStore_Private.h>
  56 #import <Accounts/ACAccountType_Private.h>
  57 #import <Accounts/ACAccountStore.h>
  58 #import <AppleAccount/AppleAccount.h>
  59 #import <AppleAccount/ACAccount+AppleAccount.h>
  60 #import <AppleAccount/ACAccountStore+AppleAccount.h>
  61
  62
  63 NSString* const SFAnalyticsSplunkTopic = @"topic";
  64 NSString* const SFAnalyticsClientId = @"clientId";
  65 NSString* const SFAnalyticsInternal = @"internal";
  66
  67 NSString* const SFAnalyticsMetricsBase = @"metricsBase";
  68 NSString* const SFAnalyticsDeviceID = @"ckdeviceID";
  69 NSString* const SFAnalyticsAltDSID = @"altDSID";
  70
  71 NSString* const SFAnalyticsEventCorrelationID = @"eventLinkID";
  72
  73 NSString* const SFAnalyticsSecondsCustomerKey = @"SecondsBetweenUploadsCustomer";
  74 NSString* const SFAnalyticsSecondsInternalKey = @"SecondsBetweenUploadsInternal";
  75 NSString* const SFAnalyticsSecondsSeedKey = @"SecondsBetweenUploadsSeed";
  76 NSString* const SFAnalyticsMaxEventsKey = @"NumberOfEvents";
  77 NSString* const SFAnalyticsDevicePercentageCustomerKey = @"DevicePercentageCustomer";
  78 NSString* const SFAnalyticsDevicePercentageInternalKey = @"DevicePercentageInternal";
  79 NSString* const SFAnalyticsDevicePercentageSeedKey = @"DevicePercentageSeed";
  80
  81 NSString* const SupdErrorDomain = @"com.apple.security.supd";
  82
  83 #define SFANALYTICS_SPLUNK_DEV 0
  84 #define OS_CRASH_TRACER_LOG_BUG_TYPE "226"
  85
  86 #if SFANALYTICS_SPLUNK_DEV
  87 NSUInteger const secondsBetweenUploadsCustomer = 10;
  88 NSUInteger const secondsBetweenUploadsInternal = 10;
  89 NSUInteger const secondsBetweenUploadsSeed = 10;
  90 #else // SFANALYTICS_SPLUNK_DEV
  91 NSUInteger const secondsBetweenUploadsCustomer = (3 * (60 * 60 * 24));
  92 NSUInteger const secondsBetweenUploadsInternal = (60 * 60 * 24);
  93 NSUInteger const secondsBetweenUploadsSeed = (60 * 60 * 24);
  94 #endif // SFANALYTICS_SPLUNK_DEV
  95
  96 @implementation SFAnalyticsReporter
  97 - (BOOL)saveReport:(NSData *)reportData fileName:(NSString *)fileName
  98 {
  99     BOOL writtenToLog = NO;
 100 #if TARGET_OS_OSX
 101     NSDictionary *optionsDictionary = @{ (__bridge NSString *)kCRProblemReportSubmissionPolicyKey: (__bridge NSString *)kCRSubmissionPolicyAlternate };
 102 #else // !TARGET_OS_OSX
 103     NSDictionary *optionsDictionary = nil; // The keys above are not defined or required on iOS.
 104 #endif // !TARGET_OS_OSX
 105
 106
 107     secdebug("saveReport", "calling out to `OSAWriteLogForSubmission`");
 108     writtenToLog = OSAWriteLogForSubmission(@OS_CRASH_TRACER_LOG_BUG_TYPE, fileName,
 109                                             nil, optionsDictionary, ^(NSFileHandle *fileHandle) {
 110                                                 secnotice("OSAWriteLogForSubmission", "Writing log data to report: %@", fileName);
 111                                                 [fileHandle writeData:reportData];
 112                                             });
 113     return writtenToLog;
 114 }
 115 @end
 116
 117 #define DEFAULT_SPLUNK_MAX_EVENTS_TO_REPORT 1000
 118 #define DEFAULT_SPLUNK_DEVICE_PERCENTAGE 100
 119
 120 static supd *_supdInstance = nil;
 121
 122 BOOL runningTests = NO;
 123 BOOL deviceAnalyticsOverride = NO;
 124 BOOL deviceAnalyticsEnabled = NO;
 125 BOOL iCloudAnalyticsOverride = NO;
 126 BOOL iCloudAnalyticsEnabled = NO;
 127
 128 static BOOL
 129 _isDeviceAnalyticsEnabled(void)
 130 {
 131     // This flag is only set during tests.
 132     if (deviceAnalyticsOverride) {
 133         return deviceAnalyticsEnabled;
 134     }
 135
 136     static BOOL dataCollectionEnabled = NO;
 137     static dispatch_once_t onceToken;
 138     dispatch_once(&onceToken, ^{
 139 #if TARGET_OS_IPHONE
 140         dataCollectionEnabled = DiagnosticLogSubmissionEnabled();
 141 #elif TARGET_OS_OSX
 142         dataCollectionEnabled = CRIsAutoSubmitEnabled();
 143 #endif
 144     });
 145     return dataCollectionEnabled;
 146 }
 147
 148 static NSString *
 149 accountAltDSID(void)
 150 {
 151     ACAccountStore *accountStore = [[ACAccountStore alloc] init];
 152     ACAccount *primaryAccount = [accountStore aa_primaryAppleAccount];
 153     if (primaryAccount == nil) {
 154         return nil;
 155     }
 156     return [primaryAccount aa_altDSID];
 157 }
 158
 159 static NSString *const kAnalyticsiCloudIdMSKey = @"com.apple.idms.config.privacy.icloud.data";
 160
 161 static NSDictionary *
 162 _getiCloudConfigurationInfoWithError(NSError **outError)
 163 {
 164     __block NSDictionary *outConfigurationInfo = nil;
 165     __block NSError *localError = nil;
 166
 167     NSString *altDSID = accountAltDSID();
 168     if (altDSID != nil) {
 169         secnotice("_getiCloudConfigurationInfoWithError", "Fetching configuration info");
 170
 171         dispatch_semaphore_t sema = dispatch_semaphore_create(0);
 172         AKAppleIDAuthenticationController *authController = [AKAppleIDAuthenticationController new];
 173         [authController configurationInfoWithIdentifiers:@[kAnalyticsiCloudIdMSKey]
 174                                               forAltDSID:altDSID
 175                                               completion:^(NSDictionary<NSString *, id<NSSecureCoding>> *configurationInfo, NSError *error) {
 176             if (error) {
 177                 secerror("_getiCloudConfigurationInfoWithError: Error fetching configurationInfo: %@", error);
 178                 localError = error;
 179             } else if (![configurationInfo isKindOfClass:[NSDictionary class]]) {
 180                 secerror("_getiCloudConfigurationInfoWithError: configurationInfo dict was not a dict, it was a %{public}@", [configurationInfo class]);
 181                 localError = error;
 182                 configurationInfo = nil;
 183             } else {
 184                 secnotice("_getiCloudConfigurationInfoWithError", "fetched configurationInfo %@", configurationInfo);
 185                 outConfigurationInfo = configurationInfo;
 186             }
 187             dispatch_semaphore_signal(sema);
 188         }];
 189         dispatch_semaphore_wait(sema, dispatch_time(DISPATCH_TIME_NOW, (uint64_t)(5 * NSEC_PER_SEC)));
 190     } else {
 191         secerror("_getiCloudConfigurationInfoWithError: Failed to fetch primary account info.");
 192     }
 193
 194     if (localError && outError) {
 195         *outError = localError;
 196     }
 197     return outConfigurationInfo;
 198 }
 199
 200 static BOOL
 201 _isiCloudAnalyticsEnabled()
 202 {
 203     // This flag is only set during tests.
 204     if (iCloudAnalyticsOverride) {
 205         return iCloudAnalyticsEnabled;
 206     }
 207
 208     static bool cachedAllowsICloudAnalytics = false;
 209
 210     static dispatch_once_t onceToken;
 211     dispatch_once(&onceToken, ^{
 212         NSError *error = nil;
 213         NSDictionary *accountConfiguration = _getiCloudConfigurationInfoWithError(&error);
 214         if (error == nil && accountConfiguration != nil) {
 215             id iCloudAnalyticsOptIn = accountConfiguration[kAnalyticsiCloudIdMSKey];
 216             if (iCloudAnalyticsOptIn != nil) {
 217                 BOOL iCloudAnalyticsOptInHasCorrectType = ([iCloudAnalyticsOptIn isKindOfClass:[NSNumber class]] || [iCloudAnalyticsOptIn isKindOfClass:[NSString class]]);
 218                 if (iCloudAnalyticsOptInHasCorrectType) {
 219                     NSNumber *iCloudAnalyticsOptInNumber = @([iCloudAnalyticsOptIn integerValue]);
 220                     cachedAllowsICloudAnalytics = ![iCloudAnalyticsOptInNumber isEqualToNumber:[NSNumber numberWithInteger:0]];
 221                 }
 222             }
 223         } else if (error != nil) {
 224             secerror("_isiCloudAnalyticsEnabled: %@", error);
 225         }
 226     });
 227
 228     return cachedAllowsICloudAnalytics;
 229 }
 230
 231 /*  NSData GZip category based on GeoKit's implementation */
 232 @interface NSData (GZip)
 233 - (NSData *)supd_gzipDeflate;
 234 @end
 235
 236 #define GZIP_OFFSET 16
 237 #define GZIP_STRIDE_LEN 16384
 238
 239 @implementation NSData (Gzip)
 240 - (NSData *)supd_gzipDeflate
 241 {
 242     if ([self length] == 0) {
 243         return self;
 244     }
 245
 246     z_stream strm;
 247     memset(&strm, 0, sizeof(strm));
 248     strm.next_in=(uint8_t *)[self bytes];
 249     strm.avail_in = (unsigned int)[self length];
 250
 251
 252     if (Z_OK != deflateInit2(&strm, Z_BEST_COMPRESSION, Z_DEFLATED,
 253                              MAX_WBITS + GZIP_OFFSET, MAX_MEM_LEVEL, Z_DEFAULT_STRATEGY)) {
 254         return nil;
 255     }
 256
 257     NSMutableData *compressed = [NSMutableData dataWithLength:GZIP_STRIDE_LEN];
 258
 259     do {
 260         if (strm.total_out >= [compressed length]) {
 261             [compressed increaseLengthBy: 16384];
 262         }
 263
 264         strm.next_out = [compressed mutableBytes] + strm.total_out;
 265         strm.avail_out = (int)[compressed length] - (int)strm.total_out;
 266
 267         deflate(&strm, Z_FINISH);
 268
 269     } while (strm.avail_out == 0);
 270
 271     deflateEnd(&strm);
 272
 273     [compressed setLength: strm.total_out];
 274     if (strm.avail_in == 0) {
 275         return [NSData dataWithData:compressed];
 276     } else {
 277         return nil;
 278     }
 279 }
 280 @end
 281
 282 @implementation SFAnalyticsClient {
 283     NSString* _path;
 284     NSString* _name;
 285     BOOL _requireDeviceAnalytics;
 286     BOOL _requireiCloudAnalytics;
 287 }
 288
 289 @synthesize storePath = _path;
 290 @synthesize name = _name;
 291
 292 - (instancetype)initWithStorePath:(NSString*)path name:(NSString*)name
 293                   deviceAnalytics:(BOOL)deviceAnalytics iCloudAnalytics:(BOOL)iCloudAnalytics {
 294     if (self = [super init]) {
 295         _path = path;
 296         _name = name;
 297         _requireDeviceAnalytics = deviceAnalytics;
 298         _requireiCloudAnalytics = iCloudAnalytics;
 299     }
 300     return self;
 301 }
 302
 303 @end
 304
 305 @interface SFAnalyticsTopic ()
 306 @property NSURL* _splunkUploadURL;
 307
 308 @property BOOL allowInsecureSplunkCert;
 309 @property BOOL ignoreServersMessagesTellingUsToGoAway;
 310 @property BOOL disableUploads;
 311 @property BOOL disableClientId;
 312
 313 @property NSUInteger secondsBetweenUploads;
 314 @property NSUInteger maxEventsToReport;
 315 @property float devicePercentage; // for sampling reporting devices
 316
 317 @property NSDictionary* metricsBase; // data the server provides and wants us to send back
 318 @property NSArray* blacklistedFields;
 319 @property NSArray* blacklistedEvents;
 320 @end
 321
 322 @implementation SFAnalyticsTopic
 323
 324 - (void)setupClientsForTopic:(NSString *)topicName
 325 {
 326     NSMutableArray<SFAnalyticsClient*>* clients = [NSMutableArray<SFAnalyticsClient*> new];
 327     if ([topicName isEqualToString:SFAnalyticsTopicKeySync]) {
 328         [clients addObject:[[SFAnalyticsClient alloc] initWithStorePath:[self.class databasePathForCKKS]
 329                                                                    name:@"ckks" deviceAnalytics:NO iCloudAnalytics:YES]];
 330         [clients addObject:[[SFAnalyticsClient alloc] initWithStorePath:[self.class databasePathForSOS]
 331                                                                    name:@"sos" deviceAnalytics:NO iCloudAnalytics:YES]];
 332         [clients addObject:[[SFAnalyticsClient alloc] initWithStorePath:[self.class databasePathForPCS]
 333                                                                    name:@"pcs" deviceAnalytics:NO iCloudAnalytics:YES]];
 334         [clients addObject:[[SFAnalyticsClient alloc] initWithStorePath:[self.class databasePathForSignIn]
 335                                                                    name:@"signins" deviceAnalytics:NO iCloudAnalytics:YES]];
 336         [clients addObject:[[SFAnalyticsClient alloc] initWithStorePath:[self.class databasePathForLocal]
 337                                                                    name:@"local" deviceAnalytics:YES iCloudAnalytics:NO]];
 338     } else if ([topicName isEqualToString:SFAnalyticsTopicCloudServices]) {
 339         [clients addObject:[[SFAnalyticsClient alloc] initWithStorePath:[self.class databasePathForCloudServices]
 340                                                                    name:@"CloudServices"
 341                                                         deviceAnalytics:YES
 342                                                         iCloudAnalytics:NO]];
 343     } else if ([topicName isEqualToString:SFAnalyticsTopicTrust]) {
 344 #if TARGET_OS_OSX
 345         _set_user_dir_suffix("com.apple.trustd"); // supd needs to read trustd's cache dir for these
 346 #endif
 347         [clients addObject:[[SFAnalyticsClient alloc] initWithStorePath:[self.class databasePathForTrust]
 348                                                                    name:@"trust" deviceAnalytics:YES iCloudAnalytics:NO]];
 349         [clients addObject:[[SFAnalyticsClient alloc] initWithStorePath:[self.class databasePathForTrustdHealth]
 350                                                                    name:@"trustdHealth" deviceAnalytics:YES iCloudAnalytics:NO]];
 351         [clients addObject:[[SFAnalyticsClient alloc] initWithStorePath:[self.class databasePathForTLS]
 352                                                                    name:@"tls" deviceAnalytics:YES iCloudAnalytics:NO]];
 353
 354 #if TARGET_OS_OSX
 355         _set_user_dir_suffix(NULL); // set back to the default cache dir
 356 #endif
 357     } else if ([topicName isEqualToString:SFAnalyticsTopicTransparency]) {
 358         [clients addObject:[[SFAnalyticsClient alloc] initWithStorePath:[self.class databasePathForTransparency]
 359                                                                    name:@"transparency" deviceAnalytics:NO iCloudAnalytics:YES]];
 360     }
 361
 362     _topicClients = clients;
 363 }
 364
 365 - (instancetype)initWithDictionary:(NSDictionary *)dictionary name:(NSString *)topicName samplingRates:(NSDictionary *)rates {
 366     if (self = [super init]) {
 367         _internalTopicName = topicName;
 368         [self setupClientsForTopic:topicName];
 369         _splunkTopicName = dictionary[@"splunk_topic"];
 370         __splunkUploadURL = [NSURL URLWithString:dictionary[@"splunk_uploadURL"]];
 371         _splunkBagURL = [NSURL URLWithString:dictionary[@"splunk_bagURL"]];
 372         _allowInsecureSplunkCert = [[dictionary valueForKey:@"splunk_allowInsecureCertificate"] boolValue];
 373         _uploadSizeLimit = [[dictionary valueForKey:@"uploadSizeLimit"] unsignedIntegerValue];
 374
 375         NSString* splunkEndpoint = dictionary[@"splunk_endpointDomain"];
 376         if (dictionary[@"disableClientId"]) {
 377             _disableClientId = YES;
 378         }
 379
 380         NSUserDefaults* defaults = [[NSUserDefaults alloc] initWithSuiteName:SFAnalyticsUserDefaultsSuite];
 381         NSString* userDefaultsSplunkTopic = [defaults stringForKey:@"splunk_topic"];
 382         if (userDefaultsSplunkTopic) {
 383             _splunkTopicName = userDefaultsSplunkTopic;
 384         }
 385
 386         NSURL* userDefaultsSplunkUploadURL = [NSURL URLWithString:[defaults stringForKey:@"splunk_uploadURL"]];
 387         if (userDefaultsSplunkUploadURL) {
 388             __splunkUploadURL = userDefaultsSplunkUploadURL;
 389         }
 390
 391         NSURL* userDefaultsSplunkBagURL = [NSURL URLWithString:[defaults stringForKey:@"splunk_bagURL"]];
 392         if (userDefaultsSplunkBagURL) {
 393             _splunkBagURL = userDefaultsSplunkBagURL;
 394         }
 395
 396         NSInteger userDefaultsUploadSizeLimit = [defaults integerForKey:@"uploadSizeLimit"];
 397         if (userDefaultsUploadSizeLimit > 0) {
 398             _uploadSizeLimit = userDefaultsUploadSizeLimit;
 399         }
 400
 401         BOOL userDefaultsAllowInsecureSplunkCert = [defaults boolForKey:@"splunk_allowInsecureCertificate"];
 402         _allowInsecureSplunkCert |= userDefaultsAllowInsecureSplunkCert;
 403
 404         NSString* userDefaultsSplunkEndpoint = [defaults stringForKey:@"splunk_endpointDomain"];
 405         if (userDefaultsSplunkEndpoint) {
 406             splunkEndpoint = userDefaultsSplunkEndpoint;
 407         }
 408
 409 #if SFANALYTICS_SPLUNK_DEV
 410         _secondsBetweenUploads = secondsBetweenUploadsInternal;
 411         _maxEventsToReport = SFAnalyticsMaxEventsToReport;
 412         _devicePercentage = DEFAULT_SPLUNK_DEVICE_PERCENTAGE;
 413 #else
 414         bool internal = os_variant_has_internal_diagnostics("com.apple.security");
 415         if (rates) {
 416 #if RC_SEED_BUILD
 417             NSNumber *secondsNum = internal ? rates[SFAnalyticsSecondsInternalKey] : rates[SFAnalyticsSecondsSeedKey];
 418             NSNumber *percentageNum = internal ? rates[SFAnalyticsDevicePercentageInternalKey] : rates[SFAnalyticsDevicePercentageSeedKey];
 419 #else
 420             NSNumber *secondsNum = internal ? rates[SFAnalyticsSecondsInternalKey] : rates[SFAnalyticsSecondsCustomerKey];
 421             NSNumber *percentageNum = internal ? rates[SFAnalyticsDevicePercentageInternalKey] : rates[SFAnalyticsDevicePercentageCustomerKey];
 422 #endif
 423             _secondsBetweenUploads = [secondsNum integerValue];
 424             _maxEventsToReport = [rates[SFAnalyticsMaxEventsKey] unsignedIntegerValue];
 425             _devicePercentage = [percentageNum floatValue];
 426         } else {
 427 #if RC_SEED_BUILD
 428             _secondsBetweenUploads = internal ? secondsBetweenUploadsInternal : secondsBetweenUploadsSeed;
 429 #else
 430             _secondsBetweenUploads = internal ? secondsBetweenUploadsInternal : secondsBetweenUploadsCustomer;
 431 #endif
 432             _maxEventsToReport = SFAnalyticsMaxEventsToReport;
 433             _devicePercentage = DEFAULT_SPLUNK_DEVICE_PERCENTAGE;
 434         }
 435 #endif
 436         secnotice("supd", "created %@ with %lu seconds between uploads, %lu max events, %f percent of uploads",
 437                   _internalTopicName, (unsigned long)_secondsBetweenUploads, (unsigned long)_maxEventsToReport, _devicePercentage);
 438
 439 #if SFANALYTICS_SPLUNK_DEV
 440         _ignoreServersMessagesTellingUsToGoAway = YES;
 441
 442         if (!_splunkUploadURL && splunkEndpoint) {
 443             NSString* urlString = [NSString stringWithFormat:@"https://%@/report/2/%@", splunkEndpoint, _splunkTopicName];
 444             _splunkUploadURL = [NSURL URLWithString:urlString];
 445         }
 446 #else
 447         (void)splunkEndpoint;
 448 #endif
 449     }
 450     return self;
 451 }
 452
 453 - (BOOL)isSampledUpload
 454 {
 455     uint32_t sample = arc4random();
 456     if ((double)_devicePercentage < ((double)1 / UINT32_MAX) * 100) {
 457         /* Requested percentage is smaller than we can sample. just do 1 out of UINT32_MAX */
 458         if (sample == 0) {
 459             return YES;
 460         }
 461     } else {
 462         if ((double)sample <= (double)UINT32_MAX * ((double)_devicePercentage / 100)) {
 463             return YES;
 464         }
 465     }
 466     return NO;
 467 }
 468
 469 - (BOOL)postJSON:(NSData*)json toEndpoint:(NSURL*)endpoint error:(NSError**)error
 470 {
 471     if (!endpoint) {
 472         if (error) {
 473             NSString *description = [NSString stringWithFormat:@"No endpoint for %@", _internalTopicName];
 474             *error = [NSError errorWithDomain:@"SupdUploadErrorDomain"
 475                                          code:-10
 476                                      userInfo:@{NSLocalizedDescriptionKey : description}];
 477         }
 478         return false;
 479     }
 480     /*
 481      * Create the NSURLSession
 482      *  We use the ephemeral session config because we don't need cookies or cache
 483      */
 484     NSURLSessionConfiguration *configuration = [NSURLSessionConfiguration ephemeralSessionConfiguration];
 485
 486     configuration.HTTPAdditionalHeaders = @{ @"User-Agent" : [NSString stringWithFormat:@"securityd/%s", SECURITY_BUILD_VERSION]};
 487
 488     NSURLSession* postSession = [NSURLSession sessionWithConfiguration:configuration
 489                                                               delegate:self
 490                                                          delegateQueue:nil];
 491
 492     NSMutableURLRequest* postRequest = [[NSMutableURLRequest alloc] init];
 493     postRequest.URL = endpoint;
 494     postRequest.HTTPMethod = @"POST";
 495     postRequest.HTTPBody = [json supd_gzipDeflate];
 496     [postRequest setValue:@"gzip" forHTTPHeaderField:@"Content-Encoding"];
 497
 498     /*
 499      * Create the upload task.
 500      */
 501     dispatch_semaphore_t sem = dispatch_semaphore_create(0);
 502     __block BOOL uploadSuccess = NO;
 503     NSURLSessionDataTask* uploadTask = [postSession dataTaskWithRequest:postRequest
 504                                                       completionHandler:^(NSData * _Nullable __unused data, NSURLResponse * _Nullable response, NSError * _Nullable requestError) {
 505         if (requestError) {
 506             secerror("Error in uploading the events to splunk for %@: %@", self->_internalTopicName, requestError);
 507         } else if (![response isKindOfClass:NSHTTPURLResponse.class]){
 508             Class class = response.class;
 509             secerror("Received the wrong kind of response for %@: %@", self->_internalTopicName, NSStringFromClass(class));
 510         } else {
 511             NSHTTPURLResponse* httpResponse = (NSHTTPURLResponse*)response;
 512             if(httpResponse.statusCode >= 200 && httpResponse.statusCode < 300) {
 513                 /* Success */
 514                 uploadSuccess = YES;
 515                 secnotice("upload", "Splunk upload success for %@", self->_internalTopicName);
 516             } else {
 517                 secnotice("upload", "Splunk upload for %@ unexpected status to URL: %@ -- status: %d",
 518                           self->_internalTopicName, endpoint, (int)(httpResponse.statusCode));
 519             }
 520         }
 521         dispatch_semaphore_signal(sem);
 522     }];
 523     secnotice("upload", "Splunk upload start for %@", self->_internalTopicName);
 524     [uploadTask resume];
 525     dispatch_semaphore_wait(sem, dispatch_time(DISPATCH_TIME_NOW, (uint64_t)(5 * 60 * NSEC_PER_SEC)));
 526     return uploadSuccess;
 527 }
 528
 529 - (BOOL)eventIsBlacklisted:(NSMutableDictionary*)event {
 530     return _blacklistedEvents ? [_blacklistedEvents containsObject:event[SFAnalyticsEventType]] : NO;
 531 }
 532
 533 - (void)removeBlacklistedFieldsFromEvent:(NSMutableDictionary*)event {
 534     for (NSString* badField in self->_blacklistedFields) {
 535         [event removeObjectForKey:badField];
 536     }
 537 }
 538
 539 - (void)addRequiredFieldsToEvent:(NSMutableDictionary*)event {
 540     [_metricsBase enumerateKeysAndObjectsUsingBlock:^(id  _Nonnull key, id  _Nonnull obj, BOOL * _Nonnull stop) {
 541         if (!event[key]) {
 542             event[key] = obj;
 543         }
 544     }];
 545 }
 546
 547 - (BOOL)prepareEventForUpload:(NSMutableDictionary*)event
 548                    linkedUUID:(NSUUID *)linkedUUID {
 549     if ([self eventIsBlacklisted:event]) {
 550         return NO;
 551     }
 552
 553     [self removeBlacklistedFieldsFromEvent:event];
 554     [self addRequiredFieldsToEvent:event];
 555     if (_disableClientId) {
 556         event[SFAnalyticsClientId] = @(0);
 557     }
 558     event[SFAnalyticsSplunkTopic] = self->_splunkTopicName ?: [NSNull null];
 559     if (linkedUUID) {
 560         event[SFAnalyticsEventCorrelationID] = [linkedUUID UUIDString];
 561     }
 562     return YES;
 563 }
 564
 565 - (void)addFailures:(NSMutableArray<NSArray*>*)failures toUploadRecords:(NSMutableArray*)records threshold:(NSUInteger)threshold linkedUUID:(NSUUID *)linkedUUID
 566 {
 567     // The first 0 through 'threshold' items are getting uploaded in any case (which might be 0 for lower priority data)
 568
 569     for (NSArray* client in failures) {
 570         [client enumerateObjectsUsingBlock:^(id  _Nonnull obj, NSUInteger idx, BOOL * _Nonnull stop) {
 571             NSMutableDictionary* event = (NSMutableDictionary*)obj;
 572             if (idx >= threshold) {
 573                 *stop = YES;
 574                 return;
 575             }
 576             if ([self prepareEventForUpload:event linkedUUID:linkedUUID]) {
 577                 if ([NSJSONSerialization isValidJSONObject:event]) {
 578                     [records addObject:event];
 579                 } else {
 580                     secerror("supd: Replacing event with errorEvent because invalid JSON: %@", event);
 581                     NSString* originalType = event[SFAnalyticsEventType];
 582                     NSDictionary* errorEvent = @{ SFAnalyticsEventType : SFAnalyticsEventTypeErrorEvent,
 583                                                   SFAnalyticsEventErrorDestription : [NSString stringWithFormat:@"JSON:%@", originalType]};
 584                     [records addObject:errorEvent];
 585                 }
 586             }
 587         }];
 588     }
 589
 590     // Are there more items than we shoved into the upload records?
 591     NSInteger excessItems = 0;
 592     for (NSArray* client in failures) {
 593         NSInteger localExcess = client.count - threshold;
 594         excessItems += localExcess > 0 ? localExcess : 0;
 595     }
 596
 597     // Then, if we have space and items left, apply a scaling factor to distribute events across clients to fill upload buffer
 598     if (records.count < _maxEventsToReport && excessItems > 0) {
 599         double scale = (_maxEventsToReport - records.count) / (double)excessItems;
 600         if (scale > 1) {
 601             scale = 1;
 602         }
 603
 604         for (NSArray* client in failures) {
 605             if (client.count > threshold) {
 606                 NSRange range = NSMakeRange(threshold, (client.count - threshold) * scale);
 607                 NSArray* sub = [client subarrayWithRange:range];
 608                 [sub enumerateObjectsUsingBlock:^(id  _Nonnull obj, NSUInteger idx, BOOL * _Nonnull stop) {
 609                     if ([self prepareEventForUpload:obj linkedUUID:linkedUUID]) {
 610                         [records addObject:obj];
 611                     }
 612                 }];
 613             }
 614         }
 615     }
 616 }
 617
 618 - (NSMutableDictionary*)sampleStatisticsForSamples:(NSArray*)samples withName:(NSString*)name
 619 {
 620     NSMutableDictionary* statistics = [NSMutableDictionary dictionary];
 621     NSUInteger count = samples.count;
 622     NSArray* sortedSamples = [samples sortedArrayUsingSelector:@selector(compare:)];
 623     NSArray* samplesAsExpressionArray = @[[NSExpression expressionForConstantValue:sortedSamples]];
 624
 625     if (count == 1) {
 626         statistics[name] = samples[0];
 627     } else {
 628         // NSExpression takes population standard deviation. Our data is a sample of whatever we sampled over time,
 629         // but the difference between the two is fairly minor (divide by N before taking sqrt versus divide by N-1).
 630         statistics[[NSString stringWithFormat:@"%@-dev", name]] = [[NSExpression expressionForFunction:@"stddev:" arguments:samplesAsExpressionArray] expressionValueWithObject:nil context:nil];
 631
 632         statistics[[NSString stringWithFormat:@"%@-min", name]] = [[NSExpression expressionForFunction:@"min:" arguments:samplesAsExpressionArray] expressionValueWithObject:nil context:nil];
 633         statistics[[NSString stringWithFormat:@"%@-max", name]] = [[NSExpression expressionForFunction:@"max:" arguments:samplesAsExpressionArray] expressionValueWithObject:nil context:nil];
 634         statistics[[NSString stringWithFormat:@"%@-avg", name]] = [[NSExpression expressionForFunction:@"average:" arguments:samplesAsExpressionArray] expressionValueWithObject:nil context:nil];
 635         statistics[[NSString stringWithFormat:@"%@-med", name]] = [[NSExpression expressionForFunction:@"median:" arguments:samplesAsExpressionArray] expressionValueWithObject:nil context:nil];
 636     }
 637
 638     if (count > 3) {
 639         NSString* q1 = [NSString stringWithFormat:@"%@-1q", name];
 640         NSString* q3 = [NSString stringWithFormat:@"%@-3q", name];
 641         // From Wikipedia, which is never wrong
 642         if (count % 2 == 0) {
 643             // The lower quartile value is the median of the lower half of the data. The upper quartile value is the median of the upper half of the data.
 644             statistics[q1] = [[NSExpression expressionForFunction:@"median:" arguments:@[[NSExpression expressionForConstantValue:[sortedSamples subarrayWithRange:NSMakeRange(0, count / 2)]]]] expressionValueWithObject:nil context:nil];
 645             statistics[q3] = [[NSExpression expressionForFunction:@"median:" arguments:@[[NSExpression expressionForConstantValue:[sortedSamples subarrayWithRange:NSMakeRange((count / 2), count / 2)]]]] expressionValueWithObject:nil context:nil];
 646         } else if (count % 4 == 1) {
 647             // If there are (4n+1) data points, then the lower quartile is 25% of the nth data value plus 75% of the (n+1)th data value;
 648             // the upper quartile is 75% of the (3n+1)th data point plus 25% of the (3n+2)th data point.
 649             // (offset n by -1 since we count from 0)
 650             NSUInteger n = count / 4;
 651             statistics[q1] = @(([sortedSamples[n - 1] doubleValue] + [sortedSamples[n] doubleValue] * 3.0) / 4.0);
 652             statistics[q3] = @(([sortedSamples[(3 * n)] doubleValue] * 3.0 + [sortedSamples[(3 * n) + 1] doubleValue]) / 4.0);
 653         } else if (count % 4 == 3){
 654             // If there are (4n+3) data points, then the lower quartile is 75% of the (n+1)th data value plus 25% of the (n+2)th data value;
 655             // the upper quartile is 25% of the (3n+2)th data point plus 75% of the (3n+3)th data point.
 656             // (offset n by -1 since we count from 0)
 657             NSUInteger n = count / 4;
 658             statistics[q1] = @(([sortedSamples[n] doubleValue] * 3.0 + [sortedSamples[n + 1] doubleValue]) / 4.0);
 659             statistics[q3] = @(([sortedSamples[(3 * n) + 1] doubleValue] + [sortedSamples[(3 * n) + 2] doubleValue] * 3.0) / 4.0);
 660         }
 661     }
 662
 663     return statistics;
 664 }
 665
 666 - (NSMutableDictionary*)healthSummaryWithName:(NSString*)name store:(SFAnalyticsSQLiteStore*)store uuid:(NSUUID *)uuid
 667 {
 668     __block NSMutableDictionary* summary = [NSMutableDictionary new];
 669
 670     // Add some events of our own before pulling in data
 671     summary[SFAnalyticsEventType] = [NSString stringWithFormat:@"%@HealthSummary", name];
 672     if ([self eventIsBlacklisted:summary]) {
 673         return nil;
 674     }
 675     summary[SFAnalyticsEventTime] = @([[NSDate date] timeIntervalSince1970] * 1000);    // Splunk wants milliseconds
 676     [SFAnalytics addOSVersionToEvent:summary];
 677     if (store.uploadDate) {
 678         summary[SFAnalyticsAttributeLastUploadTime] = @([store.uploadDate timeIntervalSince1970] * 1000);
 679     } else {
 680         summary[SFAnalyticsAttributeLastUploadTime] = @(0);
 681     }
 682
 683     // Process counters
 684     NSDictionary* successCounts = store.summaryCounts;
 685     __block NSInteger totalSuccessCount = 0;
 686     __block NSInteger totalHardFailureCount = 0;
 687     __block NSInteger totalSoftFailureCount = 0;
 688     [successCounts enumerateKeysAndObjectsUsingBlock:^(NSString* _Nonnull eventType, NSDictionary* _Nonnull counts, BOOL* _Nonnull stop) {
 689         summary[[NSString stringWithFormat:@"%@-success", eventType]] = counts[SFAnalyticsColumnSuccessCount];
 690         summary[[NSString stringWithFormat:@"%@-hardfail", eventType]] = counts[SFAnalyticsColumnHardFailureCount];
 691         summary[[NSString stringWithFormat:@"%@-softfail", eventType]] = counts[SFAnalyticsColumnSoftFailureCount];
 692         totalSuccessCount += [counts[SFAnalyticsColumnSuccessCount] integerValue];
 693         totalHardFailureCount += [counts[SFAnalyticsColumnHardFailureCount] integerValue];
 694         totalSoftFailureCount += [counts[SFAnalyticsColumnSoftFailureCount] integerValue];
 695     }];
 696
 697     summary[SFAnalyticsColumnSuccessCount] = @(totalSuccessCount);
 698     summary[SFAnalyticsColumnHardFailureCount] = @(totalHardFailureCount);
 699     summary[SFAnalyticsColumnSoftFailureCount] = @(totalSoftFailureCount);
 700     if (os_variant_has_internal_diagnostics("com.apple.security")) {
 701         summary[SFAnalyticsInternal] = @YES;
 702     }
 703
 704     // Process samples
 705     NSMutableDictionary<NSString*,NSMutableArray*>* samplesBySampler = [NSMutableDictionary<NSString*,NSMutableArray*> dictionary];
 706     for (NSDictionary* sample in [store samples]) {
 707         if (!samplesBySampler[sample[SFAnalyticsColumnSampleName]]) {
 708             samplesBySampler[sample[SFAnalyticsColumnSampleName]] = [NSMutableArray array];
 709         }
 710         [samplesBySampler[sample[SFAnalyticsColumnSampleName]] addObject:sample[SFAnalyticsColumnSampleValue]];
 711     }
 712     [samplesBySampler enumerateKeysAndObjectsUsingBlock:^(NSString * _Nonnull key, NSMutableArray * _Nonnull obj, BOOL * _Nonnull stop) {
 713         NSMutableDictionary* event = [self sampleStatisticsForSamples:obj withName:key];
 714         [summary addEntriesFromDictionary:event];
 715     }];
 716
 717     // Should always return yes because we already checked for event blacklisting specifically (unless summary itself is blacklisted)
 718     if (![self prepareEventForUpload:summary linkedUUID:uuid]) {
 719         secwarning("supd: health summary for %@ blacklisted", name);
 720         return nil;
 721     }
 722
 723     // Seems unlikely because we only insert strings, samplers only take NSNumbers and frankly, sampleStatisticsForSamples probably would have crashed
 724     if (![NSJSONSerialization isValidJSONObject:summary]) {
 725         secerror("json: health summary for client %@ is invalid JSON: %@", name, summary);
 726         return [@{ SFAnalyticsEventType : SFAnalyticsEventTypeErrorEvent,
 727                    SFAnalyticsEventErrorDestription : [NSString stringWithFormat:@"JSON:%@HealthSummary", name]} mutableCopy];
 728     }
 729
 730     return summary;
 731 }
 732
 733 - (void)updateUploadDateForClients:(NSArray<SFAnalyticsClient*>*)clients date:(NSDate *)date clearData:(BOOL)clearData
 734 {
 735     for (SFAnalyticsClient* client in clients) {
 736         SFAnalyticsSQLiteStore* store = [SFAnalyticsSQLiteStore storeWithPath:client.storePath schema:SFAnalyticsTableSchema];
 737         secnotice("postprocess", "Setting upload date (%@) for client: %@", date, client.name);
 738         store.uploadDate = date;
 739         if (clearData) {
 740             secnotice("postprocess", "Clearing collected data for client: %@", client.name);
 741             [store clearAllData];
 742         }
 743     }
 744 }
 745
 746 - (size_t)serializedEventSize:(NSObject *)event
 747                         error:(NSError**)error
 748 {
 749     if (![NSJSONSerialization isValidJSONObject:event]) {
 750         secnotice("serializedEventSize", "invalid JSON object");
 751         return 0;
 752     }
 753
 754     NSData *json = [NSJSONSerialization dataWithJSONObject:event
 755                                                    options:0
 756                                                      error:error];
 757     if (json) {
 758         return [json length];
 759     } else {
 760         secnotice("serializedEventSize", "failed to serialize event");
 761         return 0;
 762     }
 763 }
 764
 765 - (NSArray<NSArray *> *)chunkFailureSet:(size_t)sizeCapacity
 766                                  events:(NSArray<NSDictionary *> *)events
 767                                   error:(NSError **)error
 768 {
 769     const size_t postBodyLimit = 1000; // 1000 events in a single upload
 770     size_t currentSize = 0;
 771     size_t currentEventCount = 0;
 772
 773     NSMutableArray<NSArray<NSDictionary *> *> *eventChunks = [[NSMutableArray<NSArray<NSDictionary *> *> alloc] init];
 774     NSMutableArray<NSDictionary *> *currentEventChunk = [[NSMutableArray<NSDictionary *> alloc] init];
 775     for (NSDictionary *event in events) {
 776         NSError *localError = nil;
 777         size_t eventSize = [self serializedEventSize:event error:&localError];
 778         if (localError != nil) {
 779             if (error) {
 780                 *error = localError;
 781             }
 782             secemergency("Unable to serialize event JSON: %@", [localError localizedDescription]);
 783             return nil;
 784         }
 785
 786         BOOL countLessThanLimit = currentEventCount < postBodyLimit;
 787         BOOL sizeLessThanCapacity = (currentSize + eventSize) <= sizeCapacity;
 788         if (!countLessThanLimit || !sizeLessThanCapacity) {
 789             [eventChunks addObject:currentEventChunk];
 790             currentEventChunk = [[NSMutableArray<NSDictionary *> alloc] init];
 791             currentEventCount = 0;
 792             currentSize = 0;
 793         }
 794
 795         [currentEventChunk addObject:event];
 796         currentEventCount++;
 797         currentSize += eventSize;
 798     }
 799
 800     if ([currentEventChunk count] > 0) {
 801         [eventChunks addObject:currentEventChunk];
 802     }
 803
 804     return eventChunks;
 805 }
 806
 807 - (NSDictionary *)createEventDictionary:(NSArray *)healthSummaries
 808                                failures:(NSArray<NSDictionary *> *)failures
 809                                   error:(NSError **)error
 810 {
 811     NSMutableArray *events = [[NSMutableArray alloc] init];
 812     [events addObjectsFromArray:healthSummaries];
 813     if (failures) {
 814         [events addObjectsFromArray:failures];
 815     }
 816
 817     NSDictionary *eventDictionary = @{
 818         SFAnalyticsPostTime : @([[NSDate date] timeIntervalSince1970] * 1000),
 819         @"events" : events,
 820     };
 821
 822     if (![NSJSONSerialization isValidJSONObject:eventDictionary]) {
 823         secemergency("json: final dictionary invalid JSON.");
 824         if (error) {
 825             *error = [NSError errorWithDomain:SupdErrorDomain code:SupdInvalidJSONError
 826                                      userInfo:@{NSLocalizedDescriptionKey : [NSString localizedStringWithFormat:@"Final dictionary for upload is invalid JSON: %@", eventDictionary]}];
 827         }
 828         return nil;
 829     }
 830
 831     return eventDictionary;
 832 }
 833
 834 - (NSArray<NSDictionary *> *)createChunkedLoggingJSON:(NSArray<NSDictionary *> *)healthSummaries
 835                                              failures:(NSArray<NSDictionary *> *)failures
 836                                                 error:(NSError **)error
 837 {
 838     NSError *localError = nil;
 839     size_t baseSize = [self serializedEventSize:healthSummaries error:&localError];
 840     if (localError != nil) {
 841         secemergency("Unable to serialize health summary JSON");
 842         if (error) {
 843             *error = localError;
 844         }
 845         return nil;
 846     }
 847
 848     NSArray<NSArray *> *chunkedEvents = [self chunkFailureSet:(self.uploadSizeLimit - baseSize) events:failures error:&localError];
 849
 850     NSMutableArray<NSDictionary *> *jsonResults = [[NSMutableArray<NSDictionary *> alloc] init];
 851     for (NSArray<NSDictionary *> *failureSet in chunkedEvents) {
 852         NSDictionary *eventDictionary = [self createEventDictionary:healthSummaries failures:failureSet error:error];
 853         if (eventDictionary) {
 854             [jsonResults addObject:eventDictionary];
 855         } else {
 856             return nil;
 857         }
 858     }
 859
 860     if ([jsonResults count] == 0) {
 861         NSDictionary *eventDictionary = [self createEventDictionary:healthSummaries failures:nil error:error];
 862         if (eventDictionary) {
 863             [jsonResults addObject:eventDictionary];
 864         } else {
 865             return nil;
 866         }
 867     }
 868
 869     return jsonResults;
 870 }
 871
 872 - (BOOL)copyEvents:(NSMutableArray<NSDictionary *> **)healthSummaries
 873           failures:(NSMutableArray<NSDictionary *> **)failures
 874          forUpload:(BOOL)upload
 875 participatingClients:(NSMutableArray<SFAnalyticsClient*>**)clients
 876              force:(BOOL)force
 877         linkedUUID:(NSUUID *)linkedUUID
 878              error:(NSError**)error
 879 {
 880     NSMutableArray<SFAnalyticsClient*> *localClients = [[NSMutableArray alloc] init];
 881     NSMutableArray<NSDictionary *> *localHealthSummaries = [[NSMutableArray<NSDictionary *> alloc] init];
 882     NSMutableArray<NSDictionary *> *localFailures = [[NSMutableArray<NSDictionary *> alloc] init];
 883     NSMutableArray<NSArray*> *hardFailures = [[NSMutableArray alloc] init];
 884     NSMutableArray<NSArray*> *softFailures = [[NSMutableArray alloc] init];
 885     NSString *ckdeviceID = nil;
 886     NSString *accountID = nil;
 887
 888     if (os_variant_has_internal_diagnostics("com.apple.security") && [_internalTopicName isEqualToString:SFAnalyticsTopicKeySync]) {
 889         ckdeviceID = [self askSecurityForCKDeviceID];
 890         accountID = accountAltDSID();
 891     }
 892     for (SFAnalyticsClient* client in self->_topicClients) {
 893         if (!force && [client requireDeviceAnalytics] && !_isDeviceAnalyticsEnabled()) {
 894             // Client required device analytics, yet the user did not opt in.
 895             secnotice("getLoggingJSON", "Client '%@' requires device analytics yet user did not opt in.", [client name]);
 896             continue;
 897         }
 898         if (!force && [client requireiCloudAnalytics] && !_isiCloudAnalyticsEnabled()) {
 899             // Client required iCloud analytics, yet the user did not opt in.
 900             secnotice("getLoggingJSON", "Client '%@' requires iCloud analytics yet user did not opt in.", [client name]);
 901             continue;
 902         }
 903
 904         SFAnalyticsSQLiteStore* store = [SFAnalyticsSQLiteStore storeWithPath:client.storePath schema:SFAnalyticsTableSchema];
 905
 906         if (upload) {
 907             NSDate* uploadDate = store.uploadDate;
 908             if (!force && uploadDate && [[NSDate date] timeIntervalSinceDate:uploadDate] < _secondsBetweenUploads) {
 909                 secnotice("json", "ignoring client '%@' for %@ because last upload too recent: %@",
 910                           client.name, _internalTopicName, uploadDate);
 911                 continue;
 912             }
 913
 914             if (force) {
 915                 secnotice("json", "client '%@' for topic '%@' force-included", client.name, _internalTopicName);
 916             } else {
 917                 secnotice("json", "including client '%@' for topic '%@' for upload", client.name, _internalTopicName);
 918             }
 919             [localClients addObject:client];
 920         }
 921
 922         NSMutableDictionary* healthSummary = [self healthSummaryWithName:client.name store:store uuid:linkedUUID];
 923         if (healthSummary) {
 924             if (ckdeviceID) {
 925                 healthSummary[SFAnalyticsDeviceID] = ckdeviceID;
 926             }
 927             if (accountID) {
 928                 healthSummary[SFAnalyticsAltDSID] = accountID;
 929             }
 930             [localHealthSummaries addObject:healthSummary];
 931         }
 932
 933         [hardFailures addObject:store.hardFailures];
 934         [softFailures addObject:store.softFailures];
 935     }
 936
 937     if (upload && [localClients count] == 0) {
 938         if (error) {
 939             NSString *description = [NSString stringWithFormat:@"Upload too recent for all clients for %@", _internalTopicName];
 940             *error = [NSError errorWithDomain:@"SupdUploadErrorDomain"
 941                                          code:-10
 942                                      userInfo:@{NSLocalizedDescriptionKey : description}];
 943         }
 944         return NO;
 945     }
 946
 947     if (clients) {
 948         *clients = localClients;
 949     }
 950
 951     if (failures) {
 952         [self addFailures:hardFailures toUploadRecords:localFailures threshold:_maxEventsToReport/10 linkedUUID:linkedUUID];
 953         [self addFailures:softFailures toUploadRecords:localFailures threshold:0 linkedUUID:linkedUUID];
 954         [*failures addObjectsFromArray:localFailures];
 955     }
 956
 957     if (healthSummaries) {
 958         [*healthSummaries addObjectsFromArray:localHealthSummaries];
 959     }
 960
 961     return YES;
 962 }
 963
 964 - (NSArray<NSDictionary *> *)createChunkedLoggingJSON:(bool)pretty
 965                                             forUpload:(BOOL)upload
 966                                  participatingClients:(NSMutableArray<SFAnalyticsClient*>**)clients
 967                                                 force:(BOOL)force                                       // supdctl uploads ignore privacy settings and recency
 968                                                 error:(NSError**)error
 969 {
 970     NSUUID *linkedUUID = [NSUUID UUID];
 971     NSError *localError = nil;
 972     NSMutableArray *failures = [[NSMutableArray alloc] init];
 973     NSMutableArray *healthSummaries = [[NSMutableArray alloc] init];
 974     BOOL copied = [self copyEvents:&healthSummaries
 975                           failures:&failures
 976                          forUpload:upload
 977               participatingClients:clients
 978                              force:force
 979                         linkedUUID:linkedUUID
 980                              error:&localError];
 981     if (!copied || localError) {
 982         if (error) {
 983             *error = localError;
 984         }
 985         return nil;
 986     }
 987
 988     // Trim failures to the max count, based on health summary count
 989     if ([failures count] > (_maxEventsToReport - [healthSummaries count])) {
 990         NSRange range;
 991         range.location = 0;
 992         range.length = _maxEventsToReport - [healthSummaries count];
 993         failures = [[failures subarrayWithRange:range] mutableCopy];
 994     }
 995
 996     return [self createChunkedLoggingJSON:healthSummaries failures:failures error:error];
 997 }
 998
 999 - (NSDictionary *)createLoggingJSON:(bool)pretty
1000                           forUpload:(BOOL)upload
1001                participatingClients:(NSMutableArray<SFAnalyticsClient*>**)clients
1002                               force:(BOOL)force                                       // supdctl uploads ignore privacy settings and recency
1003                               error:(NSError**)error
1004 {
1005     NSError *localError = nil;
1006     NSMutableArray *failures = [[NSMutableArray alloc] init];
1007     NSMutableArray *healthSummaries = [[NSMutableArray alloc] init];
1008     BOOL copied = [self copyEvents:&healthSummaries
1009                           failures:&failures
1010                          forUpload:upload
1011               participatingClients:clients
1012                              force:force
1013                         linkedUUID:nil
1014                              error:&localError];
1015     if (!copied || localError) {
1016         if (error) {
1017             *error = localError;
1018         }
1019         return nil;
1020     }
1021
1022     // Trim failures to the max count, based on health summary count
1023     if ([failures count] > (_maxEventsToReport - [healthSummaries count])) {
1024         NSRange range;
1025         range.location = 0;
1026         range.length = _maxEventsToReport - [healthSummaries count];
1027         failures = [[failures subarrayWithRange:range] mutableCopy];
1028     }
1029
1030     return [self createEventDictionary:healthSummaries failures:failures error:error];
1031 }
1032
1033 // Is at least one client eligible for data collection based on user consent? Otherwise callers should NOT reach off-device.
1034 - (BOOL)haveEligibleClients {
1035     for (SFAnalyticsClient* client in self.topicClients) {
1036         if ((!client.requireDeviceAnalytics || _isDeviceAnalyticsEnabled()) &&
1037             (!client.requireiCloudAnalytics || _isiCloudAnalyticsEnabled())) {
1038             return YES;
1039         }
1040     }
1041     return NO;
1042 }
1043
1044 - (NSString*)askSecurityForCKDeviceID
1045 {
1046     NSError* error = nil;
1047     CKKSControl* rpc = [CKKSControl controlObject:&error];
1048     if(error || !rpc) {
1049         secerror("unable to obtain CKKS endpoint: %@", error);
1050         return nil;
1051     }
1052
1053     __block NSString* localCKDeviceID;
1054     dispatch_semaphore_t sema = dispatch_semaphore_create(0);
1055     [rpc rpcGetCKDeviceIDWithReply:^(NSString* ckdeviceID) {
1056         localCKDeviceID = ckdeviceID;
1057         dispatch_semaphore_signal(sema);
1058     }];
1059
1060     if (dispatch_semaphore_wait(sema, dispatch_time(DISPATCH_TIME_NOW, NSEC_PER_SEC * 10)) != 0) {
1061         secerror("timed out waiting for a response from security");
1062         return nil;
1063     }
1064
1065     return localCKDeviceID;
1066 }
1067
1068 // this method is kind of evil for the fact that it has side-effects in pulling other things besides the metricsURL from the server, and as such should NOT be memoized.
1069 // TODO redo this, probably to return a dictionary.
1070 - (NSURL*)splunkUploadURL:(BOOL)force
1071 {
1072     if (!force && ![self haveEligibleClients]) {    // force is true IFF called from supdctl. Customers don't have it and internal audiences must call it explicitly.
1073         secnotice("getURL", "Not going to talk to server for topic %@ because no eligible clients", [self internalTopicName]);
1074         return nil;
1075     }
1076
1077     if (__splunkUploadURL) {
1078         return __splunkUploadURL;
1079     }
1080
1081     secnotice("getURL", "Asking server for endpoint and config data for topic %@", [self internalTopicName]);
1082
1083     __weak __typeof(self) weakSelf = self;
1084     dispatch_semaphore_t sem = dispatch_semaphore_create(0);
1085
1086     __block NSError* error = nil;
1087     NSURLSessionConfiguration *configuration = [NSURLSessionConfiguration ephemeralSessionConfiguration];
1088     NSURLSession* storeBagSession = [NSURLSession sessionWithConfiguration:configuration
1089                                                                   delegate:self
1090                                                              delegateQueue:nil];
1091
1092     NSURL* requestEndpoint = _splunkBagURL;
1093     __block NSURL* result = nil;
1094     NSURLSessionDataTask* storeBagTask = [storeBagSession dataTaskWithURL:requestEndpoint completionHandler:^(NSData * _Nullable data,
1095                                                                                                               NSURLResponse * _Nullable __unused response,
1096                                                                                                               NSError * _Nullable responseError) {
1097
1098         __strong __typeof(self) strongSelf = weakSelf;
1099         if (!strongSelf) {
1100             return;
1101         }
1102
1103         if (data && !responseError) {
1104             NSData *responseData = data; // shut up compiler
1105             NSDictionary* responseDict = [NSJSONSerialization JSONObjectWithData:responseData options:0 error:&error];
1106             if([responseDict isKindOfClass:NSDictionary.class] && !error) {
1107                 if (!self->_ignoreServersMessagesTellingUsToGoAway) {
1108                     self->_disableUploads = [[responseDict valueForKey:@"sendDisabled"] boolValue];
1109                     if (self->_disableUploads) {
1110                         // then don't upload anything right now
1111                         secerror("not returning a splunk URL because uploads are disabled for %@", self->_internalTopicName);
1112                         dispatch_semaphore_signal(sem);
1113                         return;
1114                     }
1115
1116                     // backend works with milliseconds
1117                     NSUInteger secondsBetweenUploads = [[responseDict valueForKey:@"postFrequency"] unsignedIntegerValue] / 1000;
1118                     if (secondsBetweenUploads > 0) {
1119                         if (os_variant_has_internal_diagnostics("com.apple.security") &&
1120                             self->_secondsBetweenUploads < secondsBetweenUploads) {
1121                             secnotice("getURL", "Overriding server-sent post frequency because device is internal (%lu -> %lu)", (unsigned long)secondsBetweenUploads, (unsigned long)self->_secondsBetweenUploads);
1122                         } else {
1123                             strongSelf->_secondsBetweenUploads = secondsBetweenUploads;
1124                         }
1125                     }
1126
1127                     strongSelf->_blacklistedEvents = responseDict[@"blacklistedEvents"];
1128                     strongSelf->_blacklistedFields = responseDict[@"blacklistedFields"];
1129                 }
1130
1131                 strongSelf->_metricsBase = responseDict[@"metricsBase"];
1132
1133                 NSString* metricsEndpoint = responseDict[@"metricsUrl"];
1134                 if([metricsEndpoint isKindOfClass:NSString.class]) {
1135                     /* Lives our URL */
1136                     NSString* endpoint = [metricsEndpoint stringByAppendingFormat:@"/2/%@", strongSelf->_splunkTopicName];
1137                     secnotice("upload", "got metrics endpoint %@ for %@", endpoint, self->_internalTopicName);
1138                     NSURL* endpointURL = [NSURL URLWithString:endpoint];
1139                     if([endpointURL.scheme isEqualToString:@"https"]) {
1140                         result = endpointURL;
1141                     }
1142                 }
1143             }
1144         }
1145         else {
1146             error = responseError;
1147         }
1148         if (error) {
1149             secnotice("upload", "Unable to fetch splunk endpoint at URL for %@: %@ -- error: %@",
1150                       self->_internalTopicName, requestEndpoint, error.description);
1151         }
1152         else if (!result) {
1153             secnotice("upload", "Malformed iTunes config payload for %@!", self->_internalTopicName);
1154         }
1155
1156         dispatch_semaphore_signal(sem);
1157     }];
1158
1159     [storeBagTask resume];
1160     dispatch_semaphore_wait(sem, dispatch_time(DISPATCH_TIME_NOW, (uint64_t)(60 * NSEC_PER_SEC)));
1161
1162     return result;
1163 }
1164
1165 - (void)URLSession:(NSURLSession *)session didReceiveChallenge:(NSURLAuthenticationChallenge *)challenge
1166  completionHandler:(void (^)(NSURLSessionAuthChallengeDisposition, NSURLCredential *))completionHandler {
1167     assert(completionHandler);
1168     (void)session;
1169     secnotice("upload", "Splunk upload challenge for %@", _internalTopicName);
1170     NSURLCredential *cred = nil;
1171
1172     if ([challenge previousFailureCount] > 0) {
1173         // Previous failures occurred, bail
1174         completionHandler(NSURLSessionAuthChallengeCancelAuthenticationChallenge, nil);
1175
1176     } else if ([challenge.protectionSpace.authenticationMethod isEqualToString:NSURLAuthenticationMethodServerTrust]) {
1177         /*
1178          * Evaluate trust for the certificate
1179          */
1180
1181         SecTrustRef serverTrust = challenge.protectionSpace.serverTrust;
1182         // Coverity gets upset if we don't check status even though result is all we need.
1183         bool trustResult = SecTrustEvaluateWithError(serverTrust, NULL);
1184         if (_allowInsecureSplunkCert || trustResult) {
1185             /*
1186              * All is well, accept the credentials
1187              */
1188             if(_allowInsecureSplunkCert) {
1189                 secnotice("upload", "Force Accepting Splunk Credential for %@", _internalTopicName);
1190             }
1191             cred = [NSURLCredential credentialForTrust:serverTrust];
1192             completionHandler(NSURLSessionAuthChallengeUseCredential, cred);
1193
1194         } else {
1195             /*
1196              * An error occurred in evaluating trust, bail
1197              */
1198             completionHandler(NSURLSessionAuthChallengeCancelAuthenticationChallenge, nil);
1199         }
1200     } else {
1201         /*
1202          * Just perform the default handling
1203          */
1204         completionHandler(NSURLSessionAuthChallengePerformDefaultHandling, nil);
1205     }
1206 }
1207
1208 - (NSDictionary*)eventDictWithBlacklistedFieldsStrippedFrom:(NSDictionary*)eventDict
1209 {
1210     NSMutableDictionary* strippedDict = eventDict.mutableCopy;
1211     for (NSString* blacklistedField in _blacklistedFields) {
1212         [strippedDict removeObjectForKey:blacklistedField];
1213     }
1214     return strippedDict;
1215 }
1216
1217 // MARK: Database path retrieval
1218
1219 + (NSString*)databasePathForCKKS
1220 {
1221     return [(__bridge_transfer NSURL*)SecCopyURLForFileInKeychainDirectory((__bridge CFStringRef)@"Analytics/ckks_analytics.db") path];
1222 }
1223
1224 + (NSString*)databasePathForSOS
1225 {
1226     return [(__bridge_transfer NSURL*)SecCopyURLForFileInKeychainDirectory((__bridge CFStringRef)@"Analytics/sos_analytics.db") path];
1227 }
1228
1229 + (NSString*)AppSupportPath
1230 {
1231 #if TARGET_OS_IPHONE && !TARGET_OS_SIMULATOR
1232     return @"/var/mobile/Library/Application Support";
1233 #else
1234     NSArray<NSString *>*paths = NSSearchPathForDirectoriesInDomains(NSApplicationSupportDirectory, NSUserDomainMask, true);
1235     if ([paths count] < 1) {
1236         return nil;
1237     }
1238     return [NSString stringWithString: paths[0]];
1239 #endif /* TARGET_OS_IPHONE && !TARGET_OS_SIMULATOR  */
1240 }
1241
1242 + (NSString*)databasePathForPCS
1243 {
1244     NSString *appSup = [self AppSupportPath];
1245     if (!appSup) {
1246         return nil;
1247     }
1248     NSString *dbpath = [NSString stringWithFormat:@"%@/com.apple.ProtectedCloudStorage/PCSAnalytics.db", appSup];
1249     secnotice("supd", "PCS Database path (%@)", dbpath);
1250     return dbpath;
1251 }
1252
1253 + (NSString*)databasePathForLocal
1254 {
1255     return [(__bridge_transfer NSURL*)SecCopyURLForFileInKeychainDirectory((__bridge CFStringRef)@"Analytics/localkeychain.db") path];
1256 }
1257
1258 + (NSString*)databasePathForTrustdHealth
1259 {
1260 #if TARGET_OS_IPHONE
1261     return [(__bridge_transfer NSURL*)SecCopyURLForFileInKeychainDirectory(CFSTR("Analytics/trustd_health_analytics.db")) path];
1262 #else
1263     return [(__bridge_transfer NSURL*)SecCopyURLForFileInUserCacheDirectory(CFSTR("Analytics/trustd_health_analytics.db")) path];
1264 #endif
1265 }
1266
1267 + (NSString*)databasePathForTrust
1268 {
1269 #if TARGET_OS_IPHONE
1270     return [(__bridge_transfer NSURL*)SecCopyURLForFileInKeychainDirectory(CFSTR("Analytics/trust_analytics.db")) path];
1271 #else
1272     return [(__bridge_transfer NSURL*)SecCopyURLForFileInUserCacheDirectory(CFSTR("Analytics/trust_analytics.db")) path];
1273 #endif
1274 }
1275
1276 + (NSString*)databasePathForTLS
1277 {
1278 #if TARGET_OS_IPHONE
1279     return [(__bridge_transfer NSURL*)SecCopyURLForFileInKeychainDirectory(CFSTR("Analytics/TLS_analytics.db")) path];
1280 #else
1281     return [(__bridge_transfer NSURL*)SecCopyURLForFileInUserCacheDirectory(CFSTR("Analytics/TLS_analytics.db")) path];
1282 #endif
1283 }
1284
1285 + (NSString*)databasePathForSignIn
1286 {
1287     return [(__bridge_transfer NSURL*)SecCopyURLForFileInKeychainDirectory(CFSTR("Analytics/signin_metrics.db")) path];
1288 }
1289
1290 + (NSString*)databasePathForCloudServices
1291 {
1292     return [(__bridge_transfer NSURL*)SecCopyURLForFileInKeychainDirectory(CFSTR("Analytics/CloudServicesAnalytics.db")) path];
1293 }
1294
1295 + (NSString*)databasePathForTransparency
1296 {
1297     return [(__bridge_transfer NSURL*)SecCopyURLForFileInKeychainDirectory((__bridge CFStringRef)@"Analytics/TransparencyAnalytics.db") path];
1298 }
1299
1300 @end
1301
1302 @interface supd ()
1303 @property NSDictionary *topicsSamplingRates;
1304 @end
1305
1306 @implementation supd
1307 - (void)setupTopics
1308 {
1309     NSDictionary* systemDefaultValues = [NSDictionary dictionaryWithContentsOfFile:[[NSBundle bundleWithPath:@"/System/Library/Frameworks/Security.framework"] pathForResource:@"SFAnalytics" ofType:@"plist"]];
1310     NSMutableArray <SFAnalyticsTopic*>* topics = [NSMutableArray array];
1311     for (NSString *topicKey in systemDefaultValues) {
1312         NSDictionary *topicSamplingRates = _topicsSamplingRates[topicKey];
1313         SFAnalyticsTopic *topic = [[SFAnalyticsTopic alloc] initWithDictionary:systemDefaultValues[topicKey] name:topicKey samplingRates:topicSamplingRates];
1314         [topics addObject:topic];
1315     }
1316     _analyticsTopics = [NSArray arrayWithArray:topics];
1317 }
1318
1319 + (void)instantiate {
1320     [supd instance];
1321 }
1322
1323 + (instancetype)instance {
1324     if (!_supdInstance) {
1325         _supdInstance = [self new];
1326     }
1327     return _supdInstance;
1328 }
1329
1330 // Use this for testing to get rid of any state
1331 + (void)removeInstance {
1332     _supdInstance = nil;
1333 }
1334
1335
1336 static NSString *SystemTrustStorePath = @"/System/Library/Security/Certificates.bundle";
1337 static NSString *AnalyticsSamplingRatesFilename = @"AnalyticsSamplingRates";
1338 static NSString *ContentVersionKey = @"MobileAssetContentVersion";
1339 static NSString *AssetContextFilename = @"OTAPKIContext.plist";
1340
1341 static NSNumber *getSystemVersion(NSBundle *trustStoreBundle) {
1342     NSDictionary *systemVersionPlist = [NSDictionary dictionaryWithContentsOfURL:[trustStoreBundle URLForResource:@"AssetVersion"
1343                                                                                                     withExtension:@"plist"]];
1344     if (!systemVersionPlist || ![systemVersionPlist isKindOfClass:[NSDictionary class]]) {
1345         return nil;
1346     }
1347     NSNumber *systemVersion = systemVersionPlist[ContentVersionKey];
1348     if (systemVersion == nil || ![systemVersion isKindOfClass:[NSNumber class]]) {
1349         return nil;
1350     }
1351     return systemVersion;
1352 }
1353
1354 static NSNumber *getAssetVersion(NSURL *directory) {
1355     NSDictionary *assetContextPlist = [NSDictionary dictionaryWithContentsOfURL:[directory URLByAppendingPathComponent:AssetContextFilename]];
1356     if (!assetContextPlist || ![assetContextPlist isKindOfClass:[NSDictionary class]]) {
1357         return nil;
1358     }
1359     NSNumber *assetVersion = assetContextPlist[ContentVersionKey];
1360     if (assetVersion == nil || ![assetVersion isKindOfClass:[NSNumber class]]) {
1361         return nil;
1362     }
1363     return assetVersion;
1364 }
1365
1366 static bool ShouldInitializeWithAsset(NSBundle *trustStoreBundle, NSURL *directory) {
1367     NSNumber *systemVersion = getSystemVersion(trustStoreBundle);
1368     NSNumber *assetVersion = getAssetVersion(directory);
1369
1370     if (assetVersion == nil || systemVersion == nil) {
1371         return false;
1372     }
1373     if ([assetVersion compare:systemVersion] == NSOrderedDescending) {
1374         return true;
1375     }
1376     return false;
1377 }
1378
1379 - (void)setupSamplingRates {
1380     NSBundle *trustStoreBundle = [NSBundle bundleWithPath:SystemTrustStorePath];
1381
1382     NSURL *keychainsDirectory = CFBridgingRelease(SecCopyURLForFileInSystemKeychainDirectory(nil));
1383     NSURL *directory = [keychainsDirectory URLByAppendingPathComponent:@"SupplementalsAssets/" isDirectory:YES];
1384
1385     NSDictionary *analyticsSamplingRates = nil;
1386     if (ShouldInitializeWithAsset(trustStoreBundle, directory)) {
1387         /* Try to get the asset version of the sampling rates */
1388         NSURL *analyticsSamplingRateURL = [directory URLByAppendingPathComponent:[NSString stringWithFormat:@"%@.plist", AnalyticsSamplingRatesFilename]];
1389         analyticsSamplingRates = [NSDictionary dictionaryWithContentsOfURL:analyticsSamplingRateURL];
1390         secnotice("supd", "read sampling rates from SupplementalsAssets dir");
1391         if (!analyticsSamplingRates || ![analyticsSamplingRates isKindOfClass:[NSDictionary class]]) {
1392             analyticsSamplingRates = nil;
1393         }
1394     }
1395     if (!analyticsSamplingRates) {
1396         analyticsSamplingRates = [NSDictionary dictionaryWithContentsOfURL: [trustStoreBundle URLForResource:AnalyticsSamplingRatesFilename
1397                                                                                                withExtension:@"plist"]];
1398     }
1399     if (analyticsSamplingRates && [analyticsSamplingRates isKindOfClass:[NSDictionary class]]) {
1400         _topicsSamplingRates = analyticsSamplingRates[@"Topics"];
1401         if (!_topicsSamplingRates || ![analyticsSamplingRates isKindOfClass:[NSDictionary class]]) {
1402             _topicsSamplingRates = nil; // Something has gone terribly wrong, so we'll use the hardcoded defaults in this case
1403         }
1404     }
1405 }
1406
1407 - (instancetype)initWithReporter:(SFAnalyticsReporter *)reporter
1408 {
1409     if (self = [super init]) {
1410         [self setupSamplingRates];
1411         [self setupTopics];
1412         _reporter = reporter;
1413
1414         xpc_activity_register("com.apple.securityuploadd.triggerupload", XPC_ACTIVITY_CHECK_IN, ^(xpc_activity_t activity) {
1415             xpc_activity_state_t activityState = xpc_activity_get_state(activity);
1416             secnotice("supd", "hit xpc activity trigger, state: %ld", activityState);
1417             if (activityState == XPC_ACTIVITY_STATE_RUN) {
1418                 // Run our regularly scheduled scan
1419                 [self performRegularlyScheduledUpload];
1420             }
1421         });
1422     }
1423
1424     return self;
1425 }
1426
1427 - (instancetype)init {
1428     SFAnalyticsReporter *reporter = [[SFAnalyticsReporter alloc] init];
1429     return [self initWithReporter:reporter];
1430 }
1431
1432 - (void)sendNotificationForOncePerReportSamplers
1433 {
1434     notify_post(SFAnalyticsFireSamplersNotification);
1435     [NSThread sleepForTimeInterval:3.0];
1436 }
1437
1438 - (void)performRegularlyScheduledUpload {
1439     secnotice("upload", "Starting uploads in response to regular trigger");
1440     NSError *error = nil;
1441     if ([self uploadAnalyticsWithError:&error force:NO]) {
1442         secnotice("upload", "Regularly scheduled upload successful");
1443     } else {
1444         secerror("upload: Failed to complete regularly scheduled upload: %@", error);
1445     }
1446 }
1447
1448 - (NSArray<NSData *> *)serializeLoggingEvents:(NSArray<NSDictionary *> *)events
1449                                         error:(NSError **)error
1450 {
1451     if (!events) {
1452         return nil;
1453     }
1454
1455     NSMutableArray<NSData *> *serializedEvents = [[NSMutableArray<NSData *> alloc] init];
1456     for (NSDictionary *event in events) {
1457         NSError *serializationError = nil;
1458         NSData* serializedEvent = [NSJSONSerialization dataWithJSONObject:event
1459                                                                   options:0
1460                                                                     error:&serializationError];
1461         if (serializedEvent && !serializationError) {
1462             [serializedEvents addObject:serializedEvent];
1463         } else if (error) {
1464             *error = serializationError;
1465             return nil;
1466         }
1467     }
1468
1469     return serializedEvents;
1470 }
1471
1472 - (BOOL)uploadAnalyticsWithError:(NSError**)error force:(BOOL)force {
1473     [self sendNotificationForOncePerReportSamplers];
1474
1475     BOOL result = NO;
1476     NSError* localError = nil;
1477     for (SFAnalyticsTopic *topic in _analyticsTopics) {
1478         @autoreleasepool { // The logging JSONs get quite large. Ensure they're deallocated between topics.
1479             __block NSURL* endpoint = [topic splunkUploadURL:force];   // has side effects!
1480
1481             if (!endpoint) {
1482                 secnotice("upload", "Skipping upload for %@ because no endpoint", [topic internalTopicName]);
1483                 continue;
1484             }
1485
1486             if ([topic disableUploads]) {
1487                 secnotice("upload", "Aborting upload task for %@ because uploads are disabled", [topic internalTopicName]);
1488                 continue;
1489             }
1490
1491             NSMutableArray<SFAnalyticsClient*>* clients = [NSMutableArray new];
1492             NSArray<NSDictionary *> *jsonEvents = [topic createChunkedLoggingJSON:false forUpload:YES participatingClients:&clients force:force error:&localError];
1493             if (!jsonEvents || localError) {
1494                 if ([[localError domain] isEqualToString:SupdErrorDomain] && [localError code] == SupdInvalidJSONError) {
1495                     // Pretend this was a success because at least we'll get rid of bad data.
1496                     // If someone keeps logging bad data and we only catch it here then
1497                     // this causes sustained data loss for the entire topic.
1498                     [topic updateUploadDateForClients:clients date:[NSDate date] clearData:YES];
1499                 }
1500                 secerror("upload: failed to create chunked log events for logging topic %@: %@", [topic internalTopicName], localError);
1501                 continue;
1502             }
1503
1504             NSArray<NSData *> *serializedEvents = [self serializeLoggingEvents:jsonEvents error:&localError];
1505             if (!serializedEvents || localError) {
1506                 if ([[localError domain] isEqualToString:SupdErrorDomain] && [localError code] == SupdInvalidJSONError) {
1507                     // Pretend this was a success because at least we'll get rid of bad data.
1508                     // If someone keeps logging bad data and we only catch it here then
1509                     // this causes sustained data loss for the entire topic.
1510                     [topic updateUploadDateForClients:clients date:[NSDate date] clearData:YES];
1511                 }
1512                 secerror("upload: failed to serialized chunked log events for logging topic %@: %@", [topic internalTopicName], localError);
1513                 continue;
1514             }
1515
1516             if ([topic isSampledUpload]) {
1517                 for (NSData *json in serializedEvents) {
1518                     if (![self->_reporter saveReport:json fileName:[topic internalTopicName]]) {
1519                         secerror("upload: failed to write analytics data to log");
1520                     }
1521                     if ([topic postJSON:json toEndpoint:endpoint error:&localError]) {
1522                         secnotice("upload", "Successfully posted JSON for %@", [topic internalTopicName]);
1523                         result = YES;
1524                         [topic updateUploadDateForClients:clients date:[NSDate date] clearData:YES];
1525                     } else {
1526                         secerror("upload: Failed to post JSON for %@: %@", [topic internalTopicName], localError);
1527                     }
1528                 }
1529             } else {
1530                 /* If we didn't sample this report, update date to prevent trying to upload again sooner
1531                  * than we should. Clear data so that per-day calculations remain consistent. */
1532                 secnotice("upload", "skipping unsampled upload for %@ and clearing data", [topic internalTopicName]);
1533                 [topic updateUploadDateForClients:clients date:[NSDate date] clearData:YES];
1534             }
1535         }
1536         if (error && localError) {
1537             *error = localError;
1538         }
1539     }
1540     return result;
1541 }
1542
1543 - (NSString*)sysdiagnoseStringForEventRecord:(NSDictionary*)eventRecord
1544 {
1545     NSMutableDictionary* mutableEventRecord = eventRecord.mutableCopy;
1546     [mutableEventRecord removeObjectForKey:SFAnalyticsSplunkTopic];
1547
1548     NSDate* eventDate = [NSDate dateWithTimeIntervalSince1970:[[eventRecord valueForKey:SFAnalyticsEventTime] doubleValue] / 1000];
1549     [mutableEventRecord removeObjectForKey:SFAnalyticsEventTime];
1550
1551     NSString* eventName = eventRecord[SFAnalyticsEventType];
1552     [mutableEventRecord removeObjectForKey:SFAnalyticsEventType];
1553
1554     SFAnalyticsEventClass eventClass = [[eventRecord valueForKey:SFAnalyticsEventClassKey] integerValue];
1555     NSString* eventClassString = [self stringForEventClass:eventClass];
1556     [mutableEventRecord removeObjectForKey:SFAnalyticsEventClassKey];
1557
1558     NSMutableString* additionalAttributesString = [NSMutableString string];
1559     if (mutableEventRecord.count > 0) {
1560         [additionalAttributesString appendString:@" - Attributes: {" ];
1561         __block BOOL firstAttribute = YES;
1562         [mutableEventRecord enumerateKeysAndObjectsUsingBlock:^(NSString* key, id object, BOOL* stop) {
1563             NSString* openingString = firstAttribute ? @"" : @", ";
1564             [additionalAttributesString appendString:[NSString stringWithFormat:@"%@%@ : %@", openingString, key, object]];
1565             firstAttribute = NO;
1566         }];
1567         [additionalAttributesString appendString:@" }"];
1568     }
1569
1570     return [NSString stringWithFormat:@"%@ %@: %@%@", eventDate, eventClassString, eventName, additionalAttributesString];
1571 }
1572
1573 - (NSString*)getSysdiagnoseDump
1574 {
1575     NSMutableString* sysdiagnose = [[NSMutableString alloc] init];
1576
1577     for (SFAnalyticsTopic* topic in _analyticsTopics) {
1578         for (SFAnalyticsClient* client in topic.topicClients) {
1579             [sysdiagnose appendString:[NSString stringWithFormat:@"Client: %@\n", client.name]];
1580             SFAnalyticsSQLiteStore* store = [SFAnalyticsSQLiteStore storeWithPath:client.storePath schema:SFAnalyticsTableSchema];
1581             NSArray* allEvents = store.allEvents;
1582             for (NSDictionary* eventRecord in allEvents) {
1583                 [sysdiagnose appendFormat:@"%@\n", [self sysdiagnoseStringForEventRecord:eventRecord]];
1584             }
1585             if (allEvents.count == 0) {
1586                 [sysdiagnose appendString:@"No data to report for this client\n"];
1587             }
1588         }
1589     }
1590     return sysdiagnose;
1591 }
1592
1593 - (void)setUploadDateWith:(NSDate *)date reply:(void (^)(BOOL, NSError*))reply
1594 {
1595     for (SFAnalyticsTopic* topic in _analyticsTopics) {
1596         [topic updateUploadDateForClients:topic.topicClients date:date clearData:NO];
1597     }
1598     reply(YES, nil);
1599 }
1600
1601 - (void)clientStatus:(void (^)(NSDictionary<NSString *, id> *, NSError *))reply
1602 {
1603     NSMutableDictionary *info = [NSMutableDictionary dictionary];
1604     for (SFAnalyticsTopic* topic in _analyticsTopics) {
1605         for (SFAnalyticsClient *client in topic.topicClients) {
1606             SFAnalyticsSQLiteStore* store = [SFAnalyticsSQLiteStore storeWithPath:client.storePath schema:SFAnalyticsTableSchema];
1607
1608             NSMutableDictionary *clientInfo = [NSMutableDictionary dictionary];
1609             clientInfo[@"uploadDate"] = store.uploadDate;
1610             info[client.name] = clientInfo;
1611         }
1612     }
1613
1614     reply(info, nil);
1615 }
1616
1617 - (NSString*)stringForEventClass:(SFAnalyticsEventClass)eventClass
1618 {
1619     if (eventClass == SFAnalyticsEventClassNote) {
1620         return @"EventNote";
1621     }
1622     else if (eventClass == SFAnalyticsEventClassSuccess) {
1623         return @"EventSuccess";
1624     }
1625     else if (eventClass == SFAnalyticsEventClassHardFailure) {
1626         return @"EventHardFailure";
1627     }
1628     else if (eventClass == SFAnalyticsEventClassSoftFailure) {
1629         return @"EventSoftFailure";
1630     }
1631     else {
1632         return @"EventUnknown";
1633     }
1634 }
1635
1636 // MARK: XPC Procotol Handlers
1637
1638 - (void)getSysdiagnoseDumpWithReply:(void (^)(NSString*))reply {
1639     reply([self getSysdiagnoseDump]);
1640 }
1641
1642 - (void)createLoggingJSON:(bool)pretty topic:(NSString *)topicName reply:(void (^)(NSData *, NSError*))reply {
1643     secnotice("rpcCreateLoggingJSON", "Building a JSON blob resembling the one we would have uploaded");
1644     NSError* error = nil;
1645     [self sendNotificationForOncePerReportSamplers];
1646     NSDictionary *eventDictionary = nil;
1647     for (SFAnalyticsTopic* topic in self->_analyticsTopics) {
1648         if ([topic.internalTopicName isEqualToString:topicName]) {
1649             eventDictionary = [topic createLoggingJSON:pretty forUpload:NO participatingClients:nil force:!runningTests error:&error];
1650         }
1651     }
1652
1653     NSData *data = nil;
1654     if (!eventDictionary) {
1655         secerror("Unable to obtain JSON: %@", error);
1656     } else {
1657         data = [NSJSONSerialization dataWithJSONObject:eventDictionary
1658                                                options:(pretty ? NSJSONWritingPrettyPrinted : 0)
1659                                                  error:&error];
1660     }
1661
1662     reply(data, error);
1663 }
1664
1665 - (void)createChunkedLoggingJSON:(bool)pretty topic:(NSString *)topicName reply:(void (^)(NSData *, NSError*))reply
1666 {
1667     secnotice("rpcCreateChunkedLoggingJSON", "Building an array of JSON blobs resembling the one we would have uploaded");
1668     NSError* error = nil;
1669     [self sendNotificationForOncePerReportSamplers];
1670     NSArray<NSDictionary *> *events = nil;
1671     for (SFAnalyticsTopic* topic in self->_analyticsTopics) {
1672         if ([topic.internalTopicName isEqualToString:topicName]) {
1673             events = [topic createChunkedLoggingJSON:pretty forUpload:NO participatingClients:nil force:!runningTests error:&error];
1674         }
1675     }
1676
1677     NSData *data = nil;
1678     if (!events) {
1679         secerror("Unable to obtain JSON: %@", error);
1680     } else {
1681         data = [NSJSONSerialization dataWithJSONObject:events
1682                                                options:(pretty ? NSJSONWritingPrettyPrinted : 0)
1683                                                  error:&error];
1684     }
1685
1686     reply(data, error);
1687 }
1688
1689 - (void)forceUploadWithReply:(void (^)(BOOL, NSError*))reply {
1690     secnotice("upload", "Performing upload in response to rpc message");
1691     NSError* error = nil;
1692     BOOL result = [self uploadAnalyticsWithError:&error force:YES];
1693     secnotice("upload", "Result of manually triggered upload: %@, error: %@", result ? @"success" : @"failure", error);
1694     reply(result, error);
1695 }
1696
1697 @end
1698
1699 #endif // !TARGET_OS_SIMULATOR