supd/supd.m

   1 /*
   2  * Copyright (c) 2017-2018 Apple Inc. All Rights Reserved.
   3  *
   4  * @APPLE_LICENSE_HEADER_START@
   5  *
   6  * This file contains Original Code and/or Modifications of Original Code
   7  * as defined in and that are subject to the Apple Public Source License
   8  * Version 2.0 (the 'License'). You may not use this file except in
   9  * compliance with the License. Please obtain a copy of the License at
  10  * http://www.opensource.apple.com/apsl/ and read it before using this
  11  * file.
  12  *
  13  * The Original Code and all software distributed under the License are
  14  * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  15  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  16  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
  17  * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
  18  * Please see the License for the specific language governing rights and
  19  * limitations under the License.
  20  *
  21  * @APPLE_LICENSE_HEADER_END@
  22  */
  23
  24 #import "supd.h"
  25
  26 #if !TARGET_OS_SIMULATOR
  27
  28 #import "SFAnalyticsDefines.h"
  29 #import "SFAnalyticsSQLiteStore.h"
  30 #import <Security/SFAnalytics.h>
  31
  32 #include <utilities/SecFileLocations.h>
  33 #import "utilities/debugging.h"
  34 #import <os/variant_private.h>
  35 #import <xpc/xpc.h>
  36 #include <notify.h>
  37 #import "keychain/ckks/CKKSControl.h"
  38 #import <zlib.h>
  39
  40 #import <AuthKit/AKAppleIDAuthenticationContext.h>
  41 #import <AuthKit/AKAppleIDAuthenticationController.h>
  42 #import <AuthKit/AKAppleIDAuthenticationController_Private.h>
  43
  44 #if TARGET_OS_OSX
  45 #include "dirhelper_priv.h"
  46 #include <membership.h>
  47 #endif
  48
  49 #if TARGET_OS_OSX
  50 #import <CrashReporterSupport/CrashReporterSupportPrivate.h>
  51 #else
  52 #import <CrashReporterSupport/CrashReporterSupport.h>
  53 #endif
  54
  55 #import <Accounts/Accounts.h>
  56 #import <Accounts/ACAccountStore_Private.h>
  57 #import <Accounts/ACAccountType_Private.h>
  58 #import <Accounts/ACAccountStore.h>
  59 #import <AppleAccount/AppleAccount.h>
  60 #import <AppleAccount/ACAccount+AppleAccount.h>
  61 #import <AppleAccount/ACAccountStore+AppleAccount.h>
  62
  63 #import "utilities/simulatecrash_assert.h"
  64
  65
  66 NSString* const SFAnalyticsSplunkTopic = @"topic";
  67 NSString* const SFAnalyticsClientId = @"clientId";
  68 NSString* const SFAnalyticsInternal = @"internal";
  69
  70 NSString* const SFAnalyticsMetricsBase = @"metricsBase";
  71 NSString* const SFAnalyticsDeviceID = @"ckdeviceID";
  72 NSString* const SFAnalyticsAltDSID = @"altDSID";
  73
  74 NSString* const SFAnalyticsEventCorrelationID = @"eventLinkID";
  75
  76 NSString* const SFAnalyticsSecondsCustomerKey = @"SecondsBetweenUploadsCustomer";
  77 NSString* const SFAnalyticsSecondsInternalKey = @"SecondsBetweenUploadsInternal";
  78 NSString* const SFAnalyticsSecondsSeedKey = @"SecondsBetweenUploadsSeed";
  79 NSString* const SFAnalyticsMaxEventsKey = @"NumberOfEvents";
  80 NSString* const SFAnalyticsDevicePercentageCustomerKey = @"DevicePercentageCustomer";
  81 NSString* const SFAnalyticsDevicePercentageInternalKey = @"DevicePercentageInternal";
  82 NSString* const SFAnalyticsDevicePercentageSeedKey = @"DevicePercentageSeed";
  83
  84 NSString* const SupdErrorDomain = @"com.apple.security.supd";
  85
  86 #define SFANALYTICS_SPLUNK_DEV 0
  87 #define OS_CRASH_TRACER_LOG_BUG_TYPE "226"
  88
  89 #if SFANALYTICS_SPLUNK_DEV
  90 NSUInteger const secondsBetweenUploadsCustomer = 10;
  91 NSUInteger const secondsBetweenUploadsInternal = 10;
  92 NSUInteger const secondsBetweenUploadsSeed = 10;
  93 #else // SFANALYTICS_SPLUNK_DEV
  94 NSUInteger const secondsBetweenUploadsCustomer = (3 * (60 * 60 * 24));
  95 NSUInteger const secondsBetweenUploadsInternal = (60 * 60 * 24);
  96 NSUInteger const secondsBetweenUploadsSeed = (60 * 60 * 24);
  97 #endif // SFANALYTICS_SPLUNK_DEV
  98
  99 @implementation SFAnalyticsReporter
 100 - (BOOL)saveReport:(NSData *)reportData fileName:(NSString *)fileName
 101 {
 102     BOOL writtenToLog = NO;
 103 #if TARGET_OS_OSX
 104     NSDictionary *optionsDictionary = @{ (__bridge NSString *)kCRProblemReportSubmissionPolicyKey: (__bridge NSString *)kCRSubmissionPolicyAlternate };
 105 #else // !TARGET_OS_OSX
 106     NSDictionary *optionsDictionary = nil; // The keys above are not defined or required on iOS.
 107 #endif // !TARGET_OS_OSX
 108
 109
 110     secdebug("saveReport", "calling out to `OSAWriteLogForSubmission`");
 111     writtenToLog = OSAWriteLogForSubmission(@OS_CRASH_TRACER_LOG_BUG_TYPE, fileName,
 112                                             nil, optionsDictionary, ^(NSFileHandle *fileHandle) {
 113                                                 secnotice("OSAWriteLogForSubmission", "Writing log data to report: %@", fileName);
 114                                                 [fileHandle writeData:reportData];
 115                                             });
 116     return writtenToLog;
 117 }
 118 @end
 119
 120 #define DEFAULT_SPLUNK_MAX_EVENTS_TO_REPORT 1000
 121 #define DEFAULT_SPLUNK_DEVICE_PERCENTAGE 100
 122
 123 static supd *_supdInstance = nil;
 124
 125 BOOL runningTests = NO;
 126 BOOL deviceAnalyticsOverride = NO;
 127 BOOL deviceAnalyticsEnabled = NO;
 128 BOOL iCloudAnalyticsOverride = NO;
 129 BOOL iCloudAnalyticsEnabled = NO;
 130
 131 static BOOL
 132 _isDeviceAnalyticsEnabled(void)
 133 {
 134     // This flag is only set during tests.
 135     if (deviceAnalyticsOverride) {
 136         return deviceAnalyticsEnabled;
 137     }
 138
 139     static BOOL dataCollectionEnabled = NO;
 140     static dispatch_once_t onceToken;
 141     dispatch_once(&onceToken, ^{
 142 #if TARGET_OS_IPHONE
 143         dataCollectionEnabled = DiagnosticLogSubmissionEnabled();
 144 #elif TARGET_OS_OSX
 145         dataCollectionEnabled = CRIsAutoSubmitEnabled();
 146 #endif
 147     });
 148     return dataCollectionEnabled;
 149 }
 150
 151 static NSString *
 152 accountAltDSID(void)
 153 {
 154     ACAccountStore *accountStore = [[ACAccountStore alloc] init];
 155     ACAccount *primaryAccount = [accountStore aa_primaryAppleAccount];
 156     if (primaryAccount == nil) {
 157         return nil;
 158     }
 159     return [primaryAccount aa_altDSID];
 160 }
 161
 162 static NSString *const kAnalyticsiCloudIdMSKey = @"com.apple.idms.config.privacy.icloud.data";
 163
 164 static NSDictionary *
 165 _getiCloudConfigurationInfoWithError(NSError **outError)
 166 {
 167     __block NSDictionary *outConfigurationInfo = nil;
 168     __block NSError *localError = nil;
 169
 170     NSString *altDSID = accountAltDSID();
 171     if (altDSID != nil) {
 172         secnotice("_getiCloudConfigurationInfoWithError", "Fetching configuration info");
 173
 174         dispatch_semaphore_t sema = dispatch_semaphore_create(0);
 175         AKAppleIDAuthenticationController *authController = [AKAppleIDAuthenticationController new];
 176         [authController configurationInfoWithIdentifiers:@[kAnalyticsiCloudIdMSKey]
 177                                               forAltDSID:altDSID
 178                                               completion:^(NSDictionary<NSString *, id<NSSecureCoding>> *configurationInfo, NSError *error) {
 179             if (error) {
 180                 secerror("_getiCloudConfigurationInfoWithError: Error fetching configurationInfo: %@", error);
 181                 localError = error;
 182             } else if (![configurationInfo isKindOfClass:[NSDictionary class]]) {
 183                 secerror("_getiCloudConfigurationInfoWithError: configurationInfo dict was not a dict, it was a %{public}@", [configurationInfo class]);
 184                 localError = error;
 185                 configurationInfo = nil;
 186             } else {
 187                 secnotice("_getiCloudConfigurationInfoWithError", "fetched configurationInfo %@", configurationInfo);
 188                 outConfigurationInfo = configurationInfo;
 189             }
 190             dispatch_semaphore_signal(sema);
 191         }];
 192         dispatch_semaphore_wait(sema, dispatch_time(DISPATCH_TIME_NOW, (uint64_t)(5 * NSEC_PER_SEC)));
 193     } else {
 194         secerror("_getiCloudConfigurationInfoWithError: Failed to fetch primary account info.");
 195     }
 196
 197     if (localError && outError) {
 198         *outError = localError;
 199     }
 200     return outConfigurationInfo;
 201 }
 202
 203 static BOOL
 204 _isiCloudAnalyticsEnabled()
 205 {
 206     // This flag is only set during tests.
 207     if (iCloudAnalyticsOverride) {
 208         return iCloudAnalyticsEnabled;
 209     }
 210
 211     static bool cachedAllowsICloudAnalytics = false;
 212
 213     static dispatch_once_t onceToken;
 214     dispatch_once(&onceToken, ^{
 215         NSError *error = nil;
 216         NSDictionary *accountConfiguration = _getiCloudConfigurationInfoWithError(&error);
 217         if (error == nil && accountConfiguration != nil) {
 218             id iCloudAnalyticsOptIn = accountConfiguration[kAnalyticsiCloudIdMSKey];
 219             if (iCloudAnalyticsOptIn != nil) {
 220                 BOOL iCloudAnalyticsOptInHasCorrectType = ([iCloudAnalyticsOptIn isKindOfClass:[NSNumber class]] || [iCloudAnalyticsOptIn isKindOfClass:[NSString class]]);
 221                 if (iCloudAnalyticsOptInHasCorrectType) {
 222                     NSNumber *iCloudAnalyticsOptInNumber = @([iCloudAnalyticsOptIn integerValue]);
 223                     cachedAllowsICloudAnalytics = ![iCloudAnalyticsOptInNumber isEqualToNumber:[NSNumber numberWithInteger:0]];
 224                 }
 225             }
 226         } else if (error != nil) {
 227             secerror("_isiCloudAnalyticsEnabled: %@", error);
 228         }
 229     });
 230
 231     return cachedAllowsICloudAnalytics;
 232 }
 233
 234 /*  NSData GZip category based on GeoKit's implementation */
 235 @interface NSData (GZip)
 236 - (NSData *)supd_gzipDeflate;
 237 @end
 238
 239 #define GZIP_OFFSET 16
 240 #define GZIP_STRIDE_LEN 16384
 241
 242 @implementation NSData (Gzip)
 243 - (NSData *)supd_gzipDeflate
 244 {
 245     if ([self length] == 0) {
 246         return self;
 247     }
 248
 249     z_stream strm;
 250     memset(&strm, 0, sizeof(strm));
 251     strm.next_in=(uint8_t *)[self bytes];
 252     strm.avail_in = (unsigned int)[self length];
 253
 254
 255     if (Z_OK != deflateInit2(&strm, Z_BEST_COMPRESSION, Z_DEFLATED,
 256                              MAX_WBITS + GZIP_OFFSET, MAX_MEM_LEVEL, Z_DEFAULT_STRATEGY)) {
 257         return nil;
 258     }
 259
 260     NSMutableData *compressed = [NSMutableData dataWithLength:GZIP_STRIDE_LEN];
 261
 262     do {
 263         if (strm.total_out >= [compressed length]) {
 264             [compressed increaseLengthBy: 16384];
 265         }
 266
 267         strm.next_out = [compressed mutableBytes] + strm.total_out;
 268         strm.avail_out = (int)[compressed length] - (int)strm.total_out;
 269
 270         deflate(&strm, Z_FINISH);
 271
 272     } while (strm.avail_out == 0);
 273
 274     deflateEnd(&strm);
 275
 276     [compressed setLength: strm.total_out];
 277     if (strm.avail_in == 0) {
 278         return [NSData dataWithData:compressed];
 279     } else {
 280         return nil;
 281     }
 282 }
 283 @end
 284
 285 @implementation SFAnalyticsClient {
 286     NSString* _path;
 287     NSString* _name;
 288     BOOL _requireDeviceAnalytics;
 289     BOOL _requireiCloudAnalytics;
 290 }
 291
 292 @synthesize storePath = _path;
 293 @synthesize name = _name;
 294
 295 - (instancetype)initWithStorePath:(NSString*)path name:(NSString*)name
 296                   deviceAnalytics:(BOOL)deviceAnalytics iCloudAnalytics:(BOOL)iCloudAnalytics {
 297     if (self = [super init]) {
 298         _path = path;
 299         _name = name;
 300         _requireDeviceAnalytics = deviceAnalytics;
 301         _requireiCloudAnalytics = iCloudAnalytics;
 302     }
 303     return self;
 304 }
 305
 306 @end
 307
 308 @interface SFAnalyticsTopic ()
 309 @property NSURL* _splunkUploadURL;
 310
 311 @property BOOL allowInsecureSplunkCert;
 312 @property BOOL ignoreServersMessagesTellingUsToGoAway;
 313 @property BOOL disableUploads;
 314 @property BOOL disableClientId;
 315
 316 @property NSUInteger secondsBetweenUploads;
 317 @property NSUInteger maxEventsToReport;
 318 @property float devicePercentage; // for sampling reporting devices
 319
 320 @property NSDictionary* metricsBase; // data the server provides and wants us to send back
 321 @property NSArray* blacklistedFields;
 322 @property NSArray* blacklistedEvents;
 323 @end
 324
 325 @implementation SFAnalyticsTopic
 326
 327 - (void)setupClientsForTopic:(NSString *)topicName
 328 {
 329     NSMutableArray<SFAnalyticsClient*>* clients = [NSMutableArray<SFAnalyticsClient*> new];
 330     if ([topicName isEqualToString:SFAnalyticsTopicKeySync]) {
 331         [clients addObject:[[SFAnalyticsClient alloc] initWithStorePath:[self.class databasePathForCKKS]
 332                                                                    name:@"ckks" deviceAnalytics:NO iCloudAnalytics:YES]];
 333         [clients addObject:[[SFAnalyticsClient alloc] initWithStorePath:[self.class databasePathForSOS]
 334                                                                    name:@"sos" deviceAnalytics:NO iCloudAnalytics:YES]];
 335         [clients addObject:[[SFAnalyticsClient alloc] initWithStorePath:[self.class databasePathForPCS]
 336                                                                    name:@"pcs" deviceAnalytics:NO iCloudAnalytics:YES]];
 337         [clients addObject:[[SFAnalyticsClient alloc] initWithStorePath:[self.class databasePathForSignIn]
 338                                                                    name:@"signins" deviceAnalytics:NO iCloudAnalytics:YES]];
 339         [clients addObject:[[SFAnalyticsClient alloc] initWithStorePath:[self.class databasePathForLocal]
 340                                                                    name:@"local" deviceAnalytics:YES iCloudAnalytics:NO]];
 341     } else if ([topicName isEqualToString:SFAnalyticsTopicCloudServices]) {
 342         [clients addObject:[[SFAnalyticsClient alloc] initWithStorePath:[self.class databasePathForCloudServices]
 343                                                                    name:@"CloudServices"
 344                                                         deviceAnalytics:YES
 345                                                         iCloudAnalytics:NO]];
 346     } else if ([topicName isEqualToString:SFAnalyticsTopicTrust]) {
 347         [clients addObject:[[SFAnalyticsClient alloc] initWithStorePath:[self.class databasePathForTrust]
 348                                                                    name:@"trust" deviceAnalytics:YES iCloudAnalytics:NO]];
 349 #if TARGET_OS_OSX
 350         [clients addObject:[[SFAnalyticsClient alloc] initWithStorePath:[self.class databasePathForRootTrust]
 351                                                                    name:@"rootTrust" deviceAnalytics:YES iCloudAnalytics:NO]];
 352 #endif
 353     } else if ([topicName isEqualToString:SFAnalyticsTopicNetworking]) {
 354         [clients addObject:[[SFAnalyticsClient alloc] initWithStorePath:[self.class databasePathForNetworking]
 355                                                                    name:@"networking" deviceAnalytics:YES iCloudAnalytics:NO]];
 356 #if TARGET_OS_OSX
 357         [clients addObject:[[SFAnalyticsClient alloc] initWithStorePath:[self.class databasePathForRootNetworking]
 358                                                                    name:@"rootNetworking" deviceAnalytics:YES iCloudAnalytics:NO]];
 359 #endif
 360     } else if ([topicName isEqualToString:SFAnalyticsTopicTransparency]) {
 361         [clients addObject:[[SFAnalyticsClient alloc] initWithStorePath:[self.class databasePathForTransparency]
 362                                                                    name:@"transparency" deviceAnalytics:NO iCloudAnalytics:YES]];
 363     }
 364
 365     _topicClients = clients;
 366 }
 367
 368 - (instancetype)initWithDictionary:(NSDictionary *)dictionary name:(NSString *)topicName samplingRates:(NSDictionary *)rates {
 369     if (self = [super init]) {
 370         _internalTopicName = topicName;
 371         [self setupClientsForTopic:topicName];
 372         _splunkTopicName = dictionary[@"splunk_topic"];
 373         __splunkUploadURL = [NSURL URLWithString:dictionary[@"splunk_uploadURL"]];
 374         _splunkBagURL = [NSURL URLWithString:dictionary[@"splunk_bagURL"]];
 375         _allowInsecureSplunkCert = [[dictionary valueForKey:@"splunk_allowInsecureCertificate"] boolValue];
 376         _uploadSizeLimit = [[dictionary valueForKey:@"uploadSizeLimit"] unsignedIntegerValue];
 377
 378         NSString* splunkEndpoint = dictionary[@"splunk_endpointDomain"];
 379         if (dictionary[@"disableClientId"]) {
 380             _disableClientId = YES;
 381         }
 382
 383         NSUserDefaults* defaults = [[NSUserDefaults alloc] initWithSuiteName:SFAnalyticsUserDefaultsSuite];
 384         NSString* userDefaultsSplunkTopic = [defaults stringForKey:@"splunk_topic"];
 385         if (userDefaultsSplunkTopic) {
 386             _splunkTopicName = userDefaultsSplunkTopic;
 387         }
 388
 389         NSURL* userDefaultsSplunkUploadURL = [NSURL URLWithString:[defaults stringForKey:@"splunk_uploadURL"]];
 390         if (userDefaultsSplunkUploadURL) {
 391             __splunkUploadURL = userDefaultsSplunkUploadURL;
 392         }
 393
 394         NSURL* userDefaultsSplunkBagURL = [NSURL URLWithString:[defaults stringForKey:@"splunk_bagURL"]];
 395         if (userDefaultsSplunkBagURL) {
 396             _splunkBagURL = userDefaultsSplunkBagURL;
 397         }
 398
 399         NSInteger userDefaultsUploadSizeLimit = [defaults integerForKey:@"uploadSizeLimit"];
 400         if (userDefaultsUploadSizeLimit > 0) {
 401             _uploadSizeLimit = userDefaultsUploadSizeLimit;
 402         }
 403
 404         BOOL userDefaultsAllowInsecureSplunkCert = [defaults boolForKey:@"splunk_allowInsecureCertificate"];
 405         _allowInsecureSplunkCert |= userDefaultsAllowInsecureSplunkCert;
 406
 407         NSString* userDefaultsSplunkEndpoint = [defaults stringForKey:@"splunk_endpointDomain"];
 408         if (userDefaultsSplunkEndpoint) {
 409             splunkEndpoint = userDefaultsSplunkEndpoint;
 410         }
 411
 412 #if SFANALYTICS_SPLUNK_DEV
 413         _secondsBetweenUploads = secondsBetweenUploadsInternal;
 414         _maxEventsToReport = SFAnalyticsMaxEventsToReport;
 415         _devicePercentage = DEFAULT_SPLUNK_DEVICE_PERCENTAGE;
 416 #else
 417         bool internal = os_variant_has_internal_diagnostics("com.apple.security");
 418         if (rates) {
 419 #if RC_SEED_BUILD
 420             NSNumber *secondsNum = internal ? rates[SFAnalyticsSecondsInternalKey] : rates[SFAnalyticsSecondsSeedKey];
 421             NSNumber *percentageNum = internal ? rates[SFAnalyticsDevicePercentageInternalKey] : rates[SFAnalyticsDevicePercentageSeedKey];
 422 #else
 423             NSNumber *secondsNum = internal ? rates[SFAnalyticsSecondsInternalKey] : rates[SFAnalyticsSecondsCustomerKey];
 424             NSNumber *percentageNum = internal ? rates[SFAnalyticsDevicePercentageInternalKey] : rates[SFAnalyticsDevicePercentageCustomerKey];
 425 #endif
 426             _secondsBetweenUploads = [secondsNum integerValue];
 427             _maxEventsToReport = [rates[SFAnalyticsMaxEventsKey] unsignedIntegerValue];
 428             _devicePercentage = [percentageNum floatValue];
 429         } else {
 430 #if RC_SEED_BUILD
 431             _secondsBetweenUploads = internal ? secondsBetweenUploadsInternal : secondsBetweenUploadsSeed;
 432 #else
 433             _secondsBetweenUploads = internal ? secondsBetweenUploadsInternal : secondsBetweenUploadsCustomer;
 434 #endif
 435             _maxEventsToReport = SFAnalyticsMaxEventsToReport;
 436             _devicePercentage = DEFAULT_SPLUNK_DEVICE_PERCENTAGE;
 437         }
 438 #endif
 439         secnotice("supd", "created %@ with %lu seconds between uploads, %lu max events, %f percent of uploads",
 440                   _internalTopicName, (unsigned long)_secondsBetweenUploads, (unsigned long)_maxEventsToReport, _devicePercentage);
 441
 442 #if SFANALYTICS_SPLUNK_DEV
 443         _ignoreServersMessagesTellingUsToGoAway = YES;
 444
 445         if (!_splunkUploadURL && splunkEndpoint) {
 446             NSString* urlString = [NSString stringWithFormat:@"https://%@/report/2/%@", splunkEndpoint, _splunkTopicName];
 447             _splunkUploadURL = [NSURL URLWithString:urlString];
 448         }
 449 #else
 450         (void)splunkEndpoint;
 451 #endif
 452     }
 453     return self;
 454 }
 455
 456 - (BOOL)isSampledUpload
 457 {
 458     uint32_t sample = arc4random();
 459     if ((double)_devicePercentage < ((double)1 / UINT32_MAX) * 100) {
 460         /* Requested percentage is smaller than we can sample. just do 1 out of UINT32_MAX */
 461         if (sample == 0) {
 462             return YES;
 463         }
 464     } else {
 465         if ((double)sample <= (double)UINT32_MAX * ((double)_devicePercentage / 100)) {
 466             return YES;
 467         }
 468     }
 469     return NO;
 470 }
 471
 472 - (BOOL)postJSON:(NSData*)json toEndpoint:(NSURL*)endpoint error:(NSError**)error
 473 {
 474     if (!endpoint) {
 475         if (error) {
 476             NSString *description = [NSString stringWithFormat:@"No endpoint for %@", _internalTopicName];
 477             *error = [NSError errorWithDomain:@"SupdUploadErrorDomain"
 478                                          code:-10
 479                                      userInfo:@{NSLocalizedDescriptionKey : description}];
 480         }
 481         return false;
 482     }
 483     /*
 484      * Create the NSURLSession
 485      *  We use the ephemeral session config because we don't need cookies or cache
 486      */
 487     NSURLSessionConfiguration *configuration = [NSURLSessionConfiguration ephemeralSessionConfiguration];
 488
 489     configuration.HTTPAdditionalHeaders = @{ @"User-Agent" : [NSString stringWithFormat:@"securityd/%s", SECURITY_BUILD_VERSION]};
 490
 491     NSURLSession* postSession = [NSURLSession sessionWithConfiguration:configuration
 492                                                               delegate:self
 493                                                          delegateQueue:nil];
 494
 495     NSMutableURLRequest* postRequest = [[NSMutableURLRequest alloc] init];
 496     postRequest.URL = endpoint;
 497     postRequest.HTTPMethod = @"POST";
 498     postRequest.HTTPBody = [json supd_gzipDeflate];
 499     [postRequest setValue:@"gzip" forHTTPHeaderField:@"Content-Encoding"];
 500
 501     /*
 502      * Create the upload task.
 503      */
 504     dispatch_semaphore_t sem = dispatch_semaphore_create(0);
 505     __block BOOL uploadSuccess = NO;
 506     NSURLSessionDataTask* uploadTask = [postSession dataTaskWithRequest:postRequest
 507                                                       completionHandler:^(NSData * _Nullable __unused data, NSURLResponse * _Nullable response, NSError * _Nullable requestError) {
 508         if (requestError) {
 509             secerror("Error in uploading the events to splunk for %@: %@", self->_internalTopicName, requestError);
 510         } else if (![response isKindOfClass:NSHTTPURLResponse.class]){
 511             Class class = response.class;
 512             secerror("Received the wrong kind of response for %@: %@", self->_internalTopicName, NSStringFromClass(class));
 513         } else {
 514             NSHTTPURLResponse* httpResponse = (NSHTTPURLResponse*)response;
 515             if(httpResponse.statusCode >= 200 && httpResponse.statusCode < 300) {
 516                 /* Success */
 517                 uploadSuccess = YES;
 518                 secnotice("upload", "Splunk upload success for %@", self->_internalTopicName);
 519             } else {
 520                 secnotice("upload", "Splunk upload for %@ unexpected status to URL: %@ -- status: %d",
 521                           self->_internalTopicName, endpoint, (int)(httpResponse.statusCode));
 522             }
 523         }
 524         dispatch_semaphore_signal(sem);
 525     }];
 526     secnotice("upload", "Splunk upload start for %@", self->_internalTopicName);
 527     [uploadTask resume];
 528     dispatch_semaphore_wait(sem, dispatch_time(DISPATCH_TIME_NOW, (uint64_t)(5 * 60 * NSEC_PER_SEC)));
 529     return uploadSuccess;
 530 }
 531
 532 - (BOOL)eventIsBlacklisted:(NSMutableDictionary*)event {
 533     return _blacklistedEvents ? [_blacklistedEvents containsObject:event[SFAnalyticsEventType]] : NO;
 534 }
 535
 536 - (void)removeBlacklistedFieldsFromEvent:(NSMutableDictionary*)event {
 537     for (NSString* badField in self->_blacklistedFields) {
 538         [event removeObjectForKey:badField];
 539     }
 540 }
 541
 542 - (void)addRequiredFieldsToEvent:(NSMutableDictionary*)event {
 543     [_metricsBase enumerateKeysAndObjectsUsingBlock:^(id  _Nonnull key, id  _Nonnull obj, BOOL * _Nonnull stop) {
 544         if (!event[key]) {
 545             event[key] = obj;
 546         }
 547     }];
 548 }
 549
 550 - (BOOL)prepareEventForUpload:(NSMutableDictionary*)event
 551                    linkedUUID:(NSUUID *)linkedUUID {
 552     if ([self eventIsBlacklisted:event]) {
 553         return NO;
 554     }
 555
 556     [self removeBlacklistedFieldsFromEvent:event];
 557     [self addRequiredFieldsToEvent:event];
 558     if (_disableClientId) {
 559         event[SFAnalyticsClientId] = @(0);
 560     }
 561     event[SFAnalyticsSplunkTopic] = self->_splunkTopicName ?: [NSNull null];
 562     if (linkedUUID) {
 563         event[SFAnalyticsEventCorrelationID] = [linkedUUID UUIDString];
 564     }
 565     return YES;
 566 }
 567
 568 - (void)addFailures:(NSMutableArray<NSArray*>*)failures toUploadRecords:(NSMutableArray*)records threshold:(NSUInteger)threshold linkedUUID:(NSUUID *)linkedUUID
 569 {
 570     // The first 0 through 'threshold' items are getting uploaded in any case (which might be 0 for lower priority data)
 571
 572     for (NSArray* client in failures) {
 573         [client enumerateObjectsUsingBlock:^(id  _Nonnull obj, NSUInteger idx, BOOL * _Nonnull stop) {
 574             NSMutableDictionary* event = (NSMutableDictionary*)obj;
 575             if (idx >= threshold) {
 576                 *stop = YES;
 577                 return;
 578             }
 579             if ([self prepareEventForUpload:event linkedUUID:linkedUUID]) {
 580                 if ([NSJSONSerialization isValidJSONObject:event]) {
 581                     [records addObject:event];
 582                 } else {
 583                     secerror("supd: Replacing event with errorEvent because invalid JSON: %@", event);
 584                     NSString* originalType = event[SFAnalyticsEventType];
 585                     NSDictionary* errorEvent = @{ SFAnalyticsEventType : SFAnalyticsEventTypeErrorEvent,
 586                                                   SFAnalyticsEventErrorDestription : [NSString stringWithFormat:@"JSON:%@", originalType]};
 587                     [records addObject:errorEvent];
 588                 }
 589             }
 590         }];
 591     }
 592
 593     // Are there more items than we shoved into the upload records?
 594     NSInteger excessItems = 0;
 595     for (NSArray* client in failures) {
 596         NSInteger localExcess = client.count - threshold;
 597         excessItems += localExcess > 0 ? localExcess : 0;
 598     }
 599
 600     // Then, if we have space and items left, apply a scaling factor to distribute events across clients to fill upload buffer
 601     if (records.count < _maxEventsToReport && excessItems > 0) {
 602         double scale = (_maxEventsToReport - records.count) / (double)excessItems;
 603         if (scale > 1) {
 604             scale = 1;
 605         }
 606
 607         for (NSArray* client in failures) {
 608             if (client.count > threshold) {
 609                 NSRange range = NSMakeRange(threshold, (client.count - threshold) * scale);
 610                 NSArray* sub = [client subarrayWithRange:range];
 611                 [sub enumerateObjectsUsingBlock:^(id  _Nonnull obj, NSUInteger idx, BOOL * _Nonnull stop) {
 612                     if ([self prepareEventForUpload:obj linkedUUID:linkedUUID]) {
 613                         [records addObject:obj];
 614                     }
 615                 }];
 616             }
 617         }
 618     }
 619 }
 620
 621 - (NSMutableDictionary*)sampleStatisticsForSamples:(NSArray*)samples withName:(NSString*)name
 622 {
 623     NSMutableDictionary* statistics = [NSMutableDictionary dictionary];
 624     NSUInteger count = samples.count;
 625     NSArray* sortedSamples = [samples sortedArrayUsingSelector:@selector(compare:)];
 626     NSArray* samplesAsExpressionArray = @[[NSExpression expressionForConstantValue:sortedSamples]];
 627
 628     if (count == 1) {
 629         statistics[name] = samples[0];
 630     } else {
 631         // NSExpression takes population standard deviation. Our data is a sample of whatever we sampled over time,
 632         // but the difference between the two is fairly minor (divide by N before taking sqrt versus divide by N-1).
 633         statistics[[NSString stringWithFormat:@"%@-dev", name]] = [[NSExpression expressionForFunction:@"stddev:" arguments:samplesAsExpressionArray] expressionValueWithObject:nil context:nil];
 634
 635         statistics[[NSString stringWithFormat:@"%@-min", name]] = [[NSExpression expressionForFunction:@"min:" arguments:samplesAsExpressionArray] expressionValueWithObject:nil context:nil];
 636         statistics[[NSString stringWithFormat:@"%@-max", name]] = [[NSExpression expressionForFunction:@"max:" arguments:samplesAsExpressionArray] expressionValueWithObject:nil context:nil];
 637         statistics[[NSString stringWithFormat:@"%@-avg", name]] = [[NSExpression expressionForFunction:@"average:" arguments:samplesAsExpressionArray] expressionValueWithObject:nil context:nil];
 638         statistics[[NSString stringWithFormat:@"%@-med", name]] = [[NSExpression expressionForFunction:@"median:" arguments:samplesAsExpressionArray] expressionValueWithObject:nil context:nil];
 639     }
 640
 641     if (count > 3) {
 642         NSString* q1 = [NSString stringWithFormat:@"%@-1q", name];
 643         NSString* q3 = [NSString stringWithFormat:@"%@-3q", name];
 644         // From Wikipedia, which is never wrong
 645         if (count % 2 == 0) {
 646             // The lower quartile value is the median of the lower half of the data. The upper quartile value is the median of the upper half of the data.
 647             statistics[q1] = [[NSExpression expressionForFunction:@"median:" arguments:@[[NSExpression expressionForConstantValue:[sortedSamples subarrayWithRange:NSMakeRange(0, count / 2)]]]] expressionValueWithObject:nil context:nil];
 648             statistics[q3] = [[NSExpression expressionForFunction:@"median:" arguments:@[[NSExpression expressionForConstantValue:[sortedSamples subarrayWithRange:NSMakeRange((count / 2), count / 2)]]]] expressionValueWithObject:nil context:nil];
 649         } else if (count % 4 == 1) {
 650             // If there are (4n+1) data points, then the lower quartile is 25% of the nth data value plus 75% of the (n+1)th data value;
 651             // the upper quartile is 75% of the (3n+1)th data point plus 25% of the (3n+2)th data point.
 652             // (offset n by -1 since we count from 0)
 653             NSUInteger n = count / 4;
 654             statistics[q1] = @(([sortedSamples[n - 1] doubleValue] + [sortedSamples[n] doubleValue] * 3.0) / 4.0);
 655             statistics[q3] = @(([sortedSamples[(3 * n)] doubleValue] * 3.0 + [sortedSamples[(3 * n) + 1] doubleValue]) / 4.0);
 656         } else if (count % 4 == 3){
 657             // If there are (4n+3) data points, then the lower quartile is 75% of the (n+1)th data value plus 25% of the (n+2)th data value;
 658             // the upper quartile is 25% of the (3n+2)th data point plus 75% of the (3n+3)th data point.
 659             // (offset n by -1 since we count from 0)
 660             NSUInteger n = count / 4;
 661             statistics[q1] = @(([sortedSamples[n] doubleValue] * 3.0 + [sortedSamples[n + 1] doubleValue]) / 4.0);
 662             statistics[q3] = @(([sortedSamples[(3 * n) + 1] doubleValue] + [sortedSamples[(3 * n) + 2] doubleValue] * 3.0) / 4.0);
 663         }
 664     }
 665
 666     return statistics;
 667 }
 668
 669 - (NSMutableDictionary*)healthSummaryWithName:(NSString*)name store:(SFAnalyticsSQLiteStore*)store uuid:(NSUUID *)uuid
 670 {
 671     __block NSMutableDictionary* summary = [NSMutableDictionary new];
 672
 673     // Add some events of our own before pulling in data
 674     summary[SFAnalyticsEventType] = [NSString stringWithFormat:@"%@HealthSummary", name];
 675     if ([self eventIsBlacklisted:summary]) {
 676         return nil;
 677     }
 678     summary[SFAnalyticsEventTime] = @([[NSDate date] timeIntervalSince1970] * 1000);    // Splunk wants milliseconds
 679     [SFAnalytics addOSVersionToEvent:summary];
 680     if (store.uploadDate) {
 681         summary[SFAnalyticsAttributeLastUploadTime] = @([store.uploadDate timeIntervalSince1970] * 1000);
 682     } else {
 683         summary[SFAnalyticsAttributeLastUploadTime] = @(0);
 684     }
 685
 686     // Process counters
 687     NSDictionary* successCounts = store.summaryCounts;
 688     __block NSInteger totalSuccessCount = 0;
 689     __block NSInteger totalHardFailureCount = 0;
 690     __block NSInteger totalSoftFailureCount = 0;
 691     [successCounts enumerateKeysAndObjectsUsingBlock:^(NSString* _Nonnull eventType, NSDictionary* _Nonnull counts, BOOL* _Nonnull stop) {
 692         summary[[NSString stringWithFormat:@"%@-success", eventType]] = counts[SFAnalyticsColumnSuccessCount];
 693         summary[[NSString stringWithFormat:@"%@-hardfail", eventType]] = counts[SFAnalyticsColumnHardFailureCount];
 694         summary[[NSString stringWithFormat:@"%@-softfail", eventType]] = counts[SFAnalyticsColumnSoftFailureCount];
 695         totalSuccessCount += [counts[SFAnalyticsColumnSuccessCount] integerValue];
 696         totalHardFailureCount += [counts[SFAnalyticsColumnHardFailureCount] integerValue];
 697         totalSoftFailureCount += [counts[SFAnalyticsColumnSoftFailureCount] integerValue];
 698     }];
 699
 700     summary[SFAnalyticsColumnSuccessCount] = @(totalSuccessCount);
 701     summary[SFAnalyticsColumnHardFailureCount] = @(totalHardFailureCount);
 702     summary[SFAnalyticsColumnSoftFailureCount] = @(totalSoftFailureCount);
 703     if (os_variant_has_internal_diagnostics("com.apple.security")) {
 704         summary[SFAnalyticsInternal] = @YES;
 705     }
 706
 707     // Process samples
 708     NSMutableDictionary<NSString*,NSMutableArray*>* samplesBySampler = [NSMutableDictionary<NSString*,NSMutableArray*> dictionary];
 709     for (NSDictionary* sample in [store samples]) {
 710         if (!samplesBySampler[sample[SFAnalyticsColumnSampleName]]) {
 711             samplesBySampler[sample[SFAnalyticsColumnSampleName]] = [NSMutableArray array];
 712         }
 713         [samplesBySampler[sample[SFAnalyticsColumnSampleName]] addObject:sample[SFAnalyticsColumnSampleValue]];
 714     }
 715     [samplesBySampler enumerateKeysAndObjectsUsingBlock:^(NSString * _Nonnull key, NSMutableArray * _Nonnull obj, BOOL * _Nonnull stop) {
 716         NSMutableDictionary* event = [self sampleStatisticsForSamples:obj withName:key];
 717         [summary addEntriesFromDictionary:event];
 718     }];
 719
 720     // Should always return yes because we already checked for event blacklisting specifically (unless summary itself is blacklisted)
 721     if (![self prepareEventForUpload:summary linkedUUID:uuid]) {
 722         secwarning("supd: health summary for %@ blacklisted", name);
 723         return nil;
 724     }
 725
 726     // Seems unlikely because we only insert strings, samplers only take NSNumbers and frankly, sampleStatisticsForSamples probably would have crashed
 727     if (![NSJSONSerialization isValidJSONObject:summary]) {
 728         secerror("json: health summary for client %@ is invalid JSON: %@", name, summary);
 729         return [@{ SFAnalyticsEventType : SFAnalyticsEventTypeErrorEvent,
 730                    SFAnalyticsEventErrorDestription : [NSString stringWithFormat:@"JSON:%@HealthSummary", name]} mutableCopy];
 731     }
 732
 733     return summary;
 734 }
 735
 736 - (void)updateUploadDateForClients:(NSArray<SFAnalyticsClient*>*)clients date:(NSDate *)date clearData:(BOOL)clearData
 737 {
 738     for (SFAnalyticsClient* client in clients) {
 739         SFAnalyticsSQLiteStore* store = [SFAnalyticsSQLiteStore storeWithPath:client.storePath schema:SFAnalyticsTableSchema];
 740         secnotice("postprocess", "Setting upload date (%@) for client: %@", date, client.name);
 741         store.uploadDate = date;
 742         if (clearData) {
 743             secnotice("postprocess", "Clearing collected data for client: %@", client.name);
 744             [store clearAllData];
 745         }
 746     }
 747 }
 748
 749 - (size_t)serializedEventSize:(NSObject *)event
 750                         error:(NSError**)error
 751 {
 752     if (![NSJSONSerialization isValidJSONObject:event]) {
 753         secnotice("serializedEventSize", "invalid JSON object");
 754         return 0;
 755     }
 756
 757     NSData *json = [NSJSONSerialization dataWithJSONObject:event
 758                                                    options:0
 759                                                      error:error];
 760     if (json) {
 761         return [json length];
 762     } else {
 763         secnotice("serializedEventSize", "failed to serialize event");
 764         return 0;
 765     }
 766 }
 767
 768 - (NSArray<NSArray *> *)chunkFailureSet:(size_t)sizeCapacity
 769                                  events:(NSArray<NSDictionary *> *)events
 770                                   error:(NSError **)error
 771 {
 772     const size_t postBodyLimit = 1000; // 1000 events in a single upload
 773     size_t currentSize = 0;
 774     size_t currentEventCount = 0;
 775
 776     NSMutableArray<NSArray<NSDictionary *> *> *eventChunks = [[NSMutableArray<NSArray<NSDictionary *> *> alloc] init];
 777     NSMutableArray<NSDictionary *> *currentEventChunk = [[NSMutableArray<NSDictionary *> alloc] init];
 778     for (NSDictionary *event in events) {
 779         NSError *localError = nil;
 780         size_t eventSize = [self serializedEventSize:event error:&localError];
 781         if (localError != nil) {
 782             if (error) {
 783                 *error = localError;
 784             }
 785             secemergency("Unable to serialize event JSON: %@", [localError localizedDescription]);
 786             return nil;
 787         }
 788
 789         BOOL countLessThanLimit = currentEventCount < postBodyLimit;
 790         BOOL sizeLessThanCapacity = (currentSize + eventSize) <= sizeCapacity;
 791         if (!countLessThanLimit || !sizeLessThanCapacity) {
 792             [eventChunks addObject:currentEventChunk];
 793             currentEventChunk = [[NSMutableArray<NSDictionary *> alloc] init];
 794             currentEventCount = 0;
 795             currentSize = 0;
 796         }
 797
 798         [currentEventChunk addObject:event];
 799         currentEventCount++;
 800         currentSize += eventSize;
 801     }
 802
 803     if ([currentEventChunk count] > 0) {
 804         [eventChunks addObject:currentEventChunk];
 805     }
 806
 807     return eventChunks;
 808 }
 809
 810 - (NSDictionary *)createEventDictionary:(NSArray *)healthSummaries
 811                                failures:(NSArray<NSDictionary *> *)failures
 812                                   error:(NSError **)error
 813 {
 814     NSMutableArray *events = [[NSMutableArray alloc] init];
 815     [events addObjectsFromArray:healthSummaries];
 816     if (failures) {
 817         [events addObjectsFromArray:failures];
 818     }
 819
 820     NSDictionary *eventDictionary = @{
 821         SFAnalyticsPostTime : @([[NSDate date] timeIntervalSince1970] * 1000),
 822         @"events" : events,
 823     };
 824
 825     if (![NSJSONSerialization isValidJSONObject:eventDictionary]) {
 826         secemergency("json: final dictionary invalid JSON.");
 827         if (error) {
 828             *error = [NSError errorWithDomain:SupdErrorDomain code:SupdInvalidJSONError
 829                                      userInfo:@{NSLocalizedDescriptionKey : [NSString localizedStringWithFormat:@"Final dictionary for upload is invalid JSON: %@", eventDictionary]}];
 830         }
 831         return nil;
 832     }
 833
 834     return eventDictionary;
 835 }
 836
 837 - (NSArray<NSDictionary *> *)createChunkedLoggingJSON:(NSArray<NSDictionary *> *)healthSummaries
 838                                              failures:(NSArray<NSDictionary *> *)failures
 839                                                 error:(NSError **)error
 840 {
 841     NSError *localError = nil;
 842     size_t baseSize = [self serializedEventSize:healthSummaries error:&localError];
 843     if (localError != nil) {
 844         secemergency("Unable to serialize health summary JSON");
 845         if (error) {
 846             *error = localError;
 847         }
 848         return nil;
 849     }
 850
 851     NSArray<NSArray *> *chunkedEvents = [self chunkFailureSet:(self.uploadSizeLimit - baseSize) events:failures error:&localError];
 852
 853     NSMutableArray<NSDictionary *> *jsonResults = [[NSMutableArray<NSDictionary *> alloc] init];
 854     for (NSArray<NSDictionary *> *failureSet in chunkedEvents) {
 855         NSDictionary *eventDictionary = [self createEventDictionary:healthSummaries failures:failureSet error:error];
 856         if (eventDictionary) {
 857             [jsonResults addObject:eventDictionary];
 858         } else {
 859             return nil;
 860         }
 861     }
 862
 863     if ([jsonResults count] == 0) {
 864         NSDictionary *eventDictionary = [self createEventDictionary:healthSummaries failures:nil error:error];
 865         if (eventDictionary) {
 866             [jsonResults addObject:eventDictionary];
 867         } else {
 868             return nil;
 869         }
 870     }
 871
 872     return jsonResults;
 873 }
 874
 875 - (BOOL)copyEvents:(NSMutableArray<NSDictionary *> **)healthSummaries
 876           failures:(NSMutableArray<NSDictionary *> **)failures
 877          forUpload:(BOOL)upload
 878 participatingClients:(NSMutableArray<SFAnalyticsClient*>**)clients
 879              force:(BOOL)force
 880         linkedUUID:(NSUUID *)linkedUUID
 881              error:(NSError**)error
 882 {
 883     NSMutableArray<SFAnalyticsClient*> *localClients = [[NSMutableArray alloc] init];
 884     NSMutableArray<NSDictionary *> *localHealthSummaries = [[NSMutableArray<NSDictionary *> alloc] init];
 885     NSMutableArray<NSDictionary *> *localFailures = [[NSMutableArray<NSDictionary *> alloc] init];
 886     NSMutableArray<NSArray*> *hardFailures = [[NSMutableArray alloc] init];
 887     NSMutableArray<NSArray*> *softFailures = [[NSMutableArray alloc] init];
 888     NSString *ckdeviceID = nil;
 889     NSString *accountID = nil;
 890
 891     if (os_variant_has_internal_diagnostics("com.apple.security") && [_internalTopicName isEqualToString:SFAnalyticsTopicKeySync]) {
 892         ckdeviceID = [self askSecurityForCKDeviceID];
 893         accountID = accountAltDSID();
 894     }
 895     for (SFAnalyticsClient* client in self->_topicClients) {
 896         @autoreleasepool {
 897             if (!force && [client requireDeviceAnalytics] && !_isDeviceAnalyticsEnabled()) {
 898                 // Client required device analytics, yet the user did not opt in.
 899                 secnotice("getLoggingJSON", "Client '%@' requires device analytics yet user did not opt in.", [client name]);
 900                 continue;
 901             }
 902             if (!force && [client requireiCloudAnalytics] && !_isiCloudAnalyticsEnabled()) {
 903                 // Client required iCloud analytics, yet the user did not opt in.
 904                 secnotice("getLoggingJSON", "Client '%@' requires iCloud analytics yet user did not opt in.", [client name]);
 905                 continue;
 906             }
 907
 908             SFAnalyticsSQLiteStore* store = [SFAnalyticsSQLiteStore storeWithPath:client.storePath schema:SFAnalyticsTableSchema];
 909
 910             if (upload) {
 911                 NSDate* uploadDate = store.uploadDate;
 912                 if (!force && uploadDate && [[NSDate date] timeIntervalSinceDate:uploadDate] < _secondsBetweenUploads) {
 913                     secnotice("json", "ignoring client '%@' for %@ because last upload too recent: %@",
 914                               client.name, _internalTopicName, uploadDate);
 915                     continue;
 916                 }
 917
 918                 if (force) {
 919                     secnotice("json", "client '%@' for topic '%@' force-included", client.name, _internalTopicName);
 920                 } else {
 921                     secnotice("json", "including client '%@' for topic '%@' for upload", client.name, _internalTopicName);
 922                 }
 923                 [localClients addObject:client];
 924             }
 925
 926             NSMutableDictionary* healthSummary = [self healthSummaryWithName:client.name store:store uuid:linkedUUID];
 927             if (healthSummary) {
 928                 if (ckdeviceID) {
 929                     healthSummary[SFAnalyticsDeviceID] = ckdeviceID;
 930                 }
 931                 if (accountID) {
 932                     healthSummary[SFAnalyticsAltDSID] = accountID;
 933                 }
 934                 [localHealthSummaries addObject:healthSummary];
 935             }
 936
 937             [hardFailures addObject:store.hardFailures];
 938             [softFailures addObject:store.softFailures];
 939         }
 940     }
 941
 942     if (upload && [localClients count] == 0) {
 943         if (error) {
 944             NSString *description = [NSString stringWithFormat:@"Upload too recent for all clients for %@", _internalTopicName];
 945             *error = [NSError errorWithDomain:@"SupdUploadErrorDomain"
 946                                          code:-10
 947                                      userInfo:@{NSLocalizedDescriptionKey : description}];
 948         }
 949         return NO;
 950     }
 951
 952     if (clients) {
 953         *clients = localClients;
 954     }
 955
 956     if (failures) {
 957         [self addFailures:hardFailures toUploadRecords:localFailures threshold:_maxEventsToReport/10 linkedUUID:linkedUUID];
 958         [self addFailures:softFailures toUploadRecords:localFailures threshold:0 linkedUUID:linkedUUID];
 959         [*failures addObjectsFromArray:localFailures];
 960     }
 961
 962     if (healthSummaries) {
 963         [*healthSummaries addObjectsFromArray:localHealthSummaries];
 964     }
 965
 966     return YES;
 967 }
 968
 969 - (NSArray<NSDictionary *> *)createChunkedLoggingJSON:(bool)pretty
 970                                             forUpload:(BOOL)upload
 971                                  participatingClients:(NSMutableArray<SFAnalyticsClient*>**)clients
 972                                                 force:(BOOL)force                                       // supdctl uploads ignore privacy settings and recency
 973                                                 error:(NSError**)error
 974 {
 975     NSUUID *linkedUUID = [NSUUID UUID];
 976     NSError *localError = nil;
 977     NSMutableArray *failures = [[NSMutableArray alloc] init];
 978     NSMutableArray *healthSummaries = [[NSMutableArray alloc] init];
 979     BOOL copied = [self copyEvents:&healthSummaries
 980                           failures:&failures
 981                          forUpload:upload
 982               participatingClients:clients
 983                              force:force
 984                         linkedUUID:linkedUUID
 985                              error:&localError];
 986     if (!copied || localError) {
 987         if (error) {
 988             *error = localError;
 989         }
 990         return nil;
 991     }
 992
 993     // Trim failures to the max count, based on health summary count
 994     if ([failures count] > (_maxEventsToReport - [healthSummaries count])) {
 995         NSRange range;
 996         range.location = 0;
 997         range.length = _maxEventsToReport - [healthSummaries count];
 998         failures = [[failures subarrayWithRange:range] mutableCopy];
 999     }
1000
1001     return [self createChunkedLoggingJSON:healthSummaries failures:failures error:error];
1002 }
1003
1004 - (NSDictionary *)createLoggingJSON:(bool)pretty
1005                           forUpload:(BOOL)upload
1006                participatingClients:(NSMutableArray<SFAnalyticsClient*>**)clients
1007                               force:(BOOL)force                                       // supdctl uploads ignore privacy settings and recency
1008                               error:(NSError**)error
1009 {
1010     NSError *localError = nil;
1011     NSMutableArray *failures = [[NSMutableArray alloc] init];
1012     NSMutableArray *healthSummaries = [[NSMutableArray alloc] init];
1013     BOOL copied = [self copyEvents:&healthSummaries
1014                           failures:&failures
1015                          forUpload:upload
1016               participatingClients:clients
1017                              force:force
1018                         linkedUUID:nil
1019                              error:&localError];
1020     if (!copied || localError) {
1021         if (error) {
1022             *error = localError;
1023         }
1024         return nil;
1025     }
1026
1027     // Trim failures to the max count, based on health summary count
1028     if ([failures count] > (_maxEventsToReport - [healthSummaries count])) {
1029         NSRange range;
1030         range.location = 0;
1031         range.length = _maxEventsToReport - [healthSummaries count];
1032         failures = [[failures subarrayWithRange:range] mutableCopy];
1033     }
1034
1035     return [self createEventDictionary:healthSummaries failures:failures error:error];
1036 }
1037
1038 // Is at least one client eligible for data collection based on user consent? Otherwise callers should NOT reach off-device.
1039 - (BOOL)haveEligibleClients {
1040     for (SFAnalyticsClient* client in self.topicClients) {
1041         if ((!client.requireDeviceAnalytics || _isDeviceAnalyticsEnabled()) &&
1042             (!client.requireiCloudAnalytics || _isiCloudAnalyticsEnabled())) {
1043             return YES;
1044         }
1045     }
1046     return NO;
1047 }
1048
1049 - (NSString*)askSecurityForCKDeviceID
1050 {
1051     NSError* error = nil;
1052     CKKSControl* rpc = [CKKSControl controlObject:&error];
1053     if(error || !rpc) {
1054         secerror("unable to obtain CKKS endpoint: %@", error);
1055         return nil;
1056     }
1057
1058     __block NSString* localCKDeviceID;
1059     dispatch_semaphore_t sema = dispatch_semaphore_create(0);
1060     [rpc rpcGetCKDeviceIDWithReply:^(NSString* ckdeviceID) {
1061         localCKDeviceID = ckdeviceID;
1062         dispatch_semaphore_signal(sema);
1063     }];
1064
1065     if (dispatch_semaphore_wait(sema, dispatch_time(DISPATCH_TIME_NOW, NSEC_PER_SEC * 10)) != 0) {
1066         secerror("timed out waiting for a response from security");
1067         return nil;
1068     }
1069
1070     return localCKDeviceID;
1071 }
1072
1073 // this method is kind of evil for the fact that it has side-effects in pulling other things besides the metricsURL from the server, and as such should NOT be memoized.
1074 // TODO redo this, probably to return a dictionary.
1075 - (NSURL*)splunkUploadURL:(BOOL)force
1076 {
1077     if (!force && ![self haveEligibleClients]) {    // force is true IFF called from supdctl. Customers don't have it and internal audiences must call it explicitly.
1078         secnotice("getURL", "Not going to talk to server for topic %@ because no eligible clients", [self internalTopicName]);
1079         return nil;
1080     }
1081
1082     if (__splunkUploadURL) {
1083         return __splunkUploadURL;
1084     }
1085
1086     secnotice("getURL", "Asking server for endpoint and config data for topic %@", [self internalTopicName]);
1087
1088     __weak __typeof(self) weakSelf = self;
1089     dispatch_semaphore_t sem = dispatch_semaphore_create(0);
1090
1091     __block NSError* error = nil;
1092     NSURLSessionConfiguration *configuration = [NSURLSessionConfiguration ephemeralSessionConfiguration];
1093     NSURLSession* storeBagSession = [NSURLSession sessionWithConfiguration:configuration
1094                                                                   delegate:self
1095                                                              delegateQueue:nil];
1096
1097     NSURL* requestEndpoint = _splunkBagURL;
1098     __block NSURL* result = nil;
1099     NSURLSessionDataTask* storeBagTask = [storeBagSession dataTaskWithURL:requestEndpoint completionHandler:^(NSData * _Nullable data,
1100                                                                                                               NSURLResponse * _Nullable __unused response,
1101                                                                                                               NSError * _Nullable responseError) {
1102
1103         __strong __typeof(self) strongSelf = weakSelf;
1104         if (!strongSelf) {
1105             return;
1106         }
1107
1108         if (data && !responseError) {
1109             NSData *responseData = data; // shut up compiler
1110             NSDictionary* responseDict = [NSJSONSerialization JSONObjectWithData:responseData options:0 error:&error];
1111             if([responseDict isKindOfClass:NSDictionary.class] && !error) {
1112                 if (!self->_ignoreServersMessagesTellingUsToGoAway) {
1113                     self->_disableUploads = [[responseDict valueForKey:@"sendDisabled"] boolValue];
1114                     if (self->_disableUploads) {
1115                         // then don't upload anything right now
1116                         secerror("not returning a splunk URL because uploads are disabled for %@", self->_internalTopicName);
1117                         dispatch_semaphore_signal(sem);
1118                         return;
1119                     }
1120
1121                     // backend works with milliseconds
1122                     NSUInteger secondsBetweenUploads = [[responseDict valueForKey:@"postFrequency"] unsignedIntegerValue] / 1000;
1123                     if (secondsBetweenUploads > 0) {
1124                         if (os_variant_has_internal_diagnostics("com.apple.security") &&
1125                             self->_secondsBetweenUploads < secondsBetweenUploads) {
1126                             secnotice("getURL", "Overriding server-sent post frequency because device is internal (%lu -> %lu)", (unsigned long)secondsBetweenUploads, (unsigned long)self->_secondsBetweenUploads);
1127                         } else {
1128                             strongSelf->_secondsBetweenUploads = secondsBetweenUploads;
1129                         }
1130                     }
1131
1132                     strongSelf->_blacklistedEvents = responseDict[@"blacklistedEvents"];
1133                     strongSelf->_blacklistedFields = responseDict[@"blacklistedFields"];
1134                 }
1135
1136                 strongSelf->_metricsBase = responseDict[@"metricsBase"];
1137
1138                 NSString* metricsEndpoint = responseDict[@"metricsUrl"];
1139                 if([metricsEndpoint isKindOfClass:NSString.class]) {
1140                     /* Lives our URL */
1141                     NSString* endpoint = [metricsEndpoint stringByAppendingFormat:@"/2/%@", strongSelf->_splunkTopicName];
1142                     secnotice("upload", "got metrics endpoint %@ for %@", endpoint, self->_internalTopicName);
1143                     NSURL* endpointURL = [NSURL URLWithString:endpoint];
1144                     if([endpointURL.scheme isEqualToString:@"https"]) {
1145                         result = endpointURL;
1146                     }
1147                 }
1148             }
1149         }
1150         else {
1151             error = responseError;
1152         }
1153         if (error) {
1154             secnotice("upload", "Unable to fetch splunk endpoint at URL for %@: %@ -- error: %@",
1155                       self->_internalTopicName, requestEndpoint, error.description);
1156         }
1157         else if (!result) {
1158             secnotice("upload", "Malformed iTunes config payload for %@!", self->_internalTopicName);
1159         }
1160
1161         dispatch_semaphore_signal(sem);
1162     }];
1163
1164     [storeBagTask resume];
1165     dispatch_semaphore_wait(sem, dispatch_time(DISPATCH_TIME_NOW, (uint64_t)(60 * NSEC_PER_SEC)));
1166
1167     return result;
1168 }
1169
1170 - (void)URLSession:(NSURLSession *)session didReceiveChallenge:(NSURLAuthenticationChallenge *)challenge
1171  completionHandler:(void (^)(NSURLSessionAuthChallengeDisposition, NSURLCredential *))completionHandler {
1172     assert(completionHandler);
1173     (void)session;
1174     secnotice("upload", "Splunk upload challenge for %@", _internalTopicName);
1175     NSURLCredential *cred = nil;
1176
1177     if ([challenge previousFailureCount] > 0) {
1178         // Previous failures occurred, bail
1179         completionHandler(NSURLSessionAuthChallengeCancelAuthenticationChallenge, nil);
1180
1181     } else if ([challenge.protectionSpace.authenticationMethod isEqualToString:NSURLAuthenticationMethodServerTrust]) {
1182         /*
1183          * Evaluate trust for the certificate
1184          */
1185
1186         SecTrustRef serverTrust = challenge.protectionSpace.serverTrust;
1187         // Coverity gets upset if we don't check status even though result is all we need.
1188         bool trustResult = SecTrustEvaluateWithError(serverTrust, NULL);
1189         if (_allowInsecureSplunkCert || trustResult) {
1190             /*
1191              * All is well, accept the credentials
1192              */
1193             if(_allowInsecureSplunkCert) {
1194                 secnotice("upload", "Force Accepting Splunk Credential for %@", _internalTopicName);
1195             }
1196             cred = [NSURLCredential credentialForTrust:serverTrust];
1197             completionHandler(NSURLSessionAuthChallengeUseCredential, cred);
1198
1199         } else {
1200             /*
1201              * An error occurred in evaluating trust, bail
1202              */
1203             completionHandler(NSURLSessionAuthChallengeCancelAuthenticationChallenge, nil);
1204         }
1205     } else {
1206         /*
1207          * Just perform the default handling
1208          */
1209         completionHandler(NSURLSessionAuthChallengePerformDefaultHandling, nil);
1210     }
1211 }
1212
1213 - (NSDictionary*)eventDictWithBlacklistedFieldsStrippedFrom:(NSDictionary*)eventDict
1214 {
1215     NSMutableDictionary* strippedDict = eventDict.mutableCopy;
1216     for (NSString* blacklistedField in _blacklistedFields) {
1217         [strippedDict removeObjectForKey:blacklistedField];
1218     }
1219     return strippedDict;
1220 }
1221
1222 // MARK: Database path retrieval
1223
1224 + (NSString*)databasePathForCKKS
1225 {
1226     return [(__bridge_transfer NSURL*)SecCopyURLForFileInKeychainDirectory((__bridge CFStringRef)@"Analytics/ckks_analytics.db") path];
1227 }
1228
1229 + (NSString*)databasePathForSOS
1230 {
1231     return [(__bridge_transfer NSURL*)SecCopyURLForFileInKeychainDirectory((__bridge CFStringRef)@"Analytics/sos_analytics.db") path];
1232 }
1233
1234 + (NSString*)AppSupportPath
1235 {
1236 #if TARGET_OS_IPHONE && !TARGET_OS_SIMULATOR
1237     return @"/var/mobile/Library/Application Support";
1238 #else
1239     NSArray<NSString *>*paths = NSSearchPathForDirectoriesInDomains(NSApplicationSupportDirectory, NSUserDomainMask, true);
1240     if ([paths count] < 1) {
1241         return nil;
1242     }
1243     return [NSString stringWithString: paths[0]];
1244 #endif /* TARGET_OS_IPHONE && !TARGET_OS_SIMULATOR  */
1245 }
1246
1247 + (NSString*)databasePathForPCS
1248 {
1249     NSString *appSup = [self AppSupportPath];
1250     if (!appSup) {
1251         return nil;
1252     }
1253     NSString *dbpath = [NSString stringWithFormat:@"%@/com.apple.ProtectedCloudStorage/PCSAnalytics.db", appSup];
1254     secnotice("supd", "PCS Database path (%@)", dbpath);
1255     return dbpath;
1256 }
1257
1258 + (NSString*)databasePathForLocal
1259 {
1260     return [(__bridge_transfer NSURL*)SecCopyURLForFileInKeychainDirectory((__bridge CFStringRef)@"Analytics/localkeychain.db") path];
1261 }
1262
1263 + (NSString*)databasePathForTrust
1264 {
1265 #if TARGET_OS_IPHONE
1266     return [(__bridge_transfer NSURL*)SecCopyURLForFileInKeychainDirectory(CFSTR("Analytics/trust_analytics.db")) path];
1267 #else
1268     return [SFAnalytics defaultProtectedAnalyticsDatabasePath:@"trust_analytics"];
1269 #endif
1270 }
1271
1272 #if TARGET_OS_OSX
1273 + (NSUUID *)rootUUID
1274 {
1275     uuid_t rootUuid;
1276     int ret = mbr_uid_to_uuid(0, rootUuid);
1277     if (ret != 0) {
1278         return nil;
1279     }
1280     return [[NSUUID alloc] initWithUUIDBytes:rootUuid];
1281 }
1282 #endif
1283
1284 #if TARGET_OS_OSX
1285 + (NSString*)databasePathForRootTrust
1286 {
1287     return [SFAnalytics defaultProtectedAnalyticsDatabasePath:@"trust_analytics" uuid:[SFAnalyticsTopic rootUUID]];
1288 }
1289 #endif
1290
1291 + (NSString*)databasePathForNetworking
1292 {
1293 #if TARGET_OS_IPHONE
1294     return [(__bridge_transfer NSURL*)SecCopyURLForFileInKeychainDirectory(CFSTR("Analytics/networking_analytics.db")) path];
1295 #else
1296     return [SFAnalytics defaultProtectedAnalyticsDatabasePath:@"networking_analytics"];
1297 #endif
1298 }
1299
1300 #if TARGET_OS_OSX
1301 + (NSString*)databasePathForRootNetworking
1302 {
1303     return [SFAnalytics defaultProtectedAnalyticsDatabasePath:@"networking_analytics" uuid:[SFAnalyticsTopic rootUUID]];
1304 }
1305 #endif
1306
1307 + (NSString*)databasePathForSignIn
1308 {
1309     return [(__bridge_transfer NSURL*)SecCopyURLForFileInKeychainDirectory(CFSTR("Analytics/signin_metrics.db")) path];
1310 }
1311
1312 + (NSString*)databasePathForCloudServices
1313 {
1314     return [(__bridge_transfer NSURL*)SecCopyURLForFileInKeychainDirectory(CFSTR("Analytics/CloudServicesAnalytics.db")) path];
1315 }
1316
1317 + (NSString*)databasePathForTransparency
1318 {
1319     return [(__bridge_transfer NSURL*)SecCopyURLForFileInKeychainDirectory((__bridge CFStringRef)@"Analytics/TransparencyAnalytics.db") path];
1320 }
1321
1322 @end
1323
1324 @interface supd ()
1325 @property NSDictionary *topicsSamplingRates;
1326 @end
1327
1328 @implementation supd
1329 - (void)setupTopics
1330 {
1331     NSDictionary* systemDefaultValues = [NSDictionary dictionaryWithContentsOfFile:[[NSBundle bundleWithPath:@"/System/Library/Frameworks/Security.framework"] pathForResource:@"SFAnalytics" ofType:@"plist"]];
1332     NSMutableArray <SFAnalyticsTopic*>* topics = [NSMutableArray array];
1333     for (NSString *topicKey in systemDefaultValues) {
1334         NSDictionary *topicSamplingRates = _topicsSamplingRates[topicKey];
1335         SFAnalyticsTopic *topic = [[SFAnalyticsTopic alloc] initWithDictionary:systemDefaultValues[topicKey] name:topicKey samplingRates:topicSamplingRates];
1336         [topics addObject:topic];
1337     }
1338     _analyticsTopics = [NSArray arrayWithArray:topics];
1339 }
1340
1341 + (void)instantiate {
1342     [supd instance];
1343 }
1344
1345 + (instancetype)instance {
1346     if (!_supdInstance) {
1347         _supdInstance = [self new];
1348     }
1349     return _supdInstance;
1350 }
1351
1352 // Use this for testing to get rid of any state
1353 + (void)removeInstance {
1354     _supdInstance = nil;
1355 }
1356
1357
1358 static NSString *SystemTrustStorePath = @"/System/Library/Security/Certificates.bundle";
1359 static NSString *AnalyticsSamplingRatesFilename = @"AnalyticsSamplingRates";
1360 static NSString *ContentVersionKey = @"MobileAssetContentVersion";
1361 static NSString *AssetContextFilename = @"OTAPKIContext.plist";
1362
1363 static NSNumber *getSystemVersion(NSBundle *trustStoreBundle) {
1364     NSDictionary *systemVersionPlist = [NSDictionary dictionaryWithContentsOfURL:[trustStoreBundle URLForResource:@"AssetVersion"
1365                                                                                                     withExtension:@"plist"]];
1366     if (!systemVersionPlist || ![systemVersionPlist isKindOfClass:[NSDictionary class]]) {
1367         return nil;
1368     }
1369     NSNumber *systemVersion = systemVersionPlist[ContentVersionKey];
1370     if (systemVersion == nil || ![systemVersion isKindOfClass:[NSNumber class]]) {
1371         return nil;
1372     }
1373     return systemVersion;
1374 }
1375
1376 static NSNumber *getAssetVersion(NSURL *directory) {
1377     NSDictionary *assetContextPlist = [NSDictionary dictionaryWithContentsOfURL:[directory URLByAppendingPathComponent:AssetContextFilename]];
1378     if (!assetContextPlist || ![assetContextPlist isKindOfClass:[NSDictionary class]]) {
1379         return nil;
1380     }
1381     NSNumber *assetVersion = assetContextPlist[ContentVersionKey];
1382     if (assetVersion == nil || ![assetVersion isKindOfClass:[NSNumber class]]) {
1383         return nil;
1384     }
1385     return assetVersion;
1386 }
1387
1388 static bool ShouldInitializeWithAsset(NSBundle *trustStoreBundle, NSURL *directory) {
1389     NSNumber *systemVersion = getSystemVersion(trustStoreBundle);
1390     NSNumber *assetVersion = getAssetVersion(directory);
1391
1392     if (assetVersion == nil || systemVersion == nil) {
1393         return false;
1394     }
1395     if ([assetVersion compare:systemVersion] == NSOrderedDescending) {
1396         return true;
1397     }
1398     return false;
1399 }
1400
1401 - (void)setupSamplingRates {
1402     NSBundle *trustStoreBundle = [NSBundle bundleWithPath:SystemTrustStorePath];
1403
1404     NSURL *keychainsDirectory = CFBridgingRelease(SecCopyURLForFileInSystemKeychainDirectory(nil));
1405     NSURL *directory = [keychainsDirectory URLByAppendingPathComponent:@"SupplementalsAssets/" isDirectory:YES];
1406
1407     NSDictionary *analyticsSamplingRates = nil;
1408     if (ShouldInitializeWithAsset(trustStoreBundle, directory)) {
1409         /* Try to get the asset version of the sampling rates */
1410         NSURL *analyticsSamplingRateURL = [directory URLByAppendingPathComponent:[NSString stringWithFormat:@"%@.plist", AnalyticsSamplingRatesFilename]];
1411         analyticsSamplingRates = [NSDictionary dictionaryWithContentsOfURL:analyticsSamplingRateURL];
1412         secnotice("supd", "read sampling rates from SupplementalsAssets dir");
1413         if (!analyticsSamplingRates || ![analyticsSamplingRates isKindOfClass:[NSDictionary class]]) {
1414             analyticsSamplingRates = nil;
1415         }
1416     }
1417     if (!analyticsSamplingRates) {
1418         analyticsSamplingRates = [NSDictionary dictionaryWithContentsOfURL: [trustStoreBundle URLForResource:AnalyticsSamplingRatesFilename
1419                                                                                                withExtension:@"plist"]];
1420     }
1421     if (analyticsSamplingRates && [analyticsSamplingRates isKindOfClass:[NSDictionary class]]) {
1422         _topicsSamplingRates = analyticsSamplingRates[@"Topics"];
1423         if (!_topicsSamplingRates || ![analyticsSamplingRates isKindOfClass:[NSDictionary class]]) {
1424             _topicsSamplingRates = nil; // Something has gone terribly wrong, so we'll use the hardcoded defaults in this case
1425         }
1426     }
1427 }
1428
1429 - (instancetype)initWithReporter:(SFAnalyticsReporter *)reporter
1430 {
1431     if (self = [super init]) {
1432         [self setupSamplingRates];
1433         [self setupTopics];
1434         _reporter = reporter;
1435
1436         xpc_activity_register("com.apple.securityuploadd.triggerupload", XPC_ACTIVITY_CHECK_IN, ^(xpc_activity_t activity) {
1437             xpc_activity_state_t activityState = xpc_activity_get_state(activity);
1438             secnotice("supd", "hit xpc activity trigger, state: %ld", activityState);
1439             if (activityState == XPC_ACTIVITY_STATE_RUN) {
1440                 // Run our regularly scheduled scan
1441                 [self performRegularlyScheduledUpload];
1442             }
1443         });
1444     }
1445
1446     return self;
1447 }
1448
1449 - (instancetype)init {
1450     SFAnalyticsReporter *reporter = [[SFAnalyticsReporter alloc] init];
1451     return [self initWithReporter:reporter];
1452 }
1453
1454 - (void)sendNotificationForOncePerReportSamplers
1455 {
1456     notify_post(SFAnalyticsFireSamplersNotification);
1457     [NSThread sleepForTimeInterval:3.0];
1458 }
1459
1460 - (void)performRegularlyScheduledUpload {
1461     secnotice("upload", "Starting uploads in response to regular trigger");
1462     NSError *error = nil;
1463     if ([self uploadAnalyticsWithError:&error force:NO]) {
1464         secnotice("upload", "Regularly scheduled upload successful");
1465     } else {
1466         secerror("upload: Failed to complete regularly scheduled upload: %@", error);
1467     }
1468 }
1469
1470 - (NSArray<NSData *> *)serializeLoggingEvents:(NSArray<NSDictionary *> *)events
1471                                         error:(NSError **)error
1472 {
1473     if (!events) {
1474         return nil;
1475     }
1476
1477     NSMutableArray<NSData *> *serializedEvents = [[NSMutableArray<NSData *> alloc] init];
1478     for (NSDictionary *event in events) {
1479         NSError *serializationError = nil;
1480         NSData* serializedEvent = [NSJSONSerialization dataWithJSONObject:event
1481                                                                   options:0
1482                                                                     error:&serializationError];
1483         if (serializedEvent && !serializationError) {
1484             [serializedEvents addObject:serializedEvent];
1485         } else if (error) {
1486             *error = serializationError;
1487             return nil;
1488         }
1489     }
1490
1491     return serializedEvents;
1492 }
1493
1494 - (BOOL)uploadAnalyticsWithError:(NSError**)error force:(BOOL)force {
1495     [self sendNotificationForOncePerReportSamplers];
1496
1497     BOOL result = NO;
1498     NSError* localError = nil;
1499     for (SFAnalyticsTopic *topic in _analyticsTopics) {
1500         @autoreleasepool { // The logging JSONs get quite large. Ensure they're deallocated between topics.
1501             __block NSURL* endpoint = [topic splunkUploadURL:force];   // has side effects!
1502
1503             if (!endpoint) {
1504                 secnotice("upload", "Skipping upload for %@ because no endpoint", [topic internalTopicName]);
1505                 continue;
1506             }
1507
1508             if ([topic disableUploads]) {
1509                 secnotice("upload", "Aborting upload task for %@ because uploads are disabled", [topic internalTopicName]);
1510                 continue;
1511             }
1512
1513             NSMutableArray<SFAnalyticsClient*>* clients = [NSMutableArray new];
1514             NSArray<NSDictionary *> *jsonEvents = [topic createChunkedLoggingJSON:false forUpload:YES participatingClients:&clients force:force error:&localError];
1515             if (!jsonEvents || localError) {
1516                 if ([[localError domain] isEqualToString:SupdErrorDomain] && [localError code] == SupdInvalidJSONError) {
1517                     // Pretend this was a success because at least we'll get rid of bad data.
1518                     // If someone keeps logging bad data and we only catch it here then
1519                     // this causes sustained data loss for the entire topic.
1520                     [topic updateUploadDateForClients:clients date:[NSDate date] clearData:YES];
1521                 }
1522                 secerror("upload: failed to create chunked log events for logging topic %@: %@", [topic internalTopicName], localError);
1523                 continue;
1524             }
1525
1526             NSArray<NSData *> *serializedEvents = [self serializeLoggingEvents:jsonEvents error:&localError];
1527             if (!serializedEvents || localError) {
1528                 if ([[localError domain] isEqualToString:SupdErrorDomain] && [localError code] == SupdInvalidJSONError) {
1529                     // Pretend this was a success because at least we'll get rid of bad data.
1530                     // If someone keeps logging bad data and we only catch it here then
1531                     // this causes sustained data loss for the entire topic.
1532                     [topic updateUploadDateForClients:clients date:[NSDate date] clearData:YES];
1533                 }
1534                 secerror("upload: failed to serialized chunked log events for logging topic %@: %@", [topic internalTopicName], localError);
1535                 continue;
1536             }
1537
1538             if ([topic isSampledUpload]) {
1539                 for (NSData *json in serializedEvents) {
1540                     if (![self->_reporter saveReport:json fileName:[topic internalTopicName]]) {
1541                         secerror("upload: failed to write analytics data to log");
1542                     }
1543                     if ([topic postJSON:json toEndpoint:endpoint error:&localError]) {
1544                         secnotice("upload", "Successfully posted JSON for %@", [topic internalTopicName]);
1545                         result = YES;
1546                         [topic updateUploadDateForClients:clients date:[NSDate date] clearData:YES];
1547                     } else {
1548                         secerror("upload: Failed to post JSON for %@: %@", [topic internalTopicName], localError);
1549                     }
1550                 }
1551             } else {
1552                 /* If we didn't sample this report, update date to prevent trying to upload again sooner
1553                  * than we should. Clear data so that per-day calculations remain consistent. */
1554                 secnotice("upload", "skipping unsampled upload for %@ and clearing data", [topic internalTopicName]);
1555                 [topic updateUploadDateForClients:clients date:[NSDate date] clearData:YES];
1556             }
1557         }
1558         if (error && localError) {
1559             *error = localError;
1560         }
1561     }
1562     return result;
1563 }
1564
1565 - (NSString*)sysdiagnoseStringForEventRecord:(NSDictionary*)eventRecord
1566 {
1567     NSMutableDictionary* mutableEventRecord = eventRecord.mutableCopy;
1568     [mutableEventRecord removeObjectForKey:SFAnalyticsSplunkTopic];
1569
1570     NSDate* eventDate = [NSDate dateWithTimeIntervalSince1970:[[eventRecord valueForKey:SFAnalyticsEventTime] doubleValue] / 1000];
1571     [mutableEventRecord removeObjectForKey:SFAnalyticsEventTime];
1572
1573     NSString* eventName = eventRecord[SFAnalyticsEventType];
1574     [mutableEventRecord removeObjectForKey:SFAnalyticsEventType];
1575
1576     SFAnalyticsEventClass eventClass = [[eventRecord valueForKey:SFAnalyticsEventClassKey] integerValue];
1577     NSString* eventClassString = [self stringForEventClass:eventClass];
1578     [mutableEventRecord removeObjectForKey:SFAnalyticsEventClassKey];
1579
1580     NSMutableString* additionalAttributesString = [NSMutableString string];
1581     if (mutableEventRecord.count > 0) {
1582         [additionalAttributesString appendString:@" - Attributes: {" ];
1583         __block BOOL firstAttribute = YES;
1584         [mutableEventRecord enumerateKeysAndObjectsUsingBlock:^(NSString* key, id object, BOOL* stop) {
1585             NSString* openingString = firstAttribute ? @"" : @", ";
1586             [additionalAttributesString appendString:[NSString stringWithFormat:@"%@%@ : %@", openingString, key, object]];
1587             firstAttribute = NO;
1588         }];
1589         [additionalAttributesString appendString:@" }"];
1590     }
1591
1592     return [NSString stringWithFormat:@"%@ %@: %@%@", eventDate, eventClassString, eventName, additionalAttributesString];
1593 }
1594
1595 - (NSString*)getSysdiagnoseDump
1596 {
1597     NSMutableString* sysdiagnose = [[NSMutableString alloc] init];
1598
1599     for (SFAnalyticsTopic* topic in _analyticsTopics) {
1600         for (SFAnalyticsClient* client in topic.topicClients) {
1601             [sysdiagnose appendString:[NSString stringWithFormat:@"Client: %@\n", client.name]];
1602             SFAnalyticsSQLiteStore* store = [SFAnalyticsSQLiteStore storeWithPath:client.storePath schema:SFAnalyticsTableSchema];
1603             NSArray* allEvents = store.allEvents;
1604             for (NSDictionary* eventRecord in allEvents) {
1605                 [sysdiagnose appendFormat:@"%@\n", [self sysdiagnoseStringForEventRecord:eventRecord]];
1606             }
1607             if (allEvents.count == 0) {
1608                 [sysdiagnose appendString:@"No data to report for this client\n"];
1609             }
1610         }
1611     }
1612     return sysdiagnose;
1613 }
1614
1615 - (void)setUploadDateWith:(NSDate *)date reply:(void (^)(BOOL, NSError*))reply
1616 {
1617     for (SFAnalyticsTopic* topic in _analyticsTopics) {
1618         [topic updateUploadDateForClients:topic.topicClients date:date clearData:NO];
1619     }
1620     reply(YES, nil);
1621 }
1622
1623 - (void)clientStatus:(void (^)(NSDictionary<NSString *, id> *, NSError *))reply
1624 {
1625     NSMutableDictionary *info = [NSMutableDictionary dictionary];
1626     for (SFAnalyticsTopic* topic in _analyticsTopics) {
1627         for (SFAnalyticsClient *client in topic.topicClients) {
1628             SFAnalyticsSQLiteStore* store = [SFAnalyticsSQLiteStore storeWithPath:client.storePath schema:SFAnalyticsTableSchema];
1629
1630             NSMutableDictionary *clientInfo = [NSMutableDictionary dictionary];
1631             clientInfo[@"uploadDate"] = store.uploadDate;
1632             info[client.name] = clientInfo;
1633         }
1634     }
1635
1636     reply(info, nil);
1637 }
1638
1639 - (NSString*)stringForEventClass:(SFAnalyticsEventClass)eventClass
1640 {
1641     if (eventClass == SFAnalyticsEventClassNote) {
1642         return @"EventNote";
1643     }
1644     else if (eventClass == SFAnalyticsEventClassSuccess) {
1645         return @"EventSuccess";
1646     }
1647     else if (eventClass == SFAnalyticsEventClassHardFailure) {
1648         return @"EventHardFailure";
1649     }
1650     else if (eventClass == SFAnalyticsEventClassSoftFailure) {
1651         return @"EventSoftFailure";
1652     }
1653     else {
1654         return @"EventUnknown";
1655     }
1656 }
1657
1658 // MARK: XPC Procotol Handlers
1659
1660 - (void)getSysdiagnoseDumpWithReply:(void (^)(NSString*))reply {
1661     reply([self getSysdiagnoseDump]);
1662 }
1663
1664 - (void)createLoggingJSON:(bool)pretty topic:(NSString *)topicName reply:(void (^)(NSData *, NSError*))reply {
1665     secnotice("rpcCreateLoggingJSON", "Building a JSON blob resembling the one we would have uploaded");
1666     NSError* error = nil;
1667     [self sendNotificationForOncePerReportSamplers];
1668     NSDictionary *eventDictionary = nil;
1669     for (SFAnalyticsTopic* topic in self->_analyticsTopics) {
1670         if ([topic.internalTopicName isEqualToString:topicName]) {
1671             eventDictionary = [topic createLoggingJSON:pretty forUpload:NO participatingClients:nil force:!runningTests error:&error];
1672         }
1673     }
1674
1675     NSData *data = nil;
1676     if (!eventDictionary) {
1677         secerror("Unable to obtain JSON: %@", error);
1678     } else {
1679         data = [NSJSONSerialization dataWithJSONObject:eventDictionary
1680                                                options:(pretty ? NSJSONWritingPrettyPrinted : 0)
1681                                                  error:&error];
1682     }
1683
1684     reply(data, error);
1685 }
1686
1687 - (void)createChunkedLoggingJSON:(bool)pretty topic:(NSString *)topicName reply:(void (^)(NSData *, NSError*))reply
1688 {
1689     secnotice("rpcCreateChunkedLoggingJSON", "Building an array of JSON blobs resembling the one we would have uploaded");
1690     NSError* error = nil;
1691     [self sendNotificationForOncePerReportSamplers];
1692     NSArray<NSDictionary *> *events = nil;
1693     for (SFAnalyticsTopic* topic in self->_analyticsTopics) {
1694         if ([topic.internalTopicName isEqualToString:topicName]) {
1695             events = [topic createChunkedLoggingJSON:pretty forUpload:NO participatingClients:nil force:!runningTests error:&error];
1696         }
1697     }
1698
1699     NSData *data = nil;
1700     if (!events) {
1701         secerror("Unable to obtain JSON: %@", error);
1702     } else {
1703         data = [NSJSONSerialization dataWithJSONObject:events
1704                                                options:(pretty ? NSJSONWritingPrettyPrinted : 0)
1705                                                  error:&error];
1706     }
1707
1708     reply(data, error);
1709 }
1710
1711 - (void)forceUploadWithReply:(void (^)(BOOL, NSError*))reply {
1712     secnotice("upload", "Performing upload in response to rpc message");
1713     NSError* error = nil;
1714     BOOL result = [self uploadAnalyticsWithError:&error force:YES];
1715     secnotice("upload", "Result of manually triggered upload: %@, error: %@", result ? @"success" : @"failure", error);
1716     reply(result, error);
1717 }
1718
1719 @end
1720
1721 #endif // !TARGET_OS_SIMULATOR