supd/supd.m

   1 /*
   2  * Copyright (c) 2017-2018 Apple Inc. All Rights Reserved.
   3  *
   4  * @APPLE_LICENSE_HEADER_START@
   5  *
   6  * This file contains Original Code and/or Modifications of Original Code
   7  * as defined in and that are subject to the Apple Public Source License
   8  * Version 2.0 (the 'License'). You may not use this file except in
   9  * compliance with the License. Please obtain a copy of the License at
  10  * http://www.opensource.apple.com/apsl/ and read it before using this
  11  * file.
  12  *
  13  * The Original Code and all software distributed under the License are
  14  * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  15  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  16  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
  17  * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
  18  * Please see the License for the specific language governing rights and
  19  * limitations under the License.
  20  *
  21  * @APPLE_LICENSE_HEADER_END@
  22  */
  23
  24 #import "supd.h"
  25 #import "SFAnalyticsDefines.h"
  26 #import "SFAnalyticsSQLiteStore.h"
  27 #import "SFAnalytics.h"
  28
  29 #include <utilities/SecFileLocations.h>
  30 #import "utilities/debugging.h"
  31 #import <os/variant_private.h>
  32 #import <xpc/xpc.h>
  33 #include <notify.h>
  34 #import "keychain/ckks/CKKSControl.h"
  35 #import <zlib.h>
  36
  37 #import <AuthKit/AKAppleIDAuthenticationContext.h>
  38 #import <AuthKit/AKAppleIDAuthenticationController.h>
  39 #import <AuthKit/AKAppleIDAuthenticationController_Private.h>
  40
  41 #if TARGET_OS_OSX
  42 #include "dirhelper_priv.h"
  43 #import <Accounts/Accounts.h>
  44 #import <Accounts/ACAccountStore_Private.h>
  45 #import <Accounts/ACAccountType_Private.h>
  46 #import <Accounts/ACAccountStore.h>
  47 #import <AOSAccounts/ACAccountStore+iCloudAccount.h>
  48 #import <AOSAccounts/ACAccount+iCloudAccount.h>
  49 #import <AOSAccountsLite/AOSAccountsLite.h>
  50 #import <CrashReporterSupport/CrashReporterSupportPrivate.h>
  51 #else // TARGET_OS_OSX
  52 #import <Accounts/Accounts.h>
  53 #import <AppleAccount/AppleAccount.h>
  54 #import <AppleAccount/ACAccount+AppleAccount.h>
  55 #import <AppleAccount/ACAccountStore+AppleAccount.h>
  56 #if TARGET_OS_EMBEDDED
  57 #import <CrashReporterSupport/CrashReporterSupport.h>
  58 #import <CrashReporterSupport/PreferenceManager.h>
  59 #endif // TARGET_OS_EMBEDDED
  60 #endif // TARGET_OS_OSX
  61
  62 NSString* const SFAnalyticsSplunkTopic = @"topic";
  63 NSString* const SFAnalyticsSplunkPostTime = @"postTime";
  64 NSString* const SFAnalyticsClientId = @"clientId";
  65 NSString* const SFAnalyticsInternal = @"internal";
  66
  67 NSString* const SFAnalyticsMetricsBase = @"metricsBase";
  68 NSString* const SFAnalyticsDeviceID = @"ckdeviceID";
  69
  70 NSString* const SFAnalyticsSecondsCustomerKey = @"SecondsBetweenUploadsCustomer";
  71 NSString* const SFAnalyticsSecondsInternalKey = @"SecondsBetweenUploadsInternal";
  72 NSString* const SFAnalyticsMaxEventsKey = @"NumberOfEvents";
  73 NSString* const SFAnalyticsDevicePercentageCustomerKey = @"DevicePercentageCustomer";
  74 NSString* const SFAnalyticsDevicePercentageInternalKey = @"DevicePercentageInternal";
  75
  76 #define SFANALYTICS_SPLUNK_DEV 0
  77 #define OS_CRASH_TRACER_LOG_BUG_TYPE "226"
  78
  79 #if SFANALYTICS_SPLUNK_DEV
  80 NSUInteger const secondsBetweenUploadsCustomer = 10;
  81 NSUInteger const secondsBetweenUploadsInternal = 10;
  82 #else // SFANALYTICS_SPLUNK_DEV
  83 NSUInteger const secondsBetweenUploadsCustomer = (3 * (60 * 60 * 24));
  84 NSUInteger const secondsBetweenUploadsInternal = (60 * 60 * 24);
  85 #endif // SFANALYTICS_SPLUNK_DEV
  86
  87 @implementation SFAnalyticsReporter
  88 - (BOOL)saveReport:(NSData *)reportData fileName:(NSString *)fileName
  89 {
  90 #if TARGET_OS_OSX
  91     NSDictionary *optionsDictionary = @{ (__bridge NSString *)kCRProblemReportSubmissionPolicyKey: (__bridge NSString *)kCRSubmissionPolicyAlternate };
  92 #else // !TARGET_OS_OSX
  93     NSDictionary *optionsDictionary = nil; // The keys above are not defined or required on iOS.
  94 #endif // !TARGET_OS_OSX
  95
  96     BOOL writtenToLog = NO;
  97 #if !TARGET_OS_SIMULATOR
  98     secdebug("saveReport", "calling out to `OSAWriteLogForSubmission`");
  99     writtenToLog = OSAWriteLogForSubmission(@OS_CRASH_TRACER_LOG_BUG_TYPE, fileName,
 100                                             nil, optionsDictionary, ^(NSFileHandle *fileHandle) {
 101                                                 secnotice("OSAWriteLogForSubmission", "Writing log data to report: %@", fileName);
 102                                                 [fileHandle writeData:reportData];
 103                                             });
 104 #endif // !TARGET_OS_SIMULATOR
 105     return writtenToLog;
 106 }
 107 @end
 108
 109 #define DEFAULT_SPLUNK_MAX_EVENTS_TO_REPORT 1000
 110 #define DEFAULT_SPLUNK_DEVICE_PERCENTAGE 100
 111
 112 static supd *_supdInstance = nil;
 113
 114 BOOL deviceAnalyticsOverride = NO;
 115 BOOL deviceAnalyticsEnabled = NO;
 116 BOOL iCloudAnalyticsOverride = NO;
 117 BOOL iCloudAnalyticsEnabled = NO;
 118
 119 static BOOL
 120 _isDeviceAnalyticsEnabled(void)
 121 {
 122     // This flag is only set during tests.
 123     if (deviceAnalyticsOverride) {
 124         return deviceAnalyticsEnabled;
 125     }
 126
 127     static BOOL dataCollectionEnabled = NO;
 128     static dispatch_once_t onceToken;
 129     dispatch_once(&onceToken, ^{
 130 #if TARGET_OS_EMBEDDED
 131         dataCollectionEnabled = DiagnosticLogSubmissionEnabled();
 132 #elif TARGET_OS_OSX
 133         dataCollectionEnabled = CRIsAutoSubmitEnabled();
 134 #endif
 135     });
 136     return dataCollectionEnabled;
 137 }
 138
 139 static NSString *const kAnalyticsiCloudIdMSKey = @"com.apple.idms.config.privacy.icloud.data";
 140
 141 #if TARGET_OS_IPHONE
 142 static NSDictionary *
 143 _getiCloudConfigurationInfoWithError(NSError **outError)
 144 {
 145     __block NSDictionary *outConfigurationInfo = nil;
 146     __block NSError *localError = nil;
 147
 148     ACAccountStore *accountStore = [[ACAccountStore alloc] init];
 149     ACAccount *primaryAccount = [accountStore aa_primaryAppleAccount];
 150     if (primaryAccount != nil) {
 151         NSString *altDSID = [primaryAccount aa_altDSID];
 152         secnotice("_getiCloudConfigurationInfoWithError", "Fetching configuration info");
 153
 154         dispatch_semaphore_t sema = dispatch_semaphore_create(0);
 155         AKAppleIDAuthenticationController *authController = [AKAppleIDAuthenticationController new];
 156         [authController configurationInfoWithIdentifiers:@[kAnalyticsiCloudIdMSKey]
 157                                               forAltDSID:altDSID
 158                                               completion:^(NSDictionary<NSString *, id<NSSecureCoding>> *configurationInfo, NSError *error) {
 159             if (error) {
 160                 secerror("_getiCloudConfigurationInfoWithError: Error fetching configurationInfo: %@", error);
 161                 localError = error;
 162             } else if (![configurationInfo isKindOfClass:[NSDictionary class]]) {
 163                 secerror("_getiCloudConfigurationInfoWithError: configurationInfo dict was not a dict, it was a %{public}@", [configurationInfo class]);
 164                 localError = error;
 165                 configurationInfo = nil;
 166             } else {
 167                 secnotice("_getiCloudConfigurationInfoWithError", "fetched configurationInfo %@", configurationInfo);
 168                 outConfigurationInfo = configurationInfo;
 169             }
 170             dispatch_semaphore_signal(sema);
 171         }];
 172         dispatch_semaphore_wait(sema, dispatch_time(DISPATCH_TIME_NOW, (uint64_t)(5 * NSEC_PER_SEC)));
 173     } else {
 174         secerror("_getiCloudConfigurationInfoWithError: Failed to fetch primary account info.");
 175     }
 176
 177     if (localError && outError) {
 178         *outError = localError;
 179     }
 180     return outConfigurationInfo;
 181 }
 182 #endif // TARGET_OS_IPHONE
 183
 184 #if TARGET_OS_OSX
 185 static NSString *
 186 _iCloudAccount(void)
 187 {
 188     return CFBridgingRelease(MMLCopyLoggedInAccount());
 189 }
 190
 191 static NSString *
 192 _altDSIDFromAccount(void)
 193 {
 194     static CFStringRef kMMPropertyAccountAlternateDSID = CFSTR("AccountAlternateDSID");
 195     NSString *account = _iCloudAccount();
 196     if (account != nil) {
 197         return CFBridgingRelease(MMLAccountCopyProperty((__bridge CFStringRef)account, kMMPropertyAccountAlternateDSID));
 198     }
 199     secerror("_altDSIDFromAccount: failed to fetch iCloud account");
 200     return nil;
 201 }
 202 #endif // TARGET_OS_OSX
 203
 204 static BOOL
 205 _isiCloudAnalyticsEnabled()
 206 {
 207     // This flag is only set during tests.
 208     if (iCloudAnalyticsOverride) {
 209         return iCloudAnalyticsEnabled;
 210     }
 211
 212     static bool cachedAllowsICloudAnalytics = false;
 213
 214 #if TARGET_OS_OSX
 215     static dispatch_once_t onceToken;
 216     dispatch_once(&onceToken, ^{
 217         /* AOSAccounts is not mastered into the BaseSystem. Check that those classes are linked at runtime and abort if not. */
 218         if (![AKAppleIDAuthenticationController class]) {
 219             secnotice("OTATrust", "Weak-linked AOSAccounts framework missing. Are we running in the base system?");
 220             return;
 221         }
 222
 223         NSString *currentAltDSID = _altDSIDFromAccount();
 224         if (currentAltDSID != nil) {
 225             AKAppleIDAuthenticationController *authController = [AKAppleIDAuthenticationController new];
 226             __block bool allowsICloudAnalytics = false;
 227             dispatch_semaphore_t sem = dispatch_semaphore_create(0);
 228             secnotice("isiCloudAnalyticsEnabled", "fetching iCloud Analytics value from idms");
 229             [authController configurationInfoWithIdentifiers:@[kAnalyticsiCloudIdMSKey]
 230                                                   forAltDSID:currentAltDSID
 231                                                   completion:^(NSDictionary<NSString *, id> *configurationInfo, NSError *error) {
 232                                                       if (!error && configurationInfo) {
 233                                                           NSNumber *value = configurationInfo[kAnalyticsiCloudIdMSKey];
 234                                                           if (value != nil) {
 235                                                               secnotice("_isiCloudAnalyticsEnabled", "authController:configurationInfoWithIdentifiers completed with no error and configuration information");
 236                                                               allowsICloudAnalytics = [value boolValue];
 237                                                           } else {
 238                                                               secerror("%s: no iCloud Analytics value found in IDMS", __FUNCTION__);
 239                                                           }
 240                                                       } else {
 241                                                           secerror("%s: Unable to fetch iCloud Analytics value from IDMS.", __FUNCTION__);
 242                                                       }
 243                                                       secnotice("_isiCloudAnalyticsEnabled", "authController:configurationInfoWithIdentifiers completed and returning");
 244                                                       dispatch_semaphore_signal(sem);
 245                                                   }];
 246             // Wait 5 seconds before giving up and returning from the block.
 247             dispatch_semaphore_wait(sem, dispatch_time(DISPATCH_TIME_NOW, (uint64_t)(5 * NSEC_PER_SEC)));
 248             cachedAllowsICloudAnalytics = allowsICloudAnalytics;
 249         } else {
 250             secerror("_isiCloudAnalyticsEnabled: Failed to fetch alternate DSID");
 251         }
 252     });
 253 #else // TARGET_OS_OSX
 254     static dispatch_once_t onceToken;
 255     dispatch_once(&onceToken, ^{
 256         NSError *error = nil;
 257         NSDictionary *accountConfiguration = _getiCloudConfigurationInfoWithError(&error);
 258         if (error == nil && accountConfiguration != nil) {
 259             id iCloudAnalyticsOptIn = accountConfiguration[kAnalyticsiCloudIdMSKey];
 260             if (iCloudAnalyticsOptIn != nil) {
 261                 BOOL iCloudAnalyticsOptInHasCorrectType = ([iCloudAnalyticsOptIn isKindOfClass:[NSNumber class]] || [iCloudAnalyticsOptIn isKindOfClass:[NSString class]]);
 262                 if (iCloudAnalyticsOptInHasCorrectType) {
 263                     NSNumber *iCloudAnalyticsOptInNumber = @([iCloudAnalyticsOptIn integerValue]);
 264                     cachedAllowsICloudAnalytics = ![iCloudAnalyticsOptInNumber isEqualToNumber:[NSNumber numberWithInteger:0]];
 265                 }
 266             }
 267         } else if (error != nil) {
 268             secerror("_isiCloudAnalyticsEnabled: %@", error);
 269         }
 270     });
 271 #endif // TARGET_OS_OSX
 272
 273     return cachedAllowsICloudAnalytics;
 274 }
 275
 276 /*  NSData GZip category based on GeoKit's implementation */
 277 @interface NSData (GZip)
 278 - (NSData *)supd_gzipDeflate;
 279 @end
 280
 281 #define GZIP_OFFSET 16
 282 #define GZIP_STRIDE_LEN 16384
 283
 284 @implementation NSData (Gzip)
 285 - (NSData *)supd_gzipDeflate
 286 {
 287     if ([self length] == 0) {
 288         return self;
 289     }
 290
 291     z_stream strm;
 292     memset(&strm, 0, sizeof(strm));
 293     strm.next_in=(uint8_t *)[self bytes];
 294     strm.avail_in = (unsigned int)[self length];
 295
 296
 297     if (Z_OK != deflateInit2(&strm, Z_BEST_COMPRESSION, Z_DEFLATED,
 298                              MAX_WBITS + GZIP_OFFSET, MAX_MEM_LEVEL, Z_DEFAULT_STRATEGY)) {
 299         return nil;
 300     }
 301
 302     NSMutableData *compressed = [NSMutableData dataWithLength:GZIP_STRIDE_LEN];
 303
 304     do {
 305         if (strm.total_out >= [compressed length]) {
 306             [compressed increaseLengthBy: 16384];
 307         }
 308
 309         strm.next_out = [compressed mutableBytes] + strm.total_out;
 310         strm.avail_out = (int)[compressed length] - (int)strm.total_out;
 311
 312         deflate(&strm, Z_FINISH);
 313
 314     } while (strm.avail_out == 0);
 315
 316     deflateEnd(&strm);
 317
 318     [compressed setLength: strm.total_out];
 319     if (strm.avail_in == 0) {
 320         return [NSData dataWithData:compressed];
 321     } else {
 322         return nil;
 323     }
 324 }
 325 @end
 326
 327 @implementation SFAnalyticsClient {
 328     NSString* _path;
 329     NSString* _name;
 330     BOOL _requireDeviceAnalytics;
 331     BOOL _requireiCloudAnalytics;
 332 }
 333
 334 @synthesize storePath = _path;
 335 @synthesize name = _name;
 336
 337 - (instancetype)initWithStorePath:(NSString*)path name:(NSString*)name
 338                   deviceAnalytics:(BOOL)deviceAnalytics iCloudAnalytics:(BOOL)iCloudAnalytics {
 339     if (self = [super init]) {
 340         _path = path;
 341         _name = name;
 342         _requireDeviceAnalytics = deviceAnalytics;
 343         _requireiCloudAnalytics = iCloudAnalytics;
 344     }
 345     return self;
 346 }
 347
 348 @end
 349
 350 @interface SFAnalyticsTopic ()
 351 @property NSURL* _splunkUploadURL;
 352
 353 @property BOOL allowInsecureSplunkCert;
 354 @property BOOL ignoreServersMessagesTellingUsToGoAway;
 355 @property BOOL disableUploads;
 356 @property BOOL disableClientId;
 357
 358 @property NSUInteger secondsBetweenUploads;
 359 @property NSUInteger maxEventsToReport;
 360 @property float devicePercentage; // for sampling reporting devices
 361
 362 @property NSDictionary* metricsBase; // data the server provides and wants us to send back
 363 @property NSArray* blacklistedFields;
 364 @property NSArray* blacklistedEvents;
 365 @end
 366
 367 @implementation SFAnalyticsTopic
 368 - (void)setupClientsForTopic:(NSString *)topicName
 369 {
 370     NSMutableArray<SFAnalyticsClient*>* clients = [NSMutableArray<SFAnalyticsClient*> new];
 371     if ([topicName isEqualToString:SFAnalyticsTopicKeySync]) {
 372         [clients addObject:[[SFAnalyticsClient alloc] initWithStorePath:[self.class databasePathForCKKS]
 373                                                                    name:@"ckks" deviceAnalytics:NO iCloudAnalytics:YES]];
 374         [clients addObject:[[SFAnalyticsClient alloc] initWithStorePath:[self.class databasePathForSOS]
 375                                                                    name:@"sos" deviceAnalytics:NO iCloudAnalytics:YES]];
 376         [clients addObject:[[SFAnalyticsClient alloc] initWithStorePath:[self.class databasePathForPCS]
 377                                                                    name:@"pcs" deviceAnalytics:NO iCloudAnalytics:YES]];
 378     } else if ([topicName isEqualToString:SFAnaltyicsTopicTrust]) {
 379 #if TARGET_OS_OSX
 380         _set_user_dir_suffix("com.apple.trustd"); // supd needs to read trustd's cache dir for these
 381 #endif
 382         [clients addObject:[[SFAnalyticsClient alloc] initWithStorePath:[self.class databasePathForTrust]
 383                                                                    name:@"trust" deviceAnalytics:YES iCloudAnalytics:NO]];
 384         [clients addObject:[[SFAnalyticsClient alloc] initWithStorePath:[self.class databasePathForTrustdHealth]
 385                                                                    name:@"trustdHealth" deviceAnalytics:YES iCloudAnalytics:NO]];
 386         [clients addObject:[[SFAnalyticsClient alloc] initWithStorePath:[self.class databasePathForTLS]
 387                                                                    name:@"tls" deviceAnalytics:YES iCloudAnalytics:NO]];
 388 #if TARGET_OS_OSX
 389         _set_user_dir_suffix(NULL); // set back to the default cache dir
 390 #endif
 391     }
 392
 393     _topicClients = clients;
 394 }
 395
 396 - (instancetype)initWithDictionary:(NSDictionary *)dictionary name:(NSString *)topicName samplingRates:(NSDictionary *)rates {
 397     if (self = [super init]) {
 398         _internalTopicName = topicName;
 399         [self setupClientsForTopic:topicName];
 400         _splunkTopicName = dictionary[@"splunk_topic"];
 401         __splunkUploadURL = [NSURL URLWithString:dictionary[@"splunk_uploadURL"]];
 402         _splunkBagURL = [NSURL URLWithString:dictionary[@"splunk_bagURL"]];
 403         _allowInsecureSplunkCert = [[dictionary valueForKey:@"splunk_allowInsecureCertificate"] boolValue];
 404         NSString* splunkEndpoint = dictionary[@"splunk_endpointDomain"];
 405         if (dictionary[@"disableClientId"]) {
 406             _disableClientId = YES;
 407         }
 408
 409         NSUserDefaults* defaults = [[NSUserDefaults alloc] initWithSuiteName:SFAnalyticsUserDefaultsSuite];
 410         NSString* userDefaultsSplunkTopic = [defaults stringForKey:@"splunk_topic"];
 411         if (userDefaultsSplunkTopic) {
 412             _splunkTopicName = userDefaultsSplunkTopic;
 413         }
 414
 415         NSURL* userDefaultsSplunkUploadURL = [NSURL URLWithString:[defaults stringForKey:@"splunk_uploadURL"]];
 416         if (userDefaultsSplunkUploadURL) {
 417             __splunkUploadURL = userDefaultsSplunkUploadURL;
 418         }
 419
 420         NSURL* userDefaultsSplunkBagURL = [NSURL URLWithString:[defaults stringForKey:@"splunk_bagURL"]];
 421         if (userDefaultsSplunkBagURL) {
 422             _splunkBagURL = userDefaultsSplunkBagURL;
 423         }
 424
 425         BOOL userDefaultsAllowInsecureSplunkCert = [defaults boolForKey:@"splunk_allowInsecureCertificate"];
 426         _allowInsecureSplunkCert |= userDefaultsAllowInsecureSplunkCert;
 427
 428         NSString* userDefaultsSplunkEndpoint = [defaults stringForKey:@"splunk_endpointDomain"];
 429         if (userDefaultsSplunkEndpoint) {
 430             splunkEndpoint = userDefaultsSplunkEndpoint;
 431         }
 432
 433 #if SFANALYTICS_SPLUNK_DEV
 434         _secondsBetweenUploads = secondsBetweenUploadsInternal;
 435         _maxEventsToReport = SFAnalyticsMaxEventsToReport;
 436         _devicePercentage = DEFAULT_SPLUNK_DEVICE_PERCENTAGE;
 437 #else
 438         bool internal = os_variant_has_internal_diagnostics("com.apple.security");
 439         if (rates) {
 440             NSNumber *secondsNum = internal ? rates[SFAnalyticsSecondsInternalKey] : rates[SFAnalyticsSecondsCustomerKey];
 441             _secondsBetweenUploads = [secondsNum integerValue];
 442             _maxEventsToReport = [rates[SFAnalyticsMaxEventsKey] unsignedIntegerValue];
 443             NSNumber *percentageNum = internal ? rates[SFAnalyticsDevicePercentageInternalKey] : rates[SFAnalyticsDevicePercentageCustomerKey];
 444             _devicePercentage = [percentageNum floatValue];
 445         } else {
 446             _secondsBetweenUploads = internal ? secondsBetweenUploadsInternal : secondsBetweenUploadsCustomer;
 447             _maxEventsToReport = SFAnalyticsMaxEventsToReport;
 448             _devicePercentage = DEFAULT_SPLUNK_DEVICE_PERCENTAGE;
 449         }
 450 #endif
 451         secnotice("supd", "created %@ with %lu seconds between uploads, %lu max events, %f percent of uploads",
 452                   _internalTopicName, (unsigned long)_secondsBetweenUploads, (unsigned long)_maxEventsToReport, _devicePercentage);
 453
 454 #if SFANALYTICS_SPLUNK_DEV
 455         _ignoreServersMessagesTellingUsToGoAway = YES;
 456
 457         if (!_splunkUploadURL && splunkEndpoint) {
 458             NSString* urlString = [NSString stringWithFormat:@"https://%@/report/2/%@", splunkEndpoint, _splunkTopicName];
 459             _splunkUploadURL = [NSURL URLWithString:urlString];
 460         }
 461 #else
 462         (void)splunkEndpoint;
 463 #endif
 464     }
 465     return self;
 466 }
 467
 468 - (BOOL)isSampledUpload
 469 {
 470     uint32_t sample = arc4random();
 471     if ((double)_devicePercentage < ((double)1 / UINT32_MAX) * 100) {
 472         /* Requested percentage is smaller than we can sample. just do 1 out of UINT32_MAX */
 473         if (sample == 0) {
 474             return YES;
 475         }
 476     } else {
 477         if ((double)sample <= (double)UINT32_MAX * ((double)_devicePercentage / 100)) {
 478             return YES;
 479         }
 480     }
 481     return NO;
 482 }
 483
 484 - (BOOL)postJSON:(NSData*)json toEndpoint:(NSURL*)endpoint error:(NSError**)error
 485 {
 486     if (!endpoint) {
 487         if (error) {
 488             NSString *description = [NSString stringWithFormat:@"No endpoint for %@", _internalTopicName];
 489             *error = [NSError errorWithDomain:@"SupdUploadErrorDomain"
 490                                          code:-10
 491                                      userInfo:@{NSLocalizedDescriptionKey : description}];
 492         }
 493         return false;
 494     }
 495     /*
 496      * Create the NSURLSession
 497      *  We use the ephemeral session config because we don't need cookies or cache
 498      */
 499     NSURLSessionConfiguration *configuration = [NSURLSessionConfiguration ephemeralSessionConfiguration];
 500
 501     configuration.HTTPAdditionalHeaders = @{ @"User-Agent" : [NSString stringWithFormat:@"securityd/%s", SECURITY_BUILD_VERSION]};
 502
 503     NSURLSession* postSession = [NSURLSession sessionWithConfiguration:configuration
 504                                                               delegate:self
 505                                                          delegateQueue:nil];
 506
 507     NSMutableURLRequest* postRequest = [[NSMutableURLRequest alloc] init];
 508     postRequest.URL = endpoint;
 509     postRequest.HTTPMethod = @"POST";
 510     postRequest.HTTPBody = [json supd_gzipDeflate];
 511     [postRequest setValue:@"gzip" forHTTPHeaderField:@"Content-Encoding"];
 512
 513     /*
 514      * Create the upload task.
 515      */
 516     dispatch_semaphore_t sem = dispatch_semaphore_create(0);
 517     __block BOOL uploadSuccess = NO;
 518     NSURLSessionDataTask* uploadTask = [postSession dataTaskWithRequest:postRequest
 519                                                       completionHandler:^(NSData * _Nullable __unused data, NSURLResponse * _Nullable response, NSError * _Nullable requestError) {
 520         if (requestError) {
 521             secerror("Error in uploading the events to splunk for %@: %@", self->_internalTopicName, requestError);
 522         } else if (![response isKindOfClass:NSHTTPURLResponse.class]){
 523             Class class = response.class;
 524             secerror("Received the wrong kind of response for %@: %@", self->_internalTopicName, NSStringFromClass(class));
 525         } else {
 526             NSHTTPURLResponse* httpResponse = (NSHTTPURLResponse*)response;
 527             if(httpResponse.statusCode >= 200 && httpResponse.statusCode < 300) {
 528                 /* Success */
 529                 uploadSuccess = YES;
 530                 secnotice("upload", "Splunk upload success for %@", self->_internalTopicName);
 531             } else {
 532                 secnotice("upload", "Splunk upload for %@ unexpected status to URL: %@ -- status: %d",
 533                           self->_internalTopicName, endpoint, (int)(httpResponse.statusCode));
 534             }
 535         }
 536         dispatch_semaphore_signal(sem);
 537     }];
 538     secnotice("upload", "Splunk upload start for %@", self->_internalTopicName);
 539     [uploadTask resume];
 540     dispatch_semaphore_wait(sem, dispatch_time(DISPATCH_TIME_NOW, (uint64_t)(5 * 60 * NSEC_PER_SEC)));
 541     return uploadSuccess;
 542 }
 543
 544 - (BOOL)eventIsBlacklisted:(NSMutableDictionary*)event {
 545     return _blacklistedEvents ? [_blacklistedEvents containsObject:event[SFAnalyticsEventType]] : NO;
 546 }
 547
 548 - (void)removeBlacklistedFieldsFromEvent:(NSMutableDictionary*)event {
 549     for (NSString* badField in self->_blacklistedFields) {
 550         [event removeObjectForKey:badField];
 551     }
 552 }
 553
 554 - (void)addRequiredFieldsToEvent:(NSMutableDictionary*)event {
 555     [_metricsBase enumerateKeysAndObjectsUsingBlock:^(id  _Nonnull key, id  _Nonnull obj, BOOL * _Nonnull stop) {
 556         if (!event[key]) {
 557             event[key] = obj;
 558         }
 559     }];
 560 }
 561
 562 - (BOOL)prepareEventForUpload:(NSMutableDictionary*)event {
 563     if ([self eventIsBlacklisted:event]) {
 564         return NO;
 565     }
 566
 567     [self removeBlacklistedFieldsFromEvent:event];
 568     [self addRequiredFieldsToEvent:event];
 569     if (_disableClientId) {
 570         event[SFAnalyticsClientId] = @(0);
 571     }
 572     event[SFAnalyticsSplunkTopic] = self->_splunkTopicName ?: [NSNull null];
 573     return YES;
 574 }
 575
 576 - (void)addFailures:(NSMutableArray<NSArray*>*)failures toUploadRecords:(NSMutableArray*)records threshold:(NSUInteger)threshold
 577 {
 578     // The first 0 through 'threshold' items are getting uploaded in any case (which might be 0 for lower priority data)
 579
 580     for (NSArray* client in failures) {
 581         [client enumerateObjectsUsingBlock:^(id  _Nonnull obj, NSUInteger idx, BOOL * _Nonnull stop) {
 582             if (idx >= threshold) {
 583                 *stop = YES;
 584                 return;
 585             }
 586             if ([self prepareEventForUpload:obj]) {
 587                 [records addObject:obj];
 588             }
 589         }];
 590     }
 591
 592     // Are there more items than we shoved into the upload records?
 593     NSInteger excessItems = 0;
 594     for (NSArray* client in failures) {
 595         NSInteger localExcess = client.count - threshold;
 596         excessItems += localExcess > 0 ? localExcess : 0;
 597     }
 598
 599     // Then, if we have space and items left, apply a scaling factor to distribute events across clients to fill upload buffer
 600     if (records.count < _maxEventsToReport && excessItems > 0) {
 601         double scale = (_maxEventsToReport - records.count) / (double)excessItems;
 602         if (scale > 1) {
 603             scale = 1;
 604         }
 605
 606         for (NSArray* client in failures) {
 607             if (client.count > threshold) {
 608                 NSRange range = NSMakeRange(threshold, (client.count - threshold) * scale);
 609                 NSArray* sub = [client subarrayWithRange:range];
 610                 [sub enumerateObjectsUsingBlock:^(id  _Nonnull obj, NSUInteger idx, BOOL * _Nonnull stop) {
 611                     if ([self prepareEventForUpload:obj]) {
 612                         [records addObject:obj];
 613                     }
 614                 }];
 615             }
 616         }
 617     }
 618 }
 619
 620 - (NSMutableDictionary*)sampleStatisticsForSamples:(NSArray*)samples withName:(NSString*)name
 621 {
 622     NSMutableDictionary* statistics = [NSMutableDictionary dictionary];
 623     NSUInteger count = samples.count;
 624     NSArray* sortedSamples = [samples sortedArrayUsingSelector:@selector(compare:)];
 625     NSArray* samplesAsExpressionArray = @[[NSExpression expressionForConstantValue:sortedSamples]];
 626
 627     if (count == 1) {
 628         statistics[name] = samples[0];
 629     } else {
 630         // NSExpression takes population standard deviation. Our data is a sample of whatever we sampled over time,
 631         // but the difference between the two is fairly minor (divide by N before taking sqrt versus divide by N-1).
 632         statistics[[NSString stringWithFormat:@"%@-dev", name]] = [[NSExpression expressionForFunction:@"stddev:" arguments:samplesAsExpressionArray] expressionValueWithObject:nil context:nil];
 633
 634         statistics[[NSString stringWithFormat:@"%@-min", name]] = [[NSExpression expressionForFunction:@"min:" arguments:samplesAsExpressionArray] expressionValueWithObject:nil context:nil];
 635         statistics[[NSString stringWithFormat:@"%@-max", name]] = [[NSExpression expressionForFunction:@"max:" arguments:samplesAsExpressionArray] expressionValueWithObject:nil context:nil];
 636         statistics[[NSString stringWithFormat:@"%@-avg", name]] = [[NSExpression expressionForFunction:@"average:" arguments:samplesAsExpressionArray] expressionValueWithObject:nil context:nil];
 637         statistics[[NSString stringWithFormat:@"%@-med", name]] = [[NSExpression expressionForFunction:@"median:" arguments:samplesAsExpressionArray] expressionValueWithObject:nil context:nil];
 638     }
 639
 640     if (count > 3) {
 641         NSString* q1 = [NSString stringWithFormat:@"%@-1q", name];
 642         NSString* q3 = [NSString stringWithFormat:@"%@-3q", name];
 643         // From Wikipedia, which is never wrong
 644         if (count % 2 == 0) {
 645             // The lower quartile value is the median of the lower half of the data. The upper quartile value is the median of the upper half of the data.
 646             statistics[q1] = [[NSExpression expressionForFunction:@"median:" arguments:@[[NSExpression expressionForConstantValue:[sortedSamples subarrayWithRange:NSMakeRange(0, count / 2)]]]] expressionValueWithObject:nil context:nil];
 647             statistics[q3] = [[NSExpression expressionForFunction:@"median:" arguments:@[[NSExpression expressionForConstantValue:[sortedSamples subarrayWithRange:NSMakeRange((count / 2), count / 2)]]]] expressionValueWithObject:nil context:nil];
 648         } else if (count % 4 == 1) {
 649             // If there are (4n+1) data points, then the lower quartile is 25% of the nth data value plus 75% of the (n+1)th data value;
 650             // the upper quartile is 75% of the (3n+1)th data point plus 25% of the (3n+2)th data point.
 651             // (offset n by -1 since we count from 0)
 652             NSUInteger n = count / 4;
 653             statistics[q1] = @(([sortedSamples[n - 1] doubleValue] + [sortedSamples[n] doubleValue] * 3.0) / 4.0);
 654             statistics[q3] = @(([sortedSamples[(3 * n)] doubleValue] * 3.0 + [sortedSamples[(3 * n) + 1] doubleValue]) / 4.0);
 655         } else if (count % 4 == 3){
 656             // If there are (4n+3) data points, then the lower quartile is 75% of the (n+1)th data value plus 25% of the (n+2)th data value;
 657             // the upper quartile is 25% of the (3n+2)th data point plus 75% of the (3n+3)th data point.
 658             // (offset n by -1 since we count from 0)
 659             NSUInteger n = count / 4;
 660             statistics[q1] = @(([sortedSamples[n] doubleValue] * 3.0 + [sortedSamples[n + 1] doubleValue]) / 4.0);
 661             statistics[q3] = @(([sortedSamples[(3 * n) + 1] doubleValue] + [sortedSamples[(3 * n) + 2] doubleValue] * 3.0) / 4.0);
 662         }
 663     }
 664
 665     return statistics;
 666 }
 667
 668 - (NSMutableDictionary*)healthSummaryWithName:(NSString*)name store:(SFAnalyticsSQLiteStore*)store
 669 {
 670     __block NSMutableDictionary* summary = [NSMutableDictionary new];
 671
 672     // Add some events of our own before pulling in data
 673     summary[SFAnalyticsEventType] = [NSString stringWithFormat:@"%@HealthSummary", name];
 674     if ([self eventIsBlacklisted:summary]) {
 675         return nil;
 676     }
 677     summary[SFAnalyticsEventTime] = @([[NSDate date] timeIntervalSince1970] * 1000);    // Splunk wants milliseconds
 678     [SFAnalytics addOSVersionToEvent:summary];
 679
 680     // Process counters
 681     NSDictionary* successCounts = store.summaryCounts;
 682     __block NSInteger totalSuccessCount = 0;
 683     __block NSInteger totalHardFailureCount = 0;
 684     __block NSInteger totalSoftFailureCount = 0;
 685     [successCounts enumerateKeysAndObjectsUsingBlock:^(NSString* _Nonnull eventType, NSDictionary* _Nonnull counts, BOOL* _Nonnull stop) {
 686         summary[[NSString stringWithFormat:@"%@-success", eventType]] = counts[SFAnalyticsColumnSuccessCount];
 687         summary[[NSString stringWithFormat:@"%@-hardfail", eventType]] = counts[SFAnalyticsColumnHardFailureCount];
 688         summary[[NSString stringWithFormat:@"%@-softfail", eventType]] = counts[SFAnalyticsColumnSoftFailureCount];
 689         totalSuccessCount += [counts[SFAnalyticsColumnSuccessCount] integerValue];
 690         totalHardFailureCount += [counts[SFAnalyticsColumnHardFailureCount] integerValue];
 691         totalSoftFailureCount += [counts[SFAnalyticsColumnSoftFailureCount] integerValue];
 692     }];
 693
 694     summary[SFAnalyticsColumnSuccessCount] = @(totalSuccessCount);
 695     summary[SFAnalyticsColumnHardFailureCount] = @(totalHardFailureCount);
 696     summary[SFAnalyticsColumnSoftFailureCount] = @(totalSoftFailureCount);
 697     if (os_variant_has_internal_diagnostics("com.apple.security")) {
 698         summary[SFAnalyticsInternal] = @YES;
 699     }
 700
 701     // Process samples
 702     NSMutableDictionary<NSString*,NSMutableArray*>* samplesBySampler = [NSMutableDictionary<NSString*,NSMutableArray*> dictionary];
 703     for (NSDictionary* sample in [store samples]) {
 704         if (!samplesBySampler[sample[SFAnalyticsColumnSampleName]]) {
 705             samplesBySampler[sample[SFAnalyticsColumnSampleName]] = [NSMutableArray array];
 706         }
 707         [samplesBySampler[sample[SFAnalyticsColumnSampleName]] addObject:sample[SFAnalyticsColumnSampleValue]];
 708     }
 709     [samplesBySampler enumerateKeysAndObjectsUsingBlock:^(NSString * _Nonnull key, NSMutableArray * _Nonnull obj, BOOL * _Nonnull stop) {
 710         NSMutableDictionary* event = [self sampleStatisticsForSamples:obj withName:key];
 711         [summary addEntriesFromDictionary:event];
 712     }];
 713
 714     // Should always return yes because we already checked for event blacklisting specifically
 715     if (![self prepareEventForUpload:summary]) {
 716         return nil;
 717     }
 718     return summary;
 719 }
 720
 721 - (void)updateUploadDateForClients:(NSArray<SFAnalyticsClient*>*)clients clearData:(BOOL)clearData
 722 {
 723     for (SFAnalyticsClient* client in clients) {
 724         SFAnalyticsSQLiteStore* store = [SFAnalyticsSQLiteStore storeWithPath:client.storePath schema:SFAnalyticsTableSchema];
 725         secnotice("postprocess", "Setting upload date for client: %@", client.name);
 726         store.uploadDate = [NSDate date];
 727         if (clearData) {
 728             secnotice("postprocess", "Clearing collected data for client: %@", client.name);
 729             [store clearAllData];
 730         }
 731     }
 732 }
 733
 734 - (NSData*)getLoggingJSON:(bool)pretty
 735                 forUpload:(BOOL)upload
 736      participatingClients:(NSMutableArray<SFAnalyticsClient*>**)clients
 737                     error:(NSError**)error
 738 {
 739     NSMutableArray<SFAnalyticsClient*>* localClients = [NSMutableArray new];
 740     __block NSMutableArray* uploadRecords = [NSMutableArray arrayWithCapacity:_maxEventsToReport];
 741     __block NSError *localError;
 742     __block NSMutableArray<NSArray*>* hardFailures = [NSMutableArray new];
 743     __block NSMutableArray<NSArray*>* softFailures = [NSMutableArray new];
 744     NSString* ckdeviceID = nil;
 745     if ([_internalTopicName isEqualToString:SFAnalyticsTopicKeySync]) {
 746         ckdeviceID = os_variant_has_internal_diagnostics("com.apple.security") ? [self askSecurityForCKDeviceID] : nil;
 747     }
 748     for (SFAnalyticsClient* client in self->_topicClients) {
 749         if ([client requireDeviceAnalytics] && !_isDeviceAnalyticsEnabled()) {
 750             // Client required device analytics, yet the user did not opt in.
 751             secnotice("getLoggingJSON", "Client '%@' requires device analytics yet user did not opt in.", [client name]);
 752             continue;
 753         }
 754         if ([client requireiCloudAnalytics] && !_isiCloudAnalyticsEnabled()) {
 755             // Client required iCloud analytics, yet the user did not opt in.
 756             secnotice("getLoggingJSON", "Client '%@' requires iCloud analytics yet user did not opt in.", [client name]);
 757             continue;
 758         }
 759
 760         SFAnalyticsSQLiteStore* store = [SFAnalyticsSQLiteStore storeWithPath:client.storePath schema:SFAnalyticsTableSchema];
 761
 762         if (upload) {
 763             NSDate* uploadDate = store.uploadDate;
 764             if (uploadDate && [[NSDate date] timeIntervalSinceDate:uploadDate] < _secondsBetweenUploads) {
 765                 secnotice("json", "ignoring client '%@' for %@ because last upload too recent: %@",
 766                           client.name, _internalTopicName, uploadDate);
 767                 continue;
 768             }
 769
 770             if (!uploadDate) {
 771                 secnotice("json", "ignoring client '%@' because doesn't have an upload date; giving it a baseline date",
 772                           client.name);
 773                 [self updateUploadDateForClients:@[client] clearData:NO];
 774                 continue;
 775             }
 776
 777             secnotice("json", "including client '%@' for upload", client.name);
 778             [localClients addObject:client];
 779         }
 780
 781         NSMutableDictionary* healthSummary = [self healthSummaryWithName:client.name store:store];
 782         if (healthSummary) {
 783             if (ckdeviceID) {
 784                 healthSummary[SFAnalyticsDeviceID] = ckdeviceID;
 785             }
 786             [uploadRecords addObject:healthSummary];
 787         }
 788
 789         [hardFailures addObject:store.hardFailures];
 790         [softFailures addObject:store.softFailures];
 791     }
 792
 793     if (clients) {
 794         *clients = localClients;
 795     }
 796
 797     if (upload && [localClients count] == 0) {
 798         if (error) {
 799             NSString *description = [NSString stringWithFormat:@"Upload too recent for all clients for %@", _internalTopicName];
 800             *error = [NSError errorWithDomain:@"SupdUploadErrorDomain"
 801                                          code:-10
 802                                      userInfo:@{NSLocalizedDescriptionKey : description}];
 803         }
 804         return nil;
 805     }
 806
 807     [self addFailures:hardFailures toUploadRecords:uploadRecords threshold:_maxEventsToReport/10];
 808     [self addFailures:softFailures toUploadRecords:uploadRecords threshold:0];
 809
 810     NSDictionary* jsonDict = @{
 811                                SFAnalyticsSplunkPostTime : @([[NSDate date] timeIntervalSince1970] * 1000),
 812                                @"events" : uploadRecords
 813                                };
 814
 815     NSData *json = [NSJSONSerialization dataWithJSONObject:jsonDict
 816                                                    options:(pretty ? NSJSONWritingPrettyPrinted : 0)
 817                                                      error:&localError];
 818
 819     if (error) {
 820         *error = localError;
 821     }
 822     return json;
 823 }
 824
 825 // Is at least one client eligible for data collection based on user consent? Otherwise callers should NOT reach off-device.
 826 - (BOOL)haveEligibleClients {
 827     for (SFAnalyticsClient* client in self.topicClients) {
 828         if ((!client.requireDeviceAnalytics || _isDeviceAnalyticsEnabled()) &&
 829             (!client.requireiCloudAnalytics || _isiCloudAnalyticsEnabled())) {
 830             return YES;
 831         }
 832     }
 833     return NO;
 834 }
 835
 836 - (NSString*)askSecurityForCKDeviceID
 837 {
 838     NSError* error = nil;
 839     CKKSControl* rpc = [CKKSControl controlObject:&error];
 840     if(error || !rpc) {
 841         secerror("unable to obtain CKKS endpoint: %@", error);
 842         return nil;
 843     }
 844
 845     __block NSString* localCKDeviceID;
 846     dispatch_semaphore_t sema = dispatch_semaphore_create(0);
 847     [rpc rpcGetCKDeviceIDWithReply:^(NSString* ckdeviceID) {
 848         localCKDeviceID = ckdeviceID;
 849         dispatch_semaphore_signal(sema);
 850     }];
 851
 852     if (dispatch_semaphore_wait(sema, dispatch_time(DISPATCH_TIME_NOW, NSEC_PER_SEC * 10)) != 0) {
 853         secerror("timed out waiting for a response from security");
 854         return nil;
 855     }
 856
 857     return localCKDeviceID;
 858 }
 859
 860 // this method is kind of evil for the fact that it has side-effects in pulling other things besides the metricsURL from the server, and as such should NOT be memoized.
 861 // TODO redo this, probably to return a dictionary.
 862 - (NSURL*)splunkUploadURL
 863 {
 864     if (![self haveEligibleClients]) {
 865         secnotice("getURL", "Not going to talk to server for topic %@ because no eligible clients", [self internalTopicName]);
 866         return nil;
 867     }
 868
 869     if (__splunkUploadURL) {
 870         return __splunkUploadURL;
 871     }
 872
 873     secnotice("getURL", "Asking server for endpoint and config data for topic %@", [self internalTopicName]);
 874
 875     __weak __typeof(self) weakSelf = self;
 876     dispatch_semaphore_t sem = dispatch_semaphore_create(0);
 877
 878     __block NSError* error = nil;
 879     NSURLSessionConfiguration *configuration = [NSURLSessionConfiguration ephemeralSessionConfiguration];
 880     NSURLSession* storeBagSession = [NSURLSession sessionWithConfiguration:configuration
 881                                                                   delegate:self
 882                                                              delegateQueue:nil];
 883
 884     NSURL* requestEndpoint = _splunkBagURL;
 885     __block NSURL* result = nil;
 886     NSURLSessionDataTask* storeBagTask = [storeBagSession dataTaskWithURL:requestEndpoint completionHandler:^(NSData * _Nullable data,
 887                                                                                                               NSURLResponse * _Nullable __unused response,
 888                                                                                                               NSError * _Nullable responseError) {
 889
 890         __strong __typeof(self) strongSelf = weakSelf;
 891         if (!strongSelf) {
 892             return;
 893         }
 894
 895         if (data && !responseError) {
 896             NSData *responseData = data; // shut up compiler
 897             NSDictionary* responseDict = [NSJSONSerialization JSONObjectWithData:responseData options:0 error:&error];
 898             if([responseDict isKindOfClass:NSDictionary.class] && !error) {
 899                 if (!self->_ignoreServersMessagesTellingUsToGoAway) {
 900                     self->_disableUploads = [[responseDict valueForKey:@"sendDisabled"] boolValue];
 901                     if (self->_disableUploads) {
 902                         // then don't upload anything right now
 903                         secerror("not returning a splunk URL because uploads are disabled for %@", self->_internalTopicName);
 904                         dispatch_semaphore_signal(sem);
 905                         return;
 906                     }
 907
 908                     // backend works with milliseconds
 909                     NSUInteger secondsBetweenUploads = [[responseDict valueForKey:@"postFrequency"] unsignedIntegerValue] / 1000;
 910                     if (secondsBetweenUploads > 0) {
 911                         if (os_variant_has_internal_diagnostics("com.apple.security") &&
 912                             self->_secondsBetweenUploads < secondsBetweenUploads) {
 913                             secnotice("getURL", "Overriding server-sent post frequency because device is internal (%lu -> %lu)", secondsBetweenUploads, self->_secondsBetweenUploads);
 914                         } else {
 915                             strongSelf->_secondsBetweenUploads = secondsBetweenUploads;
 916                         }
 917                     }
 918
 919                     strongSelf->_blacklistedEvents = responseDict[@"blacklistedEvents"];
 920                     strongSelf->_blacklistedFields = responseDict[@"blacklistedFields"];
 921                 }
 922
 923                 strongSelf->_metricsBase = responseDict[@"metricsBase"];
 924
 925                 NSString* metricsEndpoint = responseDict[@"metricsUrl"];
 926                 if([metricsEndpoint isKindOfClass:NSString.class]) {
 927                     /* Lives our URL */
 928                     NSString* endpoint = [metricsEndpoint stringByAppendingFormat:@"/2/%@", strongSelf->_splunkTopicName];
 929                     secnotice("upload", "got metrics endpoint %@ for %@", endpoint, self->_internalTopicName);
 930                     NSURL* endpointURL = [NSURL URLWithString:endpoint];
 931                     if([endpointURL.scheme isEqualToString:@"https"]) {
 932                         result = endpointURL;
 933                     }
 934                 }
 935             }
 936         }
 937         else {
 938             error = responseError;
 939         }
 940         if (error) {
 941             secnotice("upload", "Unable to fetch splunk endpoint at URL for %@: %@ -- error: %@",
 942                       self->_internalTopicName, requestEndpoint, error.description);
 943         }
 944         else if (!result) {
 945             secnotice("upload", "Malformed iTunes config payload for %@!", self->_internalTopicName);
 946         }
 947
 948         dispatch_semaphore_signal(sem);
 949     }];
 950
 951     [storeBagTask resume];
 952     dispatch_semaphore_wait(sem, dispatch_time(DISPATCH_TIME_NOW, (uint64_t)(60 * NSEC_PER_SEC)));
 953
 954     return result;
 955 }
 956
 957 - (void)URLSession:(NSURLSession *)session didReceiveChallenge:(NSURLAuthenticationChallenge *)challenge
 958  completionHandler:(void (^)(NSURLSessionAuthChallengeDisposition, NSURLCredential *))completionHandler {
 959     assert(completionHandler);
 960     (void)session;
 961     secnotice("upload", "Splunk upload challenge for %@", _internalTopicName);
 962     NSURLCredential *cred = nil;
 963     SecTrustResultType result = kSecTrustResultInvalid;
 964
 965     if ([challenge previousFailureCount] > 0) {
 966         // Previous failures occurred, bail
 967         completionHandler(NSURLSessionAuthChallengeCancelAuthenticationChallenge, nil);
 968
 969     } else if ([challenge.protectionSpace.authenticationMethod isEqualToString:NSURLAuthenticationMethodServerTrust]) {
 970         /*
 971          * Evaluate trust for the certificate
 972          */
 973
 974         SecTrustRef serverTrust = challenge.protectionSpace.serverTrust;
 975         // Coverity gets upset if we don't check status even though result is all we need.
 976         OSStatus status = SecTrustEvaluate(serverTrust, &result);
 977         if (_allowInsecureSplunkCert || (status == errSecSuccess && ((result == kSecTrustResultProceed) || (result == kSecTrustResultUnspecified)))) {
 978             /*
 979              * All is well, accept the credentials
 980              */
 981             if(_allowInsecureSplunkCert) {
 982                 secnotice("upload", "Force Accepting Splunk Credential for %@", _internalTopicName);
 983             }
 984             cred = [NSURLCredential credentialForTrust:serverTrust];
 985             completionHandler(NSURLSessionAuthChallengeUseCredential, cred);
 986
 987         } else {
 988             /*
 989              * An error occurred in evaluating trust, bail
 990              */
 991             completionHandler(NSURLSessionAuthChallengeCancelAuthenticationChallenge, nil);
 992         }
 993     } else {
 994         /*
 995          * Just perform the default handling
 996          */
 997         completionHandler(NSURLSessionAuthChallengePerformDefaultHandling, nil);
 998     }
 999 }
1000
1001 - (NSDictionary*)eventDictWithBlacklistedFieldsStrippedFrom:(NSDictionary*)eventDict
1002 {
1003     NSMutableDictionary* strippedDict = eventDict.mutableCopy;
1004     for (NSString* blacklistedField in _blacklistedFields) {
1005         [strippedDict removeObjectForKey:blacklistedField];
1006     }
1007     return strippedDict;
1008 }
1009
1010 // MARK: Database path retrieval
1011
1012 + (NSString*)databasePathForCKKS
1013 {
1014     return [(__bridge_transfer NSURL*)SecCopyURLForFileInKeychainDirectory((__bridge CFStringRef)@"Analytics/ckks_analytics.db") path];
1015 }
1016
1017 + (NSString*)databasePathForSOS
1018 {
1019     return [(__bridge_transfer NSURL*)SecCopyURLForFileInKeychainDirectory((__bridge CFStringRef)@"Analytics/sos_analytics.db") path];
1020 }
1021
1022 + (NSString*)AppSupportPath
1023 {
1024 #if TARGET_OS_IOS
1025     return @"/var/mobile/Library/Application Support";
1026 #else
1027     NSArray<NSString *>*paths = NSSearchPathForDirectoriesInDomains(NSApplicationSupportDirectory, NSUserDomainMask, true);
1028     if ([paths count] < 1) {
1029         return nil;
1030     }
1031     return [NSString stringWithString: paths[0]];
1032 #endif /* TARGET_OS_IOS */
1033 }
1034
1035 + (NSString*)databasePathForPCS
1036 {
1037     NSString *appSup = [self AppSupportPath];
1038     if (!appSup) {
1039         return nil;
1040     }
1041     NSString *dbpath = [NSString stringWithFormat:@"%@/com.apple.ProtectedCloudStorage/PCSAnalytics.db", appSup];
1042     secnotice("supd", "PCS Database path (%@)", dbpath);
1043     return dbpath;
1044 }
1045
1046 + (NSString*)databasePathForTrustdHealth
1047 {
1048 #if TARGET_OS_IPHONE
1049     return [(__bridge_transfer NSURL*)SecCopyURLForFileInKeychainDirectory(CFSTR("Analytics/trustd_health_analytics.db")) path];
1050 #else
1051     return [(__bridge_transfer NSURL*)SecCopyURLForFileInUserCacheDirectory(CFSTR("Analytics/trustd_health_analytics.db")) path];
1052 #endif
1053 }
1054
1055 + (NSString*)databasePathForTrust
1056 {
1057 #if TARGET_OS_IPHONE
1058     return [(__bridge_transfer NSURL*)SecCopyURLForFileInKeychainDirectory(CFSTR("Analytics/trust_analytics.db")) path];
1059 #else
1060     return [(__bridge_transfer NSURL*)SecCopyURLForFileInUserCacheDirectory(CFSTR("Analytics/trust_analytics.db")) path];
1061 #endif
1062 }
1063
1064 + (NSString*)databasePathForTLS
1065 {
1066 #if TARGET_OS_IPHONE
1067     return [(__bridge_transfer NSURL*)SecCopyURLForFileInKeychainDirectory(CFSTR("Analytics/TLS_analytics.db")) path];
1068 #else
1069     return [(__bridge_transfer NSURL*)SecCopyURLForFileInUserCacheDirectory(CFSTR("Analytics/TLS_analytics.db")) path];
1070 #endif
1071 }
1072
1073 @end
1074
1075 @interface supd ()
1076 @property NSDictionary *topicsSamplingRates;
1077 @end
1078
1079 @implementation supd
1080 - (void)setupTopics
1081 {
1082     NSDictionary* systemDefaultValues = [NSDictionary dictionaryWithContentsOfFile:[[NSBundle bundleWithPath:@"/System/Library/Frameworks/Security.framework"] pathForResource:@"SFAnalytics" ofType:@"plist"]];
1083     NSMutableArray <SFAnalyticsTopic*>* topics = [NSMutableArray array];
1084     for (NSString *topicKey in systemDefaultValues) {
1085         NSDictionary *topicSamplingRates = _topicsSamplingRates[topicKey];
1086         SFAnalyticsTopic *topic = [[SFAnalyticsTopic alloc] initWithDictionary:systemDefaultValues[topicKey] name:topicKey samplingRates:topicSamplingRates];
1087         [topics addObject:topic];
1088     }
1089     _analyticsTopics = [NSArray arrayWithArray:topics];
1090 }
1091
1092 + (void)instantiate {
1093     [supd instance];
1094 }
1095
1096 + (instancetype)instance {
1097 #if TARGET_OS_SIMULATOR
1098     return nil;
1099 #else
1100     if (!_supdInstance) {
1101         _supdInstance = [self new];
1102     }
1103     return _supdInstance;
1104 #endif
1105 }
1106
1107 // Use this for testing to get rid of any state
1108 + (void)removeInstance {
1109     _supdInstance = nil;
1110 }
1111
1112
1113 static NSString *SystemTrustStorePath = @"/System/Library/Security/Certificates.bundle";
1114 static NSString *AnalyticsSamplingRatesFilename = @"AnalyticsSamplingRates";
1115 static NSString *ContentVersionKey = @"MobileAssetContentVersion";
1116 static NSString *AssetContextFilename = @"OTAPKIContext.plist";
1117
1118 static NSNumber *getSystemVersion(NSBundle *trustStoreBundle) {
1119     NSDictionary *systemVersionPlist = [NSDictionary dictionaryWithContentsOfURL:[trustStoreBundle URLForResource:@"AssetVersion"
1120                                                                                                     withExtension:@"plist"]];
1121     if (!systemVersionPlist || ![systemVersionPlist isKindOfClass:[NSDictionary class]]) {
1122         return nil;
1123     }
1124     NSNumber *systemVersion = systemVersionPlist[ContentVersionKey];
1125     if (systemVersion == nil || ![systemVersion isKindOfClass:[NSNumber class]]) {
1126         return nil;
1127     }
1128     return systemVersion;
1129 }
1130
1131 static NSNumber *getAssetVersion(NSURL *directory) {
1132     NSDictionary *assetContextPlist = [NSDictionary dictionaryWithContentsOfURL:[directory URLByAppendingPathComponent:AssetContextFilename]];
1133     if (!assetContextPlist || ![assetContextPlist isKindOfClass:[NSDictionary class]]) {
1134         return nil;
1135     }
1136     NSNumber *assetVersion = assetContextPlist[ContentVersionKey];
1137     if (assetVersion == nil || ![assetVersion isKindOfClass:[NSNumber class]]) {
1138         return nil;
1139     }
1140     return assetVersion;
1141 }
1142
1143 static bool ShouldInitializeWithAsset(NSBundle *trustStoreBundle, NSURL *directory) {
1144     NSNumber *systemVersion = getSystemVersion(trustStoreBundle);
1145     NSNumber *assetVersion = getAssetVersion(directory);
1146
1147     if (assetVersion == nil || systemVersion == nil) {
1148         return false;
1149     }
1150     if ([assetVersion compare:systemVersion] == NSOrderedDescending) {
1151         return true;
1152     }
1153     return false;
1154 }
1155
1156 - (void)setupSamplingRates {
1157 #if TARGET_OS_SIMULATOR
1158     NSBundle *trustStoreBundle = [NSBundle bundleWithPath:[NSString stringWithFormat:@"%s%@", getenv("SIMULATOR_ROOT"), SystemTrustStorePath]];
1159 #else
1160     NSBundle *trustStoreBundle = [NSBundle bundleWithPath:SystemTrustStorePath];
1161 #endif
1162
1163 #if TARGET_OS_IPHONE
1164     NSURL *keychainsDirectory = CFBridgingRelease(SecCopyURLForFileInKeychainDirectory(nil));
1165 #else
1166     NSURL *keychainsDirectory = [NSURL fileURLWithFileSystemRepresentation:"/Library/Keychains/" isDirectory:YES relativeToURL:nil];
1167 #endif
1168     NSURL *directory = [keychainsDirectory URLByAppendingPathComponent:@"SupplementalsAssets/" isDirectory:YES];
1169
1170     NSDictionary *analyticsSamplingRates = nil;
1171     if (ShouldInitializeWithAsset(trustStoreBundle, directory)) {
1172         /* Try to get the asset version of the sampling rates */
1173         NSURL *analyticsSamplingRateURL = [directory URLByAppendingPathComponent:[NSString stringWithFormat:@"%@.plist", AnalyticsSamplingRatesFilename]];
1174         analyticsSamplingRates = [NSDictionary dictionaryWithContentsOfURL:analyticsSamplingRateURL];
1175         secnotice("supd", "read sampling rates from SupplementalsAssets dir");
1176         if (!analyticsSamplingRates || ![analyticsSamplingRates isKindOfClass:[NSDictionary class]]) {
1177             analyticsSamplingRates = nil;
1178         }
1179     }
1180     if (!analyticsSamplingRates) {
1181         analyticsSamplingRates = [NSDictionary dictionaryWithContentsOfURL: [trustStoreBundle URLForResource:AnalyticsSamplingRatesFilename
1182                                                                                                withExtension:@"plist"]];
1183     }
1184     if (analyticsSamplingRates && [analyticsSamplingRates isKindOfClass:[NSDictionary class]]) {
1185         _topicsSamplingRates = analyticsSamplingRates[@"Topics"];
1186         if (!_topicsSamplingRates || ![analyticsSamplingRates isKindOfClass:[NSDictionary class]]) {
1187             _topicsSamplingRates = nil; // Something has gone terribly wrong, so we'll use the hardcoded defaults in this case
1188         }
1189     }
1190 }
1191
1192 - (instancetype)initWithReporter:(SFAnalyticsReporter *)reporter
1193 {
1194     if (self = [super init]) {
1195         [self setupSamplingRates];
1196         [self setupTopics];
1197         _reporter = reporter;
1198
1199         xpc_activity_register("com.apple.securityuploadd.triggerupload", XPC_ACTIVITY_CHECK_IN, ^(xpc_activity_t activity) {
1200             xpc_activity_state_t activityState = xpc_activity_get_state(activity);
1201             secnotice("supd", "hit xpc activity trigger, state: %ld", activityState);
1202             if (activityState == XPC_ACTIVITY_STATE_RUN) {
1203                 // Clean up the reports directory, and then run our regularly scheduled scan
1204                 [self performRegularlyScheduledUpload];
1205             }
1206         });
1207     }
1208
1209     return self;
1210 }
1211
1212 - (instancetype)init {
1213     SFAnalyticsReporter *reporter = [[SFAnalyticsReporter alloc] init];
1214     return [self initWithReporter:reporter];
1215 }
1216
1217 - (void)sendNotificationForOncePerReportSamplers
1218 {
1219     notify_post(SFAnalyticsFireSamplersNotification);
1220     [NSThread sleepForTimeInterval:3.0];
1221 }
1222
1223 - (void)performRegularlyScheduledUpload {
1224     secnotice("upload", "Starting uploads in response to regular trigger");
1225     NSError *error = nil;
1226     if ([self uploadAnalyticsWithError:&error]) {
1227         secnotice("upload", "Regularly scheduled upload successful");
1228     } else {
1229         secerror("upload: Failed to complete regularly scheduled upload: %@", error);
1230     }
1231 }
1232
1233 - (BOOL)uploadAnalyticsWithError:(NSError**)error {
1234     [self sendNotificationForOncePerReportSamplers];
1235
1236     BOOL result = NO;
1237     NSError* localError = nil;
1238     for (SFAnalyticsTopic *topic in _analyticsTopics) {
1239         @autoreleasepool { // The logging JSONs get quite large. Ensure they're deallocated between topics.
1240             __block NSURL* endpoint = [topic splunkUploadURL];   // has side effects!
1241
1242             if (!endpoint) {
1243                 secnotice("upload", "Skipping upload for %@ because no endpoint", [topic internalTopicName]);
1244                 continue;
1245             }
1246
1247             if ([topic disableUploads]) {
1248                 secnotice("upload", "Aborting upload task for %@ because uploads are disabled", [topic internalTopicName]);
1249                 continue;
1250             }
1251
1252             NSMutableArray<SFAnalyticsClient*>* clients = [NSMutableArray new];
1253             NSData* json = [topic getLoggingJSON:false forUpload:YES participatingClients:&clients error:&localError];
1254             if (json) {
1255                 if ([topic isSampledUpload]) {
1256                     if (![self->_reporter saveReport:json fileName:[topic internalTopicName]]) {
1257                         secerror("upload: failed to write analytics data to log");
1258                     }
1259                     if ([topic postJSON:json toEndpoint:endpoint error:&localError]) {
1260                         secnotice("upload", "Successfully posted JSON for %@", [topic internalTopicName]);
1261                         result = YES;
1262                         [topic updateUploadDateForClients:clients clearData:YES];
1263                     } else {
1264                         secerror("upload: Failed to post JSON for %@", [topic internalTopicName]);
1265                     }
1266                 } else {
1267                     /* If we didn't sample this report, update date to prevent trying to upload again sooner
1268                      * than we should. Clear data so that per-day calculations remain consistent. */
1269                     secnotice("upload", "skipping unsampled upload for %@ and clearing data", [topic internalTopicName]);
1270                     [topic updateUploadDateForClients:clients clearData:YES];
1271                 }
1272             } else {
1273                 secerror("upload: failed to get logging JSON");
1274             }
1275         }
1276         if (error && localError) {
1277             *error = localError;
1278         }
1279     }
1280     return result;
1281 }
1282
1283 - (NSString*)sysdiagnoseStringForEventRecord:(NSDictionary*)eventRecord
1284 {
1285     NSMutableDictionary* mutableEventRecord = eventRecord.mutableCopy;
1286     [mutableEventRecord removeObjectForKey:SFAnalyticsSplunkTopic];
1287
1288     NSDate* eventDate = [NSDate dateWithTimeIntervalSince1970:[[eventRecord valueForKey:SFAnalyticsEventTime] doubleValue] / 1000];
1289     [mutableEventRecord removeObjectForKey:SFAnalyticsEventTime];
1290
1291     NSString* eventName = eventRecord[SFAnalyticsEventType];
1292     [mutableEventRecord removeObjectForKey:SFAnalyticsEventType];
1293
1294     SFAnalyticsEventClass eventClass = [[eventRecord valueForKey:SFAnalyticsEventClassKey] integerValue];
1295     NSString* eventClassString = [self stringForEventClass:eventClass];
1296     [mutableEventRecord removeObjectForKey:SFAnalyticsEventClassKey];
1297
1298     NSMutableString* additionalAttributesString = [NSMutableString string];
1299     if (mutableEventRecord.count > 0) {
1300         [additionalAttributesString appendString:@" - Attributes: {" ];
1301         __block BOOL firstAttribute = YES;
1302         [mutableEventRecord enumerateKeysAndObjectsUsingBlock:^(NSString* key, id object, BOOL* stop) {
1303             NSString* openingString = firstAttribute ? @"" : @", ";
1304             [additionalAttributesString appendString:[NSString stringWithFormat:@"%@%@ : %@", openingString, key, object]];
1305             firstAttribute = NO;
1306         }];
1307         [additionalAttributesString appendString:@" }"];
1308     }
1309
1310     return [NSString stringWithFormat:@"%@ %@: %@%@", eventDate, eventClassString, eventName, additionalAttributesString];
1311 }
1312
1313 - (NSString*)getSysdiagnoseDump
1314 {
1315     NSMutableString* sysdiagnose = [[NSMutableString alloc] init];
1316
1317     for (SFAnalyticsTopic* topic in _analyticsTopics) {
1318         for (SFAnalyticsClient* client in topic.topicClients) {
1319             [sysdiagnose appendString:[NSString stringWithFormat:@"Client: %@\n", client.name]];
1320             SFAnalyticsSQLiteStore* store = [SFAnalyticsSQLiteStore storeWithPath:client.storePath schema:SFAnalyticsTableSchema];
1321             NSArray* allEvents = store.allEvents;
1322             for (NSDictionary* eventRecord in allEvents) {
1323                 [sysdiagnose appendFormat:@"%@\n", [self sysdiagnoseStringForEventRecord:eventRecord]];
1324             }
1325             if (allEvents.count == 0) {
1326                 [sysdiagnose appendString:@"No data to report for this client\n"];
1327             }
1328         }
1329     }
1330     return sysdiagnose;
1331 }
1332
1333 - (NSString*)stringForEventClass:(SFAnalyticsEventClass)eventClass
1334 {
1335     if (eventClass == SFAnalyticsEventClassNote) {
1336         return @"EventNote";
1337     }
1338     else if (eventClass == SFAnalyticsEventClassSuccess) {
1339         return @"EventSuccess";
1340     }
1341     else if (eventClass == SFAnalyticsEventClassHardFailure) {
1342         return @"EventHardFailure";
1343     }
1344     else if (eventClass == SFAnalyticsEventClassSoftFailure) {
1345         return @"EventSoftFailure";
1346     }
1347     else {
1348         return @"EventUnknown";
1349     }
1350 }
1351
1352 // MARK: XPC Procotol Handlers
1353
1354 - (void)getSysdiagnoseDumpWithReply:(void (^)(NSString*))reply {
1355     reply([self getSysdiagnoseDump]);
1356 }
1357
1358 - (void)getLoggingJSON:(bool)pretty topic:(NSString *)topicName reply:(void (^)(NSData*, NSError*))reply {
1359     secnotice("rpcGetLoggingJSON", "Building a JSON blob resembling the one we would have uploaded");
1360     NSError* error = nil;
1361     [self sendNotificationForOncePerReportSamplers];
1362     NSData* json = nil;
1363     for (SFAnalyticsTopic* topic in self->_analyticsTopics) {
1364         if ([topic.internalTopicName isEqualToString:topicName]) {
1365             json = [topic getLoggingJSON:pretty forUpload:NO participatingClients:nil error:&error];
1366         }
1367     }
1368     if (!json) {
1369         secerror("Unable to obtain JSON: %@", error);
1370     }
1371     reply(json, error);
1372 }
1373
1374 - (void)forceUploadWithReply:(void (^)(BOOL, NSError*))reply {
1375     secnotice("upload", "Performing upload in response to rpc message");
1376     NSError* error = nil;
1377     BOOL result = [self uploadAnalyticsWithError:&error];
1378     secnotice("upload", "Result of manually triggered upload: %@, error: %@", result ? @"success" : @"failure", error);
1379     reply(result, error);
1380 }
1381
1382 @end