supd/supd.m

   1 /*
   2  * Copyright (c) 2017-2018 Apple Inc. All Rights Reserved.
   3  *
   4  * @APPLE_LICENSE_HEADER_START@
   5  *
   6  * This file contains Original Code and/or Modifications of Original Code
   7  * as defined in and that are subject to the Apple Public Source License
   8  * Version 2.0 (the 'License'). You may not use this file except in
   9  * compliance with the License. Please obtain a copy of the License at
  10  * http://www.opensource.apple.com/apsl/ and read it before using this
  11  * file.
  12  *
  13  * The Original Code and all software distributed under the License are
  14  * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  15  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  16  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
  17  * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
  18  * Please see the License for the specific language governing rights and
  19  * limitations under the License.
  20  *
  21  * @APPLE_LICENSE_HEADER_END@
  22  */
  23
  24 #import "supd.h"
  25 #import "SFAnalyticsDefines.h"
  26 #import "SFAnalyticsSQLiteStore.h"
  27 #import <Security/SFAnalytics.h>
  28
  29 #include <utilities/SecFileLocations.h>
  30 #import "utilities/debugging.h"
  31 #import <os/variant_private.h>
  32 #import <xpc/xpc.h>
  33 #include <notify.h>
  34 #import "keychain/ckks/CKKSControl.h"
  35 #import <zlib.h>
  36
  37 #import <AuthKit/AKAppleIDAuthenticationContext.h>
  38 #import <AuthKit/AKAppleIDAuthenticationController.h>
  39 #import <AuthKit/AKAppleIDAuthenticationController_Private.h>
  40
  41 #if TARGET_OS_OSX
  42 #include "dirhelper_priv.h"
  43 #import <Accounts/Accounts.h>
  44 #import <Accounts/ACAccountStore_Private.h>
  45 #import <Accounts/ACAccountType_Private.h>
  46 #import <Accounts/ACAccountStore.h>
  47 #import <AOSAccounts/ACAccountStore+iCloudAccount.h>
  48 #import <AOSAccounts/ACAccount+iCloudAccount.h>
  49 #import <AOSAccountsLite/AOSAccountsLite.h>
  50 #import <CrashReporterSupport/CrashReporterSupportPrivate.h>
  51 #else // TARGET_OS_OSX
  52 #import <Accounts/Accounts.h>
  53 #import <AppleAccount/AppleAccount.h>
  54 #import <AppleAccount/ACAccount+AppleAccount.h>
  55 #import <AppleAccount/ACAccountStore+AppleAccount.h>
  56 #if TARGET_OS_EMBEDDED
  57 #import <CrashReporterSupport/CrashReporterSupport.h>
  58 #import <CrashReporterSupport/PreferenceManager.h>
  59 #endif // TARGET_OS_EMBEDDED
  60 #endif // TARGET_OS_OSX
  61
  62 NSString* const SFAnalyticsSplunkTopic = @"topic";
  63 NSString* const SFAnalyticsSplunkPostTime = @"postTime";
  64 NSString* const SFAnalyticsClientId = @"clientId";
  65 NSString* const SFAnalyticsInternal = @"internal";
  66
  67 NSString* const SFAnalyticsMetricsBase = @"metricsBase";
  68 NSString* const SFAnalyticsDeviceID = @"ckdeviceID";
  69
  70 NSString* const SFAnalyticsSecondsCustomerKey = @"SecondsBetweenUploadsCustomer";
  71 NSString* const SFAnalyticsSecondsInternalKey = @"SecondsBetweenUploadsInternal";
  72 NSString* const SFAnalyticsMaxEventsKey = @"NumberOfEvents";
  73 NSString* const SFAnalyticsDevicePercentageCustomerKey = @"DevicePercentageCustomer";
  74 NSString* const SFAnalyticsDevicePercentageInternalKey = @"DevicePercentageInternal";
  75
  76 #define SFANALYTICS_SPLUNK_DEV 0
  77 #define OS_CRASH_TRACER_LOG_BUG_TYPE "226"
  78
  79 #if SFANALYTICS_SPLUNK_DEV
  80 NSUInteger const secondsBetweenUploadsCustomer = 10;
  81 NSUInteger const secondsBetweenUploadsInternal = 10;
  82 #else // SFANALYTICS_SPLUNK_DEV
  83 NSUInteger const secondsBetweenUploadsCustomer = (3 * (60 * 60 * 24));
  84 NSUInteger const secondsBetweenUploadsInternal = (60 * 60 * 24);
  85 #endif // SFANALYTICS_SPLUNK_DEV
  86
  87 @implementation SFAnalyticsReporter
  88 - (BOOL)saveReport:(NSData *)reportData fileName:(NSString *)fileName
  89 {
  90 #if TARGET_OS_OSX
  91     NSDictionary *optionsDictionary = @{ (__bridge NSString *)kCRProblemReportSubmissionPolicyKey: (__bridge NSString *)kCRSubmissionPolicyAlternate };
  92 #else // !TARGET_OS_OSX
  93     NSDictionary *optionsDictionary = nil; // The keys above are not defined or required on iOS.
  94 #endif // !TARGET_OS_OSX
  95
  96     BOOL writtenToLog = NO;
  97 #if !TARGET_OS_SIMULATOR
  98     secdebug("saveReport", "calling out to `OSAWriteLogForSubmission`");
  99     writtenToLog = OSAWriteLogForSubmission(@OS_CRASH_TRACER_LOG_BUG_TYPE, fileName,
 100                                             nil, optionsDictionary, ^(NSFileHandle *fileHandle) {
 101                                                 secnotice("OSAWriteLogForSubmission", "Writing log data to report: %@", fileName);
 102                                                 [fileHandle writeData:reportData];
 103                                             });
 104 #endif // !TARGET_OS_SIMULATOR
 105     return writtenToLog;
 106 }
 107 @end
 108
 109 #define DEFAULT_SPLUNK_MAX_EVENTS_TO_REPORT 1000
 110 #define DEFAULT_SPLUNK_DEVICE_PERCENTAGE 100
 111
 112 static supd *_supdInstance = nil;
 113
 114 BOOL runningTests = NO;
 115 BOOL deviceAnalyticsOverride = NO;
 116 BOOL deviceAnalyticsEnabled = NO;
 117 BOOL iCloudAnalyticsOverride = NO;
 118 BOOL iCloudAnalyticsEnabled = NO;
 119
 120 static BOOL
 121 _isDeviceAnalyticsEnabled(void)
 122 {
 123     // This flag is only set during tests.
 124     if (deviceAnalyticsOverride) {
 125         return deviceAnalyticsEnabled;
 126     }
 127
 128     static BOOL dataCollectionEnabled = NO;
 129     static dispatch_once_t onceToken;
 130     dispatch_once(&onceToken, ^{
 131 #if TARGET_OS_EMBEDDED
 132         dataCollectionEnabled = DiagnosticLogSubmissionEnabled();
 133 #elif TARGET_OS_OSX
 134         dataCollectionEnabled = CRIsAutoSubmitEnabled();
 135 #endif
 136     });
 137     return dataCollectionEnabled;
 138 }
 139
 140 static NSString *const kAnalyticsiCloudIdMSKey = @"com.apple.idms.config.privacy.icloud.data";
 141
 142 #if TARGET_OS_IPHONE
 143 static NSDictionary *
 144 _getiCloudConfigurationInfoWithError(NSError **outError)
 145 {
 146     __block NSDictionary *outConfigurationInfo = nil;
 147     __block NSError *localError = nil;
 148
 149     ACAccountStore *accountStore = [[ACAccountStore alloc] init];
 150     ACAccount *primaryAccount = [accountStore aa_primaryAppleAccount];
 151     if (primaryAccount != nil) {
 152         NSString *altDSID = [primaryAccount aa_altDSID];
 153         secnotice("_getiCloudConfigurationInfoWithError", "Fetching configuration info");
 154
 155         dispatch_semaphore_t sema = dispatch_semaphore_create(0);
 156         AKAppleIDAuthenticationController *authController = [AKAppleIDAuthenticationController new];
 157         [authController configurationInfoWithIdentifiers:@[kAnalyticsiCloudIdMSKey]
 158                                               forAltDSID:altDSID
 159                                               completion:^(NSDictionary<NSString *, id<NSSecureCoding>> *configurationInfo, NSError *error) {
 160             if (error) {
 161                 secerror("_getiCloudConfigurationInfoWithError: Error fetching configurationInfo: %@", error);
 162                 localError = error;
 163             } else if (![configurationInfo isKindOfClass:[NSDictionary class]]) {
 164                 secerror("_getiCloudConfigurationInfoWithError: configurationInfo dict was not a dict, it was a %{public}@", [configurationInfo class]);
 165                 localError = error;
 166                 configurationInfo = nil;
 167             } else {
 168                 secnotice("_getiCloudConfigurationInfoWithError", "fetched configurationInfo %@", configurationInfo);
 169                 outConfigurationInfo = configurationInfo;
 170             }
 171             dispatch_semaphore_signal(sema);
 172         }];
 173         dispatch_semaphore_wait(sema, dispatch_time(DISPATCH_TIME_NOW, (uint64_t)(5 * NSEC_PER_SEC)));
 174     } else {
 175         secerror("_getiCloudConfigurationInfoWithError: Failed to fetch primary account info.");
 176     }
 177
 178     if (localError && outError) {
 179         *outError = localError;
 180     }
 181     return outConfigurationInfo;
 182 }
 183 #endif // TARGET_OS_IPHONE
 184
 185 #if TARGET_OS_OSX
 186 static NSString *
 187 _iCloudAccount(void)
 188 {
 189     return CFBridgingRelease(MMLCopyLoggedInAccount());
 190 }
 191
 192 static NSString *
 193 _altDSIDFromAccount(void)
 194 {
 195     static CFStringRef kMMPropertyAccountAlternateDSID = CFSTR("AccountAlternateDSID");
 196     NSString *account = _iCloudAccount();
 197     if (account != nil) {
 198         return CFBridgingRelease(MMLAccountCopyProperty((__bridge CFStringRef)account, kMMPropertyAccountAlternateDSID));
 199     }
 200     secerror("_altDSIDFromAccount: failed to fetch iCloud account");
 201     return nil;
 202 }
 203 #endif // TARGET_OS_OSX
 204
 205 static BOOL
 206 _isiCloudAnalyticsEnabled()
 207 {
 208     // This flag is only set during tests.
 209     if (iCloudAnalyticsOverride) {
 210         return iCloudAnalyticsEnabled;
 211     }
 212
 213     static bool cachedAllowsICloudAnalytics = false;
 214
 215 #if TARGET_OS_OSX
 216     static dispatch_once_t onceToken;
 217     dispatch_once(&onceToken, ^{
 218         /* AOSAccounts is not mastered into the BaseSystem. Check that those classes are linked at runtime and abort if not. */
 219         if (![AKAppleIDAuthenticationController class]) {
 220             secnotice("OTATrust", "Weak-linked AOSAccounts framework missing. Are we running in the base system?");
 221             return;
 222         }
 223
 224         NSString *currentAltDSID = _altDSIDFromAccount();
 225         if (currentAltDSID != nil) {
 226             AKAppleIDAuthenticationController *authController = [AKAppleIDAuthenticationController new];
 227             __block bool allowsICloudAnalytics = false;
 228             dispatch_semaphore_t sem = dispatch_semaphore_create(0);
 229             secnotice("isiCloudAnalyticsEnabled", "fetching iCloud Analytics value from idms");
 230             [authController configurationInfoWithIdentifiers:@[kAnalyticsiCloudIdMSKey]
 231                                                   forAltDSID:currentAltDSID
 232                                                   completion:^(NSDictionary<NSString *, id> *configurationInfo, NSError *error) {
 233                                                       if (!error && configurationInfo) {
 234                                                           NSNumber *value = configurationInfo[kAnalyticsiCloudIdMSKey];
 235                                                           if (value != nil) {
 236                                                               secnotice("_isiCloudAnalyticsEnabled", "authController:configurationInfoWithIdentifiers completed with no error and configuration information");
 237                                                               allowsICloudAnalytics = [value boolValue];
 238                                                           } else {
 239                                                               secerror("%s: no iCloud Analytics value found in IDMS", __FUNCTION__);
 240                                                           }
 241                                                       } else {
 242                                                           secerror("%s: Unable to fetch iCloud Analytics value from IDMS.", __FUNCTION__);
 243                                                       }
 244                                                       secnotice("_isiCloudAnalyticsEnabled", "authController:configurationInfoWithIdentifiers completed and returning");
 245                                                       dispatch_semaphore_signal(sem);
 246                                                   }];
 247             // Wait 5 seconds before giving up and returning from the block.
 248             dispatch_semaphore_wait(sem, dispatch_time(DISPATCH_TIME_NOW, (uint64_t)(5 * NSEC_PER_SEC)));
 249             cachedAllowsICloudAnalytics = allowsICloudAnalytics;
 250         } else {
 251             secerror("_isiCloudAnalyticsEnabled: Failed to fetch alternate DSID");
 252         }
 253     });
 254 #else // TARGET_OS_OSX
 255     static dispatch_once_t onceToken;
 256     dispatch_once(&onceToken, ^{
 257         NSError *error = nil;
 258         NSDictionary *accountConfiguration = _getiCloudConfigurationInfoWithError(&error);
 259         if (error == nil && accountConfiguration != nil) {
 260             id iCloudAnalyticsOptIn = accountConfiguration[kAnalyticsiCloudIdMSKey];
 261             if (iCloudAnalyticsOptIn != nil) {
 262                 BOOL iCloudAnalyticsOptInHasCorrectType = ([iCloudAnalyticsOptIn isKindOfClass:[NSNumber class]] || [iCloudAnalyticsOptIn isKindOfClass:[NSString class]]);
 263                 if (iCloudAnalyticsOptInHasCorrectType) {
 264                     NSNumber *iCloudAnalyticsOptInNumber = @([iCloudAnalyticsOptIn integerValue]);
 265                     cachedAllowsICloudAnalytics = ![iCloudAnalyticsOptInNumber isEqualToNumber:[NSNumber numberWithInteger:0]];
 266                 }
 267             }
 268         } else if (error != nil) {
 269             secerror("_isiCloudAnalyticsEnabled: %@", error);
 270         }
 271     });
 272 #endif // TARGET_OS_OSX
 273
 274     return cachedAllowsICloudAnalytics;
 275 }
 276
 277 /*  NSData GZip category based on GeoKit's implementation */
 278 @interface NSData (GZip)
 279 - (NSData *)supd_gzipDeflate;
 280 @end
 281
 282 #define GZIP_OFFSET 16
 283 #define GZIP_STRIDE_LEN 16384
 284
 285 @implementation NSData (Gzip)
 286 - (NSData *)supd_gzipDeflate
 287 {
 288     if ([self length] == 0) {
 289         return self;
 290     }
 291
 292     z_stream strm;
 293     memset(&strm, 0, sizeof(strm));
 294     strm.next_in=(uint8_t *)[self bytes];
 295     strm.avail_in = (unsigned int)[self length];
 296
 297
 298     if (Z_OK != deflateInit2(&strm, Z_BEST_COMPRESSION, Z_DEFLATED,
 299                              MAX_WBITS + GZIP_OFFSET, MAX_MEM_LEVEL, Z_DEFAULT_STRATEGY)) {
 300         return nil;
 301     }
 302
 303     NSMutableData *compressed = [NSMutableData dataWithLength:GZIP_STRIDE_LEN];
 304
 305     do {
 306         if (strm.total_out >= [compressed length]) {
 307             [compressed increaseLengthBy: 16384];
 308         }
 309
 310         strm.next_out = [compressed mutableBytes] + strm.total_out;
 311         strm.avail_out = (int)[compressed length] - (int)strm.total_out;
 312
 313         deflate(&strm, Z_FINISH);
 314
 315     } while (strm.avail_out == 0);
 316
 317     deflateEnd(&strm);
 318
 319     [compressed setLength: strm.total_out];
 320     if (strm.avail_in == 0) {
 321         return [NSData dataWithData:compressed];
 322     } else {
 323         return nil;
 324     }
 325 }
 326 @end
 327
 328 @implementation SFAnalyticsClient {
 329     NSString* _path;
 330     NSString* _name;
 331     BOOL _requireDeviceAnalytics;
 332     BOOL _requireiCloudAnalytics;
 333 }
 334
 335 @synthesize storePath = _path;
 336 @synthesize name = _name;
 337
 338 - (instancetype)initWithStorePath:(NSString*)path name:(NSString*)name
 339                   deviceAnalytics:(BOOL)deviceAnalytics iCloudAnalytics:(BOOL)iCloudAnalytics {
 340     if (self = [super init]) {
 341         _path = path;
 342         _name = name;
 343         _requireDeviceAnalytics = deviceAnalytics;
 344         _requireiCloudAnalytics = iCloudAnalytics;
 345     }
 346     return self;
 347 }
 348
 349 @end
 350
 351 @interface SFAnalyticsTopic ()
 352 @property NSURL* _splunkUploadURL;
 353
 354 @property BOOL allowInsecureSplunkCert;
 355 @property BOOL ignoreServersMessagesTellingUsToGoAway;
 356 @property BOOL disableUploads;
 357 @property BOOL disableClientId;
 358
 359 @property NSUInteger secondsBetweenUploads;
 360 @property NSUInteger maxEventsToReport;
 361 @property float devicePercentage; // for sampling reporting devices
 362
 363 @property NSDictionary* metricsBase; // data the server provides and wants us to send back
 364 @property NSArray* blacklistedFields;
 365 @property NSArray* blacklistedEvents;
 366 @end
 367
 368 @implementation SFAnalyticsTopic
 369
 370 - (void)setupClientsForTopic:(NSString *)topicName
 371 {
 372     NSMutableArray<SFAnalyticsClient*>* clients = [NSMutableArray<SFAnalyticsClient*> new];
 373     if ([topicName isEqualToString:SFAnalyticsTopicKeySync]) {
 374         [clients addObject:[[SFAnalyticsClient alloc] initWithStorePath:[self.class databasePathForCKKS]
 375                                                                    name:@"ckks" deviceAnalytics:NO iCloudAnalytics:YES]];
 376         [clients addObject:[[SFAnalyticsClient alloc] initWithStorePath:[self.class databasePathForSOS]
 377                                                                    name:@"sos" deviceAnalytics:NO iCloudAnalytics:YES]];
 378         [clients addObject:[[SFAnalyticsClient alloc] initWithStorePath:[self.class databasePathForPCS]
 379                                                                    name:@"pcs" deviceAnalytics:NO iCloudAnalytics:YES]];
 380         [clients addObject:[[SFAnalyticsClient alloc] initWithStorePath:[self.class databasePathForSignIn]
 381                                                                    name:@"signins" deviceAnalytics:NO iCloudAnalytics:YES]];
 382         [clients addObject:[[SFAnalyticsClient alloc] initWithStorePath:[self.class databasePathForLocal]
 383                                                                    name:@"local" deviceAnalytics:YES iCloudAnalytics:NO]];
 384     } else if ([topicName isEqualToString:SFAnaltyicsTopicTrust]) {
 385 #if TARGET_OS_OSX
 386         _set_user_dir_suffix("com.apple.trustd"); // supd needs to read trustd's cache dir for these
 387 #endif
 388         [clients addObject:[[SFAnalyticsClient alloc] initWithStorePath:[self.class databasePathForTrust]
 389                                                                    name:@"trust" deviceAnalytics:YES iCloudAnalytics:NO]];
 390         [clients addObject:[[SFAnalyticsClient alloc] initWithStorePath:[self.class databasePathForTrustdHealth]
 391                                                                    name:@"trustdHealth" deviceAnalytics:YES iCloudAnalytics:NO]];
 392         [clients addObject:[[SFAnalyticsClient alloc] initWithStorePath:[self.class databasePathForTLS]
 393                                                                    name:@"tls" deviceAnalytics:YES iCloudAnalytics:NO]];
 394
 395 #if TARGET_OS_OSX
 396         _set_user_dir_suffix(NULL); // set back to the default cache dir
 397 #endif
 398     }
 399
 400     _topicClients = clients;
 401 }
 402
 403 - (instancetype)initWithDictionary:(NSDictionary *)dictionary name:(NSString *)topicName samplingRates:(NSDictionary *)rates {
 404     if (self = [super init]) {
 405         _internalTopicName = topicName;
 406         [self setupClientsForTopic:topicName];
 407         _splunkTopicName = dictionary[@"splunk_topic"];
 408         __splunkUploadURL = [NSURL URLWithString:dictionary[@"splunk_uploadURL"]];
 409         _splunkBagURL = [NSURL URLWithString:dictionary[@"splunk_bagURL"]];
 410         _allowInsecureSplunkCert = [[dictionary valueForKey:@"splunk_allowInsecureCertificate"] boolValue];
 411         NSString* splunkEndpoint = dictionary[@"splunk_endpointDomain"];
 412         if (dictionary[@"disableClientId"]) {
 413             _disableClientId = YES;
 414         }
 415
 416         NSUserDefaults* defaults = [[NSUserDefaults alloc] initWithSuiteName:SFAnalyticsUserDefaultsSuite];
 417         NSString* userDefaultsSplunkTopic = [defaults stringForKey:@"splunk_topic"];
 418         if (userDefaultsSplunkTopic) {
 419             _splunkTopicName = userDefaultsSplunkTopic;
 420         }
 421
 422         NSURL* userDefaultsSplunkUploadURL = [NSURL URLWithString:[defaults stringForKey:@"splunk_uploadURL"]];
 423         if (userDefaultsSplunkUploadURL) {
 424             __splunkUploadURL = userDefaultsSplunkUploadURL;
 425         }
 426
 427         NSURL* userDefaultsSplunkBagURL = [NSURL URLWithString:[defaults stringForKey:@"splunk_bagURL"]];
 428         if (userDefaultsSplunkBagURL) {
 429             _splunkBagURL = userDefaultsSplunkBagURL;
 430         }
 431
 432         BOOL userDefaultsAllowInsecureSplunkCert = [defaults boolForKey:@"splunk_allowInsecureCertificate"];
 433         _allowInsecureSplunkCert |= userDefaultsAllowInsecureSplunkCert;
 434
 435         NSString* userDefaultsSplunkEndpoint = [defaults stringForKey:@"splunk_endpointDomain"];
 436         if (userDefaultsSplunkEndpoint) {
 437             splunkEndpoint = userDefaultsSplunkEndpoint;
 438         }
 439
 440 #if SFANALYTICS_SPLUNK_DEV
 441         _secondsBetweenUploads = secondsBetweenUploadsInternal;
 442         _maxEventsToReport = SFAnalyticsMaxEventsToReport;
 443         _devicePercentage = DEFAULT_SPLUNK_DEVICE_PERCENTAGE;
 444 #else
 445         bool internal = os_variant_has_internal_diagnostics("com.apple.security");
 446         if (rates) {
 447             NSNumber *secondsNum = internal ? rates[SFAnalyticsSecondsInternalKey] : rates[SFAnalyticsSecondsCustomerKey];
 448             _secondsBetweenUploads = [secondsNum integerValue];
 449             _maxEventsToReport = [rates[SFAnalyticsMaxEventsKey] unsignedIntegerValue];
 450             NSNumber *percentageNum = internal ? rates[SFAnalyticsDevicePercentageInternalKey] : rates[SFAnalyticsDevicePercentageCustomerKey];
 451             _devicePercentage = [percentageNum floatValue];
 452         } else {
 453             _secondsBetweenUploads = internal ? secondsBetweenUploadsInternal : secondsBetweenUploadsCustomer;
 454             _maxEventsToReport = SFAnalyticsMaxEventsToReport;
 455             _devicePercentage = DEFAULT_SPLUNK_DEVICE_PERCENTAGE;
 456         }
 457 #endif
 458         secnotice("supd", "created %@ with %lu seconds between uploads, %lu max events, %f percent of uploads",
 459                   _internalTopicName, (unsigned long)_secondsBetweenUploads, (unsigned long)_maxEventsToReport, _devicePercentage);
 460
 461 #if SFANALYTICS_SPLUNK_DEV
 462         _ignoreServersMessagesTellingUsToGoAway = YES;
 463
 464         if (!_splunkUploadURL && splunkEndpoint) {
 465             NSString* urlString = [NSString stringWithFormat:@"https://%@/report/2/%@", splunkEndpoint, _splunkTopicName];
 466             _splunkUploadURL = [NSURL URLWithString:urlString];
 467         }
 468 #else
 469         (void)splunkEndpoint;
 470 #endif
 471     }
 472     return self;
 473 }
 474
 475 - (BOOL)isSampledUpload
 476 {
 477     uint32_t sample = arc4random();
 478     if ((double)_devicePercentage < ((double)1 / UINT32_MAX) * 100) {
 479         /* Requested percentage is smaller than we can sample. just do 1 out of UINT32_MAX */
 480         if (sample == 0) {
 481             return YES;
 482         }
 483     } else {
 484         if ((double)sample <= (double)UINT32_MAX * ((double)_devicePercentage / 100)) {
 485             return YES;
 486         }
 487     }
 488     return NO;
 489 }
 490
 491 - (BOOL)postJSON:(NSData*)json toEndpoint:(NSURL*)endpoint error:(NSError**)error
 492 {
 493     if (!endpoint) {
 494         if (error) {
 495             NSString *description = [NSString stringWithFormat:@"No endpoint for %@", _internalTopicName];
 496             *error = [NSError errorWithDomain:@"SupdUploadErrorDomain"
 497                                          code:-10
 498                                      userInfo:@{NSLocalizedDescriptionKey : description}];
 499         }
 500         return false;
 501     }
 502     /*
 503      * Create the NSURLSession
 504      *  We use the ephemeral session config because we don't need cookies or cache
 505      */
 506     NSURLSessionConfiguration *configuration = [NSURLSessionConfiguration ephemeralSessionConfiguration];
 507
 508     configuration.HTTPAdditionalHeaders = @{ @"User-Agent" : [NSString stringWithFormat:@"securityd/%s", SECURITY_BUILD_VERSION]};
 509
 510     NSURLSession* postSession = [NSURLSession sessionWithConfiguration:configuration
 511                                                               delegate:self
 512                                                          delegateQueue:nil];
 513
 514     NSMutableURLRequest* postRequest = [[NSMutableURLRequest alloc] init];
 515     postRequest.URL = endpoint;
 516     postRequest.HTTPMethod = @"POST";
 517     postRequest.HTTPBody = [json supd_gzipDeflate];
 518     [postRequest setValue:@"gzip" forHTTPHeaderField:@"Content-Encoding"];
 519
 520     /*
 521      * Create the upload task.
 522      */
 523     dispatch_semaphore_t sem = dispatch_semaphore_create(0);
 524     __block BOOL uploadSuccess = NO;
 525     NSURLSessionDataTask* uploadTask = [postSession dataTaskWithRequest:postRequest
 526                                                       completionHandler:^(NSData * _Nullable __unused data, NSURLResponse * _Nullable response, NSError * _Nullable requestError) {
 527         if (requestError) {
 528             secerror("Error in uploading the events to splunk for %@: %@", self->_internalTopicName, requestError);
 529         } else if (![response isKindOfClass:NSHTTPURLResponse.class]){
 530             Class class = response.class;
 531             secerror("Received the wrong kind of response for %@: %@", self->_internalTopicName, NSStringFromClass(class));
 532         } else {
 533             NSHTTPURLResponse* httpResponse = (NSHTTPURLResponse*)response;
 534             if(httpResponse.statusCode >= 200 && httpResponse.statusCode < 300) {
 535                 /* Success */
 536                 uploadSuccess = YES;
 537                 secnotice("upload", "Splunk upload success for %@", self->_internalTopicName);
 538             } else {
 539                 secnotice("upload", "Splunk upload for %@ unexpected status to URL: %@ -- status: %d",
 540                           self->_internalTopicName, endpoint, (int)(httpResponse.statusCode));
 541             }
 542         }
 543         dispatch_semaphore_signal(sem);
 544     }];
 545     secnotice("upload", "Splunk upload start for %@", self->_internalTopicName);
 546     [uploadTask resume];
 547     dispatch_semaphore_wait(sem, dispatch_time(DISPATCH_TIME_NOW, (uint64_t)(5 * 60 * NSEC_PER_SEC)));
 548     return uploadSuccess;
 549 }
 550
 551 - (BOOL)eventIsBlacklisted:(NSMutableDictionary*)event {
 552     return _blacklistedEvents ? [_blacklistedEvents containsObject:event[SFAnalyticsEventType]] : NO;
 553 }
 554
 555 - (void)removeBlacklistedFieldsFromEvent:(NSMutableDictionary*)event {
 556     for (NSString* badField in self->_blacklistedFields) {
 557         [event removeObjectForKey:badField];
 558     }
 559 }
 560
 561 - (void)addRequiredFieldsToEvent:(NSMutableDictionary*)event {
 562     [_metricsBase enumerateKeysAndObjectsUsingBlock:^(id  _Nonnull key, id  _Nonnull obj, BOOL * _Nonnull stop) {
 563         if (!event[key]) {
 564             event[key] = obj;
 565         }
 566     }];
 567 }
 568
 569 - (BOOL)prepareEventForUpload:(NSMutableDictionary*)event {
 570     if ([self eventIsBlacklisted:event]) {
 571         return NO;
 572     }
 573
 574     [self removeBlacklistedFieldsFromEvent:event];
 575     [self addRequiredFieldsToEvent:event];
 576     if (_disableClientId) {
 577         event[SFAnalyticsClientId] = @(0);
 578     }
 579     event[SFAnalyticsSplunkTopic] = self->_splunkTopicName ?: [NSNull null];
 580     return YES;
 581 }
 582
 583 - (void)addFailures:(NSMutableArray<NSArray*>*)failures toUploadRecords:(NSMutableArray*)records threshold:(NSUInteger)threshold
 584 {
 585     // The first 0 through 'threshold' items are getting uploaded in any case (which might be 0 for lower priority data)
 586
 587     for (NSArray* client in failures) {
 588         [client enumerateObjectsUsingBlock:^(id  _Nonnull obj, NSUInteger idx, BOOL * _Nonnull stop) {
 589             if (idx >= threshold) {
 590                 *stop = YES;
 591                 return;
 592             }
 593             if ([self prepareEventForUpload:obj]) {
 594                 [records addObject:obj];
 595             }
 596         }];
 597     }
 598
 599     // Are there more items than we shoved into the upload records?
 600     NSInteger excessItems = 0;
 601     for (NSArray* client in failures) {
 602         NSInteger localExcess = client.count - threshold;
 603         excessItems += localExcess > 0 ? localExcess : 0;
 604     }
 605
 606     // Then, if we have space and items left, apply a scaling factor to distribute events across clients to fill upload buffer
 607     if (records.count < _maxEventsToReport && excessItems > 0) {
 608         double scale = (_maxEventsToReport - records.count) / (double)excessItems;
 609         if (scale > 1) {
 610             scale = 1;
 611         }
 612
 613         for (NSArray* client in failures) {
 614             if (client.count > threshold) {
 615                 NSRange range = NSMakeRange(threshold, (client.count - threshold) * scale);
 616                 NSArray* sub = [client subarrayWithRange:range];
 617                 [sub enumerateObjectsUsingBlock:^(id  _Nonnull obj, NSUInteger idx, BOOL * _Nonnull stop) {
 618                     if ([self prepareEventForUpload:obj]) {
 619                         [records addObject:obj];
 620                     }
 621                 }];
 622             }
 623         }
 624     }
 625 }
 626
 627 - (NSMutableDictionary*)sampleStatisticsForSamples:(NSArray*)samples withName:(NSString*)name
 628 {
 629     NSMutableDictionary* statistics = [NSMutableDictionary dictionary];
 630     NSUInteger count = samples.count;
 631     NSArray* sortedSamples = [samples sortedArrayUsingSelector:@selector(compare:)];
 632     NSArray* samplesAsExpressionArray = @[[NSExpression expressionForConstantValue:sortedSamples]];
 633
 634     if (count == 1) {
 635         statistics[name] = samples[0];
 636     } else {
 637         // NSExpression takes population standard deviation. Our data is a sample of whatever we sampled over time,
 638         // but the difference between the two is fairly minor (divide by N before taking sqrt versus divide by N-1).
 639         statistics[[NSString stringWithFormat:@"%@-dev", name]] = [[NSExpression expressionForFunction:@"stddev:" arguments:samplesAsExpressionArray] expressionValueWithObject:nil context:nil];
 640
 641         statistics[[NSString stringWithFormat:@"%@-min", name]] = [[NSExpression expressionForFunction:@"min:" arguments:samplesAsExpressionArray] expressionValueWithObject:nil context:nil];
 642         statistics[[NSString stringWithFormat:@"%@-max", name]] = [[NSExpression expressionForFunction:@"max:" arguments:samplesAsExpressionArray] expressionValueWithObject:nil context:nil];
 643         statistics[[NSString stringWithFormat:@"%@-avg", name]] = [[NSExpression expressionForFunction:@"average:" arguments:samplesAsExpressionArray] expressionValueWithObject:nil context:nil];
 644         statistics[[NSString stringWithFormat:@"%@-med", name]] = [[NSExpression expressionForFunction:@"median:" arguments:samplesAsExpressionArray] expressionValueWithObject:nil context:nil];
 645     }
 646
 647     if (count > 3) {
 648         NSString* q1 = [NSString stringWithFormat:@"%@-1q", name];
 649         NSString* q3 = [NSString stringWithFormat:@"%@-3q", name];
 650         // From Wikipedia, which is never wrong
 651         if (count % 2 == 0) {
 652             // The lower quartile value is the median of the lower half of the data. The upper quartile value is the median of the upper half of the data.
 653             statistics[q1] = [[NSExpression expressionForFunction:@"median:" arguments:@[[NSExpression expressionForConstantValue:[sortedSamples subarrayWithRange:NSMakeRange(0, count / 2)]]]] expressionValueWithObject:nil context:nil];
 654             statistics[q3] = [[NSExpression expressionForFunction:@"median:" arguments:@[[NSExpression expressionForConstantValue:[sortedSamples subarrayWithRange:NSMakeRange((count / 2), count / 2)]]]] expressionValueWithObject:nil context:nil];
 655         } else if (count % 4 == 1) {
 656             // If there are (4n+1) data points, then the lower quartile is 25% of the nth data value plus 75% of the (n+1)th data value;
 657             // the upper quartile is 75% of the (3n+1)th data point plus 25% of the (3n+2)th data point.
 658             // (offset n by -1 since we count from 0)
 659             NSUInteger n = count / 4;
 660             statistics[q1] = @(([sortedSamples[n - 1] doubleValue] + [sortedSamples[n] doubleValue] * 3.0) / 4.0);
 661             statistics[q3] = @(([sortedSamples[(3 * n)] doubleValue] * 3.0 + [sortedSamples[(3 * n) + 1] doubleValue]) / 4.0);
 662         } else if (count % 4 == 3){
 663             // If there are (4n+3) data points, then the lower quartile is 75% of the (n+1)th data value plus 25% of the (n+2)th data value;
 664             // the upper quartile is 25% of the (3n+2)th data point plus 75% of the (3n+3)th data point.
 665             // (offset n by -1 since we count from 0)
 666             NSUInteger n = count / 4;
 667             statistics[q1] = @(([sortedSamples[n] doubleValue] * 3.0 + [sortedSamples[n + 1] doubleValue]) / 4.0);
 668             statistics[q3] = @(([sortedSamples[(3 * n) + 1] doubleValue] + [sortedSamples[(3 * n) + 2] doubleValue] * 3.0) / 4.0);
 669         }
 670     }
 671
 672     return statistics;
 673 }
 674
 675 - (NSMutableDictionary*)healthSummaryWithName:(NSString*)name store:(SFAnalyticsSQLiteStore*)store
 676 {
 677     __block NSMutableDictionary* summary = [NSMutableDictionary new];
 678
 679     // Add some events of our own before pulling in data
 680     summary[SFAnalyticsEventType] = [NSString stringWithFormat:@"%@HealthSummary", name];
 681     if ([self eventIsBlacklisted:summary]) {
 682         return nil;
 683     }
 684     summary[SFAnalyticsEventTime] = @([[NSDate date] timeIntervalSince1970] * 1000);    // Splunk wants milliseconds
 685     [SFAnalytics addOSVersionToEvent:summary];
 686
 687     // Process counters
 688     NSDictionary* successCounts = store.summaryCounts;
 689     __block NSInteger totalSuccessCount = 0;
 690     __block NSInteger totalHardFailureCount = 0;
 691     __block NSInteger totalSoftFailureCount = 0;
 692     [successCounts enumerateKeysAndObjectsUsingBlock:^(NSString* _Nonnull eventType, NSDictionary* _Nonnull counts, BOOL* _Nonnull stop) {
 693         summary[[NSString stringWithFormat:@"%@-success", eventType]] = counts[SFAnalyticsColumnSuccessCount];
 694         summary[[NSString stringWithFormat:@"%@-hardfail", eventType]] = counts[SFAnalyticsColumnHardFailureCount];
 695         summary[[NSString stringWithFormat:@"%@-softfail", eventType]] = counts[SFAnalyticsColumnSoftFailureCount];
 696         totalSuccessCount += [counts[SFAnalyticsColumnSuccessCount] integerValue];
 697         totalHardFailureCount += [counts[SFAnalyticsColumnHardFailureCount] integerValue];
 698         totalSoftFailureCount += [counts[SFAnalyticsColumnSoftFailureCount] integerValue];
 699     }];
 700
 701     summary[SFAnalyticsColumnSuccessCount] = @(totalSuccessCount);
 702     summary[SFAnalyticsColumnHardFailureCount] = @(totalHardFailureCount);
 703     summary[SFAnalyticsColumnSoftFailureCount] = @(totalSoftFailureCount);
 704     if (os_variant_has_internal_diagnostics("com.apple.security")) {
 705         summary[SFAnalyticsInternal] = @YES;
 706     }
 707
 708     // Process samples
 709     NSMutableDictionary<NSString*,NSMutableArray*>* samplesBySampler = [NSMutableDictionary<NSString*,NSMutableArray*> dictionary];
 710     for (NSDictionary* sample in [store samples]) {
 711         if (!samplesBySampler[sample[SFAnalyticsColumnSampleName]]) {
 712             samplesBySampler[sample[SFAnalyticsColumnSampleName]] = [NSMutableArray array];
 713         }
 714         [samplesBySampler[sample[SFAnalyticsColumnSampleName]] addObject:sample[SFAnalyticsColumnSampleValue]];
 715     }
 716     [samplesBySampler enumerateKeysAndObjectsUsingBlock:^(NSString * _Nonnull key, NSMutableArray * _Nonnull obj, BOOL * _Nonnull stop) {
 717         NSMutableDictionary* event = [self sampleStatisticsForSamples:obj withName:key];
 718         [summary addEntriesFromDictionary:event];
 719     }];
 720
 721     // Should always return yes because we already checked for event blacklisting specifically
 722     if (![self prepareEventForUpload:summary]) {
 723         return nil;
 724     }
 725     return summary;
 726 }
 727
 728 - (void)updateUploadDateForClients:(NSArray<SFAnalyticsClient*>*)clients clearData:(BOOL)clearData
 729 {
 730     for (SFAnalyticsClient* client in clients) {
 731         SFAnalyticsSQLiteStore* store = [SFAnalyticsSQLiteStore storeWithPath:client.storePath schema:SFAnalyticsTableSchema];
 732         secnotice("postprocess", "Setting upload date for client: %@", client.name);
 733         store.uploadDate = [NSDate date];
 734         if (clearData) {
 735             secnotice("postprocess", "Clearing collected data for client: %@", client.name);
 736             [store clearAllData];
 737         }
 738     }
 739 }
 740
 741 - (NSData*)getLoggingJSON:(bool)pretty
 742                 forUpload:(BOOL)upload
 743      participatingClients:(NSMutableArray<SFAnalyticsClient*>**)clients
 744                     force:(BOOL)force                                       // supdctl uploads ignore privacy settings and recency
 745                     error:(NSError**)error
 746 {
 747     NSMutableArray<SFAnalyticsClient*>* localClients = [NSMutableArray new];
 748     __block NSMutableArray* uploadRecords = [NSMutableArray arrayWithCapacity:_maxEventsToReport];
 749     __block NSError *localError;
 750     __block NSMutableArray<NSArray*>* hardFailures = [NSMutableArray new];
 751     __block NSMutableArray<NSArray*>* softFailures = [NSMutableArray new];
 752     NSString* ckdeviceID = nil;
 753     if ([_internalTopicName isEqualToString:SFAnalyticsTopicKeySync]) {
 754         ckdeviceID = os_variant_has_internal_diagnostics("com.apple.security") ? [self askSecurityForCKDeviceID] : nil;
 755     }
 756     for (SFAnalyticsClient* client in self->_topicClients) {
 757         if (!force && [client requireDeviceAnalytics] && !_isDeviceAnalyticsEnabled()) {
 758             // Client required device analytics, yet the user did not opt in.
 759             secnotice("getLoggingJSON", "Client '%@' requires device analytics yet user did not opt in.", [client name]);
 760             continue;
 761         }
 762         if (!force && [client requireiCloudAnalytics] && !_isiCloudAnalyticsEnabled()) {
 763             // Client required iCloud analytics, yet the user did not opt in.
 764             secnotice("getLoggingJSON", "Client '%@' requires iCloud analytics yet user did not opt in.", [client name]);
 765             continue;
 766         }
 767
 768         SFAnalyticsSQLiteStore* store = [SFAnalyticsSQLiteStore storeWithPath:client.storePath schema:SFAnalyticsTableSchema];
 769
 770         if (upload) {
 771             NSDate* uploadDate = store.uploadDate;
 772             if (!force && uploadDate && [[NSDate date] timeIntervalSinceDate:uploadDate] < _secondsBetweenUploads) {
 773                 secnotice("json", "ignoring client '%@' for %@ because last upload too recent: %@",
 774                           client.name, _internalTopicName, uploadDate);
 775                 continue;
 776             }
 777
 778             if (!force && !uploadDate) {
 779                 secnotice("json", "ignoring client '%@' because doesn't have an upload date; giving it a baseline date",
 780                           client.name);
 781                 [self updateUploadDateForClients:@[client] clearData:NO];
 782                 continue;
 783             }
 784
 785             if (force) {
 786                 secnotice("json", "client '%@' for topic '%@' force-included", client.name, _internalTopicName);
 787             } else {
 788                 secnotice("json", "including client '%@' for topic '%@' for upload", client.name, _internalTopicName);
 789             }
 790             [localClients addObject:client];
 791         }
 792
 793         NSMutableDictionary* healthSummary = [self healthSummaryWithName:client.name store:store];
 794         if (healthSummary) {
 795             if (ckdeviceID) {
 796                 healthSummary[SFAnalyticsDeviceID] = ckdeviceID;
 797             }
 798             [uploadRecords addObject:healthSummary];
 799         }
 800
 801         [hardFailures addObject:store.hardFailures];
 802         [softFailures addObject:store.softFailures];
 803     }
 804
 805     if (upload && [localClients count] == 0) {
 806         if (error) {
 807             NSString *description = [NSString stringWithFormat:@"Upload too recent for all clients for %@", _internalTopicName];
 808             *error = [NSError errorWithDomain:@"SupdUploadErrorDomain"
 809                                          code:-10
 810                                      userInfo:@{NSLocalizedDescriptionKey : description}];
 811         }
 812         return nil;
 813     }
 814
 815     if (clients) {
 816         *clients = localClients;
 817     }
 818
 819     [self addFailures:hardFailures toUploadRecords:uploadRecords threshold:_maxEventsToReport/10];
 820     [self addFailures:softFailures toUploadRecords:uploadRecords threshold:0];
 821
 822     NSDictionary* jsonDict = @{
 823                                SFAnalyticsSplunkPostTime : @([[NSDate date] timeIntervalSince1970] * 1000),
 824                                @"events" : uploadRecords
 825                                };
 826
 827     NSData *json = [NSJSONSerialization dataWithJSONObject:jsonDict
 828                                                    options:(pretty ? NSJSONWritingPrettyPrinted : 0)
 829                                                      error:&localError];
 830
 831     if (error) {
 832         *error = localError;
 833     }
 834
 835     return json;
 836 }
 837
 838 // Is at least one client eligible for data collection based on user consent? Otherwise callers should NOT reach off-device.
 839 - (BOOL)haveEligibleClients {
 840     for (SFAnalyticsClient* client in self.topicClients) {
 841         if ((!client.requireDeviceAnalytics || _isDeviceAnalyticsEnabled()) &&
 842             (!client.requireiCloudAnalytics || _isiCloudAnalyticsEnabled())) {
 843             return YES;
 844         }
 845     }
 846     return NO;
 847 }
 848
 849 - (NSString*)askSecurityForCKDeviceID
 850 {
 851     NSError* error = nil;
 852     CKKSControl* rpc = [CKKSControl controlObject:&error];
 853     if(error || !rpc) {
 854         secerror("unable to obtain CKKS endpoint: %@", error);
 855         return nil;
 856     }
 857
 858     __block NSString* localCKDeviceID;
 859     dispatch_semaphore_t sema = dispatch_semaphore_create(0);
 860     [rpc rpcGetCKDeviceIDWithReply:^(NSString* ckdeviceID) {
 861         localCKDeviceID = ckdeviceID;
 862         dispatch_semaphore_signal(sema);
 863     }];
 864
 865     if (dispatch_semaphore_wait(sema, dispatch_time(DISPATCH_TIME_NOW, NSEC_PER_SEC * 10)) != 0) {
 866         secerror("timed out waiting for a response from security");
 867         return nil;
 868     }
 869
 870     return localCKDeviceID;
 871 }
 872
 873 // this method is kind of evil for the fact that it has side-effects in pulling other things besides the metricsURL from the server, and as such should NOT be memoized.
 874 // TODO redo this, probably to return a dictionary.
 875 - (NSURL*)splunkUploadURL:(BOOL)force
 876 {
 877     if (!force && ![self haveEligibleClients]) {    // force is true IFF called from supdctl. Customers don't have it and internal audiences must call it explicitly.
 878         secnotice("getURL", "Not going to talk to server for topic %@ because no eligible clients", [self internalTopicName]);
 879         return nil;
 880     }
 881
 882     if (__splunkUploadURL) {
 883         return __splunkUploadURL;
 884     }
 885
 886     secnotice("getURL", "Asking server for endpoint and config data for topic %@", [self internalTopicName]);
 887
 888     __weak __typeof(self) weakSelf = self;
 889     dispatch_semaphore_t sem = dispatch_semaphore_create(0);
 890
 891     __block NSError* error = nil;
 892     NSURLSessionConfiguration *configuration = [NSURLSessionConfiguration ephemeralSessionConfiguration];
 893     NSURLSession* storeBagSession = [NSURLSession sessionWithConfiguration:configuration
 894                                                                   delegate:self
 895                                                              delegateQueue:nil];
 896
 897     NSURL* requestEndpoint = _splunkBagURL;
 898     __block NSURL* result = nil;
 899     NSURLSessionDataTask* storeBagTask = [storeBagSession dataTaskWithURL:requestEndpoint completionHandler:^(NSData * _Nullable data,
 900                                                                                                               NSURLResponse * _Nullable __unused response,
 901                                                                                                               NSError * _Nullable responseError) {
 902
 903         __strong __typeof(self) strongSelf = weakSelf;
 904         if (!strongSelf) {
 905             return;
 906         }
 907
 908         if (data && !responseError) {
 909             NSData *responseData = data; // shut up compiler
 910             NSDictionary* responseDict = [NSJSONSerialization JSONObjectWithData:responseData options:0 error:&error];
 911             if([responseDict isKindOfClass:NSDictionary.class] && !error) {
 912                 if (!self->_ignoreServersMessagesTellingUsToGoAway) {
 913                     self->_disableUploads = [[responseDict valueForKey:@"sendDisabled"] boolValue];
 914                     if (self->_disableUploads) {
 915                         // then don't upload anything right now
 916                         secerror("not returning a splunk URL because uploads are disabled for %@", self->_internalTopicName);
 917                         dispatch_semaphore_signal(sem);
 918                         return;
 919                     }
 920
 921                     // backend works with milliseconds
 922                     NSUInteger secondsBetweenUploads = [[responseDict valueForKey:@"postFrequency"] unsignedIntegerValue] / 1000;
 923                     if (secondsBetweenUploads > 0) {
 924                         if (os_variant_has_internal_diagnostics("com.apple.security") &&
 925                             self->_secondsBetweenUploads < secondsBetweenUploads) {
 926                             secnotice("getURL", "Overriding server-sent post frequency because device is internal (%lu -> %lu)", (unsigned long)secondsBetweenUploads, (unsigned long)self->_secondsBetweenUploads);
 927                         } else {
 928                             strongSelf->_secondsBetweenUploads = secondsBetweenUploads;
 929                         }
 930                     }
 931
 932                     strongSelf->_blacklistedEvents = responseDict[@"blacklistedEvents"];
 933                     strongSelf->_blacklistedFields = responseDict[@"blacklistedFields"];
 934                 }
 935
 936                 strongSelf->_metricsBase = responseDict[@"metricsBase"];
 937
 938                 NSString* metricsEndpoint = responseDict[@"metricsUrl"];
 939                 if([metricsEndpoint isKindOfClass:NSString.class]) {
 940                     /* Lives our URL */
 941                     NSString* endpoint = [metricsEndpoint stringByAppendingFormat:@"/2/%@", strongSelf->_splunkTopicName];
 942                     secnotice("upload", "got metrics endpoint %@ for %@", endpoint, self->_internalTopicName);
 943                     NSURL* endpointURL = [NSURL URLWithString:endpoint];
 944                     if([endpointURL.scheme isEqualToString:@"https"]) {
 945                         result = endpointURL;
 946                     }
 947                 }
 948             }
 949         }
 950         else {
 951             error = responseError;
 952         }
 953         if (error) {
 954             secnotice("upload", "Unable to fetch splunk endpoint at URL for %@: %@ -- error: %@",
 955                       self->_internalTopicName, requestEndpoint, error.description);
 956         }
 957         else if (!result) {
 958             secnotice("upload", "Malformed iTunes config payload for %@!", self->_internalTopicName);
 959         }
 960
 961         dispatch_semaphore_signal(sem);
 962     }];
 963
 964     [storeBagTask resume];
 965     dispatch_semaphore_wait(sem, dispatch_time(DISPATCH_TIME_NOW, (uint64_t)(60 * NSEC_PER_SEC)));
 966
 967     return result;
 968 }
 969
 970 - (void)URLSession:(NSURLSession *)session didReceiveChallenge:(NSURLAuthenticationChallenge *)challenge
 971  completionHandler:(void (^)(NSURLSessionAuthChallengeDisposition, NSURLCredential *))completionHandler {
 972     assert(completionHandler);
 973     (void)session;
 974     secnotice("upload", "Splunk upload challenge for %@", _internalTopicName);
 975     NSURLCredential *cred = nil;
 976     SecTrustResultType result = kSecTrustResultInvalid;
 977
 978     if ([challenge previousFailureCount] > 0) {
 979         // Previous failures occurred, bail
 980         completionHandler(NSURLSessionAuthChallengeCancelAuthenticationChallenge, nil);
 981
 982     } else if ([challenge.protectionSpace.authenticationMethod isEqualToString:NSURLAuthenticationMethodServerTrust]) {
 983         /*
 984          * Evaluate trust for the certificate
 985          */
 986
 987         SecTrustRef serverTrust = challenge.protectionSpace.serverTrust;
 988         // Coverity gets upset if we don't check status even though result is all we need.
 989         OSStatus status = SecTrustEvaluate(serverTrust, &result);
 990         if (_allowInsecureSplunkCert || (status == errSecSuccess && ((result == kSecTrustResultProceed) || (result == kSecTrustResultUnspecified)))) {
 991             /*
 992              * All is well, accept the credentials
 993              */
 994             if(_allowInsecureSplunkCert) {
 995                 secnotice("upload", "Force Accepting Splunk Credential for %@", _internalTopicName);
 996             }
 997             cred = [NSURLCredential credentialForTrust:serverTrust];
 998             completionHandler(NSURLSessionAuthChallengeUseCredential, cred);
 999
1000         } else {
1001             /*
1002              * An error occurred in evaluating trust, bail
1003              */
1004             completionHandler(NSURLSessionAuthChallengeCancelAuthenticationChallenge, nil);
1005         }
1006     } else {
1007         /*
1008          * Just perform the default handling
1009          */
1010         completionHandler(NSURLSessionAuthChallengePerformDefaultHandling, nil);
1011     }
1012 }
1013
1014 - (NSDictionary*)eventDictWithBlacklistedFieldsStrippedFrom:(NSDictionary*)eventDict
1015 {
1016     NSMutableDictionary* strippedDict = eventDict.mutableCopy;
1017     for (NSString* blacklistedField in _blacklistedFields) {
1018         [strippedDict removeObjectForKey:blacklistedField];
1019     }
1020     return strippedDict;
1021 }
1022
1023 // MARK: Database path retrieval
1024
1025 + (NSString*)databasePathForCKKS
1026 {
1027     return [(__bridge_transfer NSURL*)SecCopyURLForFileInKeychainDirectory((__bridge CFStringRef)@"Analytics/ckks_analytics.db") path];
1028 }
1029
1030 + (NSString*)databasePathForSOS
1031 {
1032     return [(__bridge_transfer NSURL*)SecCopyURLForFileInKeychainDirectory((__bridge CFStringRef)@"Analytics/sos_analytics.db") path];
1033 }
1034
1035 + (NSString*)AppSupportPath
1036 {
1037 #if TARGET_OS_IOS
1038     return @"/var/mobile/Library/Application Support";
1039 #else
1040     NSArray<NSString *>*paths = NSSearchPathForDirectoriesInDomains(NSApplicationSupportDirectory, NSUserDomainMask, true);
1041     if ([paths count] < 1) {
1042         return nil;
1043     }
1044     return [NSString stringWithString: paths[0]];
1045 #endif /* TARGET_OS_IOS */
1046 }
1047
1048 + (NSString*)databasePathForPCS
1049 {
1050     NSString *appSup = [self AppSupportPath];
1051     if (!appSup) {
1052         return nil;
1053     }
1054     NSString *dbpath = [NSString stringWithFormat:@"%@/com.apple.ProtectedCloudStorage/PCSAnalytics.db", appSup];
1055     secnotice("supd", "PCS Database path (%@)", dbpath);
1056     return dbpath;
1057 }
1058
1059 + (NSString*)databasePathForLocal
1060 {
1061     return [(__bridge_transfer NSURL*)SecCopyURLForFileInKeychainDirectory((__bridge CFStringRef)@"Analytics/localkeychain.db") path];
1062 }
1063
1064 + (NSString*)databasePathForTrustdHealth
1065 {
1066 #if TARGET_OS_IPHONE
1067     return [(__bridge_transfer NSURL*)SecCopyURLForFileInKeychainDirectory(CFSTR("Analytics/trustd_health_analytics.db")) path];
1068 #else
1069     return [(__bridge_transfer NSURL*)SecCopyURLForFileInUserCacheDirectory(CFSTR("Analytics/trustd_health_analytics.db")) path];
1070 #endif
1071 }
1072
1073 + (NSString*)databasePathForTrust
1074 {
1075 #if TARGET_OS_IPHONE
1076     return [(__bridge_transfer NSURL*)SecCopyURLForFileInKeychainDirectory(CFSTR("Analytics/trust_analytics.db")) path];
1077 #else
1078     return [(__bridge_transfer NSURL*)SecCopyURLForFileInUserCacheDirectory(CFSTR("Analytics/trust_analytics.db")) path];
1079 #endif
1080 }
1081
1082 + (NSString*)databasePathForTLS
1083 {
1084 #if TARGET_OS_IPHONE
1085     return [(__bridge_transfer NSURL*)SecCopyURLForFileInKeychainDirectory(CFSTR("Analytics/TLS_analytics.db")) path];
1086 #else
1087     return [(__bridge_transfer NSURL*)SecCopyURLForFileInUserCacheDirectory(CFSTR("Analytics/TLS_analytics.db")) path];
1088 #endif
1089 }
1090
1091 + (NSString*)databasePathForSignIn
1092 {
1093     return [(__bridge_transfer NSURL*)SecCopyURLForFileInKeychainDirectory(CFSTR("Analytics/signin_metrics.db")) path];
1094 }
1095
1096 @end
1097
1098 @interface supd ()
1099 @property NSDictionary *topicsSamplingRates;
1100 @end
1101
1102 @implementation supd
1103 - (void)setupTopics
1104 {
1105     NSDictionary* systemDefaultValues = [NSDictionary dictionaryWithContentsOfFile:[[NSBundle bundleWithPath:@"/System/Library/Frameworks/Security.framework"] pathForResource:@"SFAnalytics" ofType:@"plist"]];
1106     NSMutableArray <SFAnalyticsTopic*>* topics = [NSMutableArray array];
1107     for (NSString *topicKey in systemDefaultValues) {
1108         NSDictionary *topicSamplingRates = _topicsSamplingRates[topicKey];
1109         SFAnalyticsTopic *topic = [[SFAnalyticsTopic alloc] initWithDictionary:systemDefaultValues[topicKey] name:topicKey samplingRates:topicSamplingRates];
1110         [topics addObject:topic];
1111     }
1112     _analyticsTopics = [NSArray arrayWithArray:topics];
1113 }
1114
1115 + (void)instantiate {
1116     [supd instance];
1117 }
1118
1119 + (instancetype)instance {
1120 #if TARGET_OS_SIMULATOR
1121     return nil;
1122 #else
1123     if (!_supdInstance) {
1124         _supdInstance = [self new];
1125     }
1126     return _supdInstance;
1127 #endif
1128 }
1129
1130 // Use this for testing to get rid of any state
1131 + (void)removeInstance {
1132     _supdInstance = nil;
1133 }
1134
1135
1136 static NSString *SystemTrustStorePath = @"/System/Library/Security/Certificates.bundle";
1137 static NSString *AnalyticsSamplingRatesFilename = @"AnalyticsSamplingRates";
1138 static NSString *ContentVersionKey = @"MobileAssetContentVersion";
1139 static NSString *AssetContextFilename = @"OTAPKIContext.plist";
1140
1141 static NSNumber *getSystemVersion(NSBundle *trustStoreBundle) {
1142     NSDictionary *systemVersionPlist = [NSDictionary dictionaryWithContentsOfURL:[trustStoreBundle URLForResource:@"AssetVersion"
1143                                                                                                     withExtension:@"plist"]];
1144     if (!systemVersionPlist || ![systemVersionPlist isKindOfClass:[NSDictionary class]]) {
1145         return nil;
1146     }
1147     NSNumber *systemVersion = systemVersionPlist[ContentVersionKey];
1148     if (systemVersion == nil || ![systemVersion isKindOfClass:[NSNumber class]]) {
1149         return nil;
1150     }
1151     return systemVersion;
1152 }
1153
1154 static NSNumber *getAssetVersion(NSURL *directory) {
1155     NSDictionary *assetContextPlist = [NSDictionary dictionaryWithContentsOfURL:[directory URLByAppendingPathComponent:AssetContextFilename]];
1156     if (!assetContextPlist || ![assetContextPlist isKindOfClass:[NSDictionary class]]) {
1157         return nil;
1158     }
1159     NSNumber *assetVersion = assetContextPlist[ContentVersionKey];
1160     if (assetVersion == nil || ![assetVersion isKindOfClass:[NSNumber class]]) {
1161         return nil;
1162     }
1163     return assetVersion;
1164 }
1165
1166 static bool ShouldInitializeWithAsset(NSBundle *trustStoreBundle, NSURL *directory) {
1167     NSNumber *systemVersion = getSystemVersion(trustStoreBundle);
1168     NSNumber *assetVersion = getAssetVersion(directory);
1169
1170     if (assetVersion == nil || systemVersion == nil) {
1171         return false;
1172     }
1173     if ([assetVersion compare:systemVersion] == NSOrderedDescending) {
1174         return true;
1175     }
1176     return false;
1177 }
1178
1179 - (void)setupSamplingRates {
1180 #if TARGET_OS_SIMULATOR
1181     NSBundle *trustStoreBundle = [NSBundle bundleWithPath:[NSString stringWithFormat:@"%s%@", getenv("SIMULATOR_ROOT"), SystemTrustStorePath]];
1182 #else
1183     NSBundle *trustStoreBundle = [NSBundle bundleWithPath:SystemTrustStorePath];
1184 #endif
1185
1186 #if TARGET_OS_IPHONE
1187     NSURL *keychainsDirectory = CFBridgingRelease(SecCopyURLForFileInKeychainDirectory(nil));
1188 #else
1189     NSURL *keychainsDirectory = [NSURL fileURLWithFileSystemRepresentation:"/Library/Keychains/" isDirectory:YES relativeToURL:nil];
1190 #endif
1191     NSURL *directory = [keychainsDirectory URLByAppendingPathComponent:@"SupplementalsAssets/" isDirectory:YES];
1192
1193     NSDictionary *analyticsSamplingRates = nil;
1194     if (ShouldInitializeWithAsset(trustStoreBundle, directory)) {
1195         /* Try to get the asset version of the sampling rates */
1196         NSURL *analyticsSamplingRateURL = [directory URLByAppendingPathComponent:[NSString stringWithFormat:@"%@.plist", AnalyticsSamplingRatesFilename]];
1197         analyticsSamplingRates = [NSDictionary dictionaryWithContentsOfURL:analyticsSamplingRateURL];
1198         secnotice("supd", "read sampling rates from SupplementalsAssets dir");
1199         if (!analyticsSamplingRates || ![analyticsSamplingRates isKindOfClass:[NSDictionary class]]) {
1200             analyticsSamplingRates = nil;
1201         }
1202     }
1203     if (!analyticsSamplingRates) {
1204         analyticsSamplingRates = [NSDictionary dictionaryWithContentsOfURL: [trustStoreBundle URLForResource:AnalyticsSamplingRatesFilename
1205                                                                                                withExtension:@"plist"]];
1206     }
1207     if (analyticsSamplingRates && [analyticsSamplingRates isKindOfClass:[NSDictionary class]]) {
1208         _topicsSamplingRates = analyticsSamplingRates[@"Topics"];
1209         if (!_topicsSamplingRates || ![analyticsSamplingRates isKindOfClass:[NSDictionary class]]) {
1210             _topicsSamplingRates = nil; // Something has gone terribly wrong, so we'll use the hardcoded defaults in this case
1211         }
1212     }
1213 }
1214
1215 - (instancetype)initWithReporter:(SFAnalyticsReporter *)reporter
1216 {
1217     if (self = [super init]) {
1218         [self setupSamplingRates];
1219         [self setupTopics];
1220         _reporter = reporter;
1221
1222         xpc_activity_register("com.apple.securityuploadd.triggerupload", XPC_ACTIVITY_CHECK_IN, ^(xpc_activity_t activity) {
1223             xpc_activity_state_t activityState = xpc_activity_get_state(activity);
1224             secnotice("supd", "hit xpc activity trigger, state: %ld", activityState);
1225             if (activityState == XPC_ACTIVITY_STATE_RUN) {
1226                 // Run our regularly scheduled scan
1227                 [self performRegularlyScheduledUpload];
1228             }
1229         });
1230     }
1231
1232     return self;
1233 }
1234
1235 - (instancetype)init {
1236     SFAnalyticsReporter *reporter = [[SFAnalyticsReporter alloc] init];
1237     return [self initWithReporter:reporter];
1238 }
1239
1240 - (void)sendNotificationForOncePerReportSamplers
1241 {
1242     notify_post(SFAnalyticsFireSamplersNotification);
1243     [NSThread sleepForTimeInterval:3.0];
1244 }
1245
1246 - (void)performRegularlyScheduledUpload {
1247     secnotice("upload", "Starting uploads in response to regular trigger");
1248     NSError *error = nil;
1249     if ([self uploadAnalyticsWithError:&error force:NO]) {
1250         secnotice("upload", "Regularly scheduled upload successful");
1251     } else {
1252         secerror("upload: Failed to complete regularly scheduled upload: %@", error);
1253     }
1254 }
1255
1256 - (BOOL)uploadAnalyticsWithError:(NSError**)error force:(BOOL)force {
1257     [self sendNotificationForOncePerReportSamplers];
1258
1259     BOOL result = NO;
1260     NSError* localError = nil;
1261     for (SFAnalyticsTopic *topic in _analyticsTopics) {
1262         @autoreleasepool { // The logging JSONs get quite large. Ensure they're deallocated between topics.
1263             __block NSURL* endpoint = [topic splunkUploadURL:force];   // has side effects!
1264
1265             if (!endpoint) {
1266                 secnotice("upload", "Skipping upload for %@ because no endpoint", [topic internalTopicName]);
1267                 continue;
1268             }
1269
1270             if ([topic disableUploads]) {
1271                 secnotice("upload", "Aborting upload task for %@ because uploads are disabled", [topic internalTopicName]);
1272                 continue;
1273             }
1274
1275             NSMutableArray<SFAnalyticsClient*>* clients = [NSMutableArray new];
1276             NSData* json = [topic getLoggingJSON:false forUpload:YES participatingClients:&clients force:force error:&localError];
1277             if (json) {
1278                 if ([topic isSampledUpload]) {
1279                     if (![self->_reporter saveReport:json fileName:[topic internalTopicName]]) {
1280                         secerror("upload: failed to write analytics data to log");
1281                     }
1282                     if ([topic postJSON:json toEndpoint:endpoint error:&localError]) {
1283                         secnotice("upload", "Successfully posted JSON for %@", [topic internalTopicName]);
1284                         result = YES;
1285                         [topic updateUploadDateForClients:clients clearData:YES];
1286                     } else {
1287                         secerror("upload: Failed to post JSON for %@", [topic internalTopicName]);
1288                     }
1289                 } else {
1290                     /* If we didn't sample this report, update date to prevent trying to upload again sooner
1291                      * than we should. Clear data so that per-day calculations remain consistent. */
1292                     secnotice("upload", "skipping unsampled upload for %@ and clearing data", [topic internalTopicName]);
1293                     [topic updateUploadDateForClients:clients clearData:YES];
1294                 }
1295             } else {
1296                 secerror("upload: failed to get logging JSON");
1297             }
1298         }
1299         if (error && localError) {
1300             *error = localError;
1301         }
1302     }
1303     return result;
1304 }
1305
1306 - (NSString*)sysdiagnoseStringForEventRecord:(NSDictionary*)eventRecord
1307 {
1308     NSMutableDictionary* mutableEventRecord = eventRecord.mutableCopy;
1309     [mutableEventRecord removeObjectForKey:SFAnalyticsSplunkTopic];
1310
1311     NSDate* eventDate = [NSDate dateWithTimeIntervalSince1970:[[eventRecord valueForKey:SFAnalyticsEventTime] doubleValue] / 1000];
1312     [mutableEventRecord removeObjectForKey:SFAnalyticsEventTime];
1313
1314     NSString* eventName = eventRecord[SFAnalyticsEventType];
1315     [mutableEventRecord removeObjectForKey:SFAnalyticsEventType];
1316
1317     SFAnalyticsEventClass eventClass = [[eventRecord valueForKey:SFAnalyticsEventClassKey] integerValue];
1318     NSString* eventClassString = [self stringForEventClass:eventClass];
1319     [mutableEventRecord removeObjectForKey:SFAnalyticsEventClassKey];
1320
1321     NSMutableString* additionalAttributesString = [NSMutableString string];
1322     if (mutableEventRecord.count > 0) {
1323         [additionalAttributesString appendString:@" - Attributes: {" ];
1324         __block BOOL firstAttribute = YES;
1325         [mutableEventRecord enumerateKeysAndObjectsUsingBlock:^(NSString* key, id object, BOOL* stop) {
1326             NSString* openingString = firstAttribute ? @"" : @", ";
1327             [additionalAttributesString appendString:[NSString stringWithFormat:@"%@%@ : %@", openingString, key, object]];
1328             firstAttribute = NO;
1329         }];
1330         [additionalAttributesString appendString:@" }"];
1331     }
1332
1333     return [NSString stringWithFormat:@"%@ %@: %@%@", eventDate, eventClassString, eventName, additionalAttributesString];
1334 }
1335
1336 - (NSString*)getSysdiagnoseDump
1337 {
1338     NSMutableString* sysdiagnose = [[NSMutableString alloc] init];
1339
1340     for (SFAnalyticsTopic* topic in _analyticsTopics) {
1341         for (SFAnalyticsClient* client in topic.topicClients) {
1342             [sysdiagnose appendString:[NSString stringWithFormat:@"Client: %@\n", client.name]];
1343             SFAnalyticsSQLiteStore* store = [SFAnalyticsSQLiteStore storeWithPath:client.storePath schema:SFAnalyticsTableSchema];
1344             NSArray* allEvents = store.allEvents;
1345             for (NSDictionary* eventRecord in allEvents) {
1346                 [sysdiagnose appendFormat:@"%@\n", [self sysdiagnoseStringForEventRecord:eventRecord]];
1347             }
1348             if (allEvents.count == 0) {
1349                 [sysdiagnose appendString:@"No data to report for this client\n"];
1350             }
1351         }
1352     }
1353     return sysdiagnose;
1354 }
1355
1356 - (NSString*)stringForEventClass:(SFAnalyticsEventClass)eventClass
1357 {
1358     if (eventClass == SFAnalyticsEventClassNote) {
1359         return @"EventNote";
1360     }
1361     else if (eventClass == SFAnalyticsEventClassSuccess) {
1362         return @"EventSuccess";
1363     }
1364     else if (eventClass == SFAnalyticsEventClassHardFailure) {
1365         return @"EventHardFailure";
1366     }
1367     else if (eventClass == SFAnalyticsEventClassSoftFailure) {
1368         return @"EventSoftFailure";
1369     }
1370     else {
1371         return @"EventUnknown";
1372     }
1373 }
1374
1375 // MARK: XPC Procotol Handlers
1376
1377 - (void)getSysdiagnoseDumpWithReply:(void (^)(NSString*))reply {
1378     reply([self getSysdiagnoseDump]);
1379 }
1380
1381 - (void)getLoggingJSON:(bool)pretty topic:(NSString *)topicName reply:(void (^)(NSData*, NSError*))reply {
1382     secnotice("rpcGetLoggingJSON", "Building a JSON blob resembling the one we would have uploaded");
1383     NSError* error = nil;
1384     [self sendNotificationForOncePerReportSamplers];
1385     NSData* json = nil;
1386     for (SFAnalyticsTopic* topic in self->_analyticsTopics) {
1387         if ([topic.internalTopicName isEqualToString:topicName]) {
1388             json = [topic getLoggingJSON:pretty forUpload:NO participatingClients:nil force:!runningTests error:&error];
1389         }
1390     }
1391     if (!json) {
1392         secerror("Unable to obtain JSON: %@", error);
1393     }
1394     reply(json, error);
1395 }
1396
1397 - (void)forceUploadWithReply:(void (^)(BOOL, NSError*))reply {
1398     secnotice("upload", "Performing upload in response to rpc message");
1399     NSError* error = nil;
1400     BOOL result = [self uploadAnalyticsWithError:&error force:YES];
1401     secnotice("upload", "Result of manually triggered upload: %@, error: %@", result ? @"success" : @"failure", error);
1402     reply(result, error);
1403 }
1404
1405 @end