supd/supd.m

   1 /*
   2  * Copyright (c) 2017-2018 Apple Inc. All Rights Reserved.
   3  *
   4  * @APPLE_LICENSE_HEADER_START@
   5  *
   6  * This file contains Original Code and/or Modifications of Original Code
   7  * as defined in and that are subject to the Apple Public Source License
   8  * Version 2.0 (the 'License'). You may not use this file except in
   9  * compliance with the License. Please obtain a copy of the License at
  10  * http://www.opensource.apple.com/apsl/ and read it before using this
  11  * file.
  12  *
  13  * The Original Code and all software distributed under the License are
  14  * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  15  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  16  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
  17  * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
  18  * Please see the License for the specific language governing rights and
  19  * limitations under the License.
  20  *
  21  * @APPLE_LICENSE_HEADER_END@
  22  */
  23
  24 #import "supd.h"
  25
  26 #if !TARGET_OS_SIMULATOR
  27
  28 #import "SFAnalyticsDefines.h"
  29 #import "SFAnalyticsSQLiteStore.h"
  30 #import <Security/SFAnalytics.h>
  31
  32 #include <utilities/SecFileLocations.h>
  33 #import "utilities/debugging.h"
  34 #import <os/variant_private.h>
  35 #import <xpc/xpc.h>
  36 #include <notify.h>
  37 #import "keychain/ckks/CKKSControl.h"
  38 #import <zlib.h>
  39
  40 #import <AuthKit/AKAppleIDAuthenticationContext.h>
  41 #import <AuthKit/AKAppleIDAuthenticationController.h>
  42 #import <AuthKit/AKAppleIDAuthenticationController_Private.h>
  43
  44 #if TARGET_OS_OSX
  45 #include "dirhelper_priv.h"
  46 #endif
  47
  48 #if TARGET_OS_OSX
  49 #import <CrashReporterSupport/CrashReporterSupportPrivate.h>
  50 #else
  51 #import <CrashReporterSupport/CrashReporterSupport.h>
  52 #endif
  53
  54 #import <Accounts/Accounts.h>
  55 #import <Accounts/ACAccountStore_Private.h>
  56 #import <Accounts/ACAccountType_Private.h>
  57 #import <Accounts/ACAccountStore.h>
  58 #import <AppleAccount/AppleAccount.h>
  59 #import <AppleAccount/ACAccount+AppleAccount.h>
  60 #import <AppleAccount/ACAccountStore+AppleAccount.h>
  61
  62
  63 NSString* const SFAnalyticsSplunkTopic = @"topic";
  64 NSString* const SFAnalyticsSplunkPostTime = @"postTime";
  65 NSString* const SFAnalyticsClientId = @"clientId";
  66 NSString* const SFAnalyticsInternal = @"internal";
  67
  68 NSString* const SFAnalyticsMetricsBase = @"metricsBase";
  69 NSString* const SFAnalyticsDeviceID = @"ckdeviceID";
  70 NSString* const SFAnalyticsAltDSID = @"altDSID";
  71
  72 NSString* const SFAnalyticsSecondsCustomerKey = @"SecondsBetweenUploadsCustomer";
  73 NSString* const SFAnalyticsSecondsInternalKey = @"SecondsBetweenUploadsInternal";
  74 NSString* const SFAnalyticsSecondsSeedKey = @"SecondsBetweenUploadsSeed";
  75 NSString* const SFAnalyticsMaxEventsKey = @"NumberOfEvents";
  76 NSString* const SFAnalyticsDevicePercentageCustomerKey = @"DevicePercentageCustomer";
  77 NSString* const SFAnalyticsDevicePercentageInternalKey = @"DevicePercentageInternal";
  78 NSString* const SFAnalyticsDevicePercentageSeedKey = @"DevicePercentageSeed";
  79
  80 NSString* const SupdErrorDomain = @"com.apple.security.supd";
  81
  82 #define SFANALYTICS_SPLUNK_DEV 0
  83 #define OS_CRASH_TRACER_LOG_BUG_TYPE "226"
  84
  85 #if SFANALYTICS_SPLUNK_DEV
  86 NSUInteger const secondsBetweenUploadsCustomer = 10;
  87 NSUInteger const secondsBetweenUploadsInternal = 10;
  88 NSUInteger const secondsBetweenUploadsSeed = 10;
  89 #else // SFANALYTICS_SPLUNK_DEV
  90 NSUInteger const secondsBetweenUploadsCustomer = (3 * (60 * 60 * 24));
  91 NSUInteger const secondsBetweenUploadsInternal = (60 * 60 * 24);
  92 NSUInteger const secondsBetweenUploadsSeed = (60 * 60 * 24);
  93 #endif // SFANALYTICS_SPLUNK_DEV
  94
  95 @implementation SFAnalyticsReporter
  96 - (BOOL)saveReport:(NSData *)reportData fileName:(NSString *)fileName
  97 {
  98     BOOL writtenToLog = NO;
  99 #if TARGET_OS_OSX
 100     NSDictionary *optionsDictionary = @{ (__bridge NSString *)kCRProblemReportSubmissionPolicyKey: (__bridge NSString *)kCRSubmissionPolicyAlternate };
 101 #else // !TARGET_OS_OSX
 102     NSDictionary *optionsDictionary = nil; // The keys above are not defined or required on iOS.
 103 #endif // !TARGET_OS_OSX
 104
 105
 106     secdebug("saveReport", "calling out to `OSAWriteLogForSubmission`");
 107     writtenToLog = OSAWriteLogForSubmission(@OS_CRASH_TRACER_LOG_BUG_TYPE, fileName,
 108                                             nil, optionsDictionary, ^(NSFileHandle *fileHandle) {
 109                                                 secnotice("OSAWriteLogForSubmission", "Writing log data to report: %@", fileName);
 110                                                 [fileHandle writeData:reportData];
 111                                             });
 112     return writtenToLog;
 113 }
 114 @end
 115
 116 #define DEFAULT_SPLUNK_MAX_EVENTS_TO_REPORT 1000
 117 #define DEFAULT_SPLUNK_DEVICE_PERCENTAGE 100
 118
 119 static supd *_supdInstance = nil;
 120
 121 BOOL runningTests = NO;
 122 BOOL deviceAnalyticsOverride = NO;
 123 BOOL deviceAnalyticsEnabled = NO;
 124 BOOL iCloudAnalyticsOverride = NO;
 125 BOOL iCloudAnalyticsEnabled = NO;
 126
 127 static BOOL
 128 _isDeviceAnalyticsEnabled(void)
 129 {
 130     // This flag is only set during tests.
 131     if (deviceAnalyticsOverride) {
 132         return deviceAnalyticsEnabled;
 133     }
 134
 135     static BOOL dataCollectionEnabled = NO;
 136     static dispatch_once_t onceToken;
 137     dispatch_once(&onceToken, ^{
 138 #if TARGET_OS_IPHONE
 139         dataCollectionEnabled = DiagnosticLogSubmissionEnabled();
 140 #elif TARGET_OS_OSX
 141         dataCollectionEnabled = CRIsAutoSubmitEnabled();
 142 #endif
 143     });
 144     return dataCollectionEnabled;
 145 }
 146
 147 static NSString *
 148 accountAltDSID(void)
 149 {
 150     ACAccountStore *accountStore = [[ACAccountStore alloc] init];
 151     ACAccount *primaryAccount = [accountStore aa_primaryAppleAccount];
 152     if (primaryAccount == nil) {
 153         return nil;
 154     }
 155     return [primaryAccount aa_altDSID];
 156 }
 157
 158 static NSString *const kAnalyticsiCloudIdMSKey = @"com.apple.idms.config.privacy.icloud.data";
 159
 160 static NSDictionary *
 161 _getiCloudConfigurationInfoWithError(NSError **outError)
 162 {
 163     __block NSDictionary *outConfigurationInfo = nil;
 164     __block NSError *localError = nil;
 165
 166     NSString *altDSID = accountAltDSID();
 167     if (altDSID != nil) {
 168         secnotice("_getiCloudConfigurationInfoWithError", "Fetching configuration info");
 169
 170         dispatch_semaphore_t sema = dispatch_semaphore_create(0);
 171         AKAppleIDAuthenticationController *authController = [AKAppleIDAuthenticationController new];
 172         [authController configurationInfoWithIdentifiers:@[kAnalyticsiCloudIdMSKey]
 173                                               forAltDSID:altDSID
 174                                               completion:^(NSDictionary<NSString *, id<NSSecureCoding>> *configurationInfo, NSError *error) {
 175             if (error) {
 176                 secerror("_getiCloudConfigurationInfoWithError: Error fetching configurationInfo: %@", error);
 177                 localError = error;
 178             } else if (![configurationInfo isKindOfClass:[NSDictionary class]]) {
 179                 secerror("_getiCloudConfigurationInfoWithError: configurationInfo dict was not a dict, it was a %{public}@", [configurationInfo class]);
 180                 localError = error;
 181                 configurationInfo = nil;
 182             } else {
 183                 secnotice("_getiCloudConfigurationInfoWithError", "fetched configurationInfo %@", configurationInfo);
 184                 outConfigurationInfo = configurationInfo;
 185             }
 186             dispatch_semaphore_signal(sema);
 187         }];
 188         dispatch_semaphore_wait(sema, dispatch_time(DISPATCH_TIME_NOW, (uint64_t)(5 * NSEC_PER_SEC)));
 189     } else {
 190         secerror("_getiCloudConfigurationInfoWithError: Failed to fetch primary account info.");
 191     }
 192
 193     if (localError && outError) {
 194         *outError = localError;
 195     }
 196     return outConfigurationInfo;
 197 }
 198
 199 static BOOL
 200 _isiCloudAnalyticsEnabled()
 201 {
 202     // This flag is only set during tests.
 203     if (iCloudAnalyticsOverride) {
 204         return iCloudAnalyticsEnabled;
 205     }
 206
 207     static bool cachedAllowsICloudAnalytics = false;
 208
 209     static dispatch_once_t onceToken;
 210     dispatch_once(&onceToken, ^{
 211         NSError *error = nil;
 212         NSDictionary *accountConfiguration = _getiCloudConfigurationInfoWithError(&error);
 213         if (error == nil && accountConfiguration != nil) {
 214             id iCloudAnalyticsOptIn = accountConfiguration[kAnalyticsiCloudIdMSKey];
 215             if (iCloudAnalyticsOptIn != nil) {
 216                 BOOL iCloudAnalyticsOptInHasCorrectType = ([iCloudAnalyticsOptIn isKindOfClass:[NSNumber class]] || [iCloudAnalyticsOptIn isKindOfClass:[NSString class]]);
 217                 if (iCloudAnalyticsOptInHasCorrectType) {
 218                     NSNumber *iCloudAnalyticsOptInNumber = @([iCloudAnalyticsOptIn integerValue]);
 219                     cachedAllowsICloudAnalytics = ![iCloudAnalyticsOptInNumber isEqualToNumber:[NSNumber numberWithInteger:0]];
 220                 }
 221             }
 222         } else if (error != nil) {
 223             secerror("_isiCloudAnalyticsEnabled: %@", error);
 224         }
 225     });
 226
 227     return cachedAllowsICloudAnalytics;
 228 }
 229
 230 /*  NSData GZip category based on GeoKit's implementation */
 231 @interface NSData (GZip)
 232 - (NSData *)supd_gzipDeflate;
 233 @end
 234
 235 #define GZIP_OFFSET 16
 236 #define GZIP_STRIDE_LEN 16384
 237
 238 @implementation NSData (Gzip)
 239 - (NSData *)supd_gzipDeflate
 240 {
 241     if ([self length] == 0) {
 242         return self;
 243     }
 244
 245     z_stream strm;
 246     memset(&strm, 0, sizeof(strm));
 247     strm.next_in=(uint8_t *)[self bytes];
 248     strm.avail_in = (unsigned int)[self length];
 249
 250
 251     if (Z_OK != deflateInit2(&strm, Z_BEST_COMPRESSION, Z_DEFLATED,
 252                              MAX_WBITS + GZIP_OFFSET, MAX_MEM_LEVEL, Z_DEFAULT_STRATEGY)) {
 253         return nil;
 254     }
 255
 256     NSMutableData *compressed = [NSMutableData dataWithLength:GZIP_STRIDE_LEN];
 257
 258     do {
 259         if (strm.total_out >= [compressed length]) {
 260             [compressed increaseLengthBy: 16384];
 261         }
 262
 263         strm.next_out = [compressed mutableBytes] + strm.total_out;
 264         strm.avail_out = (int)[compressed length] - (int)strm.total_out;
 265
 266         deflate(&strm, Z_FINISH);
 267
 268     } while (strm.avail_out == 0);
 269
 270     deflateEnd(&strm);
 271
 272     [compressed setLength: strm.total_out];
 273     if (strm.avail_in == 0) {
 274         return [NSData dataWithData:compressed];
 275     } else {
 276         return nil;
 277     }
 278 }
 279 @end
 280
 281 @implementation SFAnalyticsClient {
 282     NSString* _path;
 283     NSString* _name;
 284     BOOL _requireDeviceAnalytics;
 285     BOOL _requireiCloudAnalytics;
 286 }
 287
 288 @synthesize storePath = _path;
 289 @synthesize name = _name;
 290
 291 - (instancetype)initWithStorePath:(NSString*)path name:(NSString*)name
 292                   deviceAnalytics:(BOOL)deviceAnalytics iCloudAnalytics:(BOOL)iCloudAnalytics {
 293     if (self = [super init]) {
 294         _path = path;
 295         _name = name;
 296         _requireDeviceAnalytics = deviceAnalytics;
 297         _requireiCloudAnalytics = iCloudAnalytics;
 298     }
 299     return self;
 300 }
 301
 302 @end
 303
 304 @interface SFAnalyticsTopic ()
 305 @property NSURL* _splunkUploadURL;
 306
 307 @property BOOL allowInsecureSplunkCert;
 308 @property BOOL ignoreServersMessagesTellingUsToGoAway;
 309 @property BOOL disableUploads;
 310 @property BOOL disableClientId;
 311
 312 @property NSUInteger secondsBetweenUploads;
 313 @property NSUInteger maxEventsToReport;
 314 @property float devicePercentage; // for sampling reporting devices
 315
 316 @property NSDictionary* metricsBase; // data the server provides and wants us to send back
 317 @property NSArray* blacklistedFields;
 318 @property NSArray* blacklistedEvents;
 319 @end
 320
 321 @implementation SFAnalyticsTopic
 322
 323 - (void)setupClientsForTopic:(NSString *)topicName
 324 {
 325     NSMutableArray<SFAnalyticsClient*>* clients = [NSMutableArray<SFAnalyticsClient*> new];
 326     if ([topicName isEqualToString:SFAnalyticsTopicKeySync]) {
 327         [clients addObject:[[SFAnalyticsClient alloc] initWithStorePath:[self.class databasePathForCKKS]
 328                                                                    name:@"ckks" deviceAnalytics:NO iCloudAnalytics:YES]];
 329         [clients addObject:[[SFAnalyticsClient alloc] initWithStorePath:[self.class databasePathForSOS]
 330                                                                    name:@"sos" deviceAnalytics:NO iCloudAnalytics:YES]];
 331         [clients addObject:[[SFAnalyticsClient alloc] initWithStorePath:[self.class databasePathForPCS]
 332                                                                    name:@"pcs" deviceAnalytics:NO iCloudAnalytics:YES]];
 333         [clients addObject:[[SFAnalyticsClient alloc] initWithStorePath:[self.class databasePathForSignIn]
 334                                                                    name:@"signins" deviceAnalytics:NO iCloudAnalytics:YES]];
 335         [clients addObject:[[SFAnalyticsClient alloc] initWithStorePath:[self.class databasePathForLocal]
 336                                                                    name:@"local" deviceAnalytics:YES iCloudAnalytics:NO]];
 337     } else if ([topicName isEqualToString:SFAnalyticsTopicCloudServices]) {
 338         [clients addObject:[[SFAnalyticsClient alloc] initWithStorePath:[self.class databasePathForCloudServices]
 339                                                                    name:@"CloudServices"
 340                                                         deviceAnalytics:YES
 341                                                         iCloudAnalytics:NO]];
 342     } else if ([topicName isEqualToString:SFAnalyticsTopicTrust]) {
 343 #if TARGET_OS_OSX
 344         _set_user_dir_suffix("com.apple.trustd"); // supd needs to read trustd's cache dir for these
 345 #endif
 346         [clients addObject:[[SFAnalyticsClient alloc] initWithStorePath:[self.class databasePathForTrust]
 347                                                                    name:@"trust" deviceAnalytics:YES iCloudAnalytics:NO]];
 348         [clients addObject:[[SFAnalyticsClient alloc] initWithStorePath:[self.class databasePathForTrustdHealth]
 349                                                                    name:@"trustdHealth" deviceAnalytics:YES iCloudAnalytics:NO]];
 350         [clients addObject:[[SFAnalyticsClient alloc] initWithStorePath:[self.class databasePathForTLS]
 351                                                                    name:@"tls" deviceAnalytics:YES iCloudAnalytics:NO]];
 352
 353 #if TARGET_OS_OSX
 354         _set_user_dir_suffix(NULL); // set back to the default cache dir
 355 #endif
 356     } else if ([topicName isEqualToString:SFAnalyticsTopicTransparency]) {
 357         [clients addObject:[[SFAnalyticsClient alloc] initWithStorePath:[self.class databasePathForTransparency]
 358                                                                    name:@"transparency" deviceAnalytics:NO iCloudAnalytics:YES]];
 359     }
 360
 361     _topicClients = clients;
 362 }
 363
 364 - (instancetype)initWithDictionary:(NSDictionary *)dictionary name:(NSString *)topicName samplingRates:(NSDictionary *)rates {
 365     if (self = [super init]) {
 366         _internalTopicName = topicName;
 367         [self setupClientsForTopic:topicName];
 368         _splunkTopicName = dictionary[@"splunk_topic"];
 369         __splunkUploadURL = [NSURL URLWithString:dictionary[@"splunk_uploadURL"]];
 370         _splunkBagURL = [NSURL URLWithString:dictionary[@"splunk_bagURL"]];
 371         _allowInsecureSplunkCert = [[dictionary valueForKey:@"splunk_allowInsecureCertificate"] boolValue];
 372         NSString* splunkEndpoint = dictionary[@"splunk_endpointDomain"];
 373         if (dictionary[@"disableClientId"]) {
 374             _disableClientId = YES;
 375         }
 376
 377         NSUserDefaults* defaults = [[NSUserDefaults alloc] initWithSuiteName:SFAnalyticsUserDefaultsSuite];
 378         NSString* userDefaultsSplunkTopic = [defaults stringForKey:@"splunk_topic"];
 379         if (userDefaultsSplunkTopic) {
 380             _splunkTopicName = userDefaultsSplunkTopic;
 381         }
 382
 383         NSURL* userDefaultsSplunkUploadURL = [NSURL URLWithString:[defaults stringForKey:@"splunk_uploadURL"]];
 384         if (userDefaultsSplunkUploadURL) {
 385             __splunkUploadURL = userDefaultsSplunkUploadURL;
 386         }
 387
 388         NSURL* userDefaultsSplunkBagURL = [NSURL URLWithString:[defaults stringForKey:@"splunk_bagURL"]];
 389         if (userDefaultsSplunkBagURL) {
 390             _splunkBagURL = userDefaultsSplunkBagURL;
 391         }
 392
 393         BOOL userDefaultsAllowInsecureSplunkCert = [defaults boolForKey:@"splunk_allowInsecureCertificate"];
 394         _allowInsecureSplunkCert |= userDefaultsAllowInsecureSplunkCert;
 395
 396         NSString* userDefaultsSplunkEndpoint = [defaults stringForKey:@"splunk_endpointDomain"];
 397         if (userDefaultsSplunkEndpoint) {
 398             splunkEndpoint = userDefaultsSplunkEndpoint;
 399         }
 400
 401 #if SFANALYTICS_SPLUNK_DEV
 402         _secondsBetweenUploads = secondsBetweenUploadsInternal;
 403         _maxEventsToReport = SFAnalyticsMaxEventsToReport;
 404         _devicePercentage = DEFAULT_SPLUNK_DEVICE_PERCENTAGE;
 405 #else
 406         bool internal = os_variant_has_internal_diagnostics("com.apple.security");
 407         if (rates) {
 408 #if RC_SEED_BUILD
 409             NSNumber *secondsNum = internal ? rates[SFAnalyticsSecondsInternalKey] : rates[SFAnalyticsSecondsSeedKey];
 410             NSNumber *percentageNum = internal ? rates[SFAnalyticsDevicePercentageInternalKey] : rates[SFAnalyticsDevicePercentageSeedKey];
 411 #else
 412             NSNumber *secondsNum = internal ? rates[SFAnalyticsSecondsInternalKey] : rates[SFAnalyticsSecondsCustomerKey];
 413             NSNumber *percentageNum = internal ? rates[SFAnalyticsDevicePercentageInternalKey] : rates[SFAnalyticsDevicePercentageCustomerKey];
 414 #endif
 415             _secondsBetweenUploads = [secondsNum integerValue];
 416             _maxEventsToReport = [rates[SFAnalyticsMaxEventsKey] unsignedIntegerValue];
 417             _devicePercentage = [percentageNum floatValue];
 418         } else {
 419 #if RC_SEED_BUILD
 420             _secondsBetweenUploads = internal ? secondsBetweenUploadsInternal : secondsBetweenUploadsSeed;
 421 #else
 422             _secondsBetweenUploads = internal ? secondsBetweenUploadsInternal : secondsBetweenUploadsCustomer;
 423 #endif
 424             _maxEventsToReport = SFAnalyticsMaxEventsToReport;
 425             _devicePercentage = DEFAULT_SPLUNK_DEVICE_PERCENTAGE;
 426         }
 427 #endif
 428         secnotice("supd", "created %@ with %lu seconds between uploads, %lu max events, %f percent of uploads",
 429                   _internalTopicName, (unsigned long)_secondsBetweenUploads, (unsigned long)_maxEventsToReport, _devicePercentage);
 430
 431 #if SFANALYTICS_SPLUNK_DEV
 432         _ignoreServersMessagesTellingUsToGoAway = YES;
 433
 434         if (!_splunkUploadURL && splunkEndpoint) {
 435             NSString* urlString = [NSString stringWithFormat:@"https://%@/report/2/%@", splunkEndpoint, _splunkTopicName];
 436             _splunkUploadURL = [NSURL URLWithString:urlString];
 437         }
 438 #else
 439         (void)splunkEndpoint;
 440 #endif
 441     }
 442     return self;
 443 }
 444
 445 - (BOOL)isSampledUpload
 446 {
 447     uint32_t sample = arc4random();
 448     if ((double)_devicePercentage < ((double)1 / UINT32_MAX) * 100) {
 449         /* Requested percentage is smaller than we can sample. just do 1 out of UINT32_MAX */
 450         if (sample == 0) {
 451             return YES;
 452         }
 453     } else {
 454         if ((double)sample <= (double)UINT32_MAX * ((double)_devicePercentage / 100)) {
 455             return YES;
 456         }
 457     }
 458     return NO;
 459 }
 460
 461 - (BOOL)postJSON:(NSData*)json toEndpoint:(NSURL*)endpoint error:(NSError**)error
 462 {
 463     if (!endpoint) {
 464         if (error) {
 465             NSString *description = [NSString stringWithFormat:@"No endpoint for %@", _internalTopicName];
 466             *error = [NSError errorWithDomain:@"SupdUploadErrorDomain"
 467                                          code:-10
 468                                      userInfo:@{NSLocalizedDescriptionKey : description}];
 469         }
 470         return false;
 471     }
 472     /*
 473      * Create the NSURLSession
 474      *  We use the ephemeral session config because we don't need cookies or cache
 475      */
 476     NSURLSessionConfiguration *configuration = [NSURLSessionConfiguration ephemeralSessionConfiguration];
 477
 478     configuration.HTTPAdditionalHeaders = @{ @"User-Agent" : [NSString stringWithFormat:@"securityd/%s", SECURITY_BUILD_VERSION]};
 479
 480     NSURLSession* postSession = [NSURLSession sessionWithConfiguration:configuration
 481                                                               delegate:self
 482                                                          delegateQueue:nil];
 483
 484     NSMutableURLRequest* postRequest = [[NSMutableURLRequest alloc] init];
 485     postRequest.URL = endpoint;
 486     postRequest.HTTPMethod = @"POST";
 487     postRequest.HTTPBody = [json supd_gzipDeflate];
 488     [postRequest setValue:@"gzip" forHTTPHeaderField:@"Content-Encoding"];
 489
 490     /*
 491      * Create the upload task.
 492      */
 493     dispatch_semaphore_t sem = dispatch_semaphore_create(0);
 494     __block BOOL uploadSuccess = NO;
 495     NSURLSessionDataTask* uploadTask = [postSession dataTaskWithRequest:postRequest
 496                                                       completionHandler:^(NSData * _Nullable __unused data, NSURLResponse * _Nullable response, NSError * _Nullable requestError) {
 497         if (requestError) {
 498             secerror("Error in uploading the events to splunk for %@: %@", self->_internalTopicName, requestError);
 499         } else if (![response isKindOfClass:NSHTTPURLResponse.class]){
 500             Class class = response.class;
 501             secerror("Received the wrong kind of response for %@: %@", self->_internalTopicName, NSStringFromClass(class));
 502         } else {
 503             NSHTTPURLResponse* httpResponse = (NSHTTPURLResponse*)response;
 504             if(httpResponse.statusCode >= 200 && httpResponse.statusCode < 300) {
 505                 /* Success */
 506                 uploadSuccess = YES;
 507                 secnotice("upload", "Splunk upload success for %@", self->_internalTopicName);
 508             } else {
 509                 secnotice("upload", "Splunk upload for %@ unexpected status to URL: %@ -- status: %d",
 510                           self->_internalTopicName, endpoint, (int)(httpResponse.statusCode));
 511             }
 512         }
 513         dispatch_semaphore_signal(sem);
 514     }];
 515     secnotice("upload", "Splunk upload start for %@", self->_internalTopicName);
 516     [uploadTask resume];
 517     dispatch_semaphore_wait(sem, dispatch_time(DISPATCH_TIME_NOW, (uint64_t)(5 * 60 * NSEC_PER_SEC)));
 518     return uploadSuccess;
 519 }
 520
 521 - (BOOL)eventIsBlacklisted:(NSMutableDictionary*)event {
 522     return _blacklistedEvents ? [_blacklistedEvents containsObject:event[SFAnalyticsEventType]] : NO;
 523 }
 524
 525 - (void)removeBlacklistedFieldsFromEvent:(NSMutableDictionary*)event {
 526     for (NSString* badField in self->_blacklistedFields) {
 527         [event removeObjectForKey:badField];
 528     }
 529 }
 530
 531 - (void)addRequiredFieldsToEvent:(NSMutableDictionary*)event {
 532     [_metricsBase enumerateKeysAndObjectsUsingBlock:^(id  _Nonnull key, id  _Nonnull obj, BOOL * _Nonnull stop) {
 533         if (!event[key]) {
 534             event[key] = obj;
 535         }
 536     }];
 537 }
 538
 539 - (BOOL)prepareEventForUpload:(NSMutableDictionary*)event {
 540     if ([self eventIsBlacklisted:event]) {
 541         return NO;
 542     }
 543
 544     [self removeBlacklistedFieldsFromEvent:event];
 545     [self addRequiredFieldsToEvent:event];
 546     if (_disableClientId) {
 547         event[SFAnalyticsClientId] = @(0);
 548     }
 549     event[SFAnalyticsSplunkTopic] = self->_splunkTopicName ?: [NSNull null];
 550     return YES;
 551 }
 552
 553 - (void)addFailures:(NSMutableArray<NSArray*>*)failures toUploadRecords:(NSMutableArray*)records threshold:(NSUInteger)threshold
 554 {
 555     // The first 0 through 'threshold' items are getting uploaded in any case (which might be 0 for lower priority data)
 556
 557     for (NSArray* client in failures) {
 558         [client enumerateObjectsUsingBlock:^(id  _Nonnull obj, NSUInteger idx, BOOL * _Nonnull stop) {
 559             NSMutableDictionary* event = (NSMutableDictionary*)obj;
 560             if (idx >= threshold) {
 561                 *stop = YES;
 562                 return;
 563             }
 564             if ([self prepareEventForUpload:event]) {
 565                 if ([NSJSONSerialization isValidJSONObject:event]) {
 566                     [records addObject:event];
 567                 } else {
 568                     secerror("supd: Replacing event with errorEvent because invalid JSON: %@", event);
 569                     NSString* originalType = event[SFAnalyticsEventType];
 570                     NSDictionary* errorEvent = @{ SFAnalyticsEventType : SFAnalyticsEventTypeErrorEvent,
 571                                                   SFAnalyticsEventErrorDestription : [NSString stringWithFormat:@"JSON:%@", originalType]};
 572                     [records addObject:errorEvent];
 573                 }
 574             }
 575         }];
 576     }
 577
 578     // Are there more items than we shoved into the upload records?
 579     NSInteger excessItems = 0;
 580     for (NSArray* client in failures) {
 581         NSInteger localExcess = client.count - threshold;
 582         excessItems += localExcess > 0 ? localExcess : 0;
 583     }
 584
 585     // Then, if we have space and items left, apply a scaling factor to distribute events across clients to fill upload buffer
 586     if (records.count < _maxEventsToReport && excessItems > 0) {
 587         double scale = (_maxEventsToReport - records.count) / (double)excessItems;
 588         if (scale > 1) {
 589             scale = 1;
 590         }
 591
 592         for (NSArray* client in failures) {
 593             if (client.count > threshold) {
 594                 NSRange range = NSMakeRange(threshold, (client.count - threshold) * scale);
 595                 NSArray* sub = [client subarrayWithRange:range];
 596                 [sub enumerateObjectsUsingBlock:^(id  _Nonnull obj, NSUInteger idx, BOOL * _Nonnull stop) {
 597                     if ([self prepareEventForUpload:obj]) {
 598                         [records addObject:obj];
 599                     }
 600                 }];
 601             }
 602         }
 603     }
 604 }
 605
 606 - (NSMutableDictionary*)sampleStatisticsForSamples:(NSArray*)samples withName:(NSString*)name
 607 {
 608     NSMutableDictionary* statistics = [NSMutableDictionary dictionary];
 609     NSUInteger count = samples.count;
 610     NSArray* sortedSamples = [samples sortedArrayUsingSelector:@selector(compare:)];
 611     NSArray* samplesAsExpressionArray = @[[NSExpression expressionForConstantValue:sortedSamples]];
 612
 613     if (count == 1) {
 614         statistics[name] = samples[0];
 615     } else {
 616         // NSExpression takes population standard deviation. Our data is a sample of whatever we sampled over time,
 617         // but the difference between the two is fairly minor (divide by N before taking sqrt versus divide by N-1).
 618         statistics[[NSString stringWithFormat:@"%@-dev", name]] = [[NSExpression expressionForFunction:@"stddev:" arguments:samplesAsExpressionArray] expressionValueWithObject:nil context:nil];
 619
 620         statistics[[NSString stringWithFormat:@"%@-min", name]] = [[NSExpression expressionForFunction:@"min:" arguments:samplesAsExpressionArray] expressionValueWithObject:nil context:nil];
 621         statistics[[NSString stringWithFormat:@"%@-max", name]] = [[NSExpression expressionForFunction:@"max:" arguments:samplesAsExpressionArray] expressionValueWithObject:nil context:nil];
 622         statistics[[NSString stringWithFormat:@"%@-avg", name]] = [[NSExpression expressionForFunction:@"average:" arguments:samplesAsExpressionArray] expressionValueWithObject:nil context:nil];
 623         statistics[[NSString stringWithFormat:@"%@-med", name]] = [[NSExpression expressionForFunction:@"median:" arguments:samplesAsExpressionArray] expressionValueWithObject:nil context:nil];
 624     }
 625
 626     if (count > 3) {
 627         NSString* q1 = [NSString stringWithFormat:@"%@-1q", name];
 628         NSString* q3 = [NSString stringWithFormat:@"%@-3q", name];
 629         // From Wikipedia, which is never wrong
 630         if (count % 2 == 0) {
 631             // The lower quartile value is the median of the lower half of the data. The upper quartile value is the median of the upper half of the data.
 632             statistics[q1] = [[NSExpression expressionForFunction:@"median:" arguments:@[[NSExpression expressionForConstantValue:[sortedSamples subarrayWithRange:NSMakeRange(0, count / 2)]]]] expressionValueWithObject:nil context:nil];
 633             statistics[q3] = [[NSExpression expressionForFunction:@"median:" arguments:@[[NSExpression expressionForConstantValue:[sortedSamples subarrayWithRange:NSMakeRange((count / 2), count / 2)]]]] expressionValueWithObject:nil context:nil];
 634         } else if (count % 4 == 1) {
 635             // If there are (4n+1) data points, then the lower quartile is 25% of the nth data value plus 75% of the (n+1)th data value;
 636             // the upper quartile is 75% of the (3n+1)th data point plus 25% of the (3n+2)th data point.
 637             // (offset n by -1 since we count from 0)
 638             NSUInteger n = count / 4;
 639             statistics[q1] = @(([sortedSamples[n - 1] doubleValue] + [sortedSamples[n] doubleValue] * 3.0) / 4.0);
 640             statistics[q3] = @(([sortedSamples[(3 * n)] doubleValue] * 3.0 + [sortedSamples[(3 * n) + 1] doubleValue]) / 4.0);
 641         } else if (count % 4 == 3){
 642             // If there are (4n+3) data points, then the lower quartile is 75% of the (n+1)th data value plus 25% of the (n+2)th data value;
 643             // the upper quartile is 25% of the (3n+2)th data point plus 75% of the (3n+3)th data point.
 644             // (offset n by -1 since we count from 0)
 645             NSUInteger n = count / 4;
 646             statistics[q1] = @(([sortedSamples[n] doubleValue] * 3.0 + [sortedSamples[n + 1] doubleValue]) / 4.0);
 647             statistics[q3] = @(([sortedSamples[(3 * n) + 1] doubleValue] + [sortedSamples[(3 * n) + 2] doubleValue] * 3.0) / 4.0);
 648         }
 649     }
 650
 651     return statistics;
 652 }
 653
 654 - (NSMutableDictionary*)healthSummaryWithName:(NSString*)name store:(SFAnalyticsSQLiteStore*)store
 655 {
 656     __block NSMutableDictionary* summary = [NSMutableDictionary new];
 657
 658     // Add some events of our own before pulling in data
 659     summary[SFAnalyticsEventType] = [NSString stringWithFormat:@"%@HealthSummary", name];
 660     if ([self eventIsBlacklisted:summary]) {
 661         return nil;
 662     }
 663     summary[SFAnalyticsEventTime] = @([[NSDate date] timeIntervalSince1970] * 1000);    // Splunk wants milliseconds
 664     [SFAnalytics addOSVersionToEvent:summary];
 665     if (store.uploadDate) {
 666         summary[SFAnalyticsAttributeLastUploadTime] = @([store.uploadDate timeIntervalSince1970] * 1000);
 667     } else {
 668         summary[SFAnalyticsAttributeLastUploadTime] = @(0);
 669     }
 670
 671     // Process counters
 672     NSDictionary* successCounts = store.summaryCounts;
 673     __block NSInteger totalSuccessCount = 0;
 674     __block NSInteger totalHardFailureCount = 0;
 675     __block NSInteger totalSoftFailureCount = 0;
 676     [successCounts enumerateKeysAndObjectsUsingBlock:^(NSString* _Nonnull eventType, NSDictionary* _Nonnull counts, BOOL* _Nonnull stop) {
 677         summary[[NSString stringWithFormat:@"%@-success", eventType]] = counts[SFAnalyticsColumnSuccessCount];
 678         summary[[NSString stringWithFormat:@"%@-hardfail", eventType]] = counts[SFAnalyticsColumnHardFailureCount];
 679         summary[[NSString stringWithFormat:@"%@-softfail", eventType]] = counts[SFAnalyticsColumnSoftFailureCount];
 680         totalSuccessCount += [counts[SFAnalyticsColumnSuccessCount] integerValue];
 681         totalHardFailureCount += [counts[SFAnalyticsColumnHardFailureCount] integerValue];
 682         totalSoftFailureCount += [counts[SFAnalyticsColumnSoftFailureCount] integerValue];
 683     }];
 684
 685     summary[SFAnalyticsColumnSuccessCount] = @(totalSuccessCount);
 686     summary[SFAnalyticsColumnHardFailureCount] = @(totalHardFailureCount);
 687     summary[SFAnalyticsColumnSoftFailureCount] = @(totalSoftFailureCount);
 688     if (os_variant_has_internal_diagnostics("com.apple.security")) {
 689         summary[SFAnalyticsInternal] = @YES;
 690     }
 691
 692     // Process samples
 693     NSMutableDictionary<NSString*,NSMutableArray*>* samplesBySampler = [NSMutableDictionary<NSString*,NSMutableArray*> dictionary];
 694     for (NSDictionary* sample in [store samples]) {
 695         if (!samplesBySampler[sample[SFAnalyticsColumnSampleName]]) {
 696             samplesBySampler[sample[SFAnalyticsColumnSampleName]] = [NSMutableArray array];
 697         }
 698         [samplesBySampler[sample[SFAnalyticsColumnSampleName]] addObject:sample[SFAnalyticsColumnSampleValue]];
 699     }
 700     [samplesBySampler enumerateKeysAndObjectsUsingBlock:^(NSString * _Nonnull key, NSMutableArray * _Nonnull obj, BOOL * _Nonnull stop) {
 701         NSMutableDictionary* event = [self sampleStatisticsForSamples:obj withName:key];
 702         [summary addEntriesFromDictionary:event];
 703     }];
 704
 705     // Should always return yes because we already checked for event blacklisting specifically (unless summary itself is blacklisted)
 706     if (![self prepareEventForUpload:summary]) {
 707         secwarning("supd: health summary for %@ blacklisted", name);
 708         return nil;
 709     }
 710
 711     // Seems unlikely because we only insert strings, samplers only take NSNumbers and frankly, sampleStatisticsForSamples probably would have crashed
 712     if (![NSJSONSerialization isValidJSONObject:summary]) {
 713         secerror("json: health summary for client %@ is invalid JSON: %@", name, summary);
 714         return [@{ SFAnalyticsEventType : SFAnalyticsEventTypeErrorEvent,
 715                    SFAnalyticsEventErrorDestription : [NSString stringWithFormat:@"JSON:%@HealthSummary", name]} mutableCopy];
 716     }
 717
 718     return summary;
 719 }
 720
 721 - (void)updateUploadDateForClients:(NSArray<SFAnalyticsClient*>*)clients date:(NSDate *)date clearData:(BOOL)clearData
 722 {
 723     for (SFAnalyticsClient* client in clients) {
 724         SFAnalyticsSQLiteStore* store = [SFAnalyticsSQLiteStore storeWithPath:client.storePath schema:SFAnalyticsTableSchema];
 725         secnotice("postprocess", "Setting upload date (%@) for client: %@", date, client.name);
 726         store.uploadDate = date;
 727         if (clearData) {
 728             secnotice("postprocess", "Clearing collected data for client: %@", client.name);
 729             [store clearAllData];
 730         }
 731     }
 732 }
 733
 734 - (NSData*)getLoggingJSON:(bool)pretty
 735                 forUpload:(BOOL)upload
 736      participatingClients:(NSMutableArray<SFAnalyticsClient*>**)clients
 737                     force:(BOOL)force                                       // supdctl uploads ignore privacy settings and recency
 738                     error:(NSError**)error
 739 {
 740     NSMutableArray<SFAnalyticsClient*>* localClients = [NSMutableArray new];
 741     __block NSMutableArray* uploadRecords = [NSMutableArray arrayWithCapacity:_maxEventsToReport];
 742     __block NSError *localError;
 743     __block NSMutableArray<NSArray*>* hardFailures = [NSMutableArray new];
 744     __block NSMutableArray<NSArray*>* softFailures = [NSMutableArray new];
 745     NSString* ckdeviceID = nil;
 746     NSString* accountID = nil;
 747
 748     if (os_variant_has_internal_diagnostics("com.apple.security") && [_internalTopicName isEqualToString:SFAnalyticsTopicKeySync]) {
 749         ckdeviceID = [self askSecurityForCKDeviceID];
 750         accountID = accountAltDSID();
 751     }
 752     for (SFAnalyticsClient* client in self->_topicClients) {
 753         if (!force && [client requireDeviceAnalytics] && !_isDeviceAnalyticsEnabled()) {
 754             // Client required device analytics, yet the user did not opt in.
 755             secnotice("getLoggingJSON", "Client '%@' requires device analytics yet user did not opt in.", [client name]);
 756             continue;
 757         }
 758         if (!force && [client requireiCloudAnalytics] && !_isiCloudAnalyticsEnabled()) {
 759             // Client required iCloud analytics, yet the user did not opt in.
 760             secnotice("getLoggingJSON", "Client '%@' requires iCloud analytics yet user did not opt in.", [client name]);
 761             continue;
 762         }
 763
 764         SFAnalyticsSQLiteStore* store = [SFAnalyticsSQLiteStore storeWithPath:client.storePath schema:SFAnalyticsTableSchema];
 765
 766         if (upload) {
 767             NSDate* uploadDate = store.uploadDate;
 768             if (!force && uploadDate && [[NSDate date] timeIntervalSinceDate:uploadDate] < _secondsBetweenUploads) {
 769                 secnotice("json", "ignoring client '%@' for %@ because last upload too recent: %@",
 770                           client.name, _internalTopicName, uploadDate);
 771                 continue;
 772             }
 773
 774             if (force) {
 775                 secnotice("json", "client '%@' for topic '%@' force-included", client.name, _internalTopicName);
 776             } else {
 777                 secnotice("json", "including client '%@' for topic '%@' for upload", client.name, _internalTopicName);
 778             }
 779             [localClients addObject:client];
 780         }
 781
 782         NSMutableDictionary* healthSummary = [self healthSummaryWithName:client.name store:store];
 783         if (healthSummary) {
 784             if (ckdeviceID) {
 785                 healthSummary[SFAnalyticsDeviceID] = ckdeviceID;
 786             }
 787             if (accountID) {
 788                 healthSummary[SFAnalyticsAltDSID] = accountID;
 789             }
 790             [uploadRecords addObject:healthSummary];
 791         }
 792
 793         [hardFailures addObject:store.hardFailures];
 794         [softFailures addObject:store.softFailures];
 795     }
 796
 797     if (upload && [localClients count] == 0) {
 798         if (error) {
 799             NSString *description = [NSString stringWithFormat:@"Upload too recent for all clients for %@", _internalTopicName];
 800             *error = [NSError errorWithDomain:@"SupdUploadErrorDomain"
 801                                          code:-10
 802                                      userInfo:@{NSLocalizedDescriptionKey : description}];
 803         }
 804         return nil;
 805     }
 806
 807     if (clients) {
 808         *clients = localClients;
 809     }
 810
 811     [self addFailures:hardFailures toUploadRecords:uploadRecords threshold:_maxEventsToReport/10];
 812     [self addFailures:softFailures toUploadRecords:uploadRecords threshold:0];
 813
 814     NSDictionary* jsonDict = @{
 815                                SFAnalyticsSplunkPostTime : @([[NSDate date] timeIntervalSince1970] * 1000),
 816                                @"events" : uploadRecords
 817                                };
 818
 819     // This check is "belt and suspenders" because we already checked each event separately
 820     if (![NSJSONSerialization isValidJSONObject:jsonDict]) {
 821         secemergency("json: final dictionary invalid JSON. This is terrible!");
 822         if (error) {
 823             *error = [NSError errorWithDomain:SupdErrorDomain code:SupdInvalidJSONError
 824                                      userInfo:@{NSLocalizedDescriptionKey : [NSString localizedStringWithFormat:@"Final dictionary for upload is invalid JSON: %@", jsonDict]}];
 825         }
 826         return nil;
 827     }
 828
 829     NSData *json = [NSJSONSerialization dataWithJSONObject:jsonDict
 830                                                    options:(pretty ? NSJSONWritingPrettyPrinted : 0)
 831                                                      error:&localError];
 832
 833     if (error) {
 834         *error = localError;
 835     }
 836
 837     return json;
 838 }
 839
 840 // Is at least one client eligible for data collection based on user consent? Otherwise callers should NOT reach off-device.
 841 - (BOOL)haveEligibleClients {
 842     for (SFAnalyticsClient* client in self.topicClients) {
 843         if ((!client.requireDeviceAnalytics || _isDeviceAnalyticsEnabled()) &&
 844             (!client.requireiCloudAnalytics || _isiCloudAnalyticsEnabled())) {
 845             return YES;
 846         }
 847     }
 848     return NO;
 849 }
 850
 851 - (NSString*)askSecurityForCKDeviceID
 852 {
 853     NSError* error = nil;
 854     CKKSControl* rpc = [CKKSControl controlObject:&error];
 855     if(error || !rpc) {
 856         secerror("unable to obtain CKKS endpoint: %@", error);
 857         return nil;
 858     }
 859
 860     __block NSString* localCKDeviceID;
 861     dispatch_semaphore_t sema = dispatch_semaphore_create(0);
 862     [rpc rpcGetCKDeviceIDWithReply:^(NSString* ckdeviceID) {
 863         localCKDeviceID = ckdeviceID;
 864         dispatch_semaphore_signal(sema);
 865     }];
 866
 867     if (dispatch_semaphore_wait(sema, dispatch_time(DISPATCH_TIME_NOW, NSEC_PER_SEC * 10)) != 0) {
 868         secerror("timed out waiting for a response from security");
 869         return nil;
 870     }
 871
 872     return localCKDeviceID;
 873 }
 874
 875 // this method is kind of evil for the fact that it has side-effects in pulling other things besides the metricsURL from the server, and as such should NOT be memoized.
 876 // TODO redo this, probably to return a dictionary.
 877 - (NSURL*)splunkUploadURL:(BOOL)force
 878 {
 879     if (!force && ![self haveEligibleClients]) {    // force is true IFF called from supdctl. Customers don't have it and internal audiences must call it explicitly.
 880         secnotice("getURL", "Not going to talk to server for topic %@ because no eligible clients", [self internalTopicName]);
 881         return nil;
 882     }
 883
 884     if (__splunkUploadURL) {
 885         return __splunkUploadURL;
 886     }
 887
 888     secnotice("getURL", "Asking server for endpoint and config data for topic %@", [self internalTopicName]);
 889
 890     __weak __typeof(self) weakSelf = self;
 891     dispatch_semaphore_t sem = dispatch_semaphore_create(0);
 892
 893     __block NSError* error = nil;
 894     NSURLSessionConfiguration *configuration = [NSURLSessionConfiguration ephemeralSessionConfiguration];
 895     NSURLSession* storeBagSession = [NSURLSession sessionWithConfiguration:configuration
 896                                                                   delegate:self
 897                                                              delegateQueue:nil];
 898
 899     NSURL* requestEndpoint = _splunkBagURL;
 900     __block NSURL* result = nil;
 901     NSURLSessionDataTask* storeBagTask = [storeBagSession dataTaskWithURL:requestEndpoint completionHandler:^(NSData * _Nullable data,
 902                                                                                                               NSURLResponse * _Nullable __unused response,
 903                                                                                                               NSError * _Nullable responseError) {
 904
 905         __strong __typeof(self) strongSelf = weakSelf;
 906         if (!strongSelf) {
 907             return;
 908         }
 909
 910         if (data && !responseError) {
 911             NSData *responseData = data; // shut up compiler
 912             NSDictionary* responseDict = [NSJSONSerialization JSONObjectWithData:responseData options:0 error:&error];
 913             if([responseDict isKindOfClass:NSDictionary.class] && !error) {
 914                 if (!self->_ignoreServersMessagesTellingUsToGoAway) {
 915                     self->_disableUploads = [[responseDict valueForKey:@"sendDisabled"] boolValue];
 916                     if (self->_disableUploads) {
 917                         // then don't upload anything right now
 918                         secerror("not returning a splunk URL because uploads are disabled for %@", self->_internalTopicName);
 919                         dispatch_semaphore_signal(sem);
 920                         return;
 921                     }
 922
 923                     // backend works with milliseconds
 924                     NSUInteger secondsBetweenUploads = [[responseDict valueForKey:@"postFrequency"] unsignedIntegerValue] / 1000;
 925                     if (secondsBetweenUploads > 0) {
 926                         if (os_variant_has_internal_diagnostics("com.apple.security") &&
 927                             self->_secondsBetweenUploads < secondsBetweenUploads) {
 928                             secnotice("getURL", "Overriding server-sent post frequency because device is internal (%lu -> %lu)", (unsigned long)secondsBetweenUploads, (unsigned long)self->_secondsBetweenUploads);
 929                         } else {
 930                             strongSelf->_secondsBetweenUploads = secondsBetweenUploads;
 931                         }
 932                     }
 933
 934                     strongSelf->_blacklistedEvents = responseDict[@"blacklistedEvents"];
 935                     strongSelf->_blacklistedFields = responseDict[@"blacklistedFields"];
 936                 }
 937
 938                 strongSelf->_metricsBase = responseDict[@"metricsBase"];
 939
 940                 NSString* metricsEndpoint = responseDict[@"metricsUrl"];
 941                 if([metricsEndpoint isKindOfClass:NSString.class]) {
 942                     /* Lives our URL */
 943                     NSString* endpoint = [metricsEndpoint stringByAppendingFormat:@"/2/%@", strongSelf->_splunkTopicName];
 944                     secnotice("upload", "got metrics endpoint %@ for %@", endpoint, self->_internalTopicName);
 945                     NSURL* endpointURL = [NSURL URLWithString:endpoint];
 946                     if([endpointURL.scheme isEqualToString:@"https"]) {
 947                         result = endpointURL;
 948                     }
 949                 }
 950             }
 951         }
 952         else {
 953             error = responseError;
 954         }
 955         if (error) {
 956             secnotice("upload", "Unable to fetch splunk endpoint at URL for %@: %@ -- error: %@",
 957                       self->_internalTopicName, requestEndpoint, error.description);
 958         }
 959         else if (!result) {
 960             secnotice("upload", "Malformed iTunes config payload for %@!", self->_internalTopicName);
 961         }
 962
 963         dispatch_semaphore_signal(sem);
 964     }];
 965
 966     [storeBagTask resume];
 967     dispatch_semaphore_wait(sem, dispatch_time(DISPATCH_TIME_NOW, (uint64_t)(60 * NSEC_PER_SEC)));
 968
 969     return result;
 970 }
 971
 972 - (void)URLSession:(NSURLSession *)session didReceiveChallenge:(NSURLAuthenticationChallenge *)challenge
 973  completionHandler:(void (^)(NSURLSessionAuthChallengeDisposition, NSURLCredential *))completionHandler {
 974     assert(completionHandler);
 975     (void)session;
 976     secnotice("upload", "Splunk upload challenge for %@", _internalTopicName);
 977     NSURLCredential *cred = nil;
 978
 979     if ([challenge previousFailureCount] > 0) {
 980         // Previous failures occurred, bail
 981         completionHandler(NSURLSessionAuthChallengeCancelAuthenticationChallenge, nil);
 982
 983     } else if ([challenge.protectionSpace.authenticationMethod isEqualToString:NSURLAuthenticationMethodServerTrust]) {
 984         /*
 985          * Evaluate trust for the certificate
 986          */
 987
 988         SecTrustRef serverTrust = challenge.protectionSpace.serverTrust;
 989         // Coverity gets upset if we don't check status even though result is all we need.
 990         bool trustResult = SecTrustEvaluateWithError(serverTrust, NULL);
 991         if (_allowInsecureSplunkCert || trustResult) {
 992             /*
 993              * All is well, accept the credentials
 994              */
 995             if(_allowInsecureSplunkCert) {
 996                 secnotice("upload", "Force Accepting Splunk Credential for %@", _internalTopicName);
 997             }
 998             cred = [NSURLCredential credentialForTrust:serverTrust];
 999             completionHandler(NSURLSessionAuthChallengeUseCredential, cred);
1000
1001         } else {
1002             /*
1003              * An error occurred in evaluating trust, bail
1004              */
1005             completionHandler(NSURLSessionAuthChallengeCancelAuthenticationChallenge, nil);
1006         }
1007     } else {
1008         /*
1009          * Just perform the default handling
1010          */
1011         completionHandler(NSURLSessionAuthChallengePerformDefaultHandling, nil);
1012     }
1013 }
1014
1015 - (NSDictionary*)eventDictWithBlacklistedFieldsStrippedFrom:(NSDictionary*)eventDict
1016 {
1017     NSMutableDictionary* strippedDict = eventDict.mutableCopy;
1018     for (NSString* blacklistedField in _blacklistedFields) {
1019         [strippedDict removeObjectForKey:blacklistedField];
1020     }
1021     return strippedDict;
1022 }
1023
1024 // MARK: Database path retrieval
1025
1026 + (NSString*)databasePathForCKKS
1027 {
1028     return [(__bridge_transfer NSURL*)SecCopyURLForFileInKeychainDirectory((__bridge CFStringRef)@"Analytics/ckks_analytics.db") path];
1029 }
1030
1031 + (NSString*)databasePathForSOS
1032 {
1033     return [(__bridge_transfer NSURL*)SecCopyURLForFileInKeychainDirectory((__bridge CFStringRef)@"Analytics/sos_analytics.db") path];
1034 }
1035
1036 + (NSString*)AppSupportPath
1037 {
1038 #if TARGET_OS_IPHONE && !TARGET_OS_SIMULATOR
1039     return @"/var/mobile/Library/Application Support";
1040 #else
1041     NSArray<NSString *>*paths = NSSearchPathForDirectoriesInDomains(NSApplicationSupportDirectory, NSUserDomainMask, true);
1042     if ([paths count] < 1) {
1043         return nil;
1044     }
1045     return [NSString stringWithString: paths[0]];
1046 #endif /* TARGET_OS_IPHONE && !TARGET_OS_SIMULATOR  */
1047 }
1048
1049 + (NSString*)databasePathForPCS
1050 {
1051     NSString *appSup = [self AppSupportPath];
1052     if (!appSup) {
1053         return nil;
1054     }
1055     NSString *dbpath = [NSString stringWithFormat:@"%@/com.apple.ProtectedCloudStorage/PCSAnalytics.db", appSup];
1056     secnotice("supd", "PCS Database path (%@)", dbpath);
1057     return dbpath;
1058 }
1059
1060 + (NSString*)databasePathForLocal
1061 {
1062     return [(__bridge_transfer NSURL*)SecCopyURLForFileInKeychainDirectory((__bridge CFStringRef)@"Analytics/localkeychain.db") path];
1063 }
1064
1065 + (NSString*)databasePathForTrustdHealth
1066 {
1067 #if TARGET_OS_IPHONE
1068     return [(__bridge_transfer NSURL*)SecCopyURLForFileInKeychainDirectory(CFSTR("Analytics/trustd_health_analytics.db")) path];
1069 #else
1070     return [(__bridge_transfer NSURL*)SecCopyURLForFileInUserCacheDirectory(CFSTR("Analytics/trustd_health_analytics.db")) path];
1071 #endif
1072 }
1073
1074 + (NSString*)databasePathForTrust
1075 {
1076 #if TARGET_OS_IPHONE
1077     return [(__bridge_transfer NSURL*)SecCopyURLForFileInKeychainDirectory(CFSTR("Analytics/trust_analytics.db")) path];
1078 #else
1079     return [(__bridge_transfer NSURL*)SecCopyURLForFileInUserCacheDirectory(CFSTR("Analytics/trust_analytics.db")) path];
1080 #endif
1081 }
1082
1083 + (NSString*)databasePathForTLS
1084 {
1085 #if TARGET_OS_IPHONE
1086     return [(__bridge_transfer NSURL*)SecCopyURLForFileInKeychainDirectory(CFSTR("Analytics/TLS_analytics.db")) path];
1087 #else
1088     return [(__bridge_transfer NSURL*)SecCopyURLForFileInUserCacheDirectory(CFSTR("Analytics/TLS_analytics.db")) path];
1089 #endif
1090 }
1091
1092 + (NSString*)databasePathForSignIn
1093 {
1094     return [(__bridge_transfer NSURL*)SecCopyURLForFileInKeychainDirectory(CFSTR("Analytics/signin_metrics.db")) path];
1095 }
1096
1097 + (NSString*)databasePathForCloudServices
1098 {
1099     return [(__bridge_transfer NSURL*)SecCopyURLForFileInKeychainDirectory(CFSTR("Analytics/CloudServicesAnalytics.db")) path];
1100 }
1101
1102 + (NSString*)databasePathForTransparency
1103 {
1104     return [(__bridge_transfer NSURL*)SecCopyURLForFileInKeychainDirectory((__bridge CFStringRef)@"Analytics/TransparencyAnalytics.db") path];
1105 }
1106
1107 @end
1108
1109 @interface supd ()
1110 @property NSDictionary *topicsSamplingRates;
1111 @end
1112
1113 @implementation supd
1114 - (void)setupTopics
1115 {
1116     NSDictionary* systemDefaultValues = [NSDictionary dictionaryWithContentsOfFile:[[NSBundle bundleWithPath:@"/System/Library/Frameworks/Security.framework"] pathForResource:@"SFAnalytics" ofType:@"plist"]];
1117     NSMutableArray <SFAnalyticsTopic*>* topics = [NSMutableArray array];
1118     for (NSString *topicKey in systemDefaultValues) {
1119         NSDictionary *topicSamplingRates = _topicsSamplingRates[topicKey];
1120         SFAnalyticsTopic *topic = [[SFAnalyticsTopic alloc] initWithDictionary:systemDefaultValues[topicKey] name:topicKey samplingRates:topicSamplingRates];
1121         [topics addObject:topic];
1122     }
1123     _analyticsTopics = [NSArray arrayWithArray:topics];
1124 }
1125
1126 + (void)instantiate {
1127     [supd instance];
1128 }
1129
1130 + (instancetype)instance {
1131     if (!_supdInstance) {
1132         _supdInstance = [self new];
1133     }
1134     return _supdInstance;
1135 }
1136
1137 // Use this for testing to get rid of any state
1138 + (void)removeInstance {
1139     _supdInstance = nil;
1140 }
1141
1142
1143 static NSString *SystemTrustStorePath = @"/System/Library/Security/Certificates.bundle";
1144 static NSString *AnalyticsSamplingRatesFilename = @"AnalyticsSamplingRates";
1145 static NSString *ContentVersionKey = @"MobileAssetContentVersion";
1146 static NSString *AssetContextFilename = @"OTAPKIContext.plist";
1147
1148 static NSNumber *getSystemVersion(NSBundle *trustStoreBundle) {
1149     NSDictionary *systemVersionPlist = [NSDictionary dictionaryWithContentsOfURL:[trustStoreBundle URLForResource:@"AssetVersion"
1150                                                                                                     withExtension:@"plist"]];
1151     if (!systemVersionPlist || ![systemVersionPlist isKindOfClass:[NSDictionary class]]) {
1152         return nil;
1153     }
1154     NSNumber *systemVersion = systemVersionPlist[ContentVersionKey];
1155     if (systemVersion == nil || ![systemVersion isKindOfClass:[NSNumber class]]) {
1156         return nil;
1157     }
1158     return systemVersion;
1159 }
1160
1161 static NSNumber *getAssetVersion(NSURL *directory) {
1162     NSDictionary *assetContextPlist = [NSDictionary dictionaryWithContentsOfURL:[directory URLByAppendingPathComponent:AssetContextFilename]];
1163     if (!assetContextPlist || ![assetContextPlist isKindOfClass:[NSDictionary class]]) {
1164         return nil;
1165     }
1166     NSNumber *assetVersion = assetContextPlist[ContentVersionKey];
1167     if (assetVersion == nil || ![assetVersion isKindOfClass:[NSNumber class]]) {
1168         return nil;
1169     }
1170     return assetVersion;
1171 }
1172
1173 static bool ShouldInitializeWithAsset(NSBundle *trustStoreBundle, NSURL *directory) {
1174     NSNumber *systemVersion = getSystemVersion(trustStoreBundle);
1175     NSNumber *assetVersion = getAssetVersion(directory);
1176
1177     if (assetVersion == nil || systemVersion == nil) {
1178         return false;
1179     }
1180     if ([assetVersion compare:systemVersion] == NSOrderedDescending) {
1181         return true;
1182     }
1183     return false;
1184 }
1185
1186 - (void)setupSamplingRates {
1187     NSBundle *trustStoreBundle = [NSBundle bundleWithPath:SystemTrustStorePath];
1188
1189     NSURL *keychainsDirectory = CFBridgingRelease(SecCopyURLForFileInSystemKeychainDirectory(nil));
1190     NSURL *directory = [keychainsDirectory URLByAppendingPathComponent:@"SupplementalsAssets/" isDirectory:YES];
1191
1192     NSDictionary *analyticsSamplingRates = nil;
1193     if (ShouldInitializeWithAsset(trustStoreBundle, directory)) {
1194         /* Try to get the asset version of the sampling rates */
1195         NSURL *analyticsSamplingRateURL = [directory URLByAppendingPathComponent:[NSString stringWithFormat:@"%@.plist", AnalyticsSamplingRatesFilename]];
1196         analyticsSamplingRates = [NSDictionary dictionaryWithContentsOfURL:analyticsSamplingRateURL];
1197         secnotice("supd", "read sampling rates from SupplementalsAssets dir");
1198         if (!analyticsSamplingRates || ![analyticsSamplingRates isKindOfClass:[NSDictionary class]]) {
1199             analyticsSamplingRates = nil;
1200         }
1201     }
1202     if (!analyticsSamplingRates) {
1203         analyticsSamplingRates = [NSDictionary dictionaryWithContentsOfURL: [trustStoreBundle URLForResource:AnalyticsSamplingRatesFilename
1204                                                                                                withExtension:@"plist"]];
1205     }
1206     if (analyticsSamplingRates && [analyticsSamplingRates isKindOfClass:[NSDictionary class]]) {
1207         _topicsSamplingRates = analyticsSamplingRates[@"Topics"];
1208         if (!_topicsSamplingRates || ![analyticsSamplingRates isKindOfClass:[NSDictionary class]]) {
1209             _topicsSamplingRates = nil; // Something has gone terribly wrong, so we'll use the hardcoded defaults in this case
1210         }
1211     }
1212 }
1213
1214 - (instancetype)initWithReporter:(SFAnalyticsReporter *)reporter
1215 {
1216     if (self = [super init]) {
1217         [self setupSamplingRates];
1218         [self setupTopics];
1219         _reporter = reporter;
1220
1221         xpc_activity_register("com.apple.securityuploadd.triggerupload", XPC_ACTIVITY_CHECK_IN, ^(xpc_activity_t activity) {
1222             xpc_activity_state_t activityState = xpc_activity_get_state(activity);
1223             secnotice("supd", "hit xpc activity trigger, state: %ld", activityState);
1224             if (activityState == XPC_ACTIVITY_STATE_RUN) {
1225                 // Run our regularly scheduled scan
1226                 [self performRegularlyScheduledUpload];
1227             }
1228         });
1229     }
1230
1231     return self;
1232 }
1233
1234 - (instancetype)init {
1235     SFAnalyticsReporter *reporter = [[SFAnalyticsReporter alloc] init];
1236     return [self initWithReporter:reporter];
1237 }
1238
1239 - (void)sendNotificationForOncePerReportSamplers
1240 {
1241     notify_post(SFAnalyticsFireSamplersNotification);
1242     [NSThread sleepForTimeInterval:3.0];
1243 }
1244
1245 - (void)performRegularlyScheduledUpload {
1246     secnotice("upload", "Starting uploads in response to regular trigger");
1247     NSError *error = nil;
1248     if ([self uploadAnalyticsWithError:&error force:NO]) {
1249         secnotice("upload", "Regularly scheduled upload successful");
1250     } else {
1251         secerror("upload: Failed to complete regularly scheduled upload: %@", error);
1252     }
1253 }
1254
1255 - (BOOL)uploadAnalyticsWithError:(NSError**)error force:(BOOL)force {
1256     [self sendNotificationForOncePerReportSamplers];
1257
1258     BOOL result = NO;
1259     NSError* localError = nil;
1260     for (SFAnalyticsTopic *topic in _analyticsTopics) {
1261         @autoreleasepool { // The logging JSONs get quite large. Ensure they're deallocated between topics.
1262             __block NSURL* endpoint = [topic splunkUploadURL:force];   // has side effects!
1263
1264             if (!endpoint) {
1265                 secnotice("upload", "Skipping upload for %@ because no endpoint", [topic internalTopicName]);
1266                 continue;
1267             }
1268
1269             if ([topic disableUploads]) {
1270                 secnotice("upload", "Aborting upload task for %@ because uploads are disabled", [topic internalTopicName]);
1271                 continue;
1272             }
1273
1274             NSMutableArray<SFAnalyticsClient*>* clients = [NSMutableArray new];
1275             NSData* json = [topic getLoggingJSON:false forUpload:YES participatingClients:&clients force:force error:&localError];
1276             if (json) {
1277                 if ([topic isSampledUpload]) {
1278                     if (![self->_reporter saveReport:json fileName:[topic internalTopicName]]) {
1279                         secerror("upload: failed to write analytics data to log");
1280                     }
1281                     if ([topic postJSON:json toEndpoint:endpoint error:&localError]) {
1282                         secnotice("upload", "Successfully posted JSON for %@", [topic internalTopicName]);
1283                         result = YES;
1284                         [topic updateUploadDateForClients:clients date:[NSDate date] clearData:YES];
1285                     } else {
1286                         secerror("upload: Failed to post JSON for %@: %@", [topic internalTopicName], localError);
1287                     }
1288                 } else {
1289                     /* If we didn't sample this report, update date to prevent trying to upload again sooner
1290                      * than we should. Clear data so that per-day calculations remain consistent. */
1291                     secnotice("upload", "skipping unsampled upload for %@ and clearing data", [topic internalTopicName]);
1292                     [topic updateUploadDateForClients:clients date:[NSDate date] clearData:YES];
1293                 }
1294             } else {
1295                 if ([[localError domain] isEqualToString:SupdErrorDomain] && [localError code] == SupdInvalidJSONError) {
1296                     // Pretend this was a success because at least we'll get rid of bad data.
1297                     // If someone keeps logging bad data and we only catch it here then
1298                     // this causes sustained data loss for the entire topic.
1299                     [topic updateUploadDateForClients:clients date:[NSDate date] clearData:YES];
1300                 }
1301                 secerror("upload: failed to get logging JSON for topic %@: %@", [topic internalTopicName], localError);
1302             }
1303         }
1304         if (error && localError) {
1305             *error = localError;
1306         }
1307     }
1308     return result;
1309 }
1310
1311 - (NSString*)sysdiagnoseStringForEventRecord:(NSDictionary*)eventRecord
1312 {
1313     NSMutableDictionary* mutableEventRecord = eventRecord.mutableCopy;
1314     [mutableEventRecord removeObjectForKey:SFAnalyticsSplunkTopic];
1315
1316     NSDate* eventDate = [NSDate dateWithTimeIntervalSince1970:[[eventRecord valueForKey:SFAnalyticsEventTime] doubleValue] / 1000];
1317     [mutableEventRecord removeObjectForKey:SFAnalyticsEventTime];
1318
1319     NSString* eventName = eventRecord[SFAnalyticsEventType];
1320     [mutableEventRecord removeObjectForKey:SFAnalyticsEventType];
1321
1322     SFAnalyticsEventClass eventClass = [[eventRecord valueForKey:SFAnalyticsEventClassKey] integerValue];
1323     NSString* eventClassString = [self stringForEventClass:eventClass];
1324     [mutableEventRecord removeObjectForKey:SFAnalyticsEventClassKey];
1325
1326     NSMutableString* additionalAttributesString = [NSMutableString string];
1327     if (mutableEventRecord.count > 0) {
1328         [additionalAttributesString appendString:@" - Attributes: {" ];
1329         __block BOOL firstAttribute = YES;
1330         [mutableEventRecord enumerateKeysAndObjectsUsingBlock:^(NSString* key, id object, BOOL* stop) {
1331             NSString* openingString = firstAttribute ? @"" : @", ";
1332             [additionalAttributesString appendString:[NSString stringWithFormat:@"%@%@ : %@", openingString, key, object]];
1333             firstAttribute = NO;
1334         }];
1335         [additionalAttributesString appendString:@" }"];
1336     }
1337
1338     return [NSString stringWithFormat:@"%@ %@: %@%@", eventDate, eventClassString, eventName, additionalAttributesString];
1339 }
1340
1341 - (NSString*)getSysdiagnoseDump
1342 {
1343     NSMutableString* sysdiagnose = [[NSMutableString alloc] init];
1344
1345     for (SFAnalyticsTopic* topic in _analyticsTopics) {
1346         for (SFAnalyticsClient* client in topic.topicClients) {
1347             [sysdiagnose appendString:[NSString stringWithFormat:@"Client: %@\n", client.name]];
1348             SFAnalyticsSQLiteStore* store = [SFAnalyticsSQLiteStore storeWithPath:client.storePath schema:SFAnalyticsTableSchema];
1349             NSArray* allEvents = store.allEvents;
1350             for (NSDictionary* eventRecord in allEvents) {
1351                 [sysdiagnose appendFormat:@"%@\n", [self sysdiagnoseStringForEventRecord:eventRecord]];
1352             }
1353             if (allEvents.count == 0) {
1354                 [sysdiagnose appendString:@"No data to report for this client\n"];
1355             }
1356         }
1357     }
1358     return sysdiagnose;
1359 }
1360
1361 - (void)setUploadDateWith:(NSDate *)date reply:(void (^)(BOOL, NSError*))reply
1362 {
1363     for (SFAnalyticsTopic* topic in _analyticsTopics) {
1364         [topic updateUploadDateForClients:topic.topicClients date:date clearData:NO];
1365     }
1366     reply(YES, nil);
1367 }
1368
1369 - (void)clientStatus:(void (^)(NSDictionary<NSString *, id> *, NSError *))reply
1370 {
1371     NSMutableDictionary *info = [NSMutableDictionary dictionary];
1372     for (SFAnalyticsTopic* topic in _analyticsTopics) {
1373         for (SFAnalyticsClient *client in topic.topicClients) {
1374             SFAnalyticsSQLiteStore* store = [SFAnalyticsSQLiteStore storeWithPath:client.storePath schema:SFAnalyticsTableSchema];
1375
1376             NSMutableDictionary *clientInfo = [NSMutableDictionary dictionary];
1377             clientInfo[@"uploadDate"] = store.uploadDate;
1378             info[client.name] = clientInfo;
1379         }
1380     }
1381
1382     reply(info, nil);
1383 }
1384
1385
1386
1387 - (NSString*)stringForEventClass:(SFAnalyticsEventClass)eventClass
1388 {
1389     if (eventClass == SFAnalyticsEventClassNote) {
1390         return @"EventNote";
1391     }
1392     else if (eventClass == SFAnalyticsEventClassSuccess) {
1393         return @"EventSuccess";
1394     }
1395     else if (eventClass == SFAnalyticsEventClassHardFailure) {
1396         return @"EventHardFailure";
1397     }
1398     else if (eventClass == SFAnalyticsEventClassSoftFailure) {
1399         return @"EventSoftFailure";
1400     }
1401     else {
1402         return @"EventUnknown";
1403     }
1404 }
1405
1406 // MARK: XPC Procotol Handlers
1407
1408 - (void)getSysdiagnoseDumpWithReply:(void (^)(NSString*))reply {
1409     reply([self getSysdiagnoseDump]);
1410 }
1411
1412 - (void)getLoggingJSON:(bool)pretty topic:(NSString *)topicName reply:(void (^)(NSData*, NSError*))reply {
1413     secnotice("rpcGetLoggingJSON", "Building a JSON blob resembling the one we would have uploaded");
1414     NSError* error = nil;
1415     [self sendNotificationForOncePerReportSamplers];
1416     NSData* json = nil;
1417     for (SFAnalyticsTopic* topic in self->_analyticsTopics) {
1418         if ([topic.internalTopicName isEqualToString:topicName]) {
1419             json = [topic getLoggingJSON:pretty forUpload:NO participatingClients:nil force:!runningTests error:&error];
1420         }
1421     }
1422     if (!json) {
1423         secerror("Unable to obtain JSON: %@", error);
1424     }
1425     reply(json, error);
1426 }
1427
1428 - (void)forceUploadWithReply:(void (^)(BOOL, NSError*))reply {
1429     secnotice("upload", "Performing upload in response to rpc message");
1430     NSError* error = nil;
1431     BOOL result = [self uploadAnalyticsWithError:&error force:YES];
1432     secnotice("upload", "Result of manually triggered upload: %@, error: %@", result ? @"success" : @"failure", error);
1433     reply(result, error);
1434 }
1435
1436 @end
1437
1438 #endif // !TARGET_OS_SIMULATOR