2 * Copyright (c) 2017-2018 Apple Inc. All Rights Reserved.
4 * @APPLE_LICENSE_HEADER_START@
6 * This file contains Original Code and/or Modifications of Original Code
7 * as defined in and that are subject to the Apple Public Source License
8 * Version 2.0 (the 'License'). You may not use this file except in
9 * compliance with the License. Please obtain a copy of the License at
10 * http://www.opensource.apple.com/apsl/ and read it before using this
13 * The Original Code and all software distributed under the License are
14 * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
15 * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
16 * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
17 * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
18 * Please see the License for the specific language governing rights and
19 * limitations under the License.
21 * @APPLE_LICENSE_HEADER_END@
26 #if !TARGET_OS_SIMULATOR
28 #import "SFAnalyticsDefines.h"
29 #import "SFAnalyticsSQLiteStore.h"
30 #import <Security/SFAnalytics.h>
32 #include <utilities/SecFileLocations.h>
33 #import "utilities/debugging.h"
34 #import <os/variant_private.h>
37 #import "keychain/ckks/CKKSControl.h"
40 #import <AuthKit/AKAppleIDAuthenticationContext.h>
41 #import <AuthKit/AKAppleIDAuthenticationController.h>
42 #import <AuthKit/AKAppleIDAuthenticationController_Private.h>
45 #include "dirhelper_priv.h"
49 #import <CrashReporterSupport/CrashReporterSupportPrivate.h>
51 #import <CrashReporterSupport/CrashReporterSupport.h>
54 #import <Accounts/Accounts.h>
55 #import <Accounts/ACAccountStore_Private.h>
56 #import <Accounts/ACAccountType_Private.h>
57 #import <Accounts/ACAccountStore.h>
58 #import <AppleAccount/AppleAccount.h>
59 #import <AppleAccount/ACAccount+AppleAccount.h>
60 #import <AppleAccount/ACAccountStore+AppleAccount.h>
63 NSString* const SFAnalyticsSplunkTopic = @"topic";
64 NSString* const SFAnalyticsClientId = @"clientId";
65 NSString* const SFAnalyticsInternal = @"internal";
67 NSString* const SFAnalyticsMetricsBase = @"metricsBase";
68 NSString* const SFAnalyticsDeviceID = @"ckdeviceID";
69 NSString* const SFAnalyticsAltDSID = @"altDSID";
71 NSString* const SFAnalyticsEventCorrelationID = @"eventLinkID";
73 NSString* const SFAnalyticsSecondsCustomerKey = @"SecondsBetweenUploadsCustomer";
74 NSString* const SFAnalyticsSecondsInternalKey = @"SecondsBetweenUploadsInternal";
75 NSString* const SFAnalyticsSecondsSeedKey = @"SecondsBetweenUploadsSeed";
76 NSString* const SFAnalyticsMaxEventsKey = @"NumberOfEvents";
77 NSString* const SFAnalyticsDevicePercentageCustomerKey = @"DevicePercentageCustomer";
78 NSString* const SFAnalyticsDevicePercentageInternalKey = @"DevicePercentageInternal";
79 NSString* const SFAnalyticsDevicePercentageSeedKey = @"DevicePercentageSeed";
81 NSString* const SupdErrorDomain = @"com.apple.security.supd";
83 #define SFANALYTICS_SPLUNK_DEV 0
84 #define OS_CRASH_TRACER_LOG_BUG_TYPE "226"
86 #if SFANALYTICS_SPLUNK_DEV
87 NSUInteger const secondsBetweenUploadsCustomer = 10;
88 NSUInteger const secondsBetweenUploadsInternal = 10;
89 NSUInteger const secondsBetweenUploadsSeed = 10;
90 #else // SFANALYTICS_SPLUNK_DEV
91 NSUInteger const secondsBetweenUploadsCustomer = (3 * (60 * 60 * 24));
92 NSUInteger const secondsBetweenUploadsInternal = (60 * 60 * 24);
93 NSUInteger const secondsBetweenUploadsSeed = (60 * 60 * 24);
94 #endif // SFANALYTICS_SPLUNK_DEV
96 @implementation SFAnalyticsReporter
97 - (BOOL)saveReport:(NSData *)reportData fileName:(NSString *)fileName
99 BOOL writtenToLog = NO;
101 NSDictionary *optionsDictionary = @{ (__bridge NSString *)kCRProblemReportSubmissionPolicyKey: (__bridge NSString *)kCRSubmissionPolicyAlternate };
102 #else // !TARGET_OS_OSX
103 NSDictionary *optionsDictionary = nil; // The keys above are not defined or required on iOS.
104 #endif // !TARGET_OS_OSX
107 secdebug("saveReport", "calling out to `OSAWriteLogForSubmission`");
108 writtenToLog = OSAWriteLogForSubmission(@OS_CRASH_TRACER_LOG_BUG_TYPE, fileName,
109 nil, optionsDictionary, ^(NSFileHandle *fileHandle) {
110 secnotice("OSAWriteLogForSubmission", "Writing log data to report: %@", fileName);
111 [fileHandle writeData:reportData];
117 #define DEFAULT_SPLUNK_MAX_EVENTS_TO_REPORT 1000
118 #define DEFAULT_SPLUNK_DEVICE_PERCENTAGE 100
120 static supd *_supdInstance = nil;
122 BOOL runningTests = NO;
123 BOOL deviceAnalyticsOverride = NO;
124 BOOL deviceAnalyticsEnabled = NO;
125 BOOL iCloudAnalyticsOverride = NO;
126 BOOL iCloudAnalyticsEnabled = NO;
129 _isDeviceAnalyticsEnabled(void)
131 // This flag is only set during tests.
132 if (deviceAnalyticsOverride) {
133 return deviceAnalyticsEnabled;
136 static BOOL dataCollectionEnabled = NO;
137 static dispatch_once_t onceToken;
138 dispatch_once(&onceToken, ^{
140 dataCollectionEnabled = DiagnosticLogSubmissionEnabled();
142 dataCollectionEnabled = CRIsAutoSubmitEnabled();
145 return dataCollectionEnabled;
151 ACAccountStore *accountStore = [[ACAccountStore alloc] init];
152 ACAccount *primaryAccount = [accountStore aa_primaryAppleAccount];
153 if (primaryAccount == nil) {
156 return [primaryAccount aa_altDSID];
159 static NSString *const kAnalyticsiCloudIdMSKey = @"com.apple.idms.config.privacy.icloud.data";
161 static NSDictionary *
162 _getiCloudConfigurationInfoWithError(NSError **outError)
164 __block NSDictionary *outConfigurationInfo = nil;
165 __block NSError *localError = nil;
167 NSString *altDSID = accountAltDSID();
168 if (altDSID != nil) {
169 secnotice("_getiCloudConfigurationInfoWithError", "Fetching configuration info");
171 dispatch_semaphore_t sema = dispatch_semaphore_create(0);
172 AKAppleIDAuthenticationController *authController = [AKAppleIDAuthenticationController new];
173 [authController configurationInfoWithIdentifiers:@[kAnalyticsiCloudIdMSKey]
175 completion:^(NSDictionary<NSString *, id<NSSecureCoding>> *configurationInfo, NSError *error) {
177 secerror("_getiCloudConfigurationInfoWithError: Error fetching configurationInfo: %@", error);
179 } else if (![configurationInfo isKindOfClass:[NSDictionary class]]) {
180 secerror("_getiCloudConfigurationInfoWithError: configurationInfo dict was not a dict, it was a %{public}@", [configurationInfo class]);
182 configurationInfo = nil;
184 secnotice("_getiCloudConfigurationInfoWithError", "fetched configurationInfo %@", configurationInfo);
185 outConfigurationInfo = configurationInfo;
187 dispatch_semaphore_signal(sema);
189 dispatch_semaphore_wait(sema, dispatch_time(DISPATCH_TIME_NOW, (uint64_t)(5 * NSEC_PER_SEC)));
191 secerror("_getiCloudConfigurationInfoWithError: Failed to fetch primary account info.");
194 if (localError && outError) {
195 *outError = localError;
197 return outConfigurationInfo;
201 _isiCloudAnalyticsEnabled()
203 // This flag is only set during tests.
204 if (iCloudAnalyticsOverride) {
205 return iCloudAnalyticsEnabled;
208 static bool cachedAllowsICloudAnalytics = false;
210 static dispatch_once_t onceToken;
211 dispatch_once(&onceToken, ^{
212 NSError *error = nil;
213 NSDictionary *accountConfiguration = _getiCloudConfigurationInfoWithError(&error);
214 if (error == nil && accountConfiguration != nil) {
215 id iCloudAnalyticsOptIn = accountConfiguration[kAnalyticsiCloudIdMSKey];
216 if (iCloudAnalyticsOptIn != nil) {
217 BOOL iCloudAnalyticsOptInHasCorrectType = ([iCloudAnalyticsOptIn isKindOfClass:[NSNumber class]] || [iCloudAnalyticsOptIn isKindOfClass:[NSString class]]);
218 if (iCloudAnalyticsOptInHasCorrectType) {
219 NSNumber *iCloudAnalyticsOptInNumber = @([iCloudAnalyticsOptIn integerValue]);
220 cachedAllowsICloudAnalytics = ![iCloudAnalyticsOptInNumber isEqualToNumber:[NSNumber numberWithInteger:0]];
223 } else if (error != nil) {
224 secerror("_isiCloudAnalyticsEnabled: %@", error);
228 return cachedAllowsICloudAnalytics;
231 /* NSData GZip category based on GeoKit's implementation */
232 @interface NSData (GZip)
233 - (NSData *)supd_gzipDeflate;
236 #define GZIP_OFFSET 16
237 #define GZIP_STRIDE_LEN 16384
239 @implementation NSData (Gzip)
240 - (NSData *)supd_gzipDeflate
242 if ([self length] == 0) {
247 memset(&strm, 0, sizeof(strm));
248 strm.next_in=(uint8_t *)[self bytes];
249 strm.avail_in = (unsigned int)[self length];
252 if (Z_OK != deflateInit2(&strm, Z_BEST_COMPRESSION, Z_DEFLATED,
253 MAX_WBITS + GZIP_OFFSET, MAX_MEM_LEVEL, Z_DEFAULT_STRATEGY)) {
257 NSMutableData *compressed = [NSMutableData dataWithLength:GZIP_STRIDE_LEN];
260 if (strm.total_out >= [compressed length]) {
261 [compressed increaseLengthBy: 16384];
264 strm.next_out = [compressed mutableBytes] + strm.total_out;
265 strm.avail_out = (int)[compressed length] - (int)strm.total_out;
267 deflate(&strm, Z_FINISH);
269 } while (strm.avail_out == 0);
273 [compressed setLength: strm.total_out];
274 if (strm.avail_in == 0) {
275 return [NSData dataWithData:compressed];
282 @implementation SFAnalyticsClient {
285 BOOL _requireDeviceAnalytics;
286 BOOL _requireiCloudAnalytics;
289 @synthesize storePath = _path;
290 @synthesize name = _name;
292 - (instancetype)initWithStorePath:(NSString*)path name:(NSString*)name
293 deviceAnalytics:(BOOL)deviceAnalytics iCloudAnalytics:(BOOL)iCloudAnalytics {
294 if (self = [super init]) {
297 _requireDeviceAnalytics = deviceAnalytics;
298 _requireiCloudAnalytics = iCloudAnalytics;
305 @interface SFAnalyticsTopic ()
306 @property NSURL* _splunkUploadURL;
308 @property BOOL allowInsecureSplunkCert;
309 @property BOOL ignoreServersMessagesTellingUsToGoAway;
310 @property BOOL disableUploads;
311 @property BOOL disableClientId;
313 @property NSUInteger secondsBetweenUploads;
314 @property NSUInteger maxEventsToReport;
315 @property float devicePercentage; // for sampling reporting devices
317 @property NSDictionary* metricsBase; // data the server provides and wants us to send back
318 @property NSArray* blacklistedFields;
319 @property NSArray* blacklistedEvents;
322 @implementation SFAnalyticsTopic
324 - (void)setupClientsForTopic:(NSString *)topicName
326 NSMutableArray<SFAnalyticsClient*>* clients = [NSMutableArray<SFAnalyticsClient*> new];
327 if ([topicName isEqualToString:SFAnalyticsTopicKeySync]) {
328 [clients addObject:[[SFAnalyticsClient alloc] initWithStorePath:[self.class databasePathForCKKS]
329 name:@"ckks" deviceAnalytics:NO iCloudAnalytics:YES]];
330 [clients addObject:[[SFAnalyticsClient alloc] initWithStorePath:[self.class databasePathForSOS]
331 name:@"sos" deviceAnalytics:NO iCloudAnalytics:YES]];
332 [clients addObject:[[SFAnalyticsClient alloc] initWithStorePath:[self.class databasePathForPCS]
333 name:@"pcs" deviceAnalytics:NO iCloudAnalytics:YES]];
334 [clients addObject:[[SFAnalyticsClient alloc] initWithStorePath:[self.class databasePathForSignIn]
335 name:@"signins" deviceAnalytics:NO iCloudAnalytics:YES]];
336 [clients addObject:[[SFAnalyticsClient alloc] initWithStorePath:[self.class databasePathForLocal]
337 name:@"local" deviceAnalytics:YES iCloudAnalytics:NO]];
338 } else if ([topicName isEqualToString:SFAnalyticsTopicCloudServices]) {
339 [clients addObject:[[SFAnalyticsClient alloc] initWithStorePath:[self.class databasePathForCloudServices]
340 name:@"CloudServices"
342 iCloudAnalytics:NO]];
343 } else if ([topicName isEqualToString:SFAnalyticsTopicTrust]) {
345 _set_user_dir_suffix("com.apple.trustd"); // supd needs to read trustd's cache dir for these
347 [clients addObject:[[SFAnalyticsClient alloc] initWithStorePath:[self.class databasePathForTrust]
348 name:@"trust" deviceAnalytics:YES iCloudAnalytics:NO]];
349 [clients addObject:[[SFAnalyticsClient alloc] initWithStorePath:[self.class databasePathForTrustdHealth]
350 name:@"trustdHealth" deviceAnalytics:YES iCloudAnalytics:NO]];
351 [clients addObject:[[SFAnalyticsClient alloc] initWithStorePath:[self.class databasePathForTLS]
352 name:@"tls" deviceAnalytics:YES iCloudAnalytics:NO]];
355 _set_user_dir_suffix(NULL); // set back to the default cache dir
357 } else if ([topicName isEqualToString:SFAnalyticsTopicTransparency]) {
358 [clients addObject:[[SFAnalyticsClient alloc] initWithStorePath:[self.class databasePathForTransparency]
359 name:@"transparency" deviceAnalytics:NO iCloudAnalytics:YES]];
362 _topicClients = clients;
365 - (instancetype)initWithDictionary:(NSDictionary *)dictionary name:(NSString *)topicName samplingRates:(NSDictionary *)rates {
366 if (self = [super init]) {
367 _internalTopicName = topicName;
368 [self setupClientsForTopic:topicName];
369 _splunkTopicName = dictionary[@"splunk_topic"];
370 __splunkUploadURL = [NSURL URLWithString:dictionary[@"splunk_uploadURL"]];
371 _splunkBagURL = [NSURL URLWithString:dictionary[@"splunk_bagURL"]];
372 _allowInsecureSplunkCert = [[dictionary valueForKey:@"splunk_allowInsecureCertificate"] boolValue];
373 _uploadSizeLimit = [[dictionary valueForKey:@"uploadSizeLimit"] unsignedIntegerValue];
375 NSString* splunkEndpoint = dictionary[@"splunk_endpointDomain"];
376 if (dictionary[@"disableClientId"]) {
377 _disableClientId = YES;
380 NSUserDefaults* defaults = [[NSUserDefaults alloc] initWithSuiteName:SFAnalyticsUserDefaultsSuite];
381 NSString* userDefaultsSplunkTopic = [defaults stringForKey:@"splunk_topic"];
382 if (userDefaultsSplunkTopic) {
383 _splunkTopicName = userDefaultsSplunkTopic;
386 NSURL* userDefaultsSplunkUploadURL = [NSURL URLWithString:[defaults stringForKey:@"splunk_uploadURL"]];
387 if (userDefaultsSplunkUploadURL) {
388 __splunkUploadURL = userDefaultsSplunkUploadURL;
391 NSURL* userDefaultsSplunkBagURL = [NSURL URLWithString:[defaults stringForKey:@"splunk_bagURL"]];
392 if (userDefaultsSplunkBagURL) {
393 _splunkBagURL = userDefaultsSplunkBagURL;
396 NSInteger userDefaultsUploadSizeLimit = [defaults integerForKey:@"uploadSizeLimit"];
397 if (userDefaultsUploadSizeLimit > 0) {
398 _uploadSizeLimit = userDefaultsUploadSizeLimit;
401 BOOL userDefaultsAllowInsecureSplunkCert = [defaults boolForKey:@"splunk_allowInsecureCertificate"];
402 _allowInsecureSplunkCert |= userDefaultsAllowInsecureSplunkCert;
404 NSString* userDefaultsSplunkEndpoint = [defaults stringForKey:@"splunk_endpointDomain"];
405 if (userDefaultsSplunkEndpoint) {
406 splunkEndpoint = userDefaultsSplunkEndpoint;
409 #if SFANALYTICS_SPLUNK_DEV
410 _secondsBetweenUploads = secondsBetweenUploadsInternal;
411 _maxEventsToReport = SFAnalyticsMaxEventsToReport;
412 _devicePercentage = DEFAULT_SPLUNK_DEVICE_PERCENTAGE;
414 bool internal = os_variant_has_internal_diagnostics("com.apple.security");
417 NSNumber *secondsNum = internal ? rates[SFAnalyticsSecondsInternalKey] : rates[SFAnalyticsSecondsSeedKey];
418 NSNumber *percentageNum = internal ? rates[SFAnalyticsDevicePercentageInternalKey] : rates[SFAnalyticsDevicePercentageSeedKey];
420 NSNumber *secondsNum = internal ? rates[SFAnalyticsSecondsInternalKey] : rates[SFAnalyticsSecondsCustomerKey];
421 NSNumber *percentageNum = internal ? rates[SFAnalyticsDevicePercentageInternalKey] : rates[SFAnalyticsDevicePercentageCustomerKey];
423 _secondsBetweenUploads = [secondsNum integerValue];
424 _maxEventsToReport = [rates[SFAnalyticsMaxEventsKey] unsignedIntegerValue];
425 _devicePercentage = [percentageNum floatValue];
428 _secondsBetweenUploads = internal ? secondsBetweenUploadsInternal : secondsBetweenUploadsSeed;
430 _secondsBetweenUploads = internal ? secondsBetweenUploadsInternal : secondsBetweenUploadsCustomer;
432 _maxEventsToReport = SFAnalyticsMaxEventsToReport;
433 _devicePercentage = DEFAULT_SPLUNK_DEVICE_PERCENTAGE;
436 secnotice("supd", "created %@ with %lu seconds between uploads, %lu max events, %f percent of uploads",
437 _internalTopicName, (unsigned long)_secondsBetweenUploads, (unsigned long)_maxEventsToReport, _devicePercentage);
439 #if SFANALYTICS_SPLUNK_DEV
440 _ignoreServersMessagesTellingUsToGoAway = YES;
442 if (!_splunkUploadURL && splunkEndpoint) {
443 NSString* urlString = [NSString stringWithFormat:@"https://%@/report/2/%@", splunkEndpoint, _splunkTopicName];
444 _splunkUploadURL = [NSURL URLWithString:urlString];
447 (void)splunkEndpoint;
453 - (BOOL)isSampledUpload
455 uint32_t sample = arc4random();
456 if ((double)_devicePercentage < ((double)1 / UINT32_MAX) * 100) {
457 /* Requested percentage is smaller than we can sample. just do 1 out of UINT32_MAX */
462 if ((double)sample <= (double)UINT32_MAX * ((double)_devicePercentage / 100)) {
469 - (BOOL)postJSON:(NSData*)json toEndpoint:(NSURL*)endpoint error:(NSError**)error
473 NSString *description = [NSString stringWithFormat:@"No endpoint for %@", _internalTopicName];
474 *error = [NSError errorWithDomain:@"SupdUploadErrorDomain"
476 userInfo:@{NSLocalizedDescriptionKey : description}];
481 * Create the NSURLSession
482 * We use the ephemeral session config because we don't need cookies or cache
484 NSURLSessionConfiguration *configuration = [NSURLSessionConfiguration ephemeralSessionConfiguration];
486 configuration.HTTPAdditionalHeaders = @{ @"User-Agent" : [NSString stringWithFormat:@"securityd/%s", SECURITY_BUILD_VERSION]};
488 NSURLSession* postSession = [NSURLSession sessionWithConfiguration:configuration
492 NSMutableURLRequest* postRequest = [[NSMutableURLRequest alloc] init];
493 postRequest.URL = endpoint;
494 postRequest.HTTPMethod = @"POST";
495 postRequest.HTTPBody = [json supd_gzipDeflate];
496 [postRequest setValue:@"gzip" forHTTPHeaderField:@"Content-Encoding"];
499 * Create the upload task.
501 dispatch_semaphore_t sem = dispatch_semaphore_create(0);
502 __block BOOL uploadSuccess = NO;
503 NSURLSessionDataTask* uploadTask = [postSession dataTaskWithRequest:postRequest
504 completionHandler:^(NSData * _Nullable __unused data, NSURLResponse * _Nullable response, NSError * _Nullable requestError) {
506 secerror("Error in uploading the events to splunk for %@: %@", self->_internalTopicName, requestError);
507 } else if (![response isKindOfClass:NSHTTPURLResponse.class]){
508 Class class = response.class;
509 secerror("Received the wrong kind of response for %@: %@", self->_internalTopicName, NSStringFromClass(class));
511 NSHTTPURLResponse* httpResponse = (NSHTTPURLResponse*)response;
512 if(httpResponse.statusCode >= 200 && httpResponse.statusCode < 300) {
515 secnotice("upload", "Splunk upload success for %@", self->_internalTopicName);
517 secnotice("upload", "Splunk upload for %@ unexpected status to URL: %@ -- status: %d",
518 self->_internalTopicName, endpoint, (int)(httpResponse.statusCode));
521 dispatch_semaphore_signal(sem);
523 secnotice("upload", "Splunk upload start for %@", self->_internalTopicName);
525 dispatch_semaphore_wait(sem, dispatch_time(DISPATCH_TIME_NOW, (uint64_t)(5 * 60 * NSEC_PER_SEC)));
526 return uploadSuccess;
529 - (BOOL)eventIsBlacklisted:(NSMutableDictionary*)event {
530 return _blacklistedEvents ? [_blacklistedEvents containsObject:event[SFAnalyticsEventType]] : NO;
533 - (void)removeBlacklistedFieldsFromEvent:(NSMutableDictionary*)event {
534 for (NSString* badField in self->_blacklistedFields) {
535 [event removeObjectForKey:badField];
539 - (void)addRequiredFieldsToEvent:(NSMutableDictionary*)event {
540 [_metricsBase enumerateKeysAndObjectsUsingBlock:^(id _Nonnull key, id _Nonnull obj, BOOL * _Nonnull stop) {
547 - (BOOL)prepareEventForUpload:(NSMutableDictionary*)event
548 linkedUUID:(NSUUID *)linkedUUID {
549 if ([self eventIsBlacklisted:event]) {
553 [self removeBlacklistedFieldsFromEvent:event];
554 [self addRequiredFieldsToEvent:event];
555 if (_disableClientId) {
556 event[SFAnalyticsClientId] = @(0);
558 event[SFAnalyticsSplunkTopic] = self->_splunkTopicName ?: [NSNull null];
560 event[SFAnalyticsEventCorrelationID] = [linkedUUID UUIDString];
565 - (void)addFailures:(NSMutableArray<NSArray*>*)failures toUploadRecords:(NSMutableArray*)records threshold:(NSUInteger)threshold linkedUUID:(NSUUID *)linkedUUID
567 // The first 0 through 'threshold' items are getting uploaded in any case (which might be 0 for lower priority data)
569 for (NSArray* client in failures) {
570 [client enumerateObjectsUsingBlock:^(id _Nonnull obj, NSUInteger idx, BOOL * _Nonnull stop) {
571 NSMutableDictionary* event = (NSMutableDictionary*)obj;
572 if (idx >= threshold) {
576 if ([self prepareEventForUpload:event linkedUUID:linkedUUID]) {
577 if ([NSJSONSerialization isValidJSONObject:event]) {
578 [records addObject:event];
580 secerror("supd: Replacing event with errorEvent because invalid JSON: %@", event);
581 NSString* originalType = event[SFAnalyticsEventType];
582 NSDictionary* errorEvent = @{ SFAnalyticsEventType : SFAnalyticsEventTypeErrorEvent,
583 SFAnalyticsEventErrorDestription : [NSString stringWithFormat:@"JSON:%@", originalType]};
584 [records addObject:errorEvent];
590 // Are there more items than we shoved into the upload records?
591 NSInteger excessItems = 0;
592 for (NSArray* client in failures) {
593 NSInteger localExcess = client.count - threshold;
594 excessItems += localExcess > 0 ? localExcess : 0;
597 // Then, if we have space and items left, apply a scaling factor to distribute events across clients to fill upload buffer
598 if (records.count < _maxEventsToReport && excessItems > 0) {
599 double scale = (_maxEventsToReport - records.count) / (double)excessItems;
604 for (NSArray* client in failures) {
605 if (client.count > threshold) {
606 NSRange range = NSMakeRange(threshold, (client.count - threshold) * scale);
607 NSArray* sub = [client subarrayWithRange:range];
608 [sub enumerateObjectsUsingBlock:^(id _Nonnull obj, NSUInteger idx, BOOL * _Nonnull stop) {
609 if ([self prepareEventForUpload:obj linkedUUID:linkedUUID]) {
610 [records addObject:obj];
618 - (NSMutableDictionary*)sampleStatisticsForSamples:(NSArray*)samples withName:(NSString*)name
620 NSMutableDictionary* statistics = [NSMutableDictionary dictionary];
621 NSUInteger count = samples.count;
622 NSArray* sortedSamples = [samples sortedArrayUsingSelector:@selector(compare:)];
623 NSArray* samplesAsExpressionArray = @[[NSExpression expressionForConstantValue:sortedSamples]];
626 statistics[name] = samples[0];
628 // NSExpression takes population standard deviation. Our data is a sample of whatever we sampled over time,
629 // but the difference between the two is fairly minor (divide by N before taking sqrt versus divide by N-1).
630 statistics[[NSString stringWithFormat:@"%@-dev", name]] = [[NSExpression expressionForFunction:@"stddev:" arguments:samplesAsExpressionArray] expressionValueWithObject:nil context:nil];
632 statistics[[NSString stringWithFormat:@"%@-min", name]] = [[NSExpression expressionForFunction:@"min:" arguments:samplesAsExpressionArray] expressionValueWithObject:nil context:nil];
633 statistics[[NSString stringWithFormat:@"%@-max", name]] = [[NSExpression expressionForFunction:@"max:" arguments:samplesAsExpressionArray] expressionValueWithObject:nil context:nil];
634 statistics[[NSString stringWithFormat:@"%@-avg", name]] = [[NSExpression expressionForFunction:@"average:" arguments:samplesAsExpressionArray] expressionValueWithObject:nil context:nil];
635 statistics[[NSString stringWithFormat:@"%@-med", name]] = [[NSExpression expressionForFunction:@"median:" arguments:samplesAsExpressionArray] expressionValueWithObject:nil context:nil];
639 NSString* q1 = [NSString stringWithFormat:@"%@-1q", name];
640 NSString* q3 = [NSString stringWithFormat:@"%@-3q", name];
641 // From Wikipedia, which is never wrong
642 if (count % 2 == 0) {
643 // The lower quartile value is the median of the lower half of the data. The upper quartile value is the median of the upper half of the data.
644 statistics[q1] = [[NSExpression expressionForFunction:@"median:" arguments:@[[NSExpression expressionForConstantValue:[sortedSamples subarrayWithRange:NSMakeRange(0, count / 2)]]]] expressionValueWithObject:nil context:nil];
645 statistics[q3] = [[NSExpression expressionForFunction:@"median:" arguments:@[[NSExpression expressionForConstantValue:[sortedSamples subarrayWithRange:NSMakeRange((count / 2), count / 2)]]]] expressionValueWithObject:nil context:nil];
646 } else if (count % 4 == 1) {
647 // If there are (4n+1) data points, then the lower quartile is 25% of the nth data value plus 75% of the (n+1)th data value;
648 // the upper quartile is 75% of the (3n+1)th data point plus 25% of the (3n+2)th data point.
649 // (offset n by -1 since we count from 0)
650 NSUInteger n = count / 4;
651 statistics[q1] = @(([sortedSamples[n - 1] doubleValue] + [sortedSamples[n] doubleValue] * 3.0) / 4.0);
652 statistics[q3] = @(([sortedSamples[(3 * n)] doubleValue] * 3.0 + [sortedSamples[(3 * n) + 1] doubleValue]) / 4.0);
653 } else if (count % 4 == 3){
654 // If there are (4n+3) data points, then the lower quartile is 75% of the (n+1)th data value plus 25% of the (n+2)th data value;
655 // the upper quartile is 25% of the (3n+2)th data point plus 75% of the (3n+3)th data point.
656 // (offset n by -1 since we count from 0)
657 NSUInteger n = count / 4;
658 statistics[q1] = @(([sortedSamples[n] doubleValue] * 3.0 + [sortedSamples[n + 1] doubleValue]) / 4.0);
659 statistics[q3] = @(([sortedSamples[(3 * n) + 1] doubleValue] + [sortedSamples[(3 * n) + 2] doubleValue] * 3.0) / 4.0);
666 - (NSMutableDictionary*)healthSummaryWithName:(NSString*)name store:(SFAnalyticsSQLiteStore*)store uuid:(NSUUID *)uuid
668 __block NSMutableDictionary* summary = [NSMutableDictionary new];
670 // Add some events of our own before pulling in data
671 summary[SFAnalyticsEventType] = [NSString stringWithFormat:@"%@HealthSummary", name];
672 if ([self eventIsBlacklisted:summary]) {
675 summary[SFAnalyticsEventTime] = @([[NSDate date] timeIntervalSince1970] * 1000); // Splunk wants milliseconds
676 [SFAnalytics addOSVersionToEvent:summary];
677 if (store.uploadDate) {
678 summary[SFAnalyticsAttributeLastUploadTime] = @([store.uploadDate timeIntervalSince1970] * 1000);
680 summary[SFAnalyticsAttributeLastUploadTime] = @(0);
684 NSDictionary* successCounts = store.summaryCounts;
685 __block NSInteger totalSuccessCount = 0;
686 __block NSInteger totalHardFailureCount = 0;
687 __block NSInteger totalSoftFailureCount = 0;
688 [successCounts enumerateKeysAndObjectsUsingBlock:^(NSString* _Nonnull eventType, NSDictionary* _Nonnull counts, BOOL* _Nonnull stop) {
689 summary[[NSString stringWithFormat:@"%@-success", eventType]] = counts[SFAnalyticsColumnSuccessCount];
690 summary[[NSString stringWithFormat:@"%@-hardfail", eventType]] = counts[SFAnalyticsColumnHardFailureCount];
691 summary[[NSString stringWithFormat:@"%@-softfail", eventType]] = counts[SFAnalyticsColumnSoftFailureCount];
692 totalSuccessCount += [counts[SFAnalyticsColumnSuccessCount] integerValue];
693 totalHardFailureCount += [counts[SFAnalyticsColumnHardFailureCount] integerValue];
694 totalSoftFailureCount += [counts[SFAnalyticsColumnSoftFailureCount] integerValue];
697 summary[SFAnalyticsColumnSuccessCount] = @(totalSuccessCount);
698 summary[SFAnalyticsColumnHardFailureCount] = @(totalHardFailureCount);
699 summary[SFAnalyticsColumnSoftFailureCount] = @(totalSoftFailureCount);
700 if (os_variant_has_internal_diagnostics("com.apple.security")) {
701 summary[SFAnalyticsInternal] = @YES;
705 NSMutableDictionary<NSString*,NSMutableArray*>* samplesBySampler = [NSMutableDictionary<NSString*,NSMutableArray*> dictionary];
706 for (NSDictionary* sample in [store samples]) {
707 if (!samplesBySampler[sample[SFAnalyticsColumnSampleName]]) {
708 samplesBySampler[sample[SFAnalyticsColumnSampleName]] = [NSMutableArray array];
710 [samplesBySampler[sample[SFAnalyticsColumnSampleName]] addObject:sample[SFAnalyticsColumnSampleValue]];
712 [samplesBySampler enumerateKeysAndObjectsUsingBlock:^(NSString * _Nonnull key, NSMutableArray * _Nonnull obj, BOOL * _Nonnull stop) {
713 NSMutableDictionary* event = [self sampleStatisticsForSamples:obj withName:key];
714 [summary addEntriesFromDictionary:event];
717 // Should always return yes because we already checked for event blacklisting specifically (unless summary itself is blacklisted)
718 if (![self prepareEventForUpload:summary linkedUUID:uuid]) {
719 secwarning("supd: health summary for %@ blacklisted", name);
723 // Seems unlikely because we only insert strings, samplers only take NSNumbers and frankly, sampleStatisticsForSamples probably would have crashed
724 if (![NSJSONSerialization isValidJSONObject:summary]) {
725 secerror("json: health summary for client %@ is invalid JSON: %@", name, summary);
726 return [@{ SFAnalyticsEventType : SFAnalyticsEventTypeErrorEvent,
727 SFAnalyticsEventErrorDestription : [NSString stringWithFormat:@"JSON:%@HealthSummary", name]} mutableCopy];
733 - (void)updateUploadDateForClients:(NSArray<SFAnalyticsClient*>*)clients date:(NSDate *)date clearData:(BOOL)clearData
735 for (SFAnalyticsClient* client in clients) {
736 SFAnalyticsSQLiteStore* store = [SFAnalyticsSQLiteStore storeWithPath:client.storePath schema:SFAnalyticsTableSchema];
737 secnotice("postprocess", "Setting upload date (%@) for client: %@", date, client.name);
738 store.uploadDate = date;
740 secnotice("postprocess", "Clearing collected data for client: %@", client.name);
741 [store clearAllData];
746 - (size_t)serializedEventSize:(NSObject *)event
747 error:(NSError**)error
749 if (![NSJSONSerialization isValidJSONObject:event]) {
750 secnotice("serializedEventSize", "invalid JSON object");
754 NSData *json = [NSJSONSerialization dataWithJSONObject:event
758 return [json length];
760 secnotice("serializedEventSize", "failed to serialize event");
765 - (NSArray<NSArray *> *)chunkFailureSet:(size_t)sizeCapacity
766 events:(NSArray<NSDictionary *> *)events
767 error:(NSError **)error
769 const size_t postBodyLimit = 1000; // 1000 events in a single upload
770 size_t currentSize = 0;
771 size_t currentEventCount = 0;
773 NSMutableArray<NSArray<NSDictionary *> *> *eventChunks = [[NSMutableArray<NSArray<NSDictionary *> *> alloc] init];
774 NSMutableArray<NSDictionary *> *currentEventChunk = [[NSMutableArray<NSDictionary *> alloc] init];
775 for (NSDictionary *event in events) {
776 NSError *localError = nil;
777 size_t eventSize = [self serializedEventSize:event error:&localError];
778 if (localError != nil) {
782 secemergency("Unable to serialize event JSON: %@", [localError localizedDescription]);
786 BOOL countLessThanLimit = currentEventCount < postBodyLimit;
787 BOOL sizeLessThanCapacity = (currentSize + eventSize) <= sizeCapacity;
788 if (!countLessThanLimit || !sizeLessThanCapacity) {
789 [eventChunks addObject:currentEventChunk];
790 currentEventChunk = [[NSMutableArray<NSDictionary *> alloc] init];
791 currentEventCount = 0;
795 [currentEventChunk addObject:event];
797 currentSize += eventSize;
800 if ([currentEventChunk count] > 0) {
801 [eventChunks addObject:currentEventChunk];
807 - (NSDictionary *)createEventDictionary:(NSArray *)healthSummaries
808 failures:(NSArray<NSDictionary *> *)failures
809 error:(NSError **)error
811 NSMutableArray *events = [[NSMutableArray alloc] init];
812 [events addObjectsFromArray:healthSummaries];
814 [events addObjectsFromArray:failures];
817 NSDictionary *eventDictionary = @{
818 SFAnalyticsPostTime : @([[NSDate date] timeIntervalSince1970] * 1000),
822 if (![NSJSONSerialization isValidJSONObject:eventDictionary]) {
823 secemergency("json: final dictionary invalid JSON.");
825 *error = [NSError errorWithDomain:SupdErrorDomain code:SupdInvalidJSONError
826 userInfo:@{NSLocalizedDescriptionKey : [NSString localizedStringWithFormat:@"Final dictionary for upload is invalid JSON: %@", eventDictionary]}];
831 return eventDictionary;
834 - (NSArray<NSDictionary *> *)createChunkedLoggingJSON:(NSArray<NSDictionary *> *)healthSummaries
835 failures:(NSArray<NSDictionary *> *)failures
836 error:(NSError **)error
838 NSError *localError = nil;
839 size_t baseSize = [self serializedEventSize:healthSummaries error:&localError];
840 if (localError != nil) {
841 secemergency("Unable to serialize health summary JSON");
848 NSArray<NSArray *> *chunkedEvents = [self chunkFailureSet:(self.uploadSizeLimit - baseSize) events:failures error:&localError];
850 NSMutableArray<NSDictionary *> *jsonResults = [[NSMutableArray<NSDictionary *> alloc] init];
851 for (NSArray<NSDictionary *> *failureSet in chunkedEvents) {
852 NSDictionary *eventDictionary = [self createEventDictionary:healthSummaries failures:failureSet error:error];
853 if (eventDictionary) {
854 [jsonResults addObject:eventDictionary];
860 if ([jsonResults count] == 0) {
861 NSDictionary *eventDictionary = [self createEventDictionary:healthSummaries failures:nil error:error];
862 if (eventDictionary) {
863 [jsonResults addObject:eventDictionary];
872 - (BOOL)copyEvents:(NSMutableArray<NSDictionary *> **)healthSummaries
873 failures:(NSMutableArray<NSDictionary *> **)failures
874 forUpload:(BOOL)upload
875 participatingClients:(NSMutableArray<SFAnalyticsClient*>**)clients
877 linkedUUID:(NSUUID *)linkedUUID
878 error:(NSError**)error
880 NSMutableArray<SFAnalyticsClient*> *localClients = [[NSMutableArray alloc] init];
881 NSMutableArray<NSDictionary *> *localHealthSummaries = [[NSMutableArray<NSDictionary *> alloc] init];
882 NSMutableArray<NSDictionary *> *localFailures = [[NSMutableArray<NSDictionary *> alloc] init];
883 NSMutableArray<NSArray*> *hardFailures = [[NSMutableArray alloc] init];
884 NSMutableArray<NSArray*> *softFailures = [[NSMutableArray alloc] init];
885 NSString *ckdeviceID = nil;
886 NSString *accountID = nil;
888 if (os_variant_has_internal_diagnostics("com.apple.security") && [_internalTopicName isEqualToString:SFAnalyticsTopicKeySync]) {
889 ckdeviceID = [self askSecurityForCKDeviceID];
890 accountID = accountAltDSID();
892 for (SFAnalyticsClient* client in self->_topicClients) {
893 if (!force && [client requireDeviceAnalytics] && !_isDeviceAnalyticsEnabled()) {
894 // Client required device analytics, yet the user did not opt in.
895 secnotice("getLoggingJSON", "Client '%@' requires device analytics yet user did not opt in.", [client name]);
898 if (!force && [client requireiCloudAnalytics] && !_isiCloudAnalyticsEnabled()) {
899 // Client required iCloud analytics, yet the user did not opt in.
900 secnotice("getLoggingJSON", "Client '%@' requires iCloud analytics yet user did not opt in.", [client name]);
904 SFAnalyticsSQLiteStore* store = [SFAnalyticsSQLiteStore storeWithPath:client.storePath schema:SFAnalyticsTableSchema];
907 NSDate* uploadDate = store.uploadDate;
908 if (!force && uploadDate && [[NSDate date] timeIntervalSinceDate:uploadDate] < _secondsBetweenUploads) {
909 secnotice("json", "ignoring client '%@' for %@ because last upload too recent: %@",
910 client.name, _internalTopicName, uploadDate);
915 secnotice("json", "client '%@' for topic '%@' force-included", client.name, _internalTopicName);
917 secnotice("json", "including client '%@' for topic '%@' for upload", client.name, _internalTopicName);
919 [localClients addObject:client];
922 NSMutableDictionary* healthSummary = [self healthSummaryWithName:client.name store:store uuid:linkedUUID];
925 healthSummary[SFAnalyticsDeviceID] = ckdeviceID;
928 healthSummary[SFAnalyticsAltDSID] = accountID;
930 [localHealthSummaries addObject:healthSummary];
933 [hardFailures addObject:store.hardFailures];
934 [softFailures addObject:store.softFailures];
937 if (upload && [localClients count] == 0) {
939 NSString *description = [NSString stringWithFormat:@"Upload too recent for all clients for %@", _internalTopicName];
940 *error = [NSError errorWithDomain:@"SupdUploadErrorDomain"
942 userInfo:@{NSLocalizedDescriptionKey : description}];
948 *clients = localClients;
952 [self addFailures:hardFailures toUploadRecords:localFailures threshold:_maxEventsToReport/10 linkedUUID:linkedUUID];
953 [self addFailures:softFailures toUploadRecords:localFailures threshold:0 linkedUUID:linkedUUID];
954 [*failures addObjectsFromArray:localFailures];
957 if (healthSummaries) {
958 [*healthSummaries addObjectsFromArray:localHealthSummaries];
964 - (NSArray<NSDictionary *> *)createChunkedLoggingJSON:(bool)pretty
965 forUpload:(BOOL)upload
966 participatingClients:(NSMutableArray<SFAnalyticsClient*>**)clients
967 force:(BOOL)force // supdctl uploads ignore privacy settings and recency
968 error:(NSError**)error
970 NSUUID *linkedUUID = [NSUUID UUID];
971 NSError *localError = nil;
972 NSMutableArray *failures = [[NSMutableArray alloc] init];
973 NSMutableArray *healthSummaries = [[NSMutableArray alloc] init];
974 BOOL copied = [self copyEvents:&healthSummaries
977 participatingClients:clients
979 linkedUUID:linkedUUID
981 if (!copied || localError) {
988 // Trim failures to the max count, based on health summary count
989 if ([failures count] > (_maxEventsToReport - [healthSummaries count])) {
992 range.length = _maxEventsToReport - [healthSummaries count];
993 failures = [[failures subarrayWithRange:range] mutableCopy];
996 return [self createChunkedLoggingJSON:healthSummaries failures:failures error:error];
999 - (NSDictionary *)createLoggingJSON:(bool)pretty
1000 forUpload:(BOOL)upload
1001 participatingClients:(NSMutableArray<SFAnalyticsClient*>**)clients
1002 force:(BOOL)force // supdctl uploads ignore privacy settings and recency
1003 error:(NSError**)error
1005 NSError *localError = nil;
1006 NSMutableArray *failures = [[NSMutableArray alloc] init];
1007 NSMutableArray *healthSummaries = [[NSMutableArray alloc] init];
1008 BOOL copied = [self copyEvents:&healthSummaries
1011 participatingClients:clients
1015 if (!copied || localError) {
1017 *error = localError;
1022 // Trim failures to the max count, based on health summary count
1023 if ([failures count] > (_maxEventsToReport - [healthSummaries count])) {
1026 range.length = _maxEventsToReport - [healthSummaries count];
1027 failures = [[failures subarrayWithRange:range] mutableCopy];
1030 return [self createEventDictionary:healthSummaries failures:failures error:error];
1033 // Is at least one client eligible for data collection based on user consent? Otherwise callers should NOT reach off-device.
1034 - (BOOL)haveEligibleClients {
1035 for (SFAnalyticsClient* client in self.topicClients) {
1036 if ((!client.requireDeviceAnalytics || _isDeviceAnalyticsEnabled()) &&
1037 (!client.requireiCloudAnalytics || _isiCloudAnalyticsEnabled())) {
1044 - (NSString*)askSecurityForCKDeviceID
1046 NSError* error = nil;
1047 CKKSControl* rpc = [CKKSControl controlObject:&error];
1049 secerror("unable to obtain CKKS endpoint: %@", error);
1053 __block NSString* localCKDeviceID;
1054 dispatch_semaphore_t sema = dispatch_semaphore_create(0);
1055 [rpc rpcGetCKDeviceIDWithReply:^(NSString* ckdeviceID) {
1056 localCKDeviceID = ckdeviceID;
1057 dispatch_semaphore_signal(sema);
1060 if (dispatch_semaphore_wait(sema, dispatch_time(DISPATCH_TIME_NOW, NSEC_PER_SEC * 10)) != 0) {
1061 secerror("timed out waiting for a response from security");
1065 return localCKDeviceID;
1068 // this method is kind of evil for the fact that it has side-effects in pulling other things besides the metricsURL from the server, and as such should NOT be memoized.
1069 // TODO redo this, probably to return a dictionary.
1070 - (NSURL*)splunkUploadURL:(BOOL)force
1072 if (!force && ![self haveEligibleClients]) { // force is true IFF called from supdctl. Customers don't have it and internal audiences must call it explicitly.
1073 secnotice("getURL", "Not going to talk to server for topic %@ because no eligible clients", [self internalTopicName]);
1077 if (__splunkUploadURL) {
1078 return __splunkUploadURL;
1081 secnotice("getURL", "Asking server for endpoint and config data for topic %@", [self internalTopicName]);
1083 __weak __typeof(self) weakSelf = self;
1084 dispatch_semaphore_t sem = dispatch_semaphore_create(0);
1086 __block NSError* error = nil;
1087 NSURLSessionConfiguration *configuration = [NSURLSessionConfiguration ephemeralSessionConfiguration];
1088 NSURLSession* storeBagSession = [NSURLSession sessionWithConfiguration:configuration
1092 NSURL* requestEndpoint = _splunkBagURL;
1093 __block NSURL* result = nil;
1094 NSURLSessionDataTask* storeBagTask = [storeBagSession dataTaskWithURL:requestEndpoint completionHandler:^(NSData * _Nullable data,
1095 NSURLResponse * _Nullable __unused response,
1096 NSError * _Nullable responseError) {
1098 __strong __typeof(self) strongSelf = weakSelf;
1103 if (data && !responseError) {
1104 NSData *responseData = data; // shut up compiler
1105 NSDictionary* responseDict = [NSJSONSerialization JSONObjectWithData:responseData options:0 error:&error];
1106 if([responseDict isKindOfClass:NSDictionary.class] && !error) {
1107 if (!self->_ignoreServersMessagesTellingUsToGoAway) {
1108 self->_disableUploads = [[responseDict valueForKey:@"sendDisabled"] boolValue];
1109 if (self->_disableUploads) {
1110 // then don't upload anything right now
1111 secerror("not returning a splunk URL because uploads are disabled for %@", self->_internalTopicName);
1112 dispatch_semaphore_signal(sem);
1116 // backend works with milliseconds
1117 NSUInteger secondsBetweenUploads = [[responseDict valueForKey:@"postFrequency"] unsignedIntegerValue] / 1000;
1118 if (secondsBetweenUploads > 0) {
1119 if (os_variant_has_internal_diagnostics("com.apple.security") &&
1120 self->_secondsBetweenUploads < secondsBetweenUploads) {
1121 secnotice("getURL", "Overriding server-sent post frequency because device is internal (%lu -> %lu)", (unsigned long)secondsBetweenUploads, (unsigned long)self->_secondsBetweenUploads);
1123 strongSelf->_secondsBetweenUploads = secondsBetweenUploads;
1127 strongSelf->_blacklistedEvents = responseDict[@"blacklistedEvents"];
1128 strongSelf->_blacklistedFields = responseDict[@"blacklistedFields"];
1131 strongSelf->_metricsBase = responseDict[@"metricsBase"];
1133 NSString* metricsEndpoint = responseDict[@"metricsUrl"];
1134 if([metricsEndpoint isKindOfClass:NSString.class]) {
1136 NSString* endpoint = [metricsEndpoint stringByAppendingFormat:@"/2/%@", strongSelf->_splunkTopicName];
1137 secnotice("upload", "got metrics endpoint %@ for %@", endpoint, self->_internalTopicName);
1138 NSURL* endpointURL = [NSURL URLWithString:endpoint];
1139 if([endpointURL.scheme isEqualToString:@"https"]) {
1140 result = endpointURL;
1146 error = responseError;
1149 secnotice("upload", "Unable to fetch splunk endpoint at URL for %@: %@ -- error: %@",
1150 self->_internalTopicName, requestEndpoint, error.description);
1153 secnotice("upload", "Malformed iTunes config payload for %@!", self->_internalTopicName);
1156 dispatch_semaphore_signal(sem);
1159 [storeBagTask resume];
1160 dispatch_semaphore_wait(sem, dispatch_time(DISPATCH_TIME_NOW, (uint64_t)(60 * NSEC_PER_SEC)));
1165 - (void)URLSession:(NSURLSession *)session didReceiveChallenge:(NSURLAuthenticationChallenge *)challenge
1166 completionHandler:(void (^)(NSURLSessionAuthChallengeDisposition, NSURLCredential *))completionHandler {
1167 assert(completionHandler);
1169 secnotice("upload", "Splunk upload challenge for %@", _internalTopicName);
1170 NSURLCredential *cred = nil;
1172 if ([challenge previousFailureCount] > 0) {
1173 // Previous failures occurred, bail
1174 completionHandler(NSURLSessionAuthChallengeCancelAuthenticationChallenge, nil);
1176 } else if ([challenge.protectionSpace.authenticationMethod isEqualToString:NSURLAuthenticationMethodServerTrust]) {
1178 * Evaluate trust for the certificate
1181 SecTrustRef serverTrust = challenge.protectionSpace.serverTrust;
1182 // Coverity gets upset if we don't check status even though result is all we need.
1183 bool trustResult = SecTrustEvaluateWithError(serverTrust, NULL);
1184 if (_allowInsecureSplunkCert || trustResult) {
1186 * All is well, accept the credentials
1188 if(_allowInsecureSplunkCert) {
1189 secnotice("upload", "Force Accepting Splunk Credential for %@", _internalTopicName);
1191 cred = [NSURLCredential credentialForTrust:serverTrust];
1192 completionHandler(NSURLSessionAuthChallengeUseCredential, cred);
1196 * An error occurred in evaluating trust, bail
1198 completionHandler(NSURLSessionAuthChallengeCancelAuthenticationChallenge, nil);
1202 * Just perform the default handling
1204 completionHandler(NSURLSessionAuthChallengePerformDefaultHandling, nil);
1208 - (NSDictionary*)eventDictWithBlacklistedFieldsStrippedFrom:(NSDictionary*)eventDict
1210 NSMutableDictionary* strippedDict = eventDict.mutableCopy;
1211 for (NSString* blacklistedField in _blacklistedFields) {
1212 [strippedDict removeObjectForKey:blacklistedField];
1214 return strippedDict;
1217 // MARK: Database path retrieval
1219 + (NSString*)databasePathForCKKS
1221 return [(__bridge_transfer NSURL*)SecCopyURLForFileInKeychainDirectory((__bridge CFStringRef)@"Analytics/ckks_analytics.db") path];
1224 + (NSString*)databasePathForSOS
1226 return [(__bridge_transfer NSURL*)SecCopyURLForFileInKeychainDirectory((__bridge CFStringRef)@"Analytics/sos_analytics.db") path];
1229 + (NSString*)AppSupportPath
1231 #if TARGET_OS_IPHONE && !TARGET_OS_SIMULATOR
1232 return @"/var/mobile/Library/Application Support";
1234 NSArray<NSString *>*paths = NSSearchPathForDirectoriesInDomains(NSApplicationSupportDirectory, NSUserDomainMask, true);
1235 if ([paths count] < 1) {
1238 return [NSString stringWithString: paths[0]];
1239 #endif /* TARGET_OS_IPHONE && !TARGET_OS_SIMULATOR */
1242 + (NSString*)databasePathForPCS
1244 NSString *appSup = [self AppSupportPath];
1248 NSString *dbpath = [NSString stringWithFormat:@"%@/com.apple.ProtectedCloudStorage/PCSAnalytics.db", appSup];
1249 secnotice("supd", "PCS Database path (%@)", dbpath);
1253 + (NSString*)databasePathForLocal
1255 return [(__bridge_transfer NSURL*)SecCopyURLForFileInKeychainDirectory((__bridge CFStringRef)@"Analytics/localkeychain.db") path];
1258 + (NSString*)databasePathForTrustdHealth
1260 #if TARGET_OS_IPHONE
1261 return [(__bridge_transfer NSURL*)SecCopyURLForFileInKeychainDirectory(CFSTR("Analytics/trustd_health_analytics.db")) path];
1263 return [(__bridge_transfer NSURL*)SecCopyURLForFileInUserCacheDirectory(CFSTR("Analytics/trustd_health_analytics.db")) path];
1267 + (NSString*)databasePathForTrust
1269 #if TARGET_OS_IPHONE
1270 return [(__bridge_transfer NSURL*)SecCopyURLForFileInKeychainDirectory(CFSTR("Analytics/trust_analytics.db")) path];
1272 return [(__bridge_transfer NSURL*)SecCopyURLForFileInUserCacheDirectory(CFSTR("Analytics/trust_analytics.db")) path];
1276 + (NSString*)databasePathForTLS
1278 #if TARGET_OS_IPHONE
1279 return [(__bridge_transfer NSURL*)SecCopyURLForFileInKeychainDirectory(CFSTR("Analytics/TLS_analytics.db")) path];
1281 return [(__bridge_transfer NSURL*)SecCopyURLForFileInUserCacheDirectory(CFSTR("Analytics/TLS_analytics.db")) path];
1285 + (NSString*)databasePathForSignIn
1287 return [(__bridge_transfer NSURL*)SecCopyURLForFileInKeychainDirectory(CFSTR("Analytics/signin_metrics.db")) path];
1290 + (NSString*)databasePathForCloudServices
1292 return [(__bridge_transfer NSURL*)SecCopyURLForFileInKeychainDirectory(CFSTR("Analytics/CloudServicesAnalytics.db")) path];
1295 + (NSString*)databasePathForTransparency
1297 return [(__bridge_transfer NSURL*)SecCopyURLForFileInKeychainDirectory((__bridge CFStringRef)@"Analytics/TransparencyAnalytics.db") path];
1303 @property NSDictionary *topicsSamplingRates;
1306 @implementation supd
1309 NSDictionary* systemDefaultValues = [NSDictionary dictionaryWithContentsOfFile:[[NSBundle bundleWithPath:@"/System/Library/Frameworks/Security.framework"] pathForResource:@"SFAnalytics" ofType:@"plist"]];
1310 NSMutableArray <SFAnalyticsTopic*>* topics = [NSMutableArray array];
1311 for (NSString *topicKey in systemDefaultValues) {
1312 NSDictionary *topicSamplingRates = _topicsSamplingRates[topicKey];
1313 SFAnalyticsTopic *topic = [[SFAnalyticsTopic alloc] initWithDictionary:systemDefaultValues[topicKey] name:topicKey samplingRates:topicSamplingRates];
1314 [topics addObject:topic];
1316 _analyticsTopics = [NSArray arrayWithArray:topics];
1319 + (void)instantiate {
1323 + (instancetype)instance {
1324 if (!_supdInstance) {
1325 _supdInstance = [self new];
1327 return _supdInstance;
1330 // Use this for testing to get rid of any state
1331 + (void)removeInstance {
1332 _supdInstance = nil;
1336 static NSString *SystemTrustStorePath = @"/System/Library/Security/Certificates.bundle";
1337 static NSString *AnalyticsSamplingRatesFilename = @"AnalyticsSamplingRates";
1338 static NSString *ContentVersionKey = @"MobileAssetContentVersion";
1339 static NSString *AssetContextFilename = @"OTAPKIContext.plist";
1341 static NSNumber *getSystemVersion(NSBundle *trustStoreBundle) {
1342 NSDictionary *systemVersionPlist = [NSDictionary dictionaryWithContentsOfURL:[trustStoreBundle URLForResource:@"AssetVersion"
1343 withExtension:@"plist"]];
1344 if (!systemVersionPlist || ![systemVersionPlist isKindOfClass:[NSDictionary class]]) {
1347 NSNumber *systemVersion = systemVersionPlist[ContentVersionKey];
1348 if (systemVersion == nil || ![systemVersion isKindOfClass:[NSNumber class]]) {
1351 return systemVersion;
1354 static NSNumber *getAssetVersion(NSURL *directory) {
1355 NSDictionary *assetContextPlist = [NSDictionary dictionaryWithContentsOfURL:[directory URLByAppendingPathComponent:AssetContextFilename]];
1356 if (!assetContextPlist || ![assetContextPlist isKindOfClass:[NSDictionary class]]) {
1359 NSNumber *assetVersion = assetContextPlist[ContentVersionKey];
1360 if (assetVersion == nil || ![assetVersion isKindOfClass:[NSNumber class]]) {
1363 return assetVersion;
1366 static bool ShouldInitializeWithAsset(NSBundle *trustStoreBundle, NSURL *directory) {
1367 NSNumber *systemVersion = getSystemVersion(trustStoreBundle);
1368 NSNumber *assetVersion = getAssetVersion(directory);
1370 if (assetVersion == nil || systemVersion == nil) {
1373 if ([assetVersion compare:systemVersion] == NSOrderedDescending) {
1379 - (void)setupSamplingRates {
1380 NSBundle *trustStoreBundle = [NSBundle bundleWithPath:SystemTrustStorePath];
1382 NSURL *keychainsDirectory = CFBridgingRelease(SecCopyURLForFileInSystemKeychainDirectory(nil));
1383 NSURL *directory = [keychainsDirectory URLByAppendingPathComponent:@"SupplementalsAssets/" isDirectory:YES];
1385 NSDictionary *analyticsSamplingRates = nil;
1386 if (ShouldInitializeWithAsset(trustStoreBundle, directory)) {
1387 /* Try to get the asset version of the sampling rates */
1388 NSURL *analyticsSamplingRateURL = [directory URLByAppendingPathComponent:[NSString stringWithFormat:@"%@.plist", AnalyticsSamplingRatesFilename]];
1389 analyticsSamplingRates = [NSDictionary dictionaryWithContentsOfURL:analyticsSamplingRateURL];
1390 secnotice("supd", "read sampling rates from SupplementalsAssets dir");
1391 if (!analyticsSamplingRates || ![analyticsSamplingRates isKindOfClass:[NSDictionary class]]) {
1392 analyticsSamplingRates = nil;
1395 if (!analyticsSamplingRates) {
1396 analyticsSamplingRates = [NSDictionary dictionaryWithContentsOfURL: [trustStoreBundle URLForResource:AnalyticsSamplingRatesFilename
1397 withExtension:@"plist"]];
1399 if (analyticsSamplingRates && [analyticsSamplingRates isKindOfClass:[NSDictionary class]]) {
1400 _topicsSamplingRates = analyticsSamplingRates[@"Topics"];
1401 if (!_topicsSamplingRates || ![analyticsSamplingRates isKindOfClass:[NSDictionary class]]) {
1402 _topicsSamplingRates = nil; // Something has gone terribly wrong, so we'll use the hardcoded defaults in this case
1407 - (instancetype)initWithReporter:(SFAnalyticsReporter *)reporter
1409 if (self = [super init]) {
1410 [self setupSamplingRates];
1412 _reporter = reporter;
1414 xpc_activity_register("com.apple.securityuploadd.triggerupload", XPC_ACTIVITY_CHECK_IN, ^(xpc_activity_t activity) {
1415 xpc_activity_state_t activityState = xpc_activity_get_state(activity);
1416 secnotice("supd", "hit xpc activity trigger, state: %ld", activityState);
1417 if (activityState == XPC_ACTIVITY_STATE_RUN) {
1418 // Run our regularly scheduled scan
1419 [self performRegularlyScheduledUpload];
1427 - (instancetype)init {
1428 SFAnalyticsReporter *reporter = [[SFAnalyticsReporter alloc] init];
1429 return [self initWithReporter:reporter];
1432 - (void)sendNotificationForOncePerReportSamplers
1434 notify_post(SFAnalyticsFireSamplersNotification);
1435 [NSThread sleepForTimeInterval:3.0];
1438 - (void)performRegularlyScheduledUpload {
1439 secnotice("upload", "Starting uploads in response to regular trigger");
1440 NSError *error = nil;
1441 if ([self uploadAnalyticsWithError:&error force:NO]) {
1442 secnotice("upload", "Regularly scheduled upload successful");
1444 secerror("upload: Failed to complete regularly scheduled upload: %@", error);
1448 - (NSArray<NSData *> *)serializeLoggingEvents:(NSArray<NSDictionary *> *)events
1449 error:(NSError **)error
1455 NSMutableArray<NSData *> *serializedEvents = [[NSMutableArray<NSData *> alloc] init];
1456 for (NSDictionary *event in events) {
1457 NSError *serializationError = nil;
1458 NSData* serializedEvent = [NSJSONSerialization dataWithJSONObject:event
1460 error:&serializationError];
1461 if (serializedEvent && !serializationError) {
1462 [serializedEvents addObject:serializedEvent];
1464 *error = serializationError;
1469 return serializedEvents;
1472 - (BOOL)uploadAnalyticsWithError:(NSError**)error force:(BOOL)force {
1473 [self sendNotificationForOncePerReportSamplers];
1476 NSError* localError = nil;
1477 for (SFAnalyticsTopic *topic in _analyticsTopics) {
1478 @autoreleasepool { // The logging JSONs get quite large. Ensure they're deallocated between topics.
1479 __block NSURL* endpoint = [topic splunkUploadURL:force]; // has side effects!
1482 secnotice("upload", "Skipping upload for %@ because no endpoint", [topic internalTopicName]);
1486 if ([topic disableUploads]) {
1487 secnotice("upload", "Aborting upload task for %@ because uploads are disabled", [topic internalTopicName]);
1491 NSMutableArray<SFAnalyticsClient*>* clients = [NSMutableArray new];
1492 NSArray<NSDictionary *> *jsonEvents = [topic createChunkedLoggingJSON:false forUpload:YES participatingClients:&clients force:force error:&localError];
1493 if (!jsonEvents || localError) {
1494 if ([[localError domain] isEqualToString:SupdErrorDomain] && [localError code] == SupdInvalidJSONError) {
1495 // Pretend this was a success because at least we'll get rid of bad data.
1496 // If someone keeps logging bad data and we only catch it here then
1497 // this causes sustained data loss for the entire topic.
1498 [topic updateUploadDateForClients:clients date:[NSDate date] clearData:YES];
1500 secerror("upload: failed to create chunked log events for logging topic %@: %@", [topic internalTopicName], localError);
1504 NSArray<NSData *> *serializedEvents = [self serializeLoggingEvents:jsonEvents error:&localError];
1505 if (!serializedEvents || localError) {
1506 if ([[localError domain] isEqualToString:SupdErrorDomain] && [localError code] == SupdInvalidJSONError) {
1507 // Pretend this was a success because at least we'll get rid of bad data.
1508 // If someone keeps logging bad data and we only catch it here then
1509 // this causes sustained data loss for the entire topic.
1510 [topic updateUploadDateForClients:clients date:[NSDate date] clearData:YES];
1512 secerror("upload: failed to serialized chunked log events for logging topic %@: %@", [topic internalTopicName], localError);
1516 if ([topic isSampledUpload]) {
1517 for (NSData *json in serializedEvents) {
1518 if (![self->_reporter saveReport:json fileName:[topic internalTopicName]]) {
1519 secerror("upload: failed to write analytics data to log");
1521 if ([topic postJSON:json toEndpoint:endpoint error:&localError]) {
1522 secnotice("upload", "Successfully posted JSON for %@", [topic internalTopicName]);
1524 [topic updateUploadDateForClients:clients date:[NSDate date] clearData:YES];
1526 secerror("upload: Failed to post JSON for %@: %@", [topic internalTopicName], localError);
1530 /* If we didn't sample this report, update date to prevent trying to upload again sooner
1531 * than we should. Clear data so that per-day calculations remain consistent. */
1532 secnotice("upload", "skipping unsampled upload for %@ and clearing data", [topic internalTopicName]);
1533 [topic updateUploadDateForClients:clients date:[NSDate date] clearData:YES];
1536 if (error && localError) {
1537 *error = localError;
1543 - (NSString*)sysdiagnoseStringForEventRecord:(NSDictionary*)eventRecord
1545 NSMutableDictionary* mutableEventRecord = eventRecord.mutableCopy;
1546 [mutableEventRecord removeObjectForKey:SFAnalyticsSplunkTopic];
1548 NSDate* eventDate = [NSDate dateWithTimeIntervalSince1970:[[eventRecord valueForKey:SFAnalyticsEventTime] doubleValue] / 1000];
1549 [mutableEventRecord removeObjectForKey:SFAnalyticsEventTime];
1551 NSString* eventName = eventRecord[SFAnalyticsEventType];
1552 [mutableEventRecord removeObjectForKey:SFAnalyticsEventType];
1554 SFAnalyticsEventClass eventClass = [[eventRecord valueForKey:SFAnalyticsEventClassKey] integerValue];
1555 NSString* eventClassString = [self stringForEventClass:eventClass];
1556 [mutableEventRecord removeObjectForKey:SFAnalyticsEventClassKey];
1558 NSMutableString* additionalAttributesString = [NSMutableString string];
1559 if (mutableEventRecord.count > 0) {
1560 [additionalAttributesString appendString:@" - Attributes: {" ];
1561 __block BOOL firstAttribute = YES;
1562 [mutableEventRecord enumerateKeysAndObjectsUsingBlock:^(NSString* key, id object, BOOL* stop) {
1563 NSString* openingString = firstAttribute ? @"" : @", ";
1564 [additionalAttributesString appendString:[NSString stringWithFormat:@"%@%@ : %@", openingString, key, object]];
1565 firstAttribute = NO;
1567 [additionalAttributesString appendString:@" }"];
1570 return [NSString stringWithFormat:@"%@ %@: %@%@", eventDate, eventClassString, eventName, additionalAttributesString];
1573 - (NSString*)getSysdiagnoseDump
1575 NSMutableString* sysdiagnose = [[NSMutableString alloc] init];
1577 for (SFAnalyticsTopic* topic in _analyticsTopics) {
1578 for (SFAnalyticsClient* client in topic.topicClients) {
1579 [sysdiagnose appendString:[NSString stringWithFormat:@"Client: %@\n", client.name]];
1580 SFAnalyticsSQLiteStore* store = [SFAnalyticsSQLiteStore storeWithPath:client.storePath schema:SFAnalyticsTableSchema];
1581 NSArray* allEvents = store.allEvents;
1582 for (NSDictionary* eventRecord in allEvents) {
1583 [sysdiagnose appendFormat:@"%@\n", [self sysdiagnoseStringForEventRecord:eventRecord]];
1585 if (allEvents.count == 0) {
1586 [sysdiagnose appendString:@"No data to report for this client\n"];
1593 - (void)setUploadDateWith:(NSDate *)date reply:(void (^)(BOOL, NSError*))reply
1595 for (SFAnalyticsTopic* topic in _analyticsTopics) {
1596 [topic updateUploadDateForClients:topic.topicClients date:date clearData:NO];
1601 - (void)clientStatus:(void (^)(NSDictionary<NSString *, id> *, NSError *))reply
1603 NSMutableDictionary *info = [NSMutableDictionary dictionary];
1604 for (SFAnalyticsTopic* topic in _analyticsTopics) {
1605 for (SFAnalyticsClient *client in topic.topicClients) {
1606 SFAnalyticsSQLiteStore* store = [SFAnalyticsSQLiteStore storeWithPath:client.storePath schema:SFAnalyticsTableSchema];
1608 NSMutableDictionary *clientInfo = [NSMutableDictionary dictionary];
1609 clientInfo[@"uploadDate"] = store.uploadDate;
1610 info[client.name] = clientInfo;
1617 - (NSString*)stringForEventClass:(SFAnalyticsEventClass)eventClass
1619 if (eventClass == SFAnalyticsEventClassNote) {
1620 return @"EventNote";
1622 else if (eventClass == SFAnalyticsEventClassSuccess) {
1623 return @"EventSuccess";
1625 else if (eventClass == SFAnalyticsEventClassHardFailure) {
1626 return @"EventHardFailure";
1628 else if (eventClass == SFAnalyticsEventClassSoftFailure) {
1629 return @"EventSoftFailure";
1632 return @"EventUnknown";
1636 // MARK: XPC Procotol Handlers
1638 - (void)getSysdiagnoseDumpWithReply:(void (^)(NSString*))reply {
1639 reply([self getSysdiagnoseDump]);
1642 - (void)createLoggingJSON:(bool)pretty topic:(NSString *)topicName reply:(void (^)(NSData *, NSError*))reply {
1643 secnotice("rpcCreateLoggingJSON", "Building a JSON blob resembling the one we would have uploaded");
1644 NSError* error = nil;
1645 [self sendNotificationForOncePerReportSamplers];
1646 NSDictionary *eventDictionary = nil;
1647 for (SFAnalyticsTopic* topic in self->_analyticsTopics) {
1648 if ([topic.internalTopicName isEqualToString:topicName]) {
1649 eventDictionary = [topic createLoggingJSON:pretty forUpload:NO participatingClients:nil force:!runningTests error:&error];
1654 if (!eventDictionary) {
1655 secerror("Unable to obtain JSON: %@", error);
1657 data = [NSJSONSerialization dataWithJSONObject:eventDictionary
1658 options:(pretty ? NSJSONWritingPrettyPrinted : 0)
1665 - (void)createChunkedLoggingJSON:(bool)pretty topic:(NSString *)topicName reply:(void (^)(NSData *, NSError*))reply
1667 secnotice("rpcCreateChunkedLoggingJSON", "Building an array of JSON blobs resembling the one we would have uploaded");
1668 NSError* error = nil;
1669 [self sendNotificationForOncePerReportSamplers];
1670 NSArray<NSDictionary *> *events = nil;
1671 for (SFAnalyticsTopic* topic in self->_analyticsTopics) {
1672 if ([topic.internalTopicName isEqualToString:topicName]) {
1673 events = [topic createChunkedLoggingJSON:pretty forUpload:NO participatingClients:nil force:!runningTests error:&error];
1679 secerror("Unable to obtain JSON: %@", error);
1681 data = [NSJSONSerialization dataWithJSONObject:events
1682 options:(pretty ? NSJSONWritingPrettyPrinted : 0)
1689 - (void)forceUploadWithReply:(void (^)(BOOL, NSError*))reply {
1690 secnotice("upload", "Performing upload in response to rpc message");
1691 NSError* error = nil;
1692 BOOL result = [self uploadAnalyticsWithError:&error force:YES];
1693 secnotice("upload", "Result of manually triggered upload: %@, error: %@", result ? @"success" : @"failure", error);
1694 reply(result, error);
1699 #endif // !TARGET_OS_SIMULATOR