2 * Copyright (c) 2017-2018 Apple Inc. All Rights Reserved.
4 * @APPLE_LICENSE_HEADER_START@
6 * This file contains Original Code and/or Modifications of Original Code
7 * as defined in and that are subject to the Apple Public Source License
8 * Version 2.0 (the 'License'). You may not use this file except in
9 * compliance with the License. Please obtain a copy of the License at
10 * http://www.opensource.apple.com/apsl/ and read it before using this
13 * The Original Code and all software distributed under the License are
14 * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
15 * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
16 * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
17 * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
18 * Please see the License for the specific language governing rights and
19 * limitations under the License.
21 * @APPLE_LICENSE_HEADER_END@
25 #import "SFAnalyticsDefines.h"
26 #import "SFAnalyticsSQLiteStore.h"
27 #import "SFAnalytics.h"
29 #include <utilities/SecFileLocations.h>
30 #import "utilities/debugging.h"
31 #import <os/variant_private.h>
34 #import "keychain/ckks/CKKSControl.h"
37 #import <AuthKit/AKAppleIDAuthenticationContext.h>
38 #import <AuthKit/AKAppleIDAuthenticationController.h>
39 #import <AuthKit/AKAppleIDAuthenticationController_Private.h>
42 #include "dirhelper_priv.h"
43 #import <Accounts/Accounts.h>
44 #import <Accounts/ACAccountStore_Private.h>
45 #import <Accounts/ACAccountType_Private.h>
46 #import <Accounts/ACAccountStore.h>
47 #import <AOSAccounts/ACAccountStore+iCloudAccount.h>
48 #import <AOSAccounts/ACAccount+iCloudAccount.h>
49 #import <AOSAccountsLite/AOSAccountsLite.h>
50 #import <CrashReporterSupport/CrashReporterSupportPrivate.h>
51 #else // TARGET_OS_OSX
52 #import <Accounts/Accounts.h>
53 #import <AppleAccount/AppleAccount.h>
54 #import <AppleAccount/ACAccount+AppleAccount.h>
55 #import <AppleAccount/ACAccountStore+AppleAccount.h>
56 #if TARGET_OS_EMBEDDED
57 #import <CrashReporterSupport/CrashReporterSupport.h>
58 #import <CrashReporterSupport/PreferenceManager.h>
59 #endif // TARGET_OS_EMBEDDED
60 #endif // TARGET_OS_OSX
62 NSString* const SFAnalyticsSplunkTopic = @"topic";
63 NSString* const SFAnalyticsSplunkPostTime = @"postTime";
64 NSString* const SFAnalyticsClientId = @"clientId";
65 NSString* const SFAnalyticsInternal = @"internal";
67 NSString* const SFAnalyticsMetricsBase = @"metricsBase";
68 NSString* const SFAnalyticsDeviceID = @"ckdeviceID";
70 NSString* const SFAnalyticsSecondsCustomerKey = @"SecondsBetweenUploadsCustomer";
71 NSString* const SFAnalyticsSecondsInternalKey = @"SecondsBetweenUploadsInternal";
72 NSString* const SFAnalyticsMaxEventsKey = @"NumberOfEvents";
73 NSString* const SFAnalyticsDevicePercentageCustomerKey = @"DevicePercentageCustomer";
74 NSString* const SFAnalyticsDevicePercentageInternalKey = @"DevicePercentageInternal";
76 #define SFANALYTICS_SPLUNK_DEV 0
78 #if SFANALYTICS_SPLUNK_DEV
79 NSUInteger const secondsBetweenUploadsCustomer = 10;
80 NSUInteger const secondsBetweenUploadsInternal = 10;
81 #else // SFANALYTICS_SPLUNK_DEV
82 NSUInteger const secondsBetweenUploadsCustomer = (3 * (60 * 60 * 24));
83 NSUInteger const secondsBetweenUploadsInternal = (60 * 60 * 24);
84 #endif // SFANALYTICS_SPLUNK_DEV
87 static NSString * const _SFAnalyticsDatabasePath = @"/var/db/SecurityFrameworkAnalytics/";
88 #else // TARGET_OS_OSX
89 static NSString * const _SFAnalyticsDatabasePath = nil;
90 #endif // TARGET_OS_OSX
92 @implementation SFAnalyticsReporter
93 - (NSString *)databaseDirectoryPath
98 - (NSString *)reportsDirectoryPath
100 static NSString *_SFAnalyticsReportsDirectoryPath = nil;
101 static dispatch_once_t onceToken;
102 dispatch_once(&onceToken, ^{
103 if ([self databaseDirectoryPath]) {
104 _SFAnalyticsReportsDirectoryPath = [NSString stringWithFormat:@"%@%@", [self databaseDirectoryPath], @"Reports"];
107 return _SFAnalyticsReportsDirectoryPath;
110 - (id)initWithPath:(NSString *)path validity:(NSTimeInterval)validity
112 if (self = [super init]) {
113 _databasePath = path;
114 _reportValidityPeriod = validity;
121 return [self initWithPath:_SFAnalyticsDatabasePath validity:(secondsBetweenUploadsCustomer * 2)];
124 - (BOOL)setupReportsDirectory
126 NSString *databaseDirectoryPath = [self databaseDirectoryPath];
127 NSString *reportsDirectoryPath = [self reportsDirectoryPath];
128 if (!(databaseDirectoryPath != nil && reportsDirectoryPath != nil)) {
132 // Note: securityuploadd is not sandboxed on macOS, so we can operate in the system reports directory at will.
133 __block BOOL result = YES;
134 static dispatch_once_t onceToken;
135 dispatch_once(&onceToken, ^{
136 // Create database directory if needed
137 NSFileManager *fm = [NSFileManager defaultManager];
139 BOOL ok = [fm createDirectoryAtPath:databaseDirectoryPath
140 withIntermediateDirectories:YES
144 // Create reports directory if needed
145 ok = [fm createDirectoryAtPath:reportsDirectoryPath
146 withIntermediateDirectories:YES
150 secerror("Reports directory creation failed with %@", err);
155 secerror("Database directory creation failed with %@", err);
162 - (BOOL)removeFilesFrom:(NSString *)directory
163 olderThanSecond:(NSTimeInterval)seconds
165 NSDate *olderThanSecond = [NSDate dateWithTimeIntervalSinceNow:-seconds];
167 NSFileManager *fm = [NSFileManager defaultManager];
168 NSDirectoryEnumerator *dirEnum = [fm enumeratorAtPath:directory];
170 int64_t totalSize = 0;
171 while (fileName = [dirEnum nextObject]) {
172 NSString *filePath = [NSString stringWithFormat:@"%@/%@", directory, fileName];
174 if ([fm fileExistsAtPath:filePath isDirectory:&isDir]) {
176 // Do not remove sub-directories and their contents
177 [dirEnum skipDescendents];
179 NSDate *creationDate = [[fm attributesOfItemAtPath:filePath error:nil] fileCreationDate];
180 BOOL isOlder = ([creationDate compare:olderThanSecond] == NSOrderedAscending);
182 totalSize += [[[NSFileManager defaultManager] attributesOfItemAtPath:filePath error:nil] fileSize];
183 [fm removeItemAtPath:filePath error:nil];
191 - (BOOL)cleanupReportsDirectory
193 NSString *reportsDirectoryPath = [self reportsDirectoryPath];
194 if (reportsDirectoryPath != nil) {
196 [self removeFilesFrom:reportsDirectoryPath olderThanSecond:_reportValidityPeriod];
203 - (NSString *)createReportFilename
206 NSDictionary<NSString *, id> *problemReport = nil;
207 // We do not have our own CrashReporter key, so we make our own. This causes the default report extension to be ".diag".
208 // See: https://clownfish.apple.com/index.php?action=search_cached&path=CrashReporterSupport%2FCrashReporterSupport.c&version=CrashCatcher-938.3&project=CrashCatcher&q=&language=all&index=LoboElk
210 (__bridge NSString *)kCRProblemReportProblemTypeKey : @"supd",
211 (__bridge NSString *)kCRProblemReportAppNameKey : @"securityuploadd",
212 (__bridge NSString *)kCRProblemReportDescriptionKey : @"analytics",
213 (__bridge NSString *)kCRProblemReportNoUserUUIDKey : @YES,
214 (__bridge NSString *)kCRProblemReportRoutingKey : @"anon",
215 (__bridge NSString *)kCRProblemReportSubroutingKey : @"security_uploadd",
218 CFURLRef outputPathURL = NULL;
220 #pragma clang diagnostic push
221 #pragma clang diagnostic ignored "-Wdeprecated-declarations"
222 // TODO: Use <rdar://problem/29269488> Need a variant of OSAWriteLogForSubmission() that returns pathname for the log file
223 CRStatusCode status = CRSaveProblemReport((__bridge CFDictionaryRef)problemReport, &outputPathURL);
224 #pragma clang diagnostic pop
226 NSString *outputpath = [(__bridge NSURL *)outputPathURL path];
229 return @"temporary_path.supd";
230 #endif // TARGET_OS_OSX
233 - (BOOL)saveReport:(NSData *)reportData
236 NSString *reportFileName = [self createReportFilename];
237 if (reportFileName != nil) {
238 NSURL *path = [NSURL URLWithString:[self reportsDirectoryPath]];
240 NSURL *absoluteReportName = [path URLByAppendingPathComponent:reportFileName];
241 [[NSFileManager defaultManager] createFileAtPath:[absoluteReportName absoluteString] contents:reportData attributes:nil];
250 #define DEFAULT_SPLUNK_MAX_EVENTS_TO_REPORT 1000
252 #define DEFAULT_SPLUNK_DEVICE_PERCENTAGE 100
254 static supd *_supdInstance = nil;
256 BOOL deviceAnalyticsOverride = NO;
257 BOOL deviceAnalyticsEnabled = NO;
258 BOOL iCloudAnalyticsOverride = NO;
259 BOOL iCloudAnalyticsEnabled = NO;
262 _isDeviceAnalyticsEnabled(void)
264 // This flag is only set during tests.
265 if (deviceAnalyticsOverride) {
266 return deviceAnalyticsEnabled;
269 static BOOL dataCollectionEnabled = NO;
270 static dispatch_once_t onceToken;
271 dispatch_once(&onceToken, ^{
272 #if TARGET_OS_EMBEDDED
273 dataCollectionEnabled = DiagnosticLogSubmissionEnabled();
275 dataCollectionEnabled = CRIsAutoSubmitEnabled();
278 return dataCollectionEnabled;
281 static NSString *const kAnalyticsiCloudIdMSKey = @"com.apple.idms.config.privacy.icloud.data";
284 static NSDictionary *
285 _getiCloudConfigurationInfoWithError(NSError **outError)
287 __block NSDictionary *outConfigurationInfo = nil;
288 __block NSError *localError = nil;
290 ACAccountStore *accountStore = [[ACAccountStore alloc] init];
291 ACAccount *primaryAccount = [accountStore aa_primaryAppleAccount];
292 if (primaryAccount != nil) {
293 NSString *altDSID = [primaryAccount aa_altDSID];
294 secnotice("_getiCloudConfigurationInfoWithError", "Fetching configuration info");
296 dispatch_semaphore_t sema = dispatch_semaphore_create(0);
297 AKAppleIDAuthenticationController *authController = [AKAppleIDAuthenticationController new];
298 [authController configurationInfoWithIdentifiers:@[kAnalyticsiCloudIdMSKey]
300 completion:^(NSDictionary<NSString *, id<NSSecureCoding>> *configurationInfo, NSError *error) {
302 secerror("_getiCloudConfigurationInfoWithError: Error fetching configurationInfo: %@", error);
304 } else if (![configurationInfo isKindOfClass:[NSDictionary class]]) {
305 secerror("_getiCloudConfigurationInfoWithError: configurationInfo dict was not a dict, it was a %{public}@", [configurationInfo class]);
307 configurationInfo = nil;
309 secnotice("_getiCloudConfigurationInfoWithError", "fetched configurationInfo %@", configurationInfo);
310 outConfigurationInfo = configurationInfo;
312 dispatch_semaphore_signal(sema);
314 dispatch_semaphore_wait(sema, dispatch_time(DISPATCH_TIME_NOW, (uint64_t)(5 * NSEC_PER_SEC)));
316 secerror("_getiCloudConfigurationInfoWithError: Failed to fetch primary account info.");
319 if (localError && outError) {
320 *outError = localError;
322 return outConfigurationInfo;
324 #endif // TARGET_OS_IPHONE
330 return CFBridgingRelease(MMLCopyLoggedInAccount());
334 _altDSIDFromAccount(void)
336 static CFStringRef kMMPropertyAccountAlternateDSID = CFSTR("AccountAlternateDSID");
337 NSString *account = _iCloudAccount();
338 if (account != nil) {
339 return CFBridgingRelease(MMLAccountCopyProperty((__bridge CFStringRef)account, kMMPropertyAccountAlternateDSID));
341 secerror("_altDSIDFromAccount: failed to fetch iCloud account");
344 #endif // TARGET_OS_OSX
347 _isiCloudAnalyticsEnabled()
349 // This flag is only set during tests.
350 if (iCloudAnalyticsOverride) {
351 return iCloudAnalyticsEnabled;
354 static bool cachedAllowsICloudAnalytics = false;
357 static dispatch_once_t onceToken;
358 dispatch_once(&onceToken, ^{
359 /* AOSAccounts is not mastered into the BaseSystem. Check that those classes are linked at runtime and abort if not. */
360 if (![AKAppleIDAuthenticationController class]) {
361 secnotice("OTATrust", "Weak-linked AOSAccounts framework missing. Are we running in the base system?");
365 NSString *currentAltDSID = _altDSIDFromAccount();
366 if (currentAltDSID != nil) {
367 AKAppleIDAuthenticationController *authController = [AKAppleIDAuthenticationController new];
368 __block bool allowsICloudAnalytics = false;
369 dispatch_semaphore_t sem = dispatch_semaphore_create(0);
370 secnotice("isiCloudAnalyticsEnabled", "fetching iCloud Analytics value from idms");
371 [authController configurationInfoWithIdentifiers:@[kAnalyticsiCloudIdMSKey]
372 forAltDSID:currentAltDSID
373 completion:^(NSDictionary<NSString *, id> *configurationInfo, NSError *error) {
374 if (!error && configurationInfo) {
375 NSNumber *value = configurationInfo[kAnalyticsiCloudIdMSKey];
377 secnotice("_isiCloudAnalyticsEnabled", "authController:configurationInfoWithIdentifiers completed with no error and configuration information");
378 allowsICloudAnalytics = [value boolValue];
380 secerror("%s: no iCloud Analytics value found in IDMS", __FUNCTION__);
383 secerror("%s: Unable to fetch iCloud Analytics value from IDMS.", __FUNCTION__);
385 secnotice("_isiCloudAnalyticsEnabled", "authController:configurationInfoWithIdentifiers completed and returning");
386 dispatch_semaphore_signal(sem);
388 // Wait 5 seconds before giving up and returning from the block.
389 dispatch_semaphore_wait(sem, dispatch_time(DISPATCH_TIME_NOW, (uint64_t)(5 * NSEC_PER_SEC)));
390 cachedAllowsICloudAnalytics = allowsICloudAnalytics;
392 secerror("_isiCloudAnalyticsEnabled: Failed to fetch alternate DSID");
395 #else // TARGET_OS_OSX
396 static dispatch_once_t onceToken;
397 dispatch_once(&onceToken, ^{
398 NSError *error = nil;
399 NSDictionary *accountConfiguration = _getiCloudConfigurationInfoWithError(&error);
400 if (error == nil && accountConfiguration != nil) {
401 id iCloudAnalyticsOptIn = accountConfiguration[kAnalyticsiCloudIdMSKey];
402 if (iCloudAnalyticsOptIn != nil) {
403 BOOL iCloudAnalyticsOptInHasCorrectType = ([iCloudAnalyticsOptIn isKindOfClass:[NSNumber class]] || [iCloudAnalyticsOptIn isKindOfClass:[NSString class]]);
404 if (iCloudAnalyticsOptInHasCorrectType) {
405 NSNumber *iCloudAnalyticsOptInNumber = @([iCloudAnalyticsOptIn integerValue]);
406 cachedAllowsICloudAnalytics = ![iCloudAnalyticsOptInNumber isEqualToNumber:[NSNumber numberWithInteger:0]];
409 } else if (error != nil) {
410 secerror("_isiCloudAnalyticsEnabled: %@", error);
413 #endif // TARGET_OS_OSX
415 return cachedAllowsICloudAnalytics;
418 /* NSData GZip category based on GeoKit's implementation */
419 @interface NSData (GZip)
420 - (NSData *)supd_gzipDeflate;
423 #define GZIP_OFFSET 16
424 #define GZIP_STRIDE_LEN 16384
426 @implementation NSData (Gzip)
427 - (NSData *)supd_gzipDeflate
429 if ([self length] == 0) {
434 memset(&strm, 0, sizeof(strm));
435 strm.next_in=(uint8_t *)[self bytes];
436 strm.avail_in = (unsigned int)[self length];
439 if (Z_OK != deflateInit2(&strm, Z_BEST_COMPRESSION, Z_DEFLATED,
440 MAX_WBITS + GZIP_OFFSET, MAX_MEM_LEVEL, Z_DEFAULT_STRATEGY)) {
444 NSMutableData *compressed = [NSMutableData dataWithLength:GZIP_STRIDE_LEN];
447 if (strm.total_out >= [compressed length]) {
448 [compressed increaseLengthBy: 16384];
451 strm.next_out = [compressed mutableBytes] + strm.total_out;
452 strm.avail_out = (int)[compressed length] - (int)strm.total_out;
454 deflate(&strm, Z_FINISH);
456 } while (strm.avail_out == 0);
460 [compressed setLength: strm.total_out];
461 if (strm.avail_in == 0) {
462 return [NSData dataWithData:compressed];
469 @implementation SFAnalyticsClient {
472 BOOL _requireDeviceAnalytics;
473 BOOL _requireiCloudAnalytics;
476 @synthesize storePath = _path;
477 @synthesize name = _name;
479 - (instancetype)initWithStorePath:(NSString*)path name:(NSString*)name
480 deviceAnalytics:(BOOL)deviceAnalytics iCloudAnalytics:(BOOL)iCloudAnalytics {
481 if (self = [super init]) {
484 _requireDeviceAnalytics = deviceAnalytics;
485 _requireiCloudAnalytics = iCloudAnalytics;
492 @interface SFAnalyticsTopic ()
493 @property NSURL* _splunkUploadURL;
495 @property BOOL allowInsecureSplunkCert;
496 @property BOOL ignoreServersMessagesTellingUsToGoAway;
497 @property BOOL disableUploads;
498 @property BOOL disableClientId;
500 @property NSUInteger secondsBetweenUploads;
501 @property NSUInteger maxEventsToReport;
502 @property float devicePercentage; // for sampling reporting devices
504 @property NSDictionary* metricsBase; // data the server provides and wants us to send back
505 @property NSArray* blacklistedFields;
506 @property NSArray* blacklistedEvents;
509 @implementation SFAnalyticsTopic
510 - (void)setupClientsForTopic:(NSString *)topicName
512 NSMutableArray<SFAnalyticsClient*>* clients = [NSMutableArray<SFAnalyticsClient*> new];
513 if ([topicName isEqualToString:SFAnalyticsTopicKeySync]) {
514 [clients addObject:[[SFAnalyticsClient alloc] initWithStorePath:[self.class databasePathForCKKS]
515 name:@"ckks" deviceAnalytics:NO iCloudAnalytics:YES]];
516 [clients addObject:[[SFAnalyticsClient alloc] initWithStorePath:[self.class databasePathForSOS]
517 name:@"sos" deviceAnalytics:NO iCloudAnalytics:YES]];
518 [clients addObject:[[SFAnalyticsClient alloc] initWithStorePath:[self.class databasePathForPCS]
519 name:@"pcs" deviceAnalytics:NO iCloudAnalytics:YES]];
520 } else if ([topicName isEqualToString:SFAnaltyicsTopicTrust]) {
522 _set_user_dir_suffix("com.apple.trustd"); // supd needs to read trustd's cache dir for these
524 [clients addObject:[[SFAnalyticsClient alloc] initWithStorePath:[self.class databasePathForTrust]
525 name:@"trust" deviceAnalytics:YES iCloudAnalytics:NO]];
526 [clients addObject:[[SFAnalyticsClient alloc] initWithStorePath:[self.class databasePathForTrustdHealth]
527 name:@"trustdHealth" deviceAnalytics:YES iCloudAnalytics:NO]];
528 [clients addObject:[[SFAnalyticsClient alloc] initWithStorePath:[self.class databasePathForTLS]
529 name:@"tls" deviceAnalytics:YES iCloudAnalytics:NO]];
531 _set_user_dir_suffix(NULL); // set back to the default cache dir
535 _topicClients = clients;
538 - (instancetype)initWithDictionary:(NSDictionary *)dictionary name:(NSString *)topicName samplingRates:(NSDictionary *)rates {
539 if (self = [super init]) {
540 _internalTopicName = topicName;
541 [self setupClientsForTopic:topicName];
542 _splunkTopicName = dictionary[@"splunk_topic"];
543 __splunkUploadURL = [NSURL URLWithString:dictionary[@"splunk_uploadURL"]];
544 _splunkBagURL = [NSURL URLWithString:dictionary[@"splunk_bagURL"]];
545 _allowInsecureSplunkCert = [[dictionary valueForKey:@"splunk_allowInsecureCertificate"] boolValue];
546 NSString* splunkEndpoint = dictionary[@"splunk_endpointDomain"];
547 if (dictionary[@"disableClientId"]) {
548 _disableClientId = YES;
551 NSUserDefaults* defaults = [[NSUserDefaults alloc] initWithSuiteName:SFAnalyticsUserDefaultsSuite];
552 NSString* userDefaultsSplunkTopic = [defaults stringForKey:@"splunk_topic"];
553 if (userDefaultsSplunkTopic) {
554 _splunkTopicName = userDefaultsSplunkTopic;
557 NSURL* userDefaultsSplunkUploadURL = [NSURL URLWithString:[defaults stringForKey:@"splunk_uploadURL"]];
558 if (userDefaultsSplunkUploadURL) {
559 __splunkUploadURL = userDefaultsSplunkUploadURL;
562 NSURL* userDefaultsSplunkBagURL = [NSURL URLWithString:[defaults stringForKey:@"splunk_bagURL"]];
563 if (userDefaultsSplunkBagURL) {
564 _splunkBagURL = userDefaultsSplunkBagURL;
567 BOOL userDefaultsAllowInsecureSplunkCert = [defaults boolForKey:@"splunk_allowInsecureCertificate"];
568 _allowInsecureSplunkCert |= userDefaultsAllowInsecureSplunkCert;
570 NSString* userDefaultsSplunkEndpoint = [defaults stringForKey:@"splunk_endpointDomain"];
571 if (userDefaultsSplunkEndpoint) {
572 splunkEndpoint = userDefaultsSplunkEndpoint;
575 #if SFANALYTICS_SPLUNK_DEV
576 _secondsBetweenUploads = secondsBetweenUploadsInternal;
577 _maxEventsToReport = SFAnalyticsMaxEventsToReport;
578 _devicePercentage = DEFAULT_SPLUNK_DEVICE_PERCENTAGE;
580 bool internal = os_variant_has_internal_diagnostics("com.apple.security");
582 NSNumber *secondsNum = internal ? rates[SFAnalyticsSecondsInternalKey] : rates[SFAnalyticsSecondsCustomerKey];
583 _secondsBetweenUploads = [secondsNum integerValue];
584 _maxEventsToReport = [rates[SFAnalyticsMaxEventsKey] unsignedIntegerValue];
585 NSNumber *percentageNum = internal ? rates[SFAnalyticsDevicePercentageInternalKey] : rates[SFAnalyticsDevicePercentageCustomerKey];
586 _devicePercentage = [percentageNum floatValue];
588 _secondsBetweenUploads = internal ? secondsBetweenUploadsInternal : secondsBetweenUploadsCustomer;
589 _maxEventsToReport = SFAnalyticsMaxEventsToReport;
590 _devicePercentage = DEFAULT_SPLUNK_DEVICE_PERCENTAGE;
593 secnotice("supd", "created %@ with %lu seconds between uploads, %lu max events, %f percent of uploads",
594 _internalTopicName, (unsigned long)_secondsBetweenUploads, (unsigned long)_maxEventsToReport, _devicePercentage);
596 #if SFANALYTICS_SPLUNK_DEV
597 _ignoreServersMessagesTellingUsToGoAway = YES;
599 if (!_splunkUploadURL && splunkEndpoint) {
600 NSString* urlString = [NSString stringWithFormat:@"https://%@/report/2/%@", splunkEndpoint, _splunkTopicName];
601 _splunkUploadURL = [NSURL URLWithString:urlString];
604 (void)splunkEndpoint;
610 - (BOOL)isSampledUpload
612 uint32_t sample = arc4random();
613 if ((double)_devicePercentage < ((double)1 / UINT32_MAX) * 100) {
614 /* Requested percentage is smaller than we can sample. just do 1 out of UINT32_MAX */
619 if ((double)sample <= (double)UINT32_MAX * ((double)_devicePercentage / 100)) {
626 - (BOOL)postJSON:(NSData*)json toEndpoint:(NSURL*)endpoint error:(NSError**)error
630 NSString *description = [NSString stringWithFormat:@"No endpoint for %@", _internalTopicName];
631 *error = [NSError errorWithDomain:@"SupdUploadErrorDomain"
633 userInfo:@{NSLocalizedDescriptionKey : description}];
638 * Create the NSURLSession
639 * We use the ephemeral session config because we don't need cookies or cache
641 NSURLSessionConfiguration *configuration = [NSURLSessionConfiguration ephemeralSessionConfiguration];
643 configuration.HTTPAdditionalHeaders = @{ @"User-Agent" : [NSString stringWithFormat:@"securityd/%s", SECURITY_BUILD_VERSION]};
645 NSURLSession* postSession = [NSURLSession sessionWithConfiguration:configuration
649 NSMutableURLRequest* postRequest = [[NSMutableURLRequest alloc] init];
650 postRequest.URL = endpoint;
651 postRequest.HTTPMethod = @"POST";
652 postRequest.HTTPBody = [json supd_gzipDeflate];
653 [postRequest setValue:@"gzip" forHTTPHeaderField:@"Content-Encoding"];
656 * Create the upload task.
658 dispatch_semaphore_t sem = dispatch_semaphore_create(0);
659 __block BOOL uploadSuccess = NO;
660 NSURLSessionDataTask* uploadTask = [postSession dataTaskWithRequest:postRequest
661 completionHandler:^(NSData * _Nullable __unused data, NSURLResponse * _Nullable response, NSError * _Nullable requestError) {
663 secerror("Error in uploading the events to splunk for %@: %@", self->_internalTopicName, requestError);
664 } else if (![response isKindOfClass:NSHTTPURLResponse.class]){
665 Class class = response.class;
666 secerror("Received the wrong kind of response for %@: %@", self->_internalTopicName, NSStringFromClass(class));
668 NSHTTPURLResponse* httpResponse = (NSHTTPURLResponse*)response;
669 if(httpResponse.statusCode >= 200 && httpResponse.statusCode < 300) {
672 secnotice("upload", "Splunk upload success for %@", self->_internalTopicName);
674 secnotice("upload", "Splunk upload for %@ unexpected status to URL: %@ -- status: %d",
675 self->_internalTopicName, endpoint, (int)(httpResponse.statusCode));
678 dispatch_semaphore_signal(sem);
680 secnotice("upload", "Splunk upload start for %@", self->_internalTopicName);
682 dispatch_semaphore_wait(sem, dispatch_time(DISPATCH_TIME_NOW, (uint64_t)(5 * 60 * NSEC_PER_SEC)));
683 return uploadSuccess;
686 - (BOOL)eventIsBlacklisted:(NSMutableDictionary*)event {
687 return _blacklistedEvents ? [_blacklistedEvents containsObject:event[SFAnalyticsEventType]] : NO;
690 - (void)removeBlacklistedFieldsFromEvent:(NSMutableDictionary*)event {
691 for (NSString* badField in self->_blacklistedFields) {
692 [event removeObjectForKey:badField];
696 - (void)addRequiredFieldsToEvent:(NSMutableDictionary*)event {
697 [_metricsBase enumerateKeysAndObjectsUsingBlock:^(id _Nonnull key, id _Nonnull obj, BOOL * _Nonnull stop) {
704 - (BOOL)prepareEventForUpload:(NSMutableDictionary*)event {
705 if ([self eventIsBlacklisted:event]) {
709 [self removeBlacklistedFieldsFromEvent:event];
710 [self addRequiredFieldsToEvent:event];
711 if (_disableClientId) {
712 event[SFAnalyticsClientId] = @(0);
714 event[SFAnalyticsSplunkTopic] = self->_splunkTopicName ?: [NSNull null];
718 - (void)addFailures:(NSMutableArray<NSArray*>*)failures toUploadRecords:(NSMutableArray*)records threshold:(NSUInteger)threshold
720 // The first 0 through 'threshold' items are getting uploaded in any case (which might be 0 for lower priority data)
722 for (NSArray* client in failures) {
723 [client enumerateObjectsUsingBlock:^(id _Nonnull obj, NSUInteger idx, BOOL * _Nonnull stop) {
724 if (idx >= threshold) {
728 if ([self prepareEventForUpload:obj]) {
729 [records addObject:obj];
734 // Are there more items than we shoved into the upload records?
735 NSInteger excessItems = 0;
736 for (NSArray* client in failures) {
737 NSInteger localExcess = client.count - threshold;
738 excessItems += localExcess > 0 ? localExcess : 0;
741 // Then, if we have space and items left, apply a scaling factor to distribute events across clients to fill upload buffer
742 if (records.count < _maxEventsToReport && excessItems > 0) {
743 double scale = (_maxEventsToReport - records.count) / (double)excessItems;
748 for (NSArray* client in failures) {
749 if (client.count > threshold) {
750 NSRange range = NSMakeRange(threshold, (client.count - threshold) * scale);
751 NSArray* sub = [client subarrayWithRange:range];
752 [sub enumerateObjectsUsingBlock:^(id _Nonnull obj, NSUInteger idx, BOOL * _Nonnull stop) {
753 if ([self prepareEventForUpload:obj]) {
754 [records addObject:obj];
762 - (NSMutableDictionary*)sampleStatisticsForSamples:(NSArray*)samples withName:(NSString*)name
764 NSMutableDictionary* statistics = [NSMutableDictionary dictionary];
765 NSUInteger count = samples.count;
766 NSArray* sortedSamples = [samples sortedArrayUsingSelector:@selector(compare:)];
767 NSArray* samplesAsExpressionArray = @[[NSExpression expressionForConstantValue:sortedSamples]];
770 statistics[name] = samples[0];
772 // NSExpression takes population standard deviation. Our data is a sample of whatever we sampled over time,
773 // but the difference between the two is fairly minor (divide by N before taking sqrt versus divide by N-1).
774 statistics[[NSString stringWithFormat:@"%@-dev", name]] = [[NSExpression expressionForFunction:@"stddev:" arguments:samplesAsExpressionArray] expressionValueWithObject:nil context:nil];
776 statistics[[NSString stringWithFormat:@"%@-min", name]] = [[NSExpression expressionForFunction:@"min:" arguments:samplesAsExpressionArray] expressionValueWithObject:nil context:nil];
777 statistics[[NSString stringWithFormat:@"%@-max", name]] = [[NSExpression expressionForFunction:@"max:" arguments:samplesAsExpressionArray] expressionValueWithObject:nil context:nil];
778 statistics[[NSString stringWithFormat:@"%@-avg", name]] = [[NSExpression expressionForFunction:@"average:" arguments:samplesAsExpressionArray] expressionValueWithObject:nil context:nil];
779 statistics[[NSString stringWithFormat:@"%@-med", name]] = [[NSExpression expressionForFunction:@"median:" arguments:samplesAsExpressionArray] expressionValueWithObject:nil context:nil];
783 NSString* q1 = [NSString stringWithFormat:@"%@-1q", name];
784 NSString* q3 = [NSString stringWithFormat:@"%@-3q", name];
785 // From Wikipedia, which is never wrong
786 if (count % 2 == 0) {
787 // The lower quartile value is the median of the lower half of the data. The upper quartile value is the median of the upper half of the data.
788 statistics[q1] = [[NSExpression expressionForFunction:@"median:" arguments:@[[NSExpression expressionForConstantValue:[sortedSamples subarrayWithRange:NSMakeRange(0, count / 2)]]]] expressionValueWithObject:nil context:nil];
789 statistics[q3] = [[NSExpression expressionForFunction:@"median:" arguments:@[[NSExpression expressionForConstantValue:[sortedSamples subarrayWithRange:NSMakeRange((count / 2), count / 2)]]]] expressionValueWithObject:nil context:nil];
790 } else if (count % 4 == 1) {
791 // If there are (4n+1) data points, then the lower quartile is 25% of the nth data value plus 75% of the (n+1)th data value;
792 // the upper quartile is 75% of the (3n+1)th data point plus 25% of the (3n+2)th data point.
793 // (offset n by -1 since we count from 0)
794 NSUInteger n = count / 4;
795 statistics[q1] = @(([sortedSamples[n - 1] doubleValue] + [sortedSamples[n] doubleValue] * 3.0) / 4.0);
796 statistics[q3] = @(([sortedSamples[(3 * n)] doubleValue] * 3.0 + [sortedSamples[(3 * n) + 1] doubleValue]) / 4.0);
797 } else if (count % 4 == 3){
798 // If there are (4n+3) data points, then the lower quartile is 75% of the (n+1)th data value plus 25% of the (n+2)th data value;
799 // the upper quartile is 25% of the (3n+2)th data point plus 75% of the (3n+3)th data point.
800 // (offset n by -1 since we count from 0)
801 NSUInteger n = count / 4;
802 statistics[q1] = @(([sortedSamples[n] doubleValue] * 3.0 + [sortedSamples[n + 1] doubleValue]) / 4.0);
803 statistics[q3] = @(([sortedSamples[(3 * n) + 1] doubleValue] + [sortedSamples[(3 * n) + 2] doubleValue] * 3.0) / 4.0);
810 - (NSMutableDictionary*)healthSummaryWithName:(NSString*)name store:(SFAnalyticsSQLiteStore*)store
812 __block NSMutableDictionary* summary = [NSMutableDictionary new];
814 // Add some events of our own before pulling in data
815 summary[SFAnalyticsEventType] = [NSString stringWithFormat:@"%@HealthSummary", name];
816 if ([self eventIsBlacklisted:summary]) {
819 summary[SFAnalyticsEventTime] = @([[NSDate date] timeIntervalSince1970] * 1000); // Splunk wants milliseconds
820 [SFAnalytics addOSVersionToEvent:summary];
823 NSDictionary* successCounts = store.summaryCounts;
824 __block NSInteger totalSuccessCount = 0;
825 __block NSInteger totalHardFailureCount = 0;
826 __block NSInteger totalSoftFailureCount = 0;
827 [successCounts enumerateKeysAndObjectsUsingBlock:^(NSString* _Nonnull eventType, NSDictionary* _Nonnull counts, BOOL* _Nonnull stop) {
828 summary[[NSString stringWithFormat:@"%@-success", eventType]] = counts[SFAnalyticsColumnSuccessCount];
829 summary[[NSString stringWithFormat:@"%@-hardfail", eventType]] = counts[SFAnalyticsColumnHardFailureCount];
830 summary[[NSString stringWithFormat:@"%@-softfail", eventType]] = counts[SFAnalyticsColumnSoftFailureCount];
831 totalSuccessCount += [counts[SFAnalyticsColumnSuccessCount] integerValue];
832 totalHardFailureCount += [counts[SFAnalyticsColumnHardFailureCount] integerValue];
833 totalSoftFailureCount += [counts[SFAnalyticsColumnSoftFailureCount] integerValue];
836 summary[SFAnalyticsColumnSuccessCount] = @(totalSuccessCount);
837 summary[SFAnalyticsColumnHardFailureCount] = @(totalHardFailureCount);
838 summary[SFAnalyticsColumnSoftFailureCount] = @(totalSoftFailureCount);
839 if (os_variant_has_internal_diagnostics("com.apple.security")) {
840 summary[SFAnalyticsInternal] = @YES;
844 NSMutableDictionary<NSString*,NSMutableArray*>* samplesBySampler = [NSMutableDictionary<NSString*,NSMutableArray*> dictionary];
845 for (NSDictionary* sample in [store samples]) {
846 if (!samplesBySampler[sample[SFAnalyticsColumnSampleName]]) {
847 samplesBySampler[sample[SFAnalyticsColumnSampleName]] = [NSMutableArray array];
849 [samplesBySampler[sample[SFAnalyticsColumnSampleName]] addObject:sample[SFAnalyticsColumnSampleValue]];
851 [samplesBySampler enumerateKeysAndObjectsUsingBlock:^(NSString * _Nonnull key, NSMutableArray * _Nonnull obj, BOOL * _Nonnull stop) {
852 NSMutableDictionary* event = [self sampleStatisticsForSamples:obj withName:key];
853 [summary addEntriesFromDictionary:event];
856 // Should always return yes because we already checked for event blacklisting specifically
857 if (![self prepareEventForUpload:summary]) {
863 - (void)updateUploadDateForClients:(NSArray<SFAnalyticsClient*>*)clients clearData:(BOOL)clearData
865 for (SFAnalyticsClient* client in clients) {
866 SFAnalyticsSQLiteStore* store = [SFAnalyticsSQLiteStore storeWithPath:client.storePath schema:SFAnalyticsTableSchema];
867 secnotice("postprocess", "Setting upload date for client: %@", client.name);
868 store.uploadDate = [NSDate date];
870 secnotice("postprocess", "Clearing collected data for client: %@", client.name);
871 [store clearAllData];
876 - (NSData*)getLoggingJSON:(bool)pretty
877 forUpload:(BOOL)upload
878 participatingClients:(NSMutableArray<SFAnalyticsClient*>**)clients
879 error:(NSError**)error
881 __block NSMutableArray* uploadRecords = [NSMutableArray arrayWithCapacity:_maxEventsToReport];
882 __block NSError *localError;
883 __block NSMutableArray<NSArray*>* hardFailures = [NSMutableArray new];
884 __block NSMutableArray<NSArray*>* softFailures = [NSMutableArray new];
885 NSString* ckdeviceID = nil;
886 if ([_internalTopicName isEqualToString:SFAnalyticsTopicKeySync]) {
887 ckdeviceID = os_variant_has_internal_diagnostics("com.apple.security") ? [self askSecurityForCKDeviceID] : nil;
889 for (SFAnalyticsClient* client in self->_topicClients) {
890 if ([client requireDeviceAnalytics] && !_isDeviceAnalyticsEnabled()) {
891 // Client required device analytics, yet the user did not opt in.
892 secnotice("getLoggingJSON", "Client '%@' requires device analytics yet user did not opt in.", [client name]);
895 if ([client requireiCloudAnalytics] && !_isiCloudAnalyticsEnabled()) {
896 // Client required iCloud analytics, yet the user did not opt in.
897 secnotice("getLoggingJSON", "Client '%@' requires iCloud analytics yet user did not opt in.", [client name]);
901 SFAnalyticsSQLiteStore* store = [SFAnalyticsSQLiteStore storeWithPath:client.storePath schema:SFAnalyticsTableSchema];
904 NSDate* uploadDate = store.uploadDate;
905 if (uploadDate && [[NSDate date] timeIntervalSinceDate:uploadDate] < _secondsBetweenUploads) {
906 secnotice("json", "ignoring client '%@' for %@ because last upload too recent: %@",
907 client.name, _internalTopicName, uploadDate);
912 secnotice("json", "ignoring client '%@' because doesn't have an upload date; giving it a baseline date",
914 [self updateUploadDateForClients:@[client] clearData:NO];
918 secnotice("json", "including client '%@' for upload", client.name);
919 [*clients addObject:client];
922 NSMutableDictionary* healthSummary = [self healthSummaryWithName:client.name store:store];
925 healthSummary[SFAnalyticsDeviceID] = ckdeviceID;
927 [uploadRecords addObject:healthSummary];
930 [hardFailures addObject:store.hardFailures];
931 [softFailures addObject:store.softFailures];
933 if (upload && [*clients count] == 0) {
935 NSString *description = [NSString stringWithFormat:@"Upload too recent for all clients for %@", _internalTopicName];
936 *error = [NSError errorWithDomain:@"SupdUploadErrorDomain"
938 userInfo:@{NSLocalizedDescriptionKey : description}];
943 [self addFailures:hardFailures toUploadRecords:uploadRecords threshold:_maxEventsToReport/10];
944 [self addFailures:softFailures toUploadRecords:uploadRecords threshold:0];
946 NSDictionary* jsonDict = @{
947 SFAnalyticsSplunkPostTime : @([[NSDate date] timeIntervalSince1970] * 1000),
948 @"events" : uploadRecords
951 NSData *json = [NSJSONSerialization dataWithJSONObject:jsonDict
952 options:(pretty ? NSJSONWritingPrettyPrinted : 0)
961 - (NSString*)askSecurityForCKDeviceID
963 NSError* error = nil;
964 CKKSControl* rpc = [CKKSControl controlObject:&error];
966 secerror("unable to obtain CKKS endpoint: %@", error);
970 __block NSString* localCKDeviceID;
971 dispatch_semaphore_t sema = dispatch_semaphore_create(0);
972 [rpc rpcGetCKDeviceIDWithReply:^(NSString* ckdeviceID) {
973 localCKDeviceID = ckdeviceID;
974 dispatch_semaphore_signal(sema);
977 if (dispatch_semaphore_wait(sema, dispatch_time(DISPATCH_TIME_NOW, NSEC_PER_SEC * 10)) != 0) {
978 secerror("timed out waiting for a response from security");
982 return localCKDeviceID;
985 // this method is kind of evil for the fact that it has side-effects in pulling other things besides the metricsURL from the server, and as such should NOT be memoized.
986 // TODO redo this, probably to return a dictionary.
987 - (NSURL*)splunkUploadURL
989 if (__splunkUploadURL) {
990 return __splunkUploadURL;
993 __weak __typeof(self) weakSelf = self;
994 dispatch_semaphore_t sem = dispatch_semaphore_create(0);
996 __block NSError* error = nil;
997 NSURLSessionConfiguration *configuration = [NSURLSessionConfiguration ephemeralSessionConfiguration];
998 NSURLSession* storeBagSession = [NSURLSession sessionWithConfiguration:configuration
1002 NSURL* requestEndpoint = _splunkBagURL;
1003 __block NSURL* result = nil;
1004 NSURLSessionDataTask* storeBagTask = [storeBagSession dataTaskWithURL:requestEndpoint completionHandler:^(NSData * _Nullable data,
1005 NSURLResponse * _Nullable __unused response,
1006 NSError * _Nullable responseError) {
1008 __strong __typeof(self) strongSelf = weakSelf;
1013 if (data && !responseError) {
1014 NSData *responseData = data; // shut up compiler
1015 NSDictionary* responseDict = [NSJSONSerialization JSONObjectWithData:responseData options:0 error:&error];
1016 if([responseDict isKindOfClass:NSDictionary.class] && !error) {
1017 if (!self->_ignoreServersMessagesTellingUsToGoAway) {
1018 self->_disableUploads = [[responseDict valueForKey:@"sendDisabled"] boolValue];
1019 if (self->_disableUploads) {
1020 // then don't upload anything right now
1021 secerror("not returning a splunk URL because uploads are disabled for %@", self->_internalTopicName);
1022 dispatch_semaphore_signal(sem);
1026 // backend works with milliseconds
1027 NSUInteger secondsBetweenUploads = [[responseDict valueForKey:@"postFrequency"] unsignedIntegerValue] / 1000;
1028 if (secondsBetweenUploads > 0) {
1029 if (os_variant_has_internal_diagnostics("com.apple.security") &&
1030 self->_secondsBetweenUploads < secondsBetweenUploads) {
1031 secnotice("getURL", "Overriding server-sent post frequency because device is internal (%lu -> %lu)", secondsBetweenUploads, self->_secondsBetweenUploads);
1033 strongSelf->_secondsBetweenUploads = secondsBetweenUploads;
1037 strongSelf->_blacklistedEvents = responseDict[@"blacklistedEvents"];
1038 strongSelf->_blacklistedFields = responseDict[@"blacklistedFields"];
1041 strongSelf->_metricsBase = responseDict[@"metricsBase"];
1043 NSString* metricsEndpoint = responseDict[@"metricsUrl"];
1044 if([metricsEndpoint isKindOfClass:NSString.class]) {
1046 NSString* endpoint = [metricsEndpoint stringByAppendingFormat:@"/2/%@", strongSelf->_splunkTopicName];
1047 secnotice("upload", "got metrics endpoint %@ for %@", endpoint, self->_internalTopicName);
1048 NSURL* endpointURL = [NSURL URLWithString:endpoint];
1049 if([endpointURL.scheme isEqualToString:@"https"]) {
1050 result = endpointURL;
1056 error = responseError;
1059 secnotice("upload", "Unable to fetch splunk endpoint at URL for %@: %@ -- error: %@",
1060 self->_internalTopicName, requestEndpoint, error.description);
1063 secnotice("upload", "Malformed iTunes config payload for %@!", self->_internalTopicName);
1066 dispatch_semaphore_signal(sem);
1069 [storeBagTask resume];
1070 dispatch_semaphore_wait(sem, dispatch_time(DISPATCH_TIME_NOW, (uint64_t)(60 * NSEC_PER_SEC)));
1075 - (void)URLSession:(NSURLSession *)session didReceiveChallenge:(NSURLAuthenticationChallenge *)challenge
1076 completionHandler:(void (^)(NSURLSessionAuthChallengeDisposition, NSURLCredential *))completionHandler {
1077 assert(completionHandler);
1079 secnotice("upload", "Splunk upload challenge for %@", _internalTopicName);
1080 NSURLCredential *cred = nil;
1081 SecTrustResultType result = kSecTrustResultInvalid;
1083 if ([challenge previousFailureCount] > 0) {
1084 // Previous failures occurred, bail
1085 completionHandler(NSURLSessionAuthChallengeCancelAuthenticationChallenge, nil);
1087 } else if ([challenge.protectionSpace.authenticationMethod isEqualToString:NSURLAuthenticationMethodServerTrust]) {
1089 * Evaluate trust for the certificate
1092 SecTrustRef serverTrust = challenge.protectionSpace.serverTrust;
1093 // Coverity gets upset if we don't check status even though result is all we need.
1094 OSStatus status = SecTrustEvaluate(serverTrust, &result);
1095 if (_allowInsecureSplunkCert || (status == errSecSuccess && ((result == kSecTrustResultProceed) || (result == kSecTrustResultUnspecified)))) {
1097 * All is well, accept the credentials
1099 if(_allowInsecureSplunkCert) {
1100 secnotice("upload", "Force Accepting Splunk Credential for %@", _internalTopicName);
1102 cred = [NSURLCredential credentialForTrust:serverTrust];
1103 completionHandler(NSURLSessionAuthChallengeUseCredential, cred);
1107 * An error occurred in evaluating trust, bail
1109 completionHandler(NSURLSessionAuthChallengeCancelAuthenticationChallenge, nil);
1113 * Just perform the default handling
1115 completionHandler(NSURLSessionAuthChallengePerformDefaultHandling, nil);
1119 - (NSDictionary*)eventDictWithBlacklistedFieldsStrippedFrom:(NSDictionary*)eventDict
1121 NSMutableDictionary* strippedDict = eventDict.mutableCopy;
1122 for (NSString* blacklistedField in _blacklistedFields) {
1123 [strippedDict removeObjectForKey:blacklistedField];
1125 return strippedDict;
1128 // MARK: Database path retrieval
1130 + (NSString*)databasePathForCKKS
1132 return [(__bridge_transfer NSURL*)SecCopyURLForFileInKeychainDirectory((__bridge CFStringRef)@"Analytics/ckks_analytics.db") path];
1135 + (NSString*)databasePathForSOS
1137 return [(__bridge_transfer NSURL*)SecCopyURLForFileInKeychainDirectory((__bridge CFStringRef)@"Analytics/sos_analytics.db") path];
1140 + (NSString*)AppSupportPath
1143 return @"/var/mobile/Library/Application Support";
1145 NSArray<NSString *>*paths = NSSearchPathForDirectoriesInDomains(NSApplicationSupportDirectory, NSUserDomainMask, true);
1146 if ([paths count] < 1) {
1149 return [NSString stringWithString: paths[0]];
1150 #endif /* TARGET_OS_IOS */
1153 + (NSString*)databasePathForPCS
1155 NSString *appSup = [self AppSupportPath];
1159 NSString *dbpath = [NSString stringWithFormat:@"%@/com.apple.ProtectedCloudStorage/PCSAnalytics.db", appSup];
1160 secnotice("supd", "PCS Database path (%@)", dbpath);
1164 + (NSString*)databasePathForTrustdHealth
1166 #if TARGET_OS_IPHONE
1167 return [(__bridge_transfer NSURL*)SecCopyURLForFileInKeychainDirectory(CFSTR("Analytics/trustd_health_analytics.db")) path];
1169 return [(__bridge_transfer NSURL*)SecCopyURLForFileInUserCacheDirectory(CFSTR("Analytics/trustd_health_analytics.db")) path];
1173 + (NSString*)databasePathForTrust
1175 #if TARGET_OS_IPHONE
1176 return [(__bridge_transfer NSURL*)SecCopyURLForFileInKeychainDirectory(CFSTR("Analytics/trust_analytics.db")) path];
1178 return [(__bridge_transfer NSURL*)SecCopyURLForFileInUserCacheDirectory(CFSTR("Analytics/trust_analytics.db")) path];
1182 + (NSString*)databasePathForTLS
1184 #if TARGET_OS_IPHONE
1185 return [(__bridge_transfer NSURL*)SecCopyURLForFileInKeychainDirectory(CFSTR("Analytics/TLS_analytics.db")) path];
1187 return [(__bridge_transfer NSURL*)SecCopyURLForFileInUserCacheDirectory(CFSTR("Analytics/TLS_analytics.db")) path];
1194 @property NSDictionary *topicsSamplingRates;
1197 @implementation supd
1200 NSDictionary* systemDefaultValues = [NSDictionary dictionaryWithContentsOfFile:[[NSBundle bundleWithPath:@"/System/Library/Frameworks/Security.framework"] pathForResource:@"SFAnalytics" ofType:@"plist"]];
1201 NSMutableArray <SFAnalyticsTopic*>* topics = [NSMutableArray array];
1202 for (NSString *topicKey in systemDefaultValues) {
1203 NSDictionary *topicSamplingRates = _topicsSamplingRates[topicKey];
1204 SFAnalyticsTopic *topic = [[SFAnalyticsTopic alloc] initWithDictionary:systemDefaultValues[topicKey] name:topicKey samplingRates:topicSamplingRates];
1205 [topics addObject:topic];
1207 _analyticsTopics = [NSArray arrayWithArray:topics];
1210 + (void)instantiate {
1214 + (instancetype)instance {
1215 #if TARGET_OS_SIMULATOR
1218 if (!_supdInstance) {
1219 _supdInstance = [self new];
1221 return _supdInstance;
1225 // Use this for testing to get rid of any state
1226 + (void)removeInstance {
1227 _supdInstance = nil;
1231 static NSString *SystemTrustStorePath = @"/System/Library/Security/Certificates.bundle";
1232 static NSString *AnalyticsSamplingRatesFilename = @"AnalyticsSamplingRates";
1233 static NSString *ContentVersionKey = @"MobileAssetContentVersion";
1234 static NSString *AssetContextFilename = @"OTAPKIContext.plist";
1236 static NSNumber *getSystemVersion(NSBundle *trustStoreBundle) {
1237 NSDictionary *systemVersionPlist = [NSDictionary dictionaryWithContentsOfURL:[trustStoreBundle URLForResource:@"AssetVersion"
1238 withExtension:@"plist"]];
1239 if (!systemVersionPlist || ![systemVersionPlist isKindOfClass:[NSDictionary class]]) {
1242 NSNumber *systemVersion = systemVersionPlist[ContentVersionKey];
1243 if (systemVersion == nil || ![systemVersion isKindOfClass:[NSNumber class]]) {
1246 return systemVersion;
1249 static NSNumber *getAssetVersion(NSURL *directory) {
1250 NSDictionary *assetContextPlist = [NSDictionary dictionaryWithContentsOfURL:[directory URLByAppendingPathComponent:AssetContextFilename]];
1251 if (!assetContextPlist || ![assetContextPlist isKindOfClass:[NSDictionary class]]) {
1254 NSNumber *assetVersion = assetContextPlist[ContentVersionKey];
1255 if (assetVersion == nil || ![assetVersion isKindOfClass:[NSNumber class]]) {
1258 return assetVersion;
1261 static bool ShouldInitializeWithAsset(NSBundle *trustStoreBundle, NSURL *directory) {
1262 NSNumber *systemVersion = getSystemVersion(trustStoreBundle);
1263 NSNumber *assetVersion = getAssetVersion(directory);
1265 if (assetVersion == nil || systemVersion == nil) {
1268 if ([assetVersion compare:systemVersion] == NSOrderedDescending) {
1274 - (void)setupSamplingRates {
1275 #if TARGET_OS_SIMULATOR
1276 NSBundle *trustStoreBundle = [NSBundle bundleWithPath:[NSString stringWithFormat:@"%s%@", getenv("SIMULATOR_ROOT"), SystemTrustStorePath]];
1278 NSBundle *trustStoreBundle = [NSBundle bundleWithPath:SystemTrustStorePath];
1281 #if TARGET_OS_IPHONE
1282 NSURL *keychainsDirectory = CFBridgingRelease(SecCopyURLForFileInKeychainDirectory(nil));
1284 NSURL *keychainsDirectory = [NSURL fileURLWithFileSystemRepresentation:"/Library/Keychains/" isDirectory:YES relativeToURL:nil];
1286 NSURL *directory = [keychainsDirectory URLByAppendingPathComponent:@"SupplementalsAssets/" isDirectory:YES];
1288 NSDictionary *analyticsSamplingRates = nil;
1289 if (ShouldInitializeWithAsset(trustStoreBundle, directory)) {
1290 /* Try to get the asset version of the sampling rates */
1291 NSURL *analyticsSamplingRateURL = [directory URLByAppendingPathComponent:[NSString stringWithFormat:@"%@.plist", AnalyticsSamplingRatesFilename]];
1292 analyticsSamplingRates = [NSDictionary dictionaryWithContentsOfURL:analyticsSamplingRateURL];
1293 secnotice("supd", "read sampling rates from SupplementalsAssets dir");
1294 if (!analyticsSamplingRates || ![analyticsSamplingRates isKindOfClass:[NSDictionary class]]) {
1295 analyticsSamplingRates = nil;
1298 if (!analyticsSamplingRates) {
1299 analyticsSamplingRates = [NSDictionary dictionaryWithContentsOfURL: [trustStoreBundle URLForResource:AnalyticsSamplingRatesFilename
1300 withExtension:@"plist"]];
1302 if (analyticsSamplingRates && [analyticsSamplingRates isKindOfClass:[NSDictionary class]]) {
1303 _topicsSamplingRates = analyticsSamplingRates[@"Topics"];
1304 if (!_topicsSamplingRates || ![analyticsSamplingRates isKindOfClass:[NSDictionary class]]) {
1305 _topicsSamplingRates = nil; // Something has gone terribly wrong, so we'll use the hardcoded defaults in this case
1310 - (instancetype)initWithReporter:(SFAnalyticsReporter *)reporter
1312 if (self = [super init]) {
1313 [self setupSamplingRates];
1315 _reporter = reporter;
1316 [_reporter setupReportsDirectory];
1318 xpc_activity_register("com.apple.securityuploadd.triggerupload", XPC_ACTIVITY_CHECK_IN, ^(xpc_activity_t activity) {
1319 xpc_activity_state_t activityState = xpc_activity_get_state(activity);
1320 secnotice("supd", "hit xpc activity trigger, state: %ld", activityState);
1321 if (activityState == XPC_ACTIVITY_STATE_RUN) {
1322 // Clean up the reports directory, and then run our regularly scheduled scan
1323 [_reporter cleanupReportsDirectory];
1324 [self performRegularlyScheduledUpload];
1332 - (instancetype)init {
1333 SFAnalyticsReporter *reporter = [[SFAnalyticsReporter alloc] init];
1334 return [self initWithReporter:reporter];
1337 - (void)sendNotificationForOncePerReportSamplers
1339 notify_post(SFAnalyticsFireSamplersNotification);
1340 [NSThread sleepForTimeInterval:3.0];
1343 - (void)performRegularlyScheduledUpload {
1344 secnotice("upload", "Starting uploads in response to regular trigger");
1345 NSError *error = nil;
1346 if ([self uploadAnalyticsWithError:&error]) {
1347 secnotice("upload", "Regularly scheduled upload successful");
1349 secerror("upload: Failed to complete regularly scheduled upload: %@", error);
1353 - (BOOL)uploadAnalyticsWithError:(NSError**)error {
1354 [self sendNotificationForOncePerReportSamplers];
1357 NSError* localError = nil;
1358 for (SFAnalyticsTopic *topic in _analyticsTopics) {
1359 @autoreleasepool { // The logging JSONs get quite large. Ensure they're deallocated between topics.
1360 __block NSURL* endpoint = [topic splunkUploadURL]; // has side effects!
1361 if ([topic disableUploads]) {
1362 secnotice("upload", "Aborting upload task because uploads are disabled for %@", [topic internalTopicName]);
1366 NSMutableArray<SFAnalyticsClient*>* clients = [NSMutableArray new];
1367 NSData* json = [topic getLoggingJSON:false forUpload:YES participatingClients:&clients error:&localError];
1369 if ([topic isSampledUpload]) {
1370 BOOL writtenToLog = NO;
1372 // As of now, data is NOT logged for transparency on macOS, yet we upload anyway.
1374 #elif !TARGET_OS_SIMULATOR
1375 // We override the output here and always assume we write to the log. Data transparency will be fixed in F.
1377 #endif // !TARGET_OS_SIMULATOR
1378 if (!writtenToLog) {
1379 secerror("uploadAnalyticsWithError: failed to write analytics data to log");
1380 } else if ([topic postJSON:json toEndpoint:endpoint error:&localError]) {
1381 secnotice("uploadAnalyticsWithError", "Succeeded writing analytics data to log -- proceeding with upload");
1383 [topic updateUploadDateForClients:clients clearData:YES];
1386 /* If we didn't sample this report, update date to prevent trying to upload again sooner
1387 * than we should. Clear data so that per-day calculations remain consistent. */
1388 secnotice("upload", "skipping unsampled upload for %@ and clearing data", [topic internalTopicName]);
1389 [topic updateUploadDateForClients:clients clearData:YES];
1393 if (error && localError) {
1394 *error = localError;
1400 - (NSString*)sysdiagnoseStringForEventRecord:(NSDictionary*)eventRecord
1402 NSMutableDictionary* mutableEventRecord = eventRecord.mutableCopy;
1403 [mutableEventRecord removeObjectForKey:SFAnalyticsSplunkTopic];
1405 NSDate* eventDate = [NSDate dateWithTimeIntervalSince1970:[[eventRecord valueForKey:SFAnalyticsEventTime] doubleValue] / 1000];
1406 [mutableEventRecord removeObjectForKey:SFAnalyticsEventTime];
1408 NSString* eventName = eventRecord[SFAnalyticsEventType];
1409 [mutableEventRecord removeObjectForKey:SFAnalyticsEventType];
1411 SFAnalyticsEventClass eventClass = [[eventRecord valueForKey:SFAnalyticsEventClassKey] integerValue];
1412 NSString* eventClassString = [self stringForEventClass:eventClass];
1413 [mutableEventRecord removeObjectForKey:SFAnalyticsEventClassKey];
1415 NSMutableString* additionalAttributesString = [NSMutableString string];
1416 if (mutableEventRecord.count > 0) {
1417 [additionalAttributesString appendString:@" - Attributes: {" ];
1418 __block BOOL firstAttribute = YES;
1419 [mutableEventRecord enumerateKeysAndObjectsUsingBlock:^(NSString* key, id object, BOOL* stop) {
1420 NSString* openingString = firstAttribute ? @"" : @", ";
1421 [additionalAttributesString appendString:[NSString stringWithFormat:@"%@%@ : %@", openingString, key, object]];
1422 firstAttribute = NO;
1424 [additionalAttributesString appendString:@" }"];
1427 return [NSString stringWithFormat:@"%@ %@: %@%@", eventDate, eventClassString, eventName, additionalAttributesString];
1430 - (NSString*)getSysdiagnoseDump
1432 NSMutableString* sysdiagnose = [[NSMutableString alloc] init];
1434 for (SFAnalyticsTopic* topic in _analyticsTopics) {
1435 for (SFAnalyticsClient* client in topic.topicClients) {
1436 [sysdiagnose appendString:[NSString stringWithFormat:@"Client: %@\n", client.name]];
1437 SFAnalyticsSQLiteStore* store = [SFAnalyticsSQLiteStore storeWithPath:client.storePath schema:SFAnalyticsTableSchema];
1438 NSArray* allEvents = store.allEvents;
1439 for (NSDictionary* eventRecord in allEvents) {
1440 [sysdiagnose appendFormat:@"%@\n", [self sysdiagnoseStringForEventRecord:eventRecord]];
1442 if (allEvents.count == 0) {
1443 [sysdiagnose appendString:@"No data to report for this client\n"];
1450 - (NSString*)stringForEventClass:(SFAnalyticsEventClass)eventClass
1452 if (eventClass == SFAnalyticsEventClassNote) {
1453 return @"EventNote";
1455 else if (eventClass == SFAnalyticsEventClassSuccess) {
1456 return @"EventSuccess";
1458 else if (eventClass == SFAnalyticsEventClassHardFailure) {
1459 return @"EventHardFailure";
1461 else if (eventClass == SFAnalyticsEventClassSoftFailure) {
1462 return @"EventSoftFailure";
1465 return @"EventUnknown";
1469 // MARK: XPC Procotol Handlers
1471 - (void)getSysdiagnoseDumpWithReply:(void (^)(NSString*))reply {
1472 reply([self getSysdiagnoseDump]);
1475 - (void)getLoggingJSON:(bool)pretty topic:(NSString *)topicName reply:(void (^)(NSData*, NSError*))reply {
1476 secnotice("rpcGetLoggingJSON", "Building a JSON blob resembling the one we would have uploaded");
1477 NSError* error = nil;
1478 [self sendNotificationForOncePerReportSamplers];
1480 for (SFAnalyticsTopic* topic in self->_analyticsTopics) {
1481 if ([topic.internalTopicName isEqualToString:topicName]) {
1482 json = [topic getLoggingJSON:pretty forUpload:NO participatingClients:nil error:&error];
1486 secerror("Unable to obtain JSON: %@", error);
1491 - (void)forceUploadWithReply:(void (^)(BOOL, NSError*))reply {
1492 secnotice("upload", "Performing upload in response to rpc message");
1493 NSError* error = nil;
1494 BOOL result = [self uploadAnalyticsWithError:&error];
1495 secnotice("upload", "Result of manually triggered upload: %@, error: %@", result ? @"success" : @"failure", error);
1496 reply(result, error);