]> git.saurik.com Git - apple/security.git/blob - supd/supd.m
Security-58286.51.6.tar.gz
[apple/security.git] / supd / supd.m
1 /*
2 * Copyright (c) 2017-2018 Apple Inc. All Rights Reserved.
3 *
4 * @APPLE_LICENSE_HEADER_START@
5 *
6 * This file contains Original Code and/or Modifications of Original Code
7 * as defined in and that are subject to the Apple Public Source License
8 * Version 2.0 (the 'License'). You may not use this file except in
9 * compliance with the License. Please obtain a copy of the License at
10 * http://www.opensource.apple.com/apsl/ and read it before using this
11 * file.
12 *
13 * The Original Code and all software distributed under the License are
14 * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
15 * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
16 * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
17 * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
18 * Please see the License for the specific language governing rights and
19 * limitations under the License.
20 *
21 * @APPLE_LICENSE_HEADER_END@
22 */
23
24 #import "supd.h"
25 #import "SFAnalyticsDefines.h"
26 #import "SFAnalyticsSQLiteStore.h"
27 #import "SFAnalytics.h"
28
29 #include <utilities/SecFileLocations.h>
30 #import "utilities/debugging.h"
31 #import <os/variant_private.h>
32 #import <xpc/xpc.h>
33 #include <notify.h>
34 #import "keychain/ckks/CKKSControl.h"
35 #import <zlib.h>
36
37 #import <AuthKit/AKAppleIDAuthenticationContext.h>
38 #import <AuthKit/AKAppleIDAuthenticationController.h>
39 #import <AuthKit/AKAppleIDAuthenticationController_Private.h>
40
41 #if TARGET_OS_OSX
42 #include "dirhelper_priv.h"
43 #import <Accounts/Accounts.h>
44 #import <Accounts/ACAccountStore_Private.h>
45 #import <Accounts/ACAccountType_Private.h>
46 #import <Accounts/ACAccountStore.h>
47 #import <AOSAccounts/ACAccountStore+iCloudAccount.h>
48 #import <AOSAccounts/ACAccount+iCloudAccount.h>
49 #import <AOSAccountsLite/AOSAccountsLite.h>
50 #import <CrashReporterSupport/CrashReporterSupportPrivate.h>
51 #else // TARGET_OS_OSX
52 #import <Accounts/Accounts.h>
53 #import <AppleAccount/AppleAccount.h>
54 #import <AppleAccount/ACAccount+AppleAccount.h>
55 #import <AppleAccount/ACAccountStore+AppleAccount.h>
56 #if TARGET_OS_EMBEDDED
57 #import <CrashReporterSupport/CrashReporterSupport.h>
58 #import <CrashReporterSupport/PreferenceManager.h>
59 #endif // TARGET_OS_EMBEDDED
60 #endif // TARGET_OS_OSX
61
62 NSString* const SFAnalyticsSplunkTopic = @"topic";
63 NSString* const SFAnalyticsSplunkPostTime = @"postTime";
64 NSString* const SFAnalyticsClientId = @"clientId";
65 NSString* const SFAnalyticsInternal = @"internal";
66
67 NSString* const SFAnalyticsMetricsBase = @"metricsBase";
68 NSString* const SFAnalyticsDeviceID = @"ckdeviceID";
69
70 NSString* const SFAnalyticsSecondsCustomerKey = @"SecondsBetweenUploadsCustomer";
71 NSString* const SFAnalyticsSecondsInternalKey = @"SecondsBetweenUploadsInternal";
72 NSString* const SFAnalyticsMaxEventsKey = @"NumberOfEvents";
73 NSString* const SFAnalyticsDevicePercentageCustomerKey = @"DevicePercentageCustomer";
74 NSString* const SFAnalyticsDevicePercentageInternalKey = @"DevicePercentageInternal";
75
76 #define SFANALYTICS_SPLUNK_DEV 0
77
78 #if SFANALYTICS_SPLUNK_DEV
79 NSUInteger const secondsBetweenUploadsCustomer = 10;
80 NSUInteger const secondsBetweenUploadsInternal = 10;
81 #else // SFANALYTICS_SPLUNK_DEV
82 NSUInteger const secondsBetweenUploadsCustomer = (3 * (60 * 60 * 24));
83 NSUInteger const secondsBetweenUploadsInternal = (60 * 60 * 24);
84 #endif // SFANALYTICS_SPLUNK_DEV
85
86 #if TARGET_OS_OSX
87 static NSString * const _SFAnalyticsDatabasePath = @"/var/db/SecurityFrameworkAnalytics/";
88 #else // TARGET_OS_OSX
89 static NSString * const _SFAnalyticsDatabasePath = nil;
90 #endif // TARGET_OS_OSX
91
92 @implementation SFAnalyticsReporter
93 - (NSString *)databaseDirectoryPath
94 {
95 return _databasePath;
96 }
97
98 - (NSString *)reportsDirectoryPath
99 {
100 static NSString *_SFAnalyticsReportsDirectoryPath = nil;
101 static dispatch_once_t onceToken;
102 dispatch_once(&onceToken, ^{
103 if ([self databaseDirectoryPath]) {
104 _SFAnalyticsReportsDirectoryPath = [NSString stringWithFormat:@"%@%@", [self databaseDirectoryPath], @"Reports"];
105 }
106 });
107 return _SFAnalyticsReportsDirectoryPath;
108 }
109
110 - (id)initWithPath:(NSString *)path validity:(NSTimeInterval)validity
111 {
112 if (self = [super init]) {
113 _databasePath = path;
114 _reportValidityPeriod = validity;
115 }
116 return self;
117 }
118
119 - (id)init
120 {
121 return [self initWithPath:_SFAnalyticsDatabasePath validity:(secondsBetweenUploadsCustomer * 2)];
122 }
123
124 - (BOOL)setupReportsDirectory
125 {
126 NSString *databaseDirectoryPath = [self databaseDirectoryPath];
127 NSString *reportsDirectoryPath = [self reportsDirectoryPath];
128 if (!(databaseDirectoryPath != nil && reportsDirectoryPath != nil)) {
129 return NO;
130 }
131
132 // Note: securityuploadd is not sandboxed on macOS, so we can operate in the system reports directory at will.
133 __block BOOL result = YES;
134 static dispatch_once_t onceToken;
135 dispatch_once(&onceToken, ^{
136 // Create database directory if needed
137 NSFileManager *fm = [NSFileManager defaultManager];
138 NSError *err = nil;
139 BOOL ok = [fm createDirectoryAtPath:databaseDirectoryPath
140 withIntermediateDirectories:YES
141 attributes:nil
142 error:&err];
143 if (ok) {
144 // Create reports directory if needed
145 ok = [fm createDirectoryAtPath:reportsDirectoryPath
146 withIntermediateDirectories:YES
147 attributes:nil
148 error:&err];
149 if (!ok) {
150 secerror("Reports directory creation failed with %@", err);
151 result = NO;
152 }
153
154 } else {
155 secerror("Database directory creation failed with %@", err);
156 result = NO;
157 }
158 });
159 return result;
160 }
161
162 - (BOOL)removeFilesFrom:(NSString *)directory
163 olderThanSecond:(NSTimeInterval)seconds
164 {
165 NSDate *olderThanSecond = [NSDate dateWithTimeIntervalSinceNow:-seconds];
166
167 NSFileManager *fm = [NSFileManager defaultManager];
168 NSDirectoryEnumerator *dirEnum = [fm enumeratorAtPath:directory];
169 NSString *fileName;
170 int64_t totalSize = 0;
171 while (fileName = [dirEnum nextObject]) {
172 NSString *filePath = [NSString stringWithFormat:@"%@/%@", directory, fileName];
173 BOOL isDir;
174 if ([fm fileExistsAtPath:filePath isDirectory:&isDir]) {
175 if (isDir) {
176 // Do not remove sub-directories and their contents
177 [dirEnum skipDescendents];
178 } else {
179 NSDate *creationDate = [[fm attributesOfItemAtPath:filePath error:nil] fileCreationDate];
180 BOOL isOlder = ([creationDate compare:olderThanSecond] == NSOrderedAscending);
181 if (isOlder) {
182 totalSize += [[[NSFileManager defaultManager] attributesOfItemAtPath:filePath error:nil] fileSize];
183 [fm removeItemAtPath:filePath error:nil];
184 }
185 }
186 }
187 }
188 return YES;
189 }
190
191 - (BOOL)cleanupReportsDirectory
192 {
193 NSString *reportsDirectoryPath = [self reportsDirectoryPath];
194 if (reportsDirectoryPath != nil) {
195 @autoreleasepool {
196 [self removeFilesFrom:reportsDirectoryPath olderThanSecond:_reportValidityPeriod];
197 }
198 return YES;
199 }
200 return NO;
201 }
202
203 - (NSString *)createReportFilename
204 {
205 #if TARGET_OS_OSX
206 NSDictionary<NSString *, id> *problemReport = nil;
207 // We do not have our own CrashReporter key, so we make our own. This causes the default report extension to be ".diag".
208 // See: https://clownfish.apple.com/index.php?action=search_cached&path=CrashReporterSupport%2FCrashReporterSupport.c&version=CrashCatcher-938.3&project=CrashCatcher&q=&language=all&index=LoboElk
209 problemReport = @{
210 (__bridge NSString *)kCRProblemReportProblemTypeKey : @"supd",
211 (__bridge NSString *)kCRProblemReportAppNameKey : @"securityuploadd",
212 (__bridge NSString *)kCRProblemReportDescriptionKey : @"analytics",
213 (__bridge NSString *)kCRProblemReportNoUserUUIDKey : @YES,
214 (__bridge NSString *)kCRProblemReportRoutingKey : @"anon",
215 (__bridge NSString *)kCRProblemReportSubroutingKey : @"security_uploadd",
216 };
217
218 CFURLRef outputPathURL = NULL;
219
220 #pragma clang diagnostic push
221 #pragma clang diagnostic ignored "-Wdeprecated-declarations"
222 // TODO: Use <rdar://problem/29269488> Need a variant of OSAWriteLogForSubmission() that returns pathname for the log file
223 CRStatusCode status = CRSaveProblemReport((__bridge CFDictionaryRef)problemReport, &outputPathURL);
224 #pragma clang diagnostic pop
225
226 NSString *outputpath = [(__bridge NSURL *)outputPathURL path];
227 return outputpath;
228 #else
229 return @"temporary_path.supd";
230 #endif // TARGET_OS_OSX
231 }
232
233 - (BOOL)saveReport:(NSData *)reportData
234 {
235 @autoreleasepool {
236 NSString *reportFileName = [self createReportFilename];
237 if (reportFileName != nil) {
238 NSURL *path = [NSURL URLWithString:[self reportsDirectoryPath]];
239 if (path != nil) {
240 NSURL *absoluteReportName = [path URLByAppendingPathComponent:reportFileName];
241 [[NSFileManager defaultManager] createFileAtPath:[absoluteReportName absoluteString] contents:reportData attributes:nil];
242 return YES;
243 }
244 }
245 }
246 return NO;
247 }
248 @end
249
250 #define DEFAULT_SPLUNK_MAX_EVENTS_TO_REPORT 1000
251
252 #define DEFAULT_SPLUNK_DEVICE_PERCENTAGE 100
253
254 static supd *_supdInstance = nil;
255
256 BOOL deviceAnalyticsOverride = NO;
257 BOOL deviceAnalyticsEnabled = NO;
258 BOOL iCloudAnalyticsOverride = NO;
259 BOOL iCloudAnalyticsEnabled = NO;
260
261 static BOOL
262 _isDeviceAnalyticsEnabled(void)
263 {
264 // This flag is only set during tests.
265 if (deviceAnalyticsOverride) {
266 return deviceAnalyticsEnabled;
267 }
268
269 static BOOL dataCollectionEnabled = NO;
270 static dispatch_once_t onceToken;
271 dispatch_once(&onceToken, ^{
272 #if TARGET_OS_EMBEDDED
273 dataCollectionEnabled = DiagnosticLogSubmissionEnabled();
274 #elif TARGET_OS_OSX
275 dataCollectionEnabled = CRIsAutoSubmitEnabled();
276 #endif
277 });
278 return dataCollectionEnabled;
279 }
280
281 static NSString *const kAnalyticsiCloudIdMSKey = @"com.apple.idms.config.privacy.icloud.data";
282
283 #if TARGET_OS_IPHONE
284 static NSDictionary *
285 _getiCloudConfigurationInfoWithError(NSError **outError)
286 {
287 __block NSDictionary *outConfigurationInfo = nil;
288 __block NSError *localError = nil;
289
290 ACAccountStore *accountStore = [[ACAccountStore alloc] init];
291 ACAccount *primaryAccount = [accountStore aa_primaryAppleAccount];
292 if (primaryAccount != nil) {
293 NSString *altDSID = [primaryAccount aa_altDSID];
294 secnotice("_getiCloudConfigurationInfoWithError", "Fetching configuration info");
295
296 dispatch_semaphore_t sema = dispatch_semaphore_create(0);
297 AKAppleIDAuthenticationController *authController = [AKAppleIDAuthenticationController new];
298 [authController configurationInfoWithIdentifiers:@[kAnalyticsiCloudIdMSKey]
299 forAltDSID:altDSID
300 completion:^(NSDictionary<NSString *, id<NSSecureCoding>> *configurationInfo, NSError *error) {
301 if (error) {
302 secerror("_getiCloudConfigurationInfoWithError: Error fetching configurationInfo: %@", error);
303 localError = error;
304 } else if (![configurationInfo isKindOfClass:[NSDictionary class]]) {
305 secerror("_getiCloudConfigurationInfoWithError: configurationInfo dict was not a dict, it was a %{public}@", [configurationInfo class]);
306 localError = error;
307 configurationInfo = nil;
308 } else {
309 secnotice("_getiCloudConfigurationInfoWithError", "fetched configurationInfo %@", configurationInfo);
310 outConfigurationInfo = configurationInfo;
311 }
312 dispatch_semaphore_signal(sema);
313 }];
314 dispatch_semaphore_wait(sema, dispatch_time(DISPATCH_TIME_NOW, (uint64_t)(5 * NSEC_PER_SEC)));
315 } else {
316 secerror("_getiCloudConfigurationInfoWithError: Failed to fetch primary account info.");
317 }
318
319 if (localError && outError) {
320 *outError = localError;
321 }
322 return outConfigurationInfo;
323 }
324 #endif // TARGET_OS_IPHONE
325
326 #if TARGET_OS_OSX
327 static NSString *
328 _iCloudAccount(void)
329 {
330 return CFBridgingRelease(MMLCopyLoggedInAccount());
331 }
332
333 static NSString *
334 _altDSIDFromAccount(void)
335 {
336 static CFStringRef kMMPropertyAccountAlternateDSID = CFSTR("AccountAlternateDSID");
337 NSString *account = _iCloudAccount();
338 if (account != nil) {
339 return CFBridgingRelease(MMLAccountCopyProperty((__bridge CFStringRef)account, kMMPropertyAccountAlternateDSID));
340 }
341 secerror("_altDSIDFromAccount: failed to fetch iCloud account");
342 return nil;
343 }
344 #endif // TARGET_OS_OSX
345
346 static BOOL
347 _isiCloudAnalyticsEnabled()
348 {
349 // This flag is only set during tests.
350 if (iCloudAnalyticsOverride) {
351 return iCloudAnalyticsEnabled;
352 }
353
354 static bool cachedAllowsICloudAnalytics = false;
355
356 #if TARGET_OS_OSX
357 static dispatch_once_t onceToken;
358 dispatch_once(&onceToken, ^{
359 /* AOSAccounts is not mastered into the BaseSystem. Check that those classes are linked at runtime and abort if not. */
360 if (![AKAppleIDAuthenticationController class]) {
361 secnotice("OTATrust", "Weak-linked AOSAccounts framework missing. Are we running in the base system?");
362 return;
363 }
364
365 NSString *currentAltDSID = _altDSIDFromAccount();
366 if (currentAltDSID != nil) {
367 AKAppleIDAuthenticationController *authController = [AKAppleIDAuthenticationController new];
368 __block bool allowsICloudAnalytics = false;
369 dispatch_semaphore_t sem = dispatch_semaphore_create(0);
370 secnotice("isiCloudAnalyticsEnabled", "fetching iCloud Analytics value from idms");
371 [authController configurationInfoWithIdentifiers:@[kAnalyticsiCloudIdMSKey]
372 forAltDSID:currentAltDSID
373 completion:^(NSDictionary<NSString *, id> *configurationInfo, NSError *error) {
374 if (!error && configurationInfo) {
375 NSNumber *value = configurationInfo[kAnalyticsiCloudIdMSKey];
376 if (value != nil) {
377 secnotice("_isiCloudAnalyticsEnabled", "authController:configurationInfoWithIdentifiers completed with no error and configuration information");
378 allowsICloudAnalytics = [value boolValue];
379 } else {
380 secerror("%s: no iCloud Analytics value found in IDMS", __FUNCTION__);
381 }
382 } else {
383 secerror("%s: Unable to fetch iCloud Analytics value from IDMS.", __FUNCTION__);
384 }
385 secnotice("_isiCloudAnalyticsEnabled", "authController:configurationInfoWithIdentifiers completed and returning");
386 dispatch_semaphore_signal(sem);
387 }];
388 // Wait 5 seconds before giving up and returning from the block.
389 dispatch_semaphore_wait(sem, dispatch_time(DISPATCH_TIME_NOW, (uint64_t)(5 * NSEC_PER_SEC)));
390 cachedAllowsICloudAnalytics = allowsICloudAnalytics;
391 } else {
392 secerror("_isiCloudAnalyticsEnabled: Failed to fetch alternate DSID");
393 }
394 });
395 #else // TARGET_OS_OSX
396 static dispatch_once_t onceToken;
397 dispatch_once(&onceToken, ^{
398 NSError *error = nil;
399 NSDictionary *accountConfiguration = _getiCloudConfigurationInfoWithError(&error);
400 if (error == nil && accountConfiguration != nil) {
401 id iCloudAnalyticsOptIn = accountConfiguration[kAnalyticsiCloudIdMSKey];
402 if (iCloudAnalyticsOptIn != nil) {
403 BOOL iCloudAnalyticsOptInHasCorrectType = ([iCloudAnalyticsOptIn isKindOfClass:[NSNumber class]] || [iCloudAnalyticsOptIn isKindOfClass:[NSString class]]);
404 if (iCloudAnalyticsOptInHasCorrectType) {
405 NSNumber *iCloudAnalyticsOptInNumber = @([iCloudAnalyticsOptIn integerValue]);
406 cachedAllowsICloudAnalytics = ![iCloudAnalyticsOptInNumber isEqualToNumber:[NSNumber numberWithInteger:0]];
407 }
408 }
409 } else if (error != nil) {
410 secerror("_isiCloudAnalyticsEnabled: %@", error);
411 }
412 });
413 #endif // TARGET_OS_OSX
414
415 return cachedAllowsICloudAnalytics;
416 }
417
418 /* NSData GZip category based on GeoKit's implementation */
419 @interface NSData (GZip)
420 - (NSData *)supd_gzipDeflate;
421 @end
422
423 #define GZIP_OFFSET 16
424 #define GZIP_STRIDE_LEN 16384
425
426 @implementation NSData (Gzip)
427 - (NSData *)supd_gzipDeflate
428 {
429 if ([self length] == 0) {
430 return self;
431 }
432
433 z_stream strm;
434 memset(&strm, 0, sizeof(strm));
435 strm.next_in=(uint8_t *)[self bytes];
436 strm.avail_in = (unsigned int)[self length];
437
438
439 if (Z_OK != deflateInit2(&strm, Z_BEST_COMPRESSION, Z_DEFLATED,
440 MAX_WBITS + GZIP_OFFSET, MAX_MEM_LEVEL, Z_DEFAULT_STRATEGY)) {
441 return nil;
442 }
443
444 NSMutableData *compressed = [NSMutableData dataWithLength:GZIP_STRIDE_LEN];
445
446 do {
447 if (strm.total_out >= [compressed length]) {
448 [compressed increaseLengthBy: 16384];
449 }
450
451 strm.next_out = [compressed mutableBytes] + strm.total_out;
452 strm.avail_out = (int)[compressed length] - (int)strm.total_out;
453
454 deflate(&strm, Z_FINISH);
455
456 } while (strm.avail_out == 0);
457
458 deflateEnd(&strm);
459
460 [compressed setLength: strm.total_out];
461 if (strm.avail_in == 0) {
462 return [NSData dataWithData:compressed];
463 } else {
464 return nil;
465 }
466 }
467 @end
468
469 @implementation SFAnalyticsClient {
470 NSString* _path;
471 NSString* _name;
472 BOOL _requireDeviceAnalytics;
473 BOOL _requireiCloudAnalytics;
474 }
475
476 @synthesize storePath = _path;
477 @synthesize name = _name;
478
479 - (instancetype)initWithStorePath:(NSString*)path name:(NSString*)name
480 deviceAnalytics:(BOOL)deviceAnalytics iCloudAnalytics:(BOOL)iCloudAnalytics {
481 if (self = [super init]) {
482 _path = path;
483 _name = name;
484 _requireDeviceAnalytics = deviceAnalytics;
485 _requireiCloudAnalytics = iCloudAnalytics;
486 }
487 return self;
488 }
489
490 @end
491
492 @interface SFAnalyticsTopic ()
493 @property NSURL* _splunkUploadURL;
494
495 @property BOOL allowInsecureSplunkCert;
496 @property BOOL ignoreServersMessagesTellingUsToGoAway;
497 @property BOOL disableUploads;
498 @property BOOL disableClientId;
499
500 @property NSUInteger secondsBetweenUploads;
501 @property NSUInteger maxEventsToReport;
502 @property float devicePercentage; // for sampling reporting devices
503
504 @property NSDictionary* metricsBase; // data the server provides and wants us to send back
505 @property NSArray* blacklistedFields;
506 @property NSArray* blacklistedEvents;
507 @end
508
509 @implementation SFAnalyticsTopic
510 - (void)setupClientsForTopic:(NSString *)topicName
511 {
512 NSMutableArray<SFAnalyticsClient*>* clients = [NSMutableArray<SFAnalyticsClient*> new];
513 if ([topicName isEqualToString:SFAnalyticsTopicKeySync]) {
514 [clients addObject:[[SFAnalyticsClient alloc] initWithStorePath:[self.class databasePathForCKKS]
515 name:@"ckks" deviceAnalytics:NO iCloudAnalytics:YES]];
516 [clients addObject:[[SFAnalyticsClient alloc] initWithStorePath:[self.class databasePathForSOS]
517 name:@"sos" deviceAnalytics:NO iCloudAnalytics:YES]];
518 [clients addObject:[[SFAnalyticsClient alloc] initWithStorePath:[self.class databasePathForPCS]
519 name:@"pcs" deviceAnalytics:NO iCloudAnalytics:YES]];
520 } else if ([topicName isEqualToString:SFAnaltyicsTopicTrust]) {
521 #if TARGET_OS_OSX
522 _set_user_dir_suffix("com.apple.trustd"); // supd needs to read trustd's cache dir for these
523 #endif
524 [clients addObject:[[SFAnalyticsClient alloc] initWithStorePath:[self.class databasePathForTrust]
525 name:@"trust" deviceAnalytics:YES iCloudAnalytics:NO]];
526 [clients addObject:[[SFAnalyticsClient alloc] initWithStorePath:[self.class databasePathForTrustdHealth]
527 name:@"trustdHealth" deviceAnalytics:YES iCloudAnalytics:NO]];
528 [clients addObject:[[SFAnalyticsClient alloc] initWithStorePath:[self.class databasePathForTLS]
529 name:@"tls" deviceAnalytics:YES iCloudAnalytics:NO]];
530 #if TARGET_OS_OSX
531 _set_user_dir_suffix(NULL); // set back to the default cache dir
532 #endif
533 }
534
535 _topicClients = clients;
536 }
537
538 - (instancetype)initWithDictionary:(NSDictionary *)dictionary name:(NSString *)topicName samplingRates:(NSDictionary *)rates {
539 if (self = [super init]) {
540 _internalTopicName = topicName;
541 [self setupClientsForTopic:topicName];
542 _splunkTopicName = dictionary[@"splunk_topic"];
543 __splunkUploadURL = [NSURL URLWithString:dictionary[@"splunk_uploadURL"]];
544 _splunkBagURL = [NSURL URLWithString:dictionary[@"splunk_bagURL"]];
545 _allowInsecureSplunkCert = [[dictionary valueForKey:@"splunk_allowInsecureCertificate"] boolValue];
546 NSString* splunkEndpoint = dictionary[@"splunk_endpointDomain"];
547 if (dictionary[@"disableClientId"]) {
548 _disableClientId = YES;
549 }
550
551 NSUserDefaults* defaults = [[NSUserDefaults alloc] initWithSuiteName:SFAnalyticsUserDefaultsSuite];
552 NSString* userDefaultsSplunkTopic = [defaults stringForKey:@"splunk_topic"];
553 if (userDefaultsSplunkTopic) {
554 _splunkTopicName = userDefaultsSplunkTopic;
555 }
556
557 NSURL* userDefaultsSplunkUploadURL = [NSURL URLWithString:[defaults stringForKey:@"splunk_uploadURL"]];
558 if (userDefaultsSplunkUploadURL) {
559 __splunkUploadURL = userDefaultsSplunkUploadURL;
560 }
561
562 NSURL* userDefaultsSplunkBagURL = [NSURL URLWithString:[defaults stringForKey:@"splunk_bagURL"]];
563 if (userDefaultsSplunkBagURL) {
564 _splunkBagURL = userDefaultsSplunkBagURL;
565 }
566
567 BOOL userDefaultsAllowInsecureSplunkCert = [defaults boolForKey:@"splunk_allowInsecureCertificate"];
568 _allowInsecureSplunkCert |= userDefaultsAllowInsecureSplunkCert;
569
570 NSString* userDefaultsSplunkEndpoint = [defaults stringForKey:@"splunk_endpointDomain"];
571 if (userDefaultsSplunkEndpoint) {
572 splunkEndpoint = userDefaultsSplunkEndpoint;
573 }
574
575 #if SFANALYTICS_SPLUNK_DEV
576 _secondsBetweenUploads = secondsBetweenUploadsInternal;
577 _maxEventsToReport = SFAnalyticsMaxEventsToReport;
578 _devicePercentage = DEFAULT_SPLUNK_DEVICE_PERCENTAGE;
579 #else
580 bool internal = os_variant_has_internal_diagnostics("com.apple.security");
581 if (rates) {
582 NSNumber *secondsNum = internal ? rates[SFAnalyticsSecondsInternalKey] : rates[SFAnalyticsSecondsCustomerKey];
583 _secondsBetweenUploads = [secondsNum integerValue];
584 _maxEventsToReport = [rates[SFAnalyticsMaxEventsKey] unsignedIntegerValue];
585 NSNumber *percentageNum = internal ? rates[SFAnalyticsDevicePercentageInternalKey] : rates[SFAnalyticsDevicePercentageCustomerKey];
586 _devicePercentage = [percentageNum floatValue];
587 } else {
588 _secondsBetweenUploads = internal ? secondsBetweenUploadsInternal : secondsBetweenUploadsCustomer;
589 _maxEventsToReport = SFAnalyticsMaxEventsToReport;
590 _devicePercentage = DEFAULT_SPLUNK_DEVICE_PERCENTAGE;
591 }
592 #endif
593 secnotice("supd", "created %@ with %lu seconds between uploads, %lu max events, %f percent of uploads",
594 _internalTopicName, (unsigned long)_secondsBetweenUploads, (unsigned long)_maxEventsToReport, _devicePercentage);
595
596 #if SFANALYTICS_SPLUNK_DEV
597 _ignoreServersMessagesTellingUsToGoAway = YES;
598
599 if (!_splunkUploadURL && splunkEndpoint) {
600 NSString* urlString = [NSString stringWithFormat:@"https://%@/report/2/%@", splunkEndpoint, _splunkTopicName];
601 _splunkUploadURL = [NSURL URLWithString:urlString];
602 }
603 #else
604 (void)splunkEndpoint;
605 #endif
606 }
607 return self;
608 }
609
610 - (BOOL)isSampledUpload
611 {
612 uint32_t sample = arc4random();
613 if ((double)_devicePercentage < ((double)1 / UINT32_MAX) * 100) {
614 /* Requested percentage is smaller than we can sample. just do 1 out of UINT32_MAX */
615 if (sample == 0) {
616 return YES;
617 }
618 } else {
619 if ((double)sample <= (double)UINT32_MAX * ((double)_devicePercentage / 100)) {
620 return YES;
621 }
622 }
623 return NO;
624 }
625
626 - (BOOL)postJSON:(NSData*)json toEndpoint:(NSURL*)endpoint error:(NSError**)error
627 {
628 if (!endpoint) {
629 if (error) {
630 NSString *description = [NSString stringWithFormat:@"No endpoint for %@", _internalTopicName];
631 *error = [NSError errorWithDomain:@"SupdUploadErrorDomain"
632 code:-10
633 userInfo:@{NSLocalizedDescriptionKey : description}];
634 }
635 return false;
636 }
637 /*
638 * Create the NSURLSession
639 * We use the ephemeral session config because we don't need cookies or cache
640 */
641 NSURLSessionConfiguration *configuration = [NSURLSessionConfiguration ephemeralSessionConfiguration];
642
643 configuration.HTTPAdditionalHeaders = @{ @"User-Agent" : [NSString stringWithFormat:@"securityd/%s", SECURITY_BUILD_VERSION]};
644
645 NSURLSession* postSession = [NSURLSession sessionWithConfiguration:configuration
646 delegate:self
647 delegateQueue:nil];
648
649 NSMutableURLRequest* postRequest = [[NSMutableURLRequest alloc] init];
650 postRequest.URL = endpoint;
651 postRequest.HTTPMethod = @"POST";
652 postRequest.HTTPBody = [json supd_gzipDeflate];
653 [postRequest setValue:@"gzip" forHTTPHeaderField:@"Content-Encoding"];
654
655 /*
656 * Create the upload task.
657 */
658 dispatch_semaphore_t sem = dispatch_semaphore_create(0);
659 __block BOOL uploadSuccess = NO;
660 NSURLSessionDataTask* uploadTask = [postSession dataTaskWithRequest:postRequest
661 completionHandler:^(NSData * _Nullable __unused data, NSURLResponse * _Nullable response, NSError * _Nullable requestError) {
662 if (requestError) {
663 secerror("Error in uploading the events to splunk for %@: %@", self->_internalTopicName, requestError);
664 } else if (![response isKindOfClass:NSHTTPURLResponse.class]){
665 Class class = response.class;
666 secerror("Received the wrong kind of response for %@: %@", self->_internalTopicName, NSStringFromClass(class));
667 } else {
668 NSHTTPURLResponse* httpResponse = (NSHTTPURLResponse*)response;
669 if(httpResponse.statusCode >= 200 && httpResponse.statusCode < 300) {
670 /* Success */
671 uploadSuccess = YES;
672 secnotice("upload", "Splunk upload success for %@", self->_internalTopicName);
673 } else {
674 secnotice("upload", "Splunk upload for %@ unexpected status to URL: %@ -- status: %d",
675 self->_internalTopicName, endpoint, (int)(httpResponse.statusCode));
676 }
677 }
678 dispatch_semaphore_signal(sem);
679 }];
680 secnotice("upload", "Splunk upload start for %@", self->_internalTopicName);
681 [uploadTask resume];
682 dispatch_semaphore_wait(sem, dispatch_time(DISPATCH_TIME_NOW, (uint64_t)(5 * 60 * NSEC_PER_SEC)));
683 return uploadSuccess;
684 }
685
686 - (BOOL)eventIsBlacklisted:(NSMutableDictionary*)event {
687 return _blacklistedEvents ? [_blacklistedEvents containsObject:event[SFAnalyticsEventType]] : NO;
688 }
689
690 - (void)removeBlacklistedFieldsFromEvent:(NSMutableDictionary*)event {
691 for (NSString* badField in self->_blacklistedFields) {
692 [event removeObjectForKey:badField];
693 }
694 }
695
696 - (void)addRequiredFieldsToEvent:(NSMutableDictionary*)event {
697 [_metricsBase enumerateKeysAndObjectsUsingBlock:^(id _Nonnull key, id _Nonnull obj, BOOL * _Nonnull stop) {
698 if (!event[key]) {
699 event[key] = obj;
700 }
701 }];
702 }
703
704 - (BOOL)prepareEventForUpload:(NSMutableDictionary*)event {
705 if ([self eventIsBlacklisted:event]) {
706 return NO;
707 }
708
709 [self removeBlacklistedFieldsFromEvent:event];
710 [self addRequiredFieldsToEvent:event];
711 if (_disableClientId) {
712 event[SFAnalyticsClientId] = @(0);
713 }
714 event[SFAnalyticsSplunkTopic] = self->_splunkTopicName ?: [NSNull null];
715 return YES;
716 }
717
718 - (void)addFailures:(NSMutableArray<NSArray*>*)failures toUploadRecords:(NSMutableArray*)records threshold:(NSUInteger)threshold
719 {
720 // The first 0 through 'threshold' items are getting uploaded in any case (which might be 0 for lower priority data)
721
722 for (NSArray* client in failures) {
723 [client enumerateObjectsUsingBlock:^(id _Nonnull obj, NSUInteger idx, BOOL * _Nonnull stop) {
724 if (idx >= threshold) {
725 *stop = YES;
726 return;
727 }
728 if ([self prepareEventForUpload:obj]) {
729 [records addObject:obj];
730 }
731 }];
732 }
733
734 // Are there more items than we shoved into the upload records?
735 NSInteger excessItems = 0;
736 for (NSArray* client in failures) {
737 NSInteger localExcess = client.count - threshold;
738 excessItems += localExcess > 0 ? localExcess : 0;
739 }
740
741 // Then, if we have space and items left, apply a scaling factor to distribute events across clients to fill upload buffer
742 if (records.count < _maxEventsToReport && excessItems > 0) {
743 double scale = (_maxEventsToReport - records.count) / (double)excessItems;
744 if (scale > 1) {
745 scale = 1;
746 }
747
748 for (NSArray* client in failures) {
749 if (client.count > threshold) {
750 NSRange range = NSMakeRange(threshold, (client.count - threshold) * scale);
751 NSArray* sub = [client subarrayWithRange:range];
752 [sub enumerateObjectsUsingBlock:^(id _Nonnull obj, NSUInteger idx, BOOL * _Nonnull stop) {
753 if ([self prepareEventForUpload:obj]) {
754 [records addObject:obj];
755 }
756 }];
757 }
758 }
759 }
760 }
761
762 - (NSMutableDictionary*)sampleStatisticsForSamples:(NSArray*)samples withName:(NSString*)name
763 {
764 NSMutableDictionary* statistics = [NSMutableDictionary dictionary];
765 NSUInteger count = samples.count;
766 NSArray* sortedSamples = [samples sortedArrayUsingSelector:@selector(compare:)];
767 NSArray* samplesAsExpressionArray = @[[NSExpression expressionForConstantValue:sortedSamples]];
768
769 if (count == 1) {
770 statistics[name] = samples[0];
771 } else {
772 // NSExpression takes population standard deviation. Our data is a sample of whatever we sampled over time,
773 // but the difference between the two is fairly minor (divide by N before taking sqrt versus divide by N-1).
774 statistics[[NSString stringWithFormat:@"%@-dev", name]] = [[NSExpression expressionForFunction:@"stddev:" arguments:samplesAsExpressionArray] expressionValueWithObject:nil context:nil];
775
776 statistics[[NSString stringWithFormat:@"%@-min", name]] = [[NSExpression expressionForFunction:@"min:" arguments:samplesAsExpressionArray] expressionValueWithObject:nil context:nil];
777 statistics[[NSString stringWithFormat:@"%@-max", name]] = [[NSExpression expressionForFunction:@"max:" arguments:samplesAsExpressionArray] expressionValueWithObject:nil context:nil];
778 statistics[[NSString stringWithFormat:@"%@-avg", name]] = [[NSExpression expressionForFunction:@"average:" arguments:samplesAsExpressionArray] expressionValueWithObject:nil context:nil];
779 statistics[[NSString stringWithFormat:@"%@-med", name]] = [[NSExpression expressionForFunction:@"median:" arguments:samplesAsExpressionArray] expressionValueWithObject:nil context:nil];
780 }
781
782 if (count > 3) {
783 NSString* q1 = [NSString stringWithFormat:@"%@-1q", name];
784 NSString* q3 = [NSString stringWithFormat:@"%@-3q", name];
785 // From Wikipedia, which is never wrong
786 if (count % 2 == 0) {
787 // The lower quartile value is the median of the lower half of the data. The upper quartile value is the median of the upper half of the data.
788 statistics[q1] = [[NSExpression expressionForFunction:@"median:" arguments:@[[NSExpression expressionForConstantValue:[sortedSamples subarrayWithRange:NSMakeRange(0, count / 2)]]]] expressionValueWithObject:nil context:nil];
789 statistics[q3] = [[NSExpression expressionForFunction:@"median:" arguments:@[[NSExpression expressionForConstantValue:[sortedSamples subarrayWithRange:NSMakeRange((count / 2), count / 2)]]]] expressionValueWithObject:nil context:nil];
790 } else if (count % 4 == 1) {
791 // If there are (4n+1) data points, then the lower quartile is 25% of the nth data value plus 75% of the (n+1)th data value;
792 // the upper quartile is 75% of the (3n+1)th data point plus 25% of the (3n+2)th data point.
793 // (offset n by -1 since we count from 0)
794 NSUInteger n = count / 4;
795 statistics[q1] = @(([sortedSamples[n - 1] doubleValue] + [sortedSamples[n] doubleValue] * 3.0) / 4.0);
796 statistics[q3] = @(([sortedSamples[(3 * n)] doubleValue] * 3.0 + [sortedSamples[(3 * n) + 1] doubleValue]) / 4.0);
797 } else if (count % 4 == 3){
798 // If there are (4n+3) data points, then the lower quartile is 75% of the (n+1)th data value plus 25% of the (n+2)th data value;
799 // the upper quartile is 25% of the (3n+2)th data point plus 75% of the (3n+3)th data point.
800 // (offset n by -1 since we count from 0)
801 NSUInteger n = count / 4;
802 statistics[q1] = @(([sortedSamples[n] doubleValue] * 3.0 + [sortedSamples[n + 1] doubleValue]) / 4.0);
803 statistics[q3] = @(([sortedSamples[(3 * n) + 1] doubleValue] + [sortedSamples[(3 * n) + 2] doubleValue] * 3.0) / 4.0);
804 }
805 }
806
807 return statistics;
808 }
809
810 - (NSMutableDictionary*)healthSummaryWithName:(NSString*)name store:(SFAnalyticsSQLiteStore*)store
811 {
812 __block NSMutableDictionary* summary = [NSMutableDictionary new];
813
814 // Add some events of our own before pulling in data
815 summary[SFAnalyticsEventType] = [NSString stringWithFormat:@"%@HealthSummary", name];
816 if ([self eventIsBlacklisted:summary]) {
817 return nil;
818 }
819 summary[SFAnalyticsEventTime] = @([[NSDate date] timeIntervalSince1970] * 1000); // Splunk wants milliseconds
820 [SFAnalytics addOSVersionToEvent:summary];
821
822 // Process counters
823 NSDictionary* successCounts = store.summaryCounts;
824 __block NSInteger totalSuccessCount = 0;
825 __block NSInteger totalHardFailureCount = 0;
826 __block NSInteger totalSoftFailureCount = 0;
827 [successCounts enumerateKeysAndObjectsUsingBlock:^(NSString* _Nonnull eventType, NSDictionary* _Nonnull counts, BOOL* _Nonnull stop) {
828 summary[[NSString stringWithFormat:@"%@-success", eventType]] = counts[SFAnalyticsColumnSuccessCount];
829 summary[[NSString stringWithFormat:@"%@-hardfail", eventType]] = counts[SFAnalyticsColumnHardFailureCount];
830 summary[[NSString stringWithFormat:@"%@-softfail", eventType]] = counts[SFAnalyticsColumnSoftFailureCount];
831 totalSuccessCount += [counts[SFAnalyticsColumnSuccessCount] integerValue];
832 totalHardFailureCount += [counts[SFAnalyticsColumnHardFailureCount] integerValue];
833 totalSoftFailureCount += [counts[SFAnalyticsColumnSoftFailureCount] integerValue];
834 }];
835
836 summary[SFAnalyticsColumnSuccessCount] = @(totalSuccessCount);
837 summary[SFAnalyticsColumnHardFailureCount] = @(totalHardFailureCount);
838 summary[SFAnalyticsColumnSoftFailureCount] = @(totalSoftFailureCount);
839 if (os_variant_has_internal_diagnostics("com.apple.security")) {
840 summary[SFAnalyticsInternal] = @YES;
841 }
842
843 // Process samples
844 NSMutableDictionary<NSString*,NSMutableArray*>* samplesBySampler = [NSMutableDictionary<NSString*,NSMutableArray*> dictionary];
845 for (NSDictionary* sample in [store samples]) {
846 if (!samplesBySampler[sample[SFAnalyticsColumnSampleName]]) {
847 samplesBySampler[sample[SFAnalyticsColumnSampleName]] = [NSMutableArray array];
848 }
849 [samplesBySampler[sample[SFAnalyticsColumnSampleName]] addObject:sample[SFAnalyticsColumnSampleValue]];
850 }
851 [samplesBySampler enumerateKeysAndObjectsUsingBlock:^(NSString * _Nonnull key, NSMutableArray * _Nonnull obj, BOOL * _Nonnull stop) {
852 NSMutableDictionary* event = [self sampleStatisticsForSamples:obj withName:key];
853 [summary addEntriesFromDictionary:event];
854 }];
855
856 // Should always return yes because we already checked for event blacklisting specifically
857 if (![self prepareEventForUpload:summary]) {
858 return nil;
859 }
860 return summary;
861 }
862
863 - (void)updateUploadDateForClients:(NSArray<SFAnalyticsClient*>*)clients clearData:(BOOL)clearData
864 {
865 for (SFAnalyticsClient* client in clients) {
866 SFAnalyticsSQLiteStore* store = [SFAnalyticsSQLiteStore storeWithPath:client.storePath schema:SFAnalyticsTableSchema];
867 secnotice("postprocess", "Setting upload date for client: %@", client.name);
868 store.uploadDate = [NSDate date];
869 if (clearData) {
870 secnotice("postprocess", "Clearing collected data for client: %@", client.name);
871 [store clearAllData];
872 }
873 }
874 }
875
876 - (NSData*)getLoggingJSON:(bool)pretty
877 forUpload:(BOOL)upload
878 participatingClients:(NSMutableArray<SFAnalyticsClient*>**)clients
879 error:(NSError**)error
880 {
881 __block NSMutableArray* uploadRecords = [NSMutableArray arrayWithCapacity:_maxEventsToReport];
882 __block NSError *localError;
883 __block NSMutableArray<NSArray*>* hardFailures = [NSMutableArray new];
884 __block NSMutableArray<NSArray*>* softFailures = [NSMutableArray new];
885 NSString* ckdeviceID = nil;
886 if ([_internalTopicName isEqualToString:SFAnalyticsTopicKeySync]) {
887 ckdeviceID = os_variant_has_internal_diagnostics("com.apple.security") ? [self askSecurityForCKDeviceID] : nil;
888 }
889 for (SFAnalyticsClient* client in self->_topicClients) {
890 if ([client requireDeviceAnalytics] && !_isDeviceAnalyticsEnabled()) {
891 // Client required device analytics, yet the user did not opt in.
892 secnotice("getLoggingJSON", "Client '%@' requires device analytics yet user did not opt in.", [client name]);
893 continue;
894 }
895 if ([client requireiCloudAnalytics] && !_isiCloudAnalyticsEnabled()) {
896 // Client required iCloud analytics, yet the user did not opt in.
897 secnotice("getLoggingJSON", "Client '%@' requires iCloud analytics yet user did not opt in.", [client name]);
898 continue;
899 }
900
901 SFAnalyticsSQLiteStore* store = [SFAnalyticsSQLiteStore storeWithPath:client.storePath schema:SFAnalyticsTableSchema];
902
903 if (upload) {
904 NSDate* uploadDate = store.uploadDate;
905 if (uploadDate && [[NSDate date] timeIntervalSinceDate:uploadDate] < _secondsBetweenUploads) {
906 secnotice("json", "ignoring client '%@' for %@ because last upload too recent: %@",
907 client.name, _internalTopicName, uploadDate);
908 continue;
909 }
910
911 if (!uploadDate) {
912 secnotice("json", "ignoring client '%@' because doesn't have an upload date; giving it a baseline date",
913 client.name);
914 [self updateUploadDateForClients:@[client] clearData:NO];
915 continue;
916 }
917
918 secnotice("json", "including client '%@' for upload", client.name);
919 [*clients addObject:client];
920 }
921
922 NSMutableDictionary* healthSummary = [self healthSummaryWithName:client.name store:store];
923 if (healthSummary) {
924 if (ckdeviceID) {
925 healthSummary[SFAnalyticsDeviceID] = ckdeviceID;
926 }
927 [uploadRecords addObject:healthSummary];
928 }
929
930 [hardFailures addObject:store.hardFailures];
931 [softFailures addObject:store.softFailures];
932 }
933 if (upload && [*clients count] == 0) {
934 if (error) {
935 NSString *description = [NSString stringWithFormat:@"Upload too recent for all clients for %@", _internalTopicName];
936 *error = [NSError errorWithDomain:@"SupdUploadErrorDomain"
937 code:-10
938 userInfo:@{NSLocalizedDescriptionKey : description}];
939 }
940 return nil;
941 }
942
943 [self addFailures:hardFailures toUploadRecords:uploadRecords threshold:_maxEventsToReport/10];
944 [self addFailures:softFailures toUploadRecords:uploadRecords threshold:0];
945
946 NSDictionary* jsonDict = @{
947 SFAnalyticsSplunkPostTime : @([[NSDate date] timeIntervalSince1970] * 1000),
948 @"events" : uploadRecords
949 };
950
951 NSData *json = [NSJSONSerialization dataWithJSONObject:jsonDict
952 options:(pretty ? NSJSONWritingPrettyPrinted : 0)
953 error:&localError];
954
955 if (error) {
956 *error = localError;
957 }
958 return json;
959 }
960
961 - (NSString*)askSecurityForCKDeviceID
962 {
963 NSError* error = nil;
964 CKKSControl* rpc = [CKKSControl controlObject:&error];
965 if(error || !rpc) {
966 secerror("unable to obtain CKKS endpoint: %@", error);
967 return nil;
968 }
969
970 __block NSString* localCKDeviceID;
971 dispatch_semaphore_t sema = dispatch_semaphore_create(0);
972 [rpc rpcGetCKDeviceIDWithReply:^(NSString* ckdeviceID) {
973 localCKDeviceID = ckdeviceID;
974 dispatch_semaphore_signal(sema);
975 }];
976
977 if (dispatch_semaphore_wait(sema, dispatch_time(DISPATCH_TIME_NOW, NSEC_PER_SEC * 10)) != 0) {
978 secerror("timed out waiting for a response from security");
979 return nil;
980 }
981
982 return localCKDeviceID;
983 }
984
985 // this method is kind of evil for the fact that it has side-effects in pulling other things besides the metricsURL from the server, and as such should NOT be memoized.
986 // TODO redo this, probably to return a dictionary.
987 - (NSURL*)splunkUploadURL
988 {
989 if (__splunkUploadURL) {
990 return __splunkUploadURL;
991 }
992
993 __weak __typeof(self) weakSelf = self;
994 dispatch_semaphore_t sem = dispatch_semaphore_create(0);
995
996 __block NSError* error = nil;
997 NSURLSessionConfiguration *configuration = [NSURLSessionConfiguration ephemeralSessionConfiguration];
998 NSURLSession* storeBagSession = [NSURLSession sessionWithConfiguration:configuration
999 delegate:self
1000 delegateQueue:nil];
1001
1002 NSURL* requestEndpoint = _splunkBagURL;
1003 __block NSURL* result = nil;
1004 NSURLSessionDataTask* storeBagTask = [storeBagSession dataTaskWithURL:requestEndpoint completionHandler:^(NSData * _Nullable data,
1005 NSURLResponse * _Nullable __unused response,
1006 NSError * _Nullable responseError) {
1007
1008 __strong __typeof(self) strongSelf = weakSelf;
1009 if (!strongSelf) {
1010 return;
1011 }
1012
1013 if (data && !responseError) {
1014 NSData *responseData = data; // shut up compiler
1015 NSDictionary* responseDict = [NSJSONSerialization JSONObjectWithData:responseData options:0 error:&error];
1016 if([responseDict isKindOfClass:NSDictionary.class] && !error) {
1017 if (!self->_ignoreServersMessagesTellingUsToGoAway) {
1018 self->_disableUploads = [[responseDict valueForKey:@"sendDisabled"] boolValue];
1019 if (self->_disableUploads) {
1020 // then don't upload anything right now
1021 secerror("not returning a splunk URL because uploads are disabled for %@", self->_internalTopicName);
1022 dispatch_semaphore_signal(sem);
1023 return;
1024 }
1025
1026 // backend works with milliseconds
1027 NSUInteger secondsBetweenUploads = [[responseDict valueForKey:@"postFrequency"] unsignedIntegerValue] / 1000;
1028 if (secondsBetweenUploads > 0) {
1029 if (os_variant_has_internal_diagnostics("com.apple.security") &&
1030 self->_secondsBetweenUploads < secondsBetweenUploads) {
1031 secnotice("getURL", "Overriding server-sent post frequency because device is internal (%lu -> %lu)", secondsBetweenUploads, self->_secondsBetweenUploads);
1032 } else {
1033 strongSelf->_secondsBetweenUploads = secondsBetweenUploads;
1034 }
1035 }
1036
1037 strongSelf->_blacklistedEvents = responseDict[@"blacklistedEvents"];
1038 strongSelf->_blacklistedFields = responseDict[@"blacklistedFields"];
1039 }
1040
1041 strongSelf->_metricsBase = responseDict[@"metricsBase"];
1042
1043 NSString* metricsEndpoint = responseDict[@"metricsUrl"];
1044 if([metricsEndpoint isKindOfClass:NSString.class]) {
1045 /* Lives our URL */
1046 NSString* endpoint = [metricsEndpoint stringByAppendingFormat:@"/2/%@", strongSelf->_splunkTopicName];
1047 secnotice("upload", "got metrics endpoint %@ for %@", endpoint, self->_internalTopicName);
1048 NSURL* endpointURL = [NSURL URLWithString:endpoint];
1049 if([endpointURL.scheme isEqualToString:@"https"]) {
1050 result = endpointURL;
1051 }
1052 }
1053 }
1054 }
1055 else {
1056 error = responseError;
1057 }
1058 if (error) {
1059 secnotice("upload", "Unable to fetch splunk endpoint at URL for %@: %@ -- error: %@",
1060 self->_internalTopicName, requestEndpoint, error.description);
1061 }
1062 else if (!result) {
1063 secnotice("upload", "Malformed iTunes config payload for %@!", self->_internalTopicName);
1064 }
1065
1066 dispatch_semaphore_signal(sem);
1067 }];
1068
1069 [storeBagTask resume];
1070 dispatch_semaphore_wait(sem, dispatch_time(DISPATCH_TIME_NOW, (uint64_t)(60 * NSEC_PER_SEC)));
1071
1072 return result;
1073 }
1074
1075 - (void)URLSession:(NSURLSession *)session didReceiveChallenge:(NSURLAuthenticationChallenge *)challenge
1076 completionHandler:(void (^)(NSURLSessionAuthChallengeDisposition, NSURLCredential *))completionHandler {
1077 assert(completionHandler);
1078 (void)session;
1079 secnotice("upload", "Splunk upload challenge for %@", _internalTopicName);
1080 NSURLCredential *cred = nil;
1081 SecTrustResultType result = kSecTrustResultInvalid;
1082
1083 if ([challenge previousFailureCount] > 0) {
1084 // Previous failures occurred, bail
1085 completionHandler(NSURLSessionAuthChallengeCancelAuthenticationChallenge, nil);
1086
1087 } else if ([challenge.protectionSpace.authenticationMethod isEqualToString:NSURLAuthenticationMethodServerTrust]) {
1088 /*
1089 * Evaluate trust for the certificate
1090 */
1091
1092 SecTrustRef serverTrust = challenge.protectionSpace.serverTrust;
1093 // Coverity gets upset if we don't check status even though result is all we need.
1094 OSStatus status = SecTrustEvaluate(serverTrust, &result);
1095 if (_allowInsecureSplunkCert || (status == errSecSuccess && ((result == kSecTrustResultProceed) || (result == kSecTrustResultUnspecified)))) {
1096 /*
1097 * All is well, accept the credentials
1098 */
1099 if(_allowInsecureSplunkCert) {
1100 secnotice("upload", "Force Accepting Splunk Credential for %@", _internalTopicName);
1101 }
1102 cred = [NSURLCredential credentialForTrust:serverTrust];
1103 completionHandler(NSURLSessionAuthChallengeUseCredential, cred);
1104
1105 } else {
1106 /*
1107 * An error occurred in evaluating trust, bail
1108 */
1109 completionHandler(NSURLSessionAuthChallengeCancelAuthenticationChallenge, nil);
1110 }
1111 } else {
1112 /*
1113 * Just perform the default handling
1114 */
1115 completionHandler(NSURLSessionAuthChallengePerformDefaultHandling, nil);
1116 }
1117 }
1118
1119 - (NSDictionary*)eventDictWithBlacklistedFieldsStrippedFrom:(NSDictionary*)eventDict
1120 {
1121 NSMutableDictionary* strippedDict = eventDict.mutableCopy;
1122 for (NSString* blacklistedField in _blacklistedFields) {
1123 [strippedDict removeObjectForKey:blacklistedField];
1124 }
1125 return strippedDict;
1126 }
1127
1128 // MARK: Database path retrieval
1129
1130 + (NSString*)databasePathForCKKS
1131 {
1132 return [(__bridge_transfer NSURL*)SecCopyURLForFileInKeychainDirectory((__bridge CFStringRef)@"Analytics/ckks_analytics.db") path];
1133 }
1134
1135 + (NSString*)databasePathForSOS
1136 {
1137 return [(__bridge_transfer NSURL*)SecCopyURLForFileInKeychainDirectory((__bridge CFStringRef)@"Analytics/sos_analytics.db") path];
1138 }
1139
1140 + (NSString*)AppSupportPath
1141 {
1142 #if TARGET_OS_IOS
1143 return @"/var/mobile/Library/Application Support";
1144 #else
1145 NSArray<NSString *>*paths = NSSearchPathForDirectoriesInDomains(NSApplicationSupportDirectory, NSUserDomainMask, true);
1146 if ([paths count] < 1) {
1147 return nil;
1148 }
1149 return [NSString stringWithString: paths[0]];
1150 #endif /* TARGET_OS_IOS */
1151 }
1152
1153 + (NSString*)databasePathForPCS
1154 {
1155 NSString *appSup = [self AppSupportPath];
1156 if (!appSup) {
1157 return nil;
1158 }
1159 NSString *dbpath = [NSString stringWithFormat:@"%@/com.apple.ProtectedCloudStorage/PCSAnalytics.db", appSup];
1160 secnotice("supd", "PCS Database path (%@)", dbpath);
1161 return dbpath;
1162 }
1163
1164 + (NSString*)databasePathForTrustdHealth
1165 {
1166 #if TARGET_OS_IPHONE
1167 return [(__bridge_transfer NSURL*)SecCopyURLForFileInKeychainDirectory(CFSTR("Analytics/trustd_health_analytics.db")) path];
1168 #else
1169 return [(__bridge_transfer NSURL*)SecCopyURLForFileInUserCacheDirectory(CFSTR("Analytics/trustd_health_analytics.db")) path];
1170 #endif
1171 }
1172
1173 + (NSString*)databasePathForTrust
1174 {
1175 #if TARGET_OS_IPHONE
1176 return [(__bridge_transfer NSURL*)SecCopyURLForFileInKeychainDirectory(CFSTR("Analytics/trust_analytics.db")) path];
1177 #else
1178 return [(__bridge_transfer NSURL*)SecCopyURLForFileInUserCacheDirectory(CFSTR("Analytics/trust_analytics.db")) path];
1179 #endif
1180 }
1181
1182 + (NSString*)databasePathForTLS
1183 {
1184 #if TARGET_OS_IPHONE
1185 return [(__bridge_transfer NSURL*)SecCopyURLForFileInKeychainDirectory(CFSTR("Analytics/TLS_analytics.db")) path];
1186 #else
1187 return [(__bridge_transfer NSURL*)SecCopyURLForFileInUserCacheDirectory(CFSTR("Analytics/TLS_analytics.db")) path];
1188 #endif
1189 }
1190
1191 @end
1192
1193 @interface supd ()
1194 @property NSDictionary *topicsSamplingRates;
1195 @end
1196
1197 @implementation supd
1198 - (void)setupTopics
1199 {
1200 NSDictionary* systemDefaultValues = [NSDictionary dictionaryWithContentsOfFile:[[NSBundle bundleWithPath:@"/System/Library/Frameworks/Security.framework"] pathForResource:@"SFAnalytics" ofType:@"plist"]];
1201 NSMutableArray <SFAnalyticsTopic*>* topics = [NSMutableArray array];
1202 for (NSString *topicKey in systemDefaultValues) {
1203 NSDictionary *topicSamplingRates = _topicsSamplingRates[topicKey];
1204 SFAnalyticsTopic *topic = [[SFAnalyticsTopic alloc] initWithDictionary:systemDefaultValues[topicKey] name:topicKey samplingRates:topicSamplingRates];
1205 [topics addObject:topic];
1206 }
1207 _analyticsTopics = [NSArray arrayWithArray:topics];
1208 }
1209
1210 + (void)instantiate {
1211 [supd instance];
1212 }
1213
1214 + (instancetype)instance {
1215 #if TARGET_OS_SIMULATOR
1216 return nil;
1217 #else
1218 if (!_supdInstance) {
1219 _supdInstance = [self new];
1220 }
1221 return _supdInstance;
1222 #endif
1223 }
1224
1225 // Use this for testing to get rid of any state
1226 + (void)removeInstance {
1227 _supdInstance = nil;
1228 }
1229
1230
1231 static NSString *SystemTrustStorePath = @"/System/Library/Security/Certificates.bundle";
1232 static NSString *AnalyticsSamplingRatesFilename = @"AnalyticsSamplingRates";
1233 static NSString *ContentVersionKey = @"MobileAssetContentVersion";
1234 static NSString *AssetContextFilename = @"OTAPKIContext.plist";
1235
1236 static NSNumber *getSystemVersion(NSBundle *trustStoreBundle) {
1237 NSDictionary *systemVersionPlist = [NSDictionary dictionaryWithContentsOfURL:[trustStoreBundle URLForResource:@"AssetVersion"
1238 withExtension:@"plist"]];
1239 if (!systemVersionPlist || ![systemVersionPlist isKindOfClass:[NSDictionary class]]) {
1240 return nil;
1241 }
1242 NSNumber *systemVersion = systemVersionPlist[ContentVersionKey];
1243 if (systemVersion == nil || ![systemVersion isKindOfClass:[NSNumber class]]) {
1244 return nil;
1245 }
1246 return systemVersion;
1247 }
1248
1249 static NSNumber *getAssetVersion(NSURL *directory) {
1250 NSDictionary *assetContextPlist = [NSDictionary dictionaryWithContentsOfURL:[directory URLByAppendingPathComponent:AssetContextFilename]];
1251 if (!assetContextPlist || ![assetContextPlist isKindOfClass:[NSDictionary class]]) {
1252 return nil;
1253 }
1254 NSNumber *assetVersion = assetContextPlist[ContentVersionKey];
1255 if (assetVersion == nil || ![assetVersion isKindOfClass:[NSNumber class]]) {
1256 return nil;
1257 }
1258 return assetVersion;
1259 }
1260
1261 static bool ShouldInitializeWithAsset(NSBundle *trustStoreBundle, NSURL *directory) {
1262 NSNumber *systemVersion = getSystemVersion(trustStoreBundle);
1263 NSNumber *assetVersion = getAssetVersion(directory);
1264
1265 if (assetVersion == nil || systemVersion == nil) {
1266 return false;
1267 }
1268 if ([assetVersion compare:systemVersion] == NSOrderedDescending) {
1269 return true;
1270 }
1271 return false;
1272 }
1273
1274 - (void)setupSamplingRates {
1275 #if TARGET_OS_SIMULATOR
1276 NSBundle *trustStoreBundle = [NSBundle bundleWithPath:[NSString stringWithFormat:@"%s%@", getenv("SIMULATOR_ROOT"), SystemTrustStorePath]];
1277 #else
1278 NSBundle *trustStoreBundle = [NSBundle bundleWithPath:SystemTrustStorePath];
1279 #endif
1280
1281 #if TARGET_OS_IPHONE
1282 NSURL *keychainsDirectory = CFBridgingRelease(SecCopyURLForFileInKeychainDirectory(nil));
1283 #else
1284 NSURL *keychainsDirectory = [NSURL fileURLWithFileSystemRepresentation:"/Library/Keychains/" isDirectory:YES relativeToURL:nil];
1285 #endif
1286 NSURL *directory = [keychainsDirectory URLByAppendingPathComponent:@"SupplementalsAssets/" isDirectory:YES];
1287
1288 NSDictionary *analyticsSamplingRates = nil;
1289 if (ShouldInitializeWithAsset(trustStoreBundle, directory)) {
1290 /* Try to get the asset version of the sampling rates */
1291 NSURL *analyticsSamplingRateURL = [directory URLByAppendingPathComponent:[NSString stringWithFormat:@"%@.plist", AnalyticsSamplingRatesFilename]];
1292 analyticsSamplingRates = [NSDictionary dictionaryWithContentsOfURL:analyticsSamplingRateURL];
1293 secnotice("supd", "read sampling rates from SupplementalsAssets dir");
1294 if (!analyticsSamplingRates || ![analyticsSamplingRates isKindOfClass:[NSDictionary class]]) {
1295 analyticsSamplingRates = nil;
1296 }
1297 }
1298 if (!analyticsSamplingRates) {
1299 analyticsSamplingRates = [NSDictionary dictionaryWithContentsOfURL: [trustStoreBundle URLForResource:AnalyticsSamplingRatesFilename
1300 withExtension:@"plist"]];
1301 }
1302 if (analyticsSamplingRates && [analyticsSamplingRates isKindOfClass:[NSDictionary class]]) {
1303 _topicsSamplingRates = analyticsSamplingRates[@"Topics"];
1304 if (!_topicsSamplingRates || ![analyticsSamplingRates isKindOfClass:[NSDictionary class]]) {
1305 _topicsSamplingRates = nil; // Something has gone terribly wrong, so we'll use the hardcoded defaults in this case
1306 }
1307 }
1308 }
1309
1310 - (instancetype)initWithReporter:(SFAnalyticsReporter *)reporter
1311 {
1312 if (self = [super init]) {
1313 [self setupSamplingRates];
1314 [self setupTopics];
1315 _reporter = reporter;
1316 [_reporter setupReportsDirectory];
1317
1318 xpc_activity_register("com.apple.securityuploadd.triggerupload", XPC_ACTIVITY_CHECK_IN, ^(xpc_activity_t activity) {
1319 xpc_activity_state_t activityState = xpc_activity_get_state(activity);
1320 secnotice("supd", "hit xpc activity trigger, state: %ld", activityState);
1321 if (activityState == XPC_ACTIVITY_STATE_RUN) {
1322 // Clean up the reports directory, and then run our regularly scheduled scan
1323 [_reporter cleanupReportsDirectory];
1324 [self performRegularlyScheduledUpload];
1325 }
1326 });
1327 }
1328
1329 return self;
1330 }
1331
1332 - (instancetype)init {
1333 SFAnalyticsReporter *reporter = [[SFAnalyticsReporter alloc] init];
1334 return [self initWithReporter:reporter];
1335 }
1336
1337 - (void)sendNotificationForOncePerReportSamplers
1338 {
1339 notify_post(SFAnalyticsFireSamplersNotification);
1340 [NSThread sleepForTimeInterval:3.0];
1341 }
1342
1343 - (void)performRegularlyScheduledUpload {
1344 secnotice("upload", "Starting uploads in response to regular trigger");
1345 NSError *error = nil;
1346 if ([self uploadAnalyticsWithError:&error]) {
1347 secnotice("upload", "Regularly scheduled upload successful");
1348 } else {
1349 secerror("upload: Failed to complete regularly scheduled upload: %@", error);
1350 }
1351 }
1352
1353 - (BOOL)uploadAnalyticsWithError:(NSError**)error {
1354 [self sendNotificationForOncePerReportSamplers];
1355
1356 BOOL result = NO;
1357 NSError* localError = nil;
1358 for (SFAnalyticsTopic *topic in _analyticsTopics) {
1359 @autoreleasepool { // The logging JSONs get quite large. Ensure they're deallocated between topics.
1360 __block NSURL* endpoint = [topic splunkUploadURL]; // has side effects!
1361 if ([topic disableUploads]) {
1362 secnotice("upload", "Aborting upload task because uploads are disabled for %@", [topic internalTopicName]);
1363 return NO;
1364 }
1365
1366 NSMutableArray<SFAnalyticsClient*>* clients = [NSMutableArray new];
1367 NSData* json = [topic getLoggingJSON:false forUpload:YES participatingClients:&clients error:&localError];
1368 if (json) {
1369 if ([topic isSampledUpload]) {
1370 BOOL writtenToLog = NO;
1371 #if TARGET_OS_OSX
1372 // As of now, data is NOT logged for transparency on macOS, yet we upload anyway.
1373 writtenToLog = YES;
1374 #elif !TARGET_OS_SIMULATOR
1375 // We override the output here and always assume we write to the log. Data transparency will be fixed in F.
1376 writtenToLog = YES;
1377 #endif // !TARGET_OS_SIMULATOR
1378 if (!writtenToLog) {
1379 secerror("uploadAnalyticsWithError: failed to write analytics data to log");
1380 } else if ([topic postJSON:json toEndpoint:endpoint error:&localError]) {
1381 secnotice("uploadAnalyticsWithError", "Succeeded writing analytics data to log -- proceeding with upload");
1382 result = YES;
1383 [topic updateUploadDateForClients:clients clearData:YES];
1384 }
1385 } else {
1386 /* If we didn't sample this report, update date to prevent trying to upload again sooner
1387 * than we should. Clear data so that per-day calculations remain consistent. */
1388 secnotice("upload", "skipping unsampled upload for %@ and clearing data", [topic internalTopicName]);
1389 [topic updateUploadDateForClients:clients clearData:YES];
1390 }
1391 }
1392 }
1393 if (error && localError) {
1394 *error = localError;
1395 }
1396 }
1397 return result;
1398 }
1399
1400 - (NSString*)sysdiagnoseStringForEventRecord:(NSDictionary*)eventRecord
1401 {
1402 NSMutableDictionary* mutableEventRecord = eventRecord.mutableCopy;
1403 [mutableEventRecord removeObjectForKey:SFAnalyticsSplunkTopic];
1404
1405 NSDate* eventDate = [NSDate dateWithTimeIntervalSince1970:[[eventRecord valueForKey:SFAnalyticsEventTime] doubleValue] / 1000];
1406 [mutableEventRecord removeObjectForKey:SFAnalyticsEventTime];
1407
1408 NSString* eventName = eventRecord[SFAnalyticsEventType];
1409 [mutableEventRecord removeObjectForKey:SFAnalyticsEventType];
1410
1411 SFAnalyticsEventClass eventClass = [[eventRecord valueForKey:SFAnalyticsEventClassKey] integerValue];
1412 NSString* eventClassString = [self stringForEventClass:eventClass];
1413 [mutableEventRecord removeObjectForKey:SFAnalyticsEventClassKey];
1414
1415 NSMutableString* additionalAttributesString = [NSMutableString string];
1416 if (mutableEventRecord.count > 0) {
1417 [additionalAttributesString appendString:@" - Attributes: {" ];
1418 __block BOOL firstAttribute = YES;
1419 [mutableEventRecord enumerateKeysAndObjectsUsingBlock:^(NSString* key, id object, BOOL* stop) {
1420 NSString* openingString = firstAttribute ? @"" : @", ";
1421 [additionalAttributesString appendString:[NSString stringWithFormat:@"%@%@ : %@", openingString, key, object]];
1422 firstAttribute = NO;
1423 }];
1424 [additionalAttributesString appendString:@" }"];
1425 }
1426
1427 return [NSString stringWithFormat:@"%@ %@: %@%@", eventDate, eventClassString, eventName, additionalAttributesString];
1428 }
1429
1430 - (NSString*)getSysdiagnoseDump
1431 {
1432 NSMutableString* sysdiagnose = [[NSMutableString alloc] init];
1433
1434 for (SFAnalyticsTopic* topic in _analyticsTopics) {
1435 for (SFAnalyticsClient* client in topic.topicClients) {
1436 [sysdiagnose appendString:[NSString stringWithFormat:@"Client: %@\n", client.name]];
1437 SFAnalyticsSQLiteStore* store = [SFAnalyticsSQLiteStore storeWithPath:client.storePath schema:SFAnalyticsTableSchema];
1438 NSArray* allEvents = store.allEvents;
1439 for (NSDictionary* eventRecord in allEvents) {
1440 [sysdiagnose appendFormat:@"%@\n", [self sysdiagnoseStringForEventRecord:eventRecord]];
1441 }
1442 if (allEvents.count == 0) {
1443 [sysdiagnose appendString:@"No data to report for this client\n"];
1444 }
1445 }
1446 }
1447 return sysdiagnose;
1448 }
1449
1450 - (NSString*)stringForEventClass:(SFAnalyticsEventClass)eventClass
1451 {
1452 if (eventClass == SFAnalyticsEventClassNote) {
1453 return @"EventNote";
1454 }
1455 else if (eventClass == SFAnalyticsEventClassSuccess) {
1456 return @"EventSuccess";
1457 }
1458 else if (eventClass == SFAnalyticsEventClassHardFailure) {
1459 return @"EventHardFailure";
1460 }
1461 else if (eventClass == SFAnalyticsEventClassSoftFailure) {
1462 return @"EventSoftFailure";
1463 }
1464 else {
1465 return @"EventUnknown";
1466 }
1467 }
1468
1469 // MARK: XPC Procotol Handlers
1470
1471 - (void)getSysdiagnoseDumpWithReply:(void (^)(NSString*))reply {
1472 reply([self getSysdiagnoseDump]);
1473 }
1474
1475 - (void)getLoggingJSON:(bool)pretty topic:(NSString *)topicName reply:(void (^)(NSData*, NSError*))reply {
1476 secnotice("rpcGetLoggingJSON", "Building a JSON blob resembling the one we would have uploaded");
1477 NSError* error = nil;
1478 [self sendNotificationForOncePerReportSamplers];
1479 NSData* json = nil;
1480 for (SFAnalyticsTopic* topic in self->_analyticsTopics) {
1481 if ([topic.internalTopicName isEqualToString:topicName]) {
1482 json = [topic getLoggingJSON:pretty forUpload:NO participatingClients:nil error:&error];
1483 }
1484 }
1485 if (!json) {
1486 secerror("Unable to obtain JSON: %@", error);
1487 }
1488 reply(json, error);
1489 }
1490
1491 - (void)forceUploadWithReply:(void (^)(BOOL, NSError*))reply {
1492 secnotice("upload", "Performing upload in response to rpc message");
1493 NSError* error = nil;
1494 BOOL result = [self uploadAnalyticsWithError:&error];
1495 secnotice("upload", "Result of manually triggered upload: %@, error: %@", result ? @"success" : @"failure", error);
1496 reply(result, error);
1497 }
1498
1499 @end