KVSKeychainSyncingProxy/CKDKVSProxy.m

   1 /*
   2  * Copyright (c) 2012-2014,2016 Apple Inc. All Rights Reserved.
   3  *
   4  * @APPLE_LICENSE_HEADER_START@
   5  *
   6  * This file contains Original Code and/or Modifications of Original Code
   7  * as defined in and that are subject to the Apple Public Source License
   8  * Version 2.0 (the 'License'). You may not use this file except in
   9  * compliance with the License. Please obtain a copy of the License at
  10  * http://www.opensource.apple.com/apsl/ and read it before using this
  11  * file.
  12  *
  13  * The Original Code and all software distributed under the License are
  14  * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  15  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  16  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
  17  * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
  18  * Please see the License for the specific language governing rights and
  19  * limitations under the License.
  20  *
  21  * @APPLE_LICENSE_HEADER_END@
  22  */
  23
  24 //
  25 //  CKDKVSProxy.m
  26 //  ckd-xpc
  27 //
  28
  29 #import <Foundation/Foundation.h>
  30
  31 #import <utilities/debugging.h>
  32 #import <os/activity.h>
  33
  34 #import "CKDKVSProxy.h"
  35 #import "CKDKVSStore.h"
  36 #import "CKDAKSLockMonitor.h"
  37 #import "CKDSecuritydAccount.h"
  38 #import "NSURL+SOSPlistStore.h"
  39
  40 #include <Security/SecureObjectSync/SOSARCDefines.h>
  41 #include <utilities/SecCFWrappers.h>
  42
  43 #include "SOSCloudKeychainConstants.h"
  44
  45 #include <utilities/SecAKSWrappers.h>
  46 #include <utilities/SecADWrapper.h>
  47
  48 #import "XPCNotificationDispatcher.h"
  49
  50 /*
  51  The total space available in your app’s iCloud key-value storage is 1 MB.
  52  The maximum number of keys you can specify is 1024, and the size limit for
  53  each value associated with a key is 1 MB. So, for example, if you store a
  54  single large value of 1 MB for a single key, that consumes your total
  55  available storage. If you store 1 KB of data for each key, you can use
  56  1,000 key-value pairs.
  57  */
  58
  59 static NSString *kKeyKeyParameterKeys = @"KeyParameterKeys";
  60 static NSString *kKeyCircleKeys = @"CircleKeys";
  61 static NSString *kKeyMessageKeys = @"MessageKeys";
  62
  63 static NSString *kKeyAlwaysKeys = @"AlwaysKeys";
  64 static NSString *kKeyFirstUnlockKeys = @"FirstUnlockKeys";
  65 static NSString *kKeyUnlockedKeys = @"UnlockedKeys";
  66 static NSString *kKeyPendingKeys = @"PendingKeys";
  67 static NSString *kKeyUnsentChangedKeys = @"unsentChangedKeys";
  68 static NSString *kKeyUnlockNotificationRequested = @"unlockNotificationRequested";
  69 static NSString *kKeySyncWithPeersPending = @"SyncWithPeersPending";
  70
  71 static NSString *kKeyPendingSyncPeerIDs = @"SyncPeerIDs";
  72 static NSString *kKeyPendingSyncBackupPeerIDs = @"SyncBackupPeerIDs";
  73
  74 static NSString *kKeyEnsurePeerRegistration = @"EnsurePeerRegistration";
  75 static NSString *kKeyDSID = @"DSID";
  76 static NSString *kMonitorState = @"MonitorState";
  77 static NSString *kKeyAccountUUID = @"MonitorState";
  78
  79 static NSString *kMonitorPenaltyBoxKey = @"Penalty";
  80 static NSString *kMonitorMessageKey = @"Message";
  81 static NSString *kMonitorConsecutiveWrites = @"ConsecutiveWrites";
  82 static NSString *kMonitorLastWriteTimestamp = @"LastWriteTimestamp";
  83 static NSString *kMonitorMessageQueue = @"MessageQueue";
  84 static NSString *kMonitorPenaltyTimer = @"PenaltyTimer";
  85 static NSString *kMonitorDidWriteDuringPenalty = @"DidWriteDuringPenalty";
  86
  87 static NSString *kMonitorTimeTable = @"TimeTable";
  88 static NSString *kMonitorFirstMinute = @"AFirstMinute";
  89 static NSString *kMonitorSecondMinute = @"BSecondMinute";
  90 static NSString *kMonitorThirdMinute = @"CThirdMinute";
  91 static NSString *kMonitorFourthMinute = @"DFourthMinute";
  92 static NSString *kMonitorFifthMinute = @"EFifthMinute";
  93 static NSString *kMonitorWroteInTimeSlice = @"TimeSlice";
  94 const CFStringRef kSOSKVSKeyParametersKey = CFSTR(">KeyParameters");
  95 const CFStringRef kSOSKVSInitialSyncKey = CFSTR("^InitialSync");
  96 const CFStringRef kSOSKVSAccountChangedKey = CFSTR("^AccountChanged");
  97 const CFStringRef kSOSKVSRequiredKey = CFSTR("^Required");
  98 const CFStringRef kSOSKVSOfficialDSIDKey = CFSTR("^OfficialDSID");
  99
 100 #define kSecServerKeychainChangedNotification "com.apple.security.keychainchanged"
 101
 102 static int max_penalty_timeout = 32;
 103 static int seconds_per_minute = 60;
 104
 105 static const int64_t kSyncTimerLeeway = (NSEC_PER_MSEC * 250);      // 250ms leeway for sync events.
 106
 107 static NSString* asNSString(NSObject* object) {
 108     return [object isKindOfClass:[NSString class]] ? (NSString*) object : nil;
 109 }
 110
 111 @interface NSMutableDictionary (FindAndRemove)
 112 -(NSObject*)extractObjectForKey:(NSString*)key;
 113 @end
 114
 115 @implementation NSMutableDictionary (FindAndRemove)
 116 -(NSObject*)extractObjectForKey:(NSString*)key
 117 {
 118     NSObject* result = [self objectForKey:key];
 119     [self removeObjectForKey: key];
 120     return result;
 121 }
 122 @end
 123
 124 @interface NSSet (Emptiness)
 125 - (bool) isEmpty;
 126 @end
 127
 128 @implementation NSSet (Emptiness)
 129 - (bool) isEmpty
 130 {
 131     return [self count] == 0;
 132 }
 133 @end
 134
 135 @interface NSSet (HasElements)
 136 - (bool) containsElementsNotIn: (NSSet*) other;
 137 @end
 138
 139 @implementation NSSet (HasElements)
 140 - (bool) containsElementsNotIn: (NSSet*) other
 141 {
 142     __block bool hasElements = false;
 143     [self enumerateObjectsUsingBlock:^(id  _Nonnull obj, BOOL * _Nonnull stop) {
 144         if (![other containsObject:obj]) {
 145             hasElements = true;
 146             *stop = true;
 147         }
 148     }];
 149     return hasElements;
 150 }
 151
 152 @end
 153
 154 @implementation NSSet (Stringizing)
 155 - (NSString*) sortedElementsJoinedByString: (NSString*) separator {
 156     return [self sortedElementsTruncated: 0 JoinedByString: separator];
 157 }
 158
 159 - (NSString*) sortedElementsTruncated: (NSUInteger) length JoinedByString: (NSString*) separator
 160 {
 161     NSMutableArray* strings = [NSMutableArray array];
 162
 163     [self enumerateObjectsUsingBlock:^(id  _Nonnull obj, BOOL * _Nonnull stop) {
 164         NSString *stringToInsert = nil;
 165         if ([obj isNSString__]) {
 166             stringToInsert = obj;
 167         } else {
 168             stringToInsert = [obj description];
 169         }
 170
 171         if (length > 0 && length < stringToInsert.length) {
 172             stringToInsert = [stringToInsert substringToIndex:length];
 173         }
 174
 175         [strings insertObject:stringToInsert atIndex:0];
 176     }];
 177
 178     [strings sortUsingSelector: @selector(compare:)];
 179
 180     return [strings componentsJoinedByString:separator];
 181 }
 182 @end
 183
 184 @implementation NSSet (CKDLogging)
 185 - (NSString*) logKeys {
 186     return [self sortedElementsJoinedByString:@" "];
 187 }
 188
 189 - (NSString*) logIDs {
 190     return [self sortedElementsTruncated:8 JoinedByString:@" "];
 191 }
 192 @end
 193
 194
 195 @interface NSDictionary (SOSDictionaryFormat)
 196 - (NSString*) compactDescription;
 197 @end
 198
 199 @implementation NSDictionary (SOSDictionaryFormat)
 200 - (NSString*) compactDescription
 201 {
 202     NSMutableArray *elements = [NSMutableArray array];
 203     [self enumerateKeysAndObjectsUsingBlock: ^(NSString *key, id obj, BOOL *stop) {
 204         [elements addObject: [key stringByAppendingString: @":"]];
 205         if ([obj isKindOfClass:[NSArray class]]) {
 206             [elements addObject: [(NSArray *)obj componentsJoinedByString: @" "]];
 207         } else {
 208             [elements addObject: [NSString stringWithFormat:@"%@", obj]];
 209         }
 210     }];
 211     return [elements componentsJoinedByString: @" "];
 212 }
 213 @end
 214
 215 @interface UbiqitousKVSProxy ()
 216 @property (nonatomic) NSDictionary* persistentData;
 217 - (void) doSyncWithAllPeers;
 218 - (void) persistState;
 219 @end
 220
 221 @implementation UbiqitousKVSProxy
 222
 223 + (instancetype)withAccount:(NSObject<CKDAccount>*) account
 224                       store:(NSObject<CKDStore>*) store
 225                 lockMonitor:(NSObject<CKDLockMonitor>*) lockMonitor
 226                 persistence:(NSURL*) localPersistence
 227 {
 228     return [[self alloc] initWithAccount:account
 229                                    store:store
 230                              lockMonitor:lockMonitor
 231                              persistence:localPersistence];
 232 }
 233
 234 - (instancetype)initWithAccount:(NSObject<CKDAccount>*) account
 235                           store:(NSObject<CKDStore>*) store
 236                     lockMonitor:(NSObject<CKDLockMonitor>*) lockMonitor
 237                     persistence:(NSURL*) localPersistence
 238 {
 239     if (self = [super init])
 240     {
 241         secnotice("event", "%@ start", self);
 242
 243 #if !(TARGET_OS_EMBEDDED)
 244         // rdar://problem/26247270
 245         if (geteuid() == 0) {
 246             secerror("Cannot run CloudKeychainProxy as root");
 247             return NULL;
 248         }
 249 #endif
 250         _ensurePeerRegistration = NO;
 251
 252         _pendingSyncPeerIDs = [NSMutableSet set];
 253         _pendingSyncBackupPeerIDs = [NSMutableSet set];
 254         _shadowPendingSyncPeerIDs = nil;
 255         _shadowPendingSyncBackupPeerIDs = nil;
 256
 257         _persistenceURL = localPersistence;
 258
 259         _account = account;
 260         _store = store;
 261         _lockMonitor = lockMonitor;
 262
 263
 264         _calloutQueue = dispatch_queue_create("CKDCallout", DISPATCH_QUEUE_SERIAL);
 265         _ckdkvsproxy_queue = dispatch_queue_create("CKDKVSProxy", DISPATCH_QUEUE_SERIAL);
 266
 267         _freshnessCompletions = [NSMutableArray<FreshnessResponseBlock> array];
 268
 269         _monitor = [NSMutableDictionary dictionary];
 270
 271         [[XPCNotificationDispatcher dispatcher] addListener: self];
 272
 273         int notificationToken;
 274         notify_register_dispatch(kSecServerKeychainChangedNotification, &notificationToken, _ckdkvsproxy_queue,
 275                                  ^ (int token __unused)
 276                                  {
 277                                      secinfo("backoff", "keychain changed, wiping backoff monitor state");
 278                                      self->_monitor = [NSMutableDictionary dictionary];
 279                                  });
 280
 281         [self setPersistentData: [self.persistenceURL readPlist]];
 282
 283         _dsid =  @"";
 284         _accountUUID = @"";
 285
 286         [[self store] connectToProxy: self];
 287         [[self lockMonitor] connectTo:self];
 288
 289         secdebug(XPROXYSCOPE, "%@ done", self);
 290     }
 291     return self;
 292 }
 293
 294 - (NSString *)description
 295 {
 296     return [NSString stringWithFormat:@"<%s%s%s%s%s%s%s%s%s%s>",
 297             [[self lockMonitor] locked] ? "L" : "U",
 298             [[self lockMonitor] unlockedSinceBoot] ? "B" : "-",
 299             _seenKVSStoreChange ? "K" : "-",
 300             [self hasPendingNonShadowSyncIDs] ? "s" : "-",
 301             _ensurePeerRegistration ? "e" : "-",
 302             [_pendingKeys count] ? "p" : "-",
 303             _inCallout ? "C" : "-",
 304             [self hasPendingShadowSyncIDs] ? "S" : "-",
 305             _shadowEnsurePeerRegistration ? "E" : "-",
 306             [_shadowPendingKeys count] ? "P" : "-"];
 307 }
 308
 309 //
 310 // MARK: XPC Function commands
 311 //
 312 - (void) clearStore {
 313     [self.store removeAllObjects];
 314 }
 315
 316 - (void)synchronizeStore {
 317     [self.store pushWrites];
 318 }
 319
 320 - (id) objectForKey: (NSString*) key {
 321     return [self.store objectForKey: key];
 322 }
 323 - (NSDictionary<NSString *, id>*) copyAsDictionary {
 324     return [self.store copyAsDictionary];
 325 }
 326
 327 - (void)_queue_processAllItems
 328 {
 329     dispatch_assert_queue(_ckdkvsproxy_queue);
 330
 331     NSDictionary *allItems = [self.store copyAsDictionary];
 332     if (allItems)
 333     {
 334         secnotice("event", "%@ sending: %@", self, [[allItems allKeys] componentsJoinedByString: @" "]);
 335         [self processKeyChangedEvent:allItems];
 336     }
 337     else
 338         secdebug(XPROXYSCOPE, "%@ No items in KVS", self);
 339 }
 340
 341 - (void)dealloc
 342 {
 343     secdebug(XPROXYSCOPE, "%@", self);
 344     [[NSNotificationCenter defaultCenter] removeObserver:self
 345                                                     name:NSUbiquitousKeyValueStoreDidChangeExternallyNotification
 346                                                   object:nil];
 347
 348     [[NSNotificationCenter defaultCenter] removeObserver:self
 349                                                     name:NSUbiquityIdentityDidChangeNotification
 350                                                   object:nil];
 351 }
 352
 353 // MARK: Persistence
 354
 355 - (NSDictionary*) persistentData
 356 {
 357     return @{ kKeyAlwaysKeys:[_alwaysKeys allObjects],
 358               kKeyFirstUnlockKeys:[_firstUnlockKeys allObjects],
 359               kKeyUnlockedKeys:[_unlockedKeys allObjects],
 360               kKeyPendingKeys:[_pendingKeys allObjects],
 361               kKeyPendingSyncPeerIDs:[_pendingSyncPeerIDs allObjects],
 362               kKeyPendingSyncBackupPeerIDs:[_pendingSyncBackupPeerIDs allObjects],
 363               kMonitorState:_monitor,
 364               kKeyEnsurePeerRegistration:[NSNumber numberWithBool:_ensurePeerRegistration],
 365               kKeyDSID:_dsid,
 366               kKeyAccountUUID:_accountUUID
 367               };
 368 }
 369
 370 - (void) setPersistentData: (NSDictionary*) interests
 371 {
 372     _alwaysKeys = [NSMutableSet setWithArray: interests[kKeyAlwaysKeys]];
 373     _firstUnlockKeys = [NSMutableSet setWithArray: interests[kKeyFirstUnlockKeys]];
 374     _unlockedKeys = [NSMutableSet setWithArray: interests[kKeyUnlockedKeys]];
 375
 376     _pendingKeys = [NSMutableSet setWithArray: interests[kKeyPendingKeys]];
 377
 378     _pendingSyncPeerIDs = [NSMutableSet setWithArray: interests[kKeyPendingSyncPeerIDs]];
 379     _pendingSyncBackupPeerIDs = [NSMutableSet setWithArray: interests[kKeyPendingSyncBackupPeerIDs]];
 380
 381     _monitor = interests[kMonitorState];
 382     if(_monitor == nil)
 383         _monitor = [NSMutableDictionary dictionary];
 384
 385     _ensurePeerRegistration = [interests[kKeyEnsurePeerRegistration] boolValue];
 386
 387     _dsid = interests[kKeyDSID];
 388     _accountUUID = interests[kKeyAccountUUID];
 389
 390     // If we had a sync pending, we kick it off and migrate to sync with these peers
 391     if ([interests[kKeySyncWithPeersPending] boolValue]) {
 392         [self doSyncWithAllPeers];
 393     }
 394 }
 395
 396 - (void)persistState
 397 {
 398     NSDictionary* dataToSave = self.persistentData;
 399
 400     secdebug("persistence", "Writing registeredKeys: %@", [dataToSave compactDescription]);
 401     if (![self.persistenceURL writePlist:dataToSave]) {
 402         secerror("Failed to write persistence data to %@", self.persistenceURL);
 403     }
 404 }
 405
 406
 407 #if TARGET_OS_EMBEDDED && !TARGET_IPHONE_SIMULATOR
 408 CFStringRef const CKDAggdIncreaseThrottlingKey = CFSTR("com.apple.cloudkeychainproxy.backoff.increase");
 409 CFStringRef const CKDAggdDecreaseThrottlingKey = CFSTR("com.apple.cloudkeychainproxy.backoff.decrease");
 410 #endif
 411
 412 // MARK: Penalty measurement and handling
 413 -(dispatch_source_t)setNewTimer:(int)timeout key:(NSString*)key
 414 {
 415     __block dispatch_source_t timer = dispatch_source_create(DISPATCH_SOURCE_TYPE_TIMER, 0, 0, _ckdkvsproxy_queue);
 416     dispatch_source_set_timer(timer, dispatch_time(DISPATCH_TIME_NOW, timeout * NSEC_PER_SEC * seconds_per_minute), DISPATCH_TIME_FOREVER, kSyncTimerLeeway);
 417     dispatch_source_set_event_handler(timer, ^{
 418         [self penaltyTimerFired:key];
 419     });
 420     dispatch_resume(timer);
 421     return timer;
 422 }
 423
 424 -(void) increasePenalty:(NSNumber*)currentPenalty key:(NSString*)key keyEntry:(NSMutableDictionary**)keyEntry
 425 {
 426 #if TARGET_OS_EMBEDDED && !TARGET_IPHONE_SIMULATOR
 427     SecADAddValueForScalarKey(CKDAggdIncreaseThrottlingKey, 1);
 428 #endif
 429
 430     secnotice("backoff", "increasing penalty!");
 431     int newPenalty = 0;
 432     if([currentPenalty intValue] == max_penalty_timeout){
 433         newPenalty = max_penalty_timeout;
 434     }
 435     else if ([currentPenalty intValue] == 0)
 436         newPenalty = 1;
 437     else
 438         newPenalty = [currentPenalty intValue]*2;
 439
 440     secnotice("backoff", "key %@, waiting %d minutes long to send next messages", key, newPenalty);
 441
 442     NSNumber* penalty_timeout = [[NSNumber alloc]initWithInt:newPenalty];
 443     dispatch_source_t existingTimer = [*keyEntry valueForKey:kMonitorPenaltyTimer];
 444
 445     if(existingTimer != nil){
 446         [*keyEntry removeObjectForKey:kMonitorPenaltyTimer];
 447         dispatch_suspend(existingTimer);
 448         dispatch_source_set_timer(existingTimer,dispatch_time(DISPATCH_TIME_NOW, newPenalty * NSEC_PER_SEC * seconds_per_minute), DISPATCH_TIME_FOREVER, kSyncTimerLeeway);
 449         dispatch_resume(existingTimer);
 450         [*keyEntry setObject:existingTimer forKey:kMonitorPenaltyTimer];
 451     }
 452     else{
 453         dispatch_source_t timer = [self setNewTimer:newPenalty key:key];
 454         [*keyEntry setObject:timer forKey:kMonitorPenaltyTimer];
 455     }
 456
 457     [*keyEntry setObject:penalty_timeout forKey:kMonitorPenaltyBoxKey];
 458     [_monitor setObject:*keyEntry forKey:key];
 459 }
 460
 461 -(void) decreasePenalty:(NSNumber*)currentPenalty key:(NSString*)key keyEntry:(NSMutableDictionary**)keyEntry
 462 {
 463 #if TARGET_OS_EMBEDDED && !TARGET_IPHONE_SIMULATOR
 464     SecADAddValueForScalarKey(CKDAggdDecreaseThrottlingKey, 1);
 465 #endif
 466
 467     int newPenalty = 0;
 468     secnotice("backoff","decreasing penalty!");
 469     if([currentPenalty intValue] == 0 || [currentPenalty intValue] == 1)
 470         newPenalty = 0;
 471     else
 472         newPenalty = [currentPenalty intValue]/2;
 473
 474     secnotice("backoff","key %@, waiting %d minutes long to send next messages", key, newPenalty);
 475
 476     NSNumber* penalty_timeout = [[NSNumber alloc]initWithInt:newPenalty];
 477
 478     dispatch_source_t existingTimer = [*keyEntry valueForKey:kMonitorPenaltyTimer];
 479     if(existingTimer != nil){
 480         [*keyEntry removeObjectForKey:kMonitorPenaltyTimer];
 481         dispatch_suspend(existingTimer);
 482         if(newPenalty != 0){
 483             dispatch_source_set_timer(existingTimer,dispatch_time(DISPATCH_TIME_NOW, newPenalty * NSEC_PER_SEC * seconds_per_minute), DISPATCH_TIME_FOREVER, kSyncTimerLeeway);
 484             dispatch_resume(existingTimer);
 485             [*keyEntry setObject:existingTimer forKey:kMonitorPenaltyTimer];
 486         }
 487         else{
 488             dispatch_resume(existingTimer);
 489             dispatch_source_cancel(existingTimer);
 490         }
 491     }
 492     else{
 493         if(newPenalty != 0){
 494             dispatch_source_t timer = [self setNewTimer:newPenalty key:key];
 495             [*keyEntry setObject:timer forKey:kMonitorPenaltyTimer];
 496         }
 497     }
 498
 499     [*keyEntry setObject:penalty_timeout forKey:kMonitorPenaltyBoxKey];
 500     [_monitor setObject:*keyEntry forKey:key];
 501
 502 }
 503
 504 - (void)penaltyTimerFired:(NSString*)key
 505 {
 506     secnotice("backoff", "key: %@, !!!!!!!!!!!!!!!!penalty timeout is up!!!!!!!!!!!!", key);
 507
 508     NSMutableDictionary *keyEntry = [_monitor objectForKey:key];
 509     NSMutableDictionary *queuedMessages = [keyEntry objectForKey:kMonitorMessageQueue];
 510     secnotice("backoff","key: %@, queuedMessages: %@", key, queuedMessages);
 511     if(queuedMessages && [queuedMessages count] != 0){
 512         secnotice("backoff","key: %@, message queue not empty, writing to KVS!", key);
 513         [self setObjectsFromDictionary:queuedMessages];
 514         [keyEntry setObject:[NSMutableDictionary dictionary] forKey:kMonitorMessageQueue];
 515     }
 516
 517     NSNumber *penalty_timeout = [keyEntry valueForKey:kMonitorPenaltyBoxKey];
 518     secnotice("backoff", "key: %@, current penalty timeout: %@", key, penalty_timeout);
 519
 520     NSString* didWriteDuringTimeout = [keyEntry objectForKey:kMonitorDidWriteDuringPenalty];
 521     if( didWriteDuringTimeout && [didWriteDuringTimeout isEqualToString:@"YES"] )
 522     {
 523         //increase timeout since we wrote during out penalty timeout
 524         [self increasePenalty:penalty_timeout key:key keyEntry:&keyEntry];
 525     }
 526     else{
 527         //decrease timeout since we successfully wrote messages out
 528         [self decreasePenalty:penalty_timeout key:key keyEntry:&keyEntry];
 529     }
 530
 531     //resetting the check
 532     [keyEntry setObject: @"NO" forKey:kMonitorDidWriteDuringPenalty];
 533
 534     //recompute the timetable and number of consecutive writes to KVS
 535     NSMutableDictionary *timetable = [keyEntry valueForKey:kMonitorTimeTable];
 536     NSNumber *consecutiveWrites = [keyEntry valueForKey:kMonitorConsecutiveWrites];
 537     [self recordTimestampForAppropriateInterval:&timetable key:key consecutiveWrites:&consecutiveWrites];
 538
 539     [keyEntry setObject:consecutiveWrites forKey:kMonitorConsecutiveWrites];
 540     [keyEntry setObject:timetable forKey:kMonitorTimeTable];
 541     [_monitor setObject:keyEntry forKey:key];
 542 }
 543
 544 -(NSMutableDictionary*)initializeTimeTable:(NSString*)key
 545 {
 546     NSDate *currentTime = [NSDate date];
 547     NSMutableDictionary *firstMinute = [NSMutableDictionary dictionaryWithObjectsAndKeys:[currentTime dateByAddingTimeInterval: seconds_per_minute], kMonitorFirstMinute, @"YES", kMonitorWroteInTimeSlice, nil];
 548     NSMutableDictionary *secondMinute = [NSMutableDictionary dictionaryWithObjectsAndKeys:[currentTime dateByAddingTimeInterval: seconds_per_minute * 2],kMonitorSecondMinute, @"NO", kMonitorWroteInTimeSlice, nil];
 549     NSMutableDictionary *thirdMinute = [NSMutableDictionary dictionaryWithObjectsAndKeys:[currentTime dateByAddingTimeInterval: seconds_per_minute * 3], kMonitorThirdMinute, @"NO",kMonitorWroteInTimeSlice, nil];
 550     NSMutableDictionary *fourthMinute = [NSMutableDictionary dictionaryWithObjectsAndKeys:[currentTime dateByAddingTimeInterval: seconds_per_minute * 4],kMonitorFourthMinute, @"NO", kMonitorWroteInTimeSlice, nil];
 551     NSMutableDictionary *fifthMinute = [NSMutableDictionary dictionaryWithObjectsAndKeys:[currentTime dateByAddingTimeInterval: seconds_per_minute * 5], kMonitorFifthMinute, @"NO", kMonitorWroteInTimeSlice, nil];
 552
 553     NSMutableDictionary *timeTable = [NSMutableDictionary dictionaryWithObjectsAndKeys: firstMinute, kMonitorFirstMinute,
 554                                       secondMinute, kMonitorSecondMinute,
 555                                       thirdMinute, kMonitorThirdMinute,
 556                                       fourthMinute, kMonitorFourthMinute,
 557                                       fifthMinute, kMonitorFifthMinute, nil];
 558     return timeTable;
 559 }
 560
 561 - (void)initializeKeyEntry:(NSString*)key
 562 {
 563     NSMutableDictionary *timeTable = [self initializeTimeTable:key];
 564     NSDate *currentTime = [NSDate date];
 565
 566     NSMutableDictionary *keyEntry = [NSMutableDictionary dictionaryWithObjectsAndKeys: key, kMonitorMessageKey, @0, kMonitorConsecutiveWrites, currentTime, kMonitorLastWriteTimestamp, @0, kMonitorPenaltyBoxKey, timeTable, kMonitorTimeTable,[NSMutableDictionary dictionary], kMonitorMessageQueue, nil];
 567
 568     [_monitor setObject:keyEntry forKey:key];
 569
 570 }
 571
 572 - (void)recordTimestampForAppropriateInterval:(NSMutableDictionary**)timeTable key:(NSString*)key consecutiveWrites:(NSNumber**)consecutiveWrites
 573 {
 574     NSDate *currentTime = [NSDate date];
 575     __block int cWrites = [*consecutiveWrites intValue];
 576     __block BOOL foundTimeSlot = NO;
 577     __block NSMutableDictionary *previousTable = nil;
 578     NSArray *sorted = [[*timeTable allKeys] sortedArrayUsingSelector:@selector(compare:)];
 579     [sorted enumerateObjectsUsingBlock:^(id sortedKey, NSUInteger idx, BOOL *stop)
 580      {
 581          if(foundTimeSlot == YES)
 582              return;
 583          [*timeTable enumerateKeysAndObjectsUsingBlock: ^(id minute, id obj, BOOL *stop2)
 584           {
 585               if(foundTimeSlot == YES)
 586                   return;
 587               if([sortedKey isEqualToString:minute]){
 588                   NSMutableDictionary *minutesTable = (NSMutableDictionary*)obj;
 589                   NSString *minuteKey = (NSString*)minute;
 590                   NSDate *date = [minutesTable valueForKey:minuteKey];
 591                   if([date compare:currentTime] == NSOrderedDescending){
 592                       foundTimeSlot = YES;
 593                       NSString* written = [minutesTable valueForKey:kMonitorWroteInTimeSlice];
 594                       if([written isEqualToString:@"NO"]){
 595                           [minutesTable setObject:@"YES" forKey:kMonitorWroteInTimeSlice];
 596                           if(previousTable != nil){
 597                               written = [previousTable valueForKey:kMonitorWroteInTimeSlice];
 598                               if([written isEqualToString:@"YES"]){
 599                                   cWrites++;
 600                               }
 601                               else if ([written isEqualToString:@"NO"]){
 602                                   cWrites = 0;
 603                               }
 604                           }
 605                       }
 606                       return;
 607                   }
 608                   previousTable = minutesTable;
 609               }
 610           }];
 611      }];
 612
 613     if(foundTimeSlot == NO){
 614         //reset the time table
 615         secnotice("backoff","didn't find a time slot, resetting the table");
 616         NSMutableDictionary *lastTable = [*timeTable valueForKey:kMonitorFifthMinute];
 617         NSDate *lastDate = [lastTable valueForKey:kMonitorFifthMinute];
 618
 619         if((double)[currentTime timeIntervalSinceDate: lastDate] >= seconds_per_minute){
 620             *consecutiveWrites = [[NSNumber alloc]initWithInt:0];
 621         }
 622         else{
 623             NSString* written = [lastTable valueForKey:kMonitorWroteInTimeSlice];
 624             if([written isEqualToString:@"YES"]){
 625                 cWrites++;
 626                 *consecutiveWrites = [[NSNumber alloc]initWithInt:cWrites];
 627             }
 628             else{
 629                 *consecutiveWrites = [[NSNumber alloc]initWithInt:0];
 630             }
 631         }
 632
 633         *timeTable  = [self initializeTimeTable:key];
 634         return;
 635     }
 636     *consecutiveWrites = [[NSNumber alloc]initWithInt:cWrites];
 637 }
 638 - (void)recordWriteToKVS:(NSDictionary *)values
 639 {
 640     if([_monitor count] == 0){
 641         [values enumerateKeysAndObjectsUsingBlock: ^(id key, id obj, BOOL *stop)
 642          {
 643              [self initializeKeyEntry: key];
 644          }];
 645     }
 646     else{
 647         [values enumerateKeysAndObjectsUsingBlock: ^(id key, id obj, BOOL *stop)
 648          {
 649              NSMutableDictionary *keyEntry = [self->_monitor objectForKey:key];
 650              if(keyEntry == nil){
 651                  [self initializeKeyEntry: key];
 652              }
 653              else{
 654                  NSNumber *penalty_timeout = [keyEntry objectForKey:kMonitorPenaltyBoxKey];
 655                  NSDate *lastWriteTimestamp = [keyEntry objectForKey:kMonitorLastWriteTimestamp];
 656                  NSMutableDictionary *timeTable = [keyEntry objectForKey: kMonitorTimeTable];
 657                  NSNumber *existingWrites = [keyEntry objectForKey: kMonitorConsecutiveWrites];
 658                  NSDate *currentTime = [NSDate date];
 659
 660                  [self recordTimestampForAppropriateInterval:&timeTable key:key consecutiveWrites:&existingWrites];
 661
 662                  int consecutiveWrites = [existingWrites intValue];
 663                  secnotice("backoff","consecutive writes: %d", consecutiveWrites);
 664                  [keyEntry setObject:existingWrites forKey:kMonitorConsecutiveWrites];
 665                  [keyEntry setObject:timeTable forKey:kMonitorTimeTable];
 666                  [keyEntry setObject:currentTime forKey:kMonitorLastWriteTimestamp];
 667                  [self->_monitor setObject:keyEntry forKey:key];
 668
 669                  if([penalty_timeout intValue] != 0 || ((double)[currentTime timeIntervalSinceDate: lastWriteTimestamp] <= 60 && consecutiveWrites >= 5)){
 670                      if([penalty_timeout intValue] != 0 && consecutiveWrites == 5){
 671                          secnotice("backoff","written for 5 consecutive minutes, time to start throttling");
 672                          [self increasePenalty:penalty_timeout key:key keyEntry:&keyEntry];
 673                      }
 674                      else
 675                          secnotice("backoff","monitor: keys have been written for 5 or more minutes, recording we wrote during timeout");
 676
 677                      //record we wrote during a timeout
 678                      [keyEntry setObject: @"YES" forKey:kMonitorDidWriteDuringPenalty];
 679                  }
 680                  //keep writing freely but record it
 681                  else if((double)[currentTime timeIntervalSinceDate: lastWriteTimestamp] <= 60 && consecutiveWrites < 5){
 682                      secnotice("backoff","monitor: still writing freely");
 683                  }
 684              }
 685          }];
 686     }
 687 }
 688
 689 - (NSDictionary*)recordHaltedValuesAndReturnValuesToSafelyWrite:(NSDictionary *)values
 690 {
 691     NSMutableDictionary *SafeMessages = [NSMutableDictionary dictionary];
 692     [values enumerateKeysAndObjectsUsingBlock: ^(id key, id obj, BOOL *stop)
 693      {
 694          NSMutableDictionary *keyEntry = [self->_monitor objectForKey:key];
 695          NSNumber *penalty = [keyEntry objectForKey:kMonitorPenaltyBoxKey];
 696          if([penalty intValue] != 0){
 697              NSMutableDictionary* existingQueue = [keyEntry valueForKey:kMonitorMessageQueue];
 698
 699              [existingQueue setObject:obj forKey:key];
 700
 701              [keyEntry setObject:existingQueue forKey:kMonitorMessageQueue];
 702              [self->_monitor setObject:keyEntry forKey:key];
 703          }
 704          else{
 705              [SafeMessages setObject:obj forKey:key];
 706          }
 707      }];
 708     return SafeMessages;
 709 }
 710
 711 // MARK: Object setting
 712
 713
 714 - (void)setStoreObjectsFromDictionary:(NSDictionary *)values
 715 {
 716     if (values == nil) {
 717         secdebug(XPROXYSCOPE, "%@ NULL? values: %@", self, values);
 718         return;
 719     }
 720
 721     NSMutableDictionary<NSString*, NSObject*> *mutableValues = [values mutableCopy];
 722     NSString* newDSID = asNSString([mutableValues extractObjectForKey:(__bridge NSString*) kSOSKVSOfficialDSIDKey]);
 723     if (newDSID) {
 724         _dsid = newDSID;
 725     }
 726
 727     NSString* requiredDSID = asNSString([mutableValues extractObjectForKey:(__bridge NSString*) kSOSKVSRequiredKey]);
 728     if (requiredDSID) {
 729         if (_dsid == nil || [_dsid isEqualToString: @""]) {
 730             secdebug("dsid", "CloudKeychainProxy setting dsid to :%@ from securityd", requiredDSID);
 731             _dsid = requiredDSID;
 732         } else if (![_dsid isEqual: requiredDSID]) {
 733             secerror("Account DSIDs do not match, cloud keychain proxy: %@, securityd: %@", _dsid, requiredDSID);
 734             secerror("Not going to write these: %@ into KVS!", values);
 735             return;
 736         } else {
 737             secnoticeq("dsid", "DSIDs match, writing");
 738         }
 739     }
 740
 741     secnoticeq("keytrace", "%@ sending: %@", self, [[mutableValues allKeys] componentsJoinedByString: @" "]);
 742     [mutableValues enumerateKeysAndObjectsUsingBlock: ^(id key, id obj, BOOL *stop)
 743      {
 744          if (obj == NULL || obj == [NSNull null]) {
 745              [self.store removeObjectForKey:key];
 746          } else {
 747              if ([key hasPrefix:@"ak|"]) {  // TODO: somewhat of a hack
 748                  id oldObj = [self.store objectForKey:key];
 749                  if ([oldObj isEqual: obj]) {
 750                      // Fix KVS repeated message undelivery by sending a NULL first (deafness)
 751                      secnoticeq("keytrace", "forcing resend of key write: %@", key);
 752                      [self.store removeObjectForKey:key];
 753                  }
 754              }
 755              [self.store setObject:obj forKey:key];
 756          }
 757      }];
 758
 759     [self.store pushWrites];
 760 }
 761
 762 - (void)setObjectsFromDictionary:(NSDictionary<NSString*, NSObject*> *)values
 763 {
 764     [self recordWriteToKVS: values];
 765     NSDictionary *safeValues = [self recordHaltedValuesAndReturnValuesToSafelyWrite: values];
 766     if([safeValues count] !=0){
 767         [self setStoreObjectsFromDictionary:safeValues];
 768     }
 769 }
 770
 771 - (void)waitForSynchronization:(void (^)(NSDictionary<NSString*, NSObject*> *results, NSError *err))handler
 772 {
 773     secnoticeq("fresh", "%s Requesting WFS", kWAIT2MINID);
 774
 775     [_freshnessCompletions addObject: ^(bool success, NSError *error){
 776         secnoticeq("fresh", "%s WFS Done", kWAIT2MINID);
 777         handler(nil, error);
 778     }];
 779
 780     if ([self.freshnessCompletions count] == 1) {
 781         // We can't talk to synchronize on the _ckdkvsproxy_queue or we deadlock,
 782         // bounce to a global concurrent queue
 783         dispatch_after(_nextFreshnessTime, dispatch_get_global_queue(DISPATCH_QUEUE_PRIORITY_DEFAULT, 0), ^{
 784             NSError *error = nil;
 785             bool success = [self.store pullUpdates:&error];
 786
 787             dispatch_async(self->_ckdkvsproxy_queue, ^{
 788                 [self waitForSyncDone: success error: error];
 789             });
 790         });
 791     }
 792 }
 793
 794 - (void) waitForSyncDone: (bool) success error: (NSError*) error{
 795     if (success) {
 796         const uint64_t delayBeforeCallingAgainInSeconds = 5ull * NSEC_PER_SEC;
 797         _nextFreshnessTime  = dispatch_time(DISPATCH_TIME_NOW, delayBeforeCallingAgainInSeconds);
 798     }
 799
 800     secnoticeq("fresh", "%s Completing WFS", kWAIT2MINID);
 801     [_freshnessCompletions enumerateObjectsUsingBlock:^(FreshnessResponseBlock _Nonnull block,
 802                                                         NSUInteger idx,
 803                                                         BOOL * _Nonnull stop) {
 804         block(success, error);
 805     }];
 806     [_freshnessCompletions removeAllObjects];
 807
 808 }
 809
 810 //
 811 // MARK: ----- KVS key lists -----
 812 //
 813
 814 - (NSMutableSet *)copyAllKeyInterests
 815 {
 816     NSMutableSet *allKeys = [NSMutableSet setWithSet: _alwaysKeys];
 817     [allKeys unionSet: _firstUnlockKeys];
 818     [allKeys unionSet: _unlockedKeys];
 819     return allKeys;
 820 }
 821
 822 -(void)registerAtTimeKeys:(NSDictionary*)keyparms
 823 {
 824     if (keyparms == nil)
 825         return;
 826
 827     NSArray *alwaysArray = [keyparms valueForKey: kKeyAlwaysKeys];
 828     NSArray *firstUnlockedKeysArray = [keyparms valueForKey: kKeyFirstUnlockKeys];
 829     NSArray *whenUnlockedKeysArray = [keyparms valueForKey: kKeyUnlockedKeys];
 830
 831     if(alwaysArray)
 832         [_alwaysKeys unionSet: [NSMutableSet setWithArray: alwaysArray]];
 833     if(firstUnlockedKeysArray)
 834         [_firstUnlockKeys unionSet: [NSMutableSet setWithArray: firstUnlockedKeysArray]];
 835     if(whenUnlockedKeysArray)
 836         [_unlockedKeys unionSet: [NSMutableSet setWithArray: whenUnlockedKeysArray]];
 837 }
 838
 839
 840 - (void)registerKeys: (NSDictionary*)keys forAccount: (NSString*) accountUUID
 841 {
 842     secdebug(XPROXYSCOPE, "registerKeys: keys: %@", keys);
 843
 844     // We only reset when we know the ID and they send the ID and it changes.
 845     bool newAccount = accountUUID != nil && self.accountUUID != nil && ![accountUUID isEqualToString: self.accountUUID];
 846
 847     if (accountUUID) {
 848         self.accountUUID = accountUUID;
 849     }
 850
 851     // If we're a new account we don't exclude the old keys
 852     NSMutableSet *allOldKeys = newAccount ? [NSMutableSet set] : [self copyAllKeyInterests];
 853
 854
 855     NSDictionary *keyparms = [keys valueForKey: [NSString stringWithUTF8String: kMessageKeyParameter]];
 856     NSDictionary *circles = [keys valueForKey: [NSString stringWithUTF8String: kMessageCircle]];
 857     NSDictionary *messages = [keys valueForKey: [NSString stringWithUTF8String: kMessageMessage]];
 858
 859     _alwaysKeys = [NSMutableSet set];
 860     _firstUnlockKeys = [NSMutableSet set];
 861     _unlockedKeys = [NSMutableSet set];
 862
 863     [self registerAtTimeKeys: keyparms];
 864     [self registerAtTimeKeys: circles];
 865     [self registerAtTimeKeys: messages];
 866
 867     NSMutableSet *allNewKeys = [self copyAllKeyInterests];
 868
 869     // Make sure keys we no longer care about are not pending
 870     [_pendingKeys intersectSet:allNewKeys];
 871     if (_shadowPendingKeys) {
 872         [_shadowPendingKeys intersectSet:allNewKeys];
 873     }
 874
 875     // All new keys only is new keys (remove old keys)
 876     [allNewKeys minusSet:allOldKeys];
 877
 878     // Mark new keys pending, they're new!
 879     NSMutableSet *newKeysForCurrentLockState = [self pendKeysAndGetNewlyPended:allNewKeys];
 880
 881     [self persistState]; // Before we might call out, save our state so we recover if we crash
 882
 883     [self intersectWithCurrentLockState: newKeysForCurrentLockState];
 884     // TODO: Don't processPendingKeysForCurrentLockState if none of the new keys have values.
 885     if ([newKeysForCurrentLockState count] != 0) {
 886         [self processPendingKeysForCurrentLockState];
 887     }
 888 }
 889
 890 // MARK: ----- Event Handling -----
 891
 892 - (void)_queue_handleNotification:(const char *) name
 893 {
 894     dispatch_assert_queue(_ckdkvsproxy_queue);
 895
 896     if (strcmp(name, kNotifyTokenForceUpdate)==0) {
 897         // DEBUG -- Possibly remove in future
 898         [self _queue_processAllItems];
 899     } else if (strcmp(name, kCloudKeychainStorechangeChangeNotification)==0) {
 900         // DEBUG -- Possibly remove in future
 901         [self _queue_kvsStoreChange];
 902     }
 903 }
 904
 905 - (void)_queue_storeKeysChanged: (NSSet<NSString*>*) changedKeys initial: (bool) initial
 906 {
 907     dispatch_assert_queue(_ckdkvsproxy_queue);
 908
 909     // Mark that our store is talking to us, so we don't have to make up for missing anything previous.
 910     _seenKVSStoreChange = YES;
 911
 912     // Unmark them as pending as they have just changed and we'll process them.
 913     [_pendingKeys minusSet:changedKeys];
 914
 915     // Only send values that we're currently interested in.
 916     NSSet *keysOfInterestThatChanged = [self pendKeysAndGetPendingForCurrentLockState:changedKeys];
 917     NSMutableDictionary *changedValues = [self copyValues:keysOfInterestThatChanged];
 918     if (initial)
 919         changedValues[(__bridge NSString*)kSOSKVSInitialSyncKey] =  @"true";
 920
 921     secnotice("event", "%@ keysChangedInCloud: %@ keysOfInterest: %@ initial: %@",
 922               self,
 923               [[changedKeys allObjects] componentsJoinedByString: @" "],
 924               [[changedValues allKeys] componentsJoinedByString: @" "],
 925               initial ? @"YES" : @"NO");
 926
 927     if ([changedValues count])
 928         [self processKeyChangedEvent:changedValues];
 929 }
 930
 931 - (void)_queue_storeAccountChanged
 932 {
 933     dispatch_assert_queue(_ckdkvsproxy_queue);
 934
 935     secnotice("event", "%@", self);
 936
 937     NSDictionary *changedValues = nil;
 938     if(_dsid)
 939         changedValues = @{ (__bridge NSString*)kSOSKVSAccountChangedKey: _dsid };
 940     else
 941         changedValues = @{ (__bridge NSString*)kSOSKVSAccountChangedKey: @"true" };
 942
 943     [self processKeyChangedEvent:changedValues];
 944 }
 945
 946 - (void) doAfterFlush: (dispatch_block_t) block
 947 {
 948     //Flush any pending communication to Securityd.
 949     if(!_inCallout)
 950         dispatch_async(_calloutQueue, block);
 951     else
 952         _shadowFlushBlock = block;
 953 }
 954
 955 - (void) calloutWith: (void(^)(NSSet *pending, NSSet* pendingSyncIDs, NSSet* pendingBackupSyncIDs, bool ensurePeerRegistration, dispatch_queue_t queue, void(^done)(NSSet *handledKeys, NSSet *handledSyncs, bool handledEnsurePeerRegistration, NSError* error))) callout
 956 {
 957     // In CKDKVSProxy's serial queue
 958
 959     // dispatch_get_global_queue - well-known global concurrent queue
 960     // dispatch_get_main_queue   - default queue that is bound to the main thread
 961     xpc_transaction_begin();
 962     dispatch_async(_calloutQueue, ^{
 963         __block NSSet *myPending;
 964         __block NSSet *mySyncPeerIDs;
 965         __block NSSet *mySyncBackupPeerIDs;
 966         __block bool myEnsurePeerRegistration;
 967         __block bool wasLocked;
 968         dispatch_sync(self->_ckdkvsproxy_queue, ^{
 969             myPending = [self->_pendingKeys copy];
 970             mySyncPeerIDs = [self->_pendingSyncPeerIDs copy];
 971             mySyncBackupPeerIDs = [self->_pendingSyncBackupPeerIDs copy];
 972
 973             myEnsurePeerRegistration = self->_ensurePeerRegistration;
 974             wasLocked = [self.lockMonitor locked];
 975
 976             self->_inCallout = YES;
 977
 978             self->_shadowPendingKeys = [NSMutableSet set];
 979             self->_shadowPendingSyncPeerIDs = [NSMutableSet set];
 980             self->_shadowPendingSyncBackupPeerIDs = [NSMutableSet set];
 981         });
 982
 983         callout(myPending, mySyncPeerIDs, mySyncBackupPeerIDs, myEnsurePeerRegistration, self->_ckdkvsproxy_queue, ^(NSSet *handledKeys, NSSet *handledSyncs, bool handledEnsurePeerRegistration, NSError* failure) {
 984             secdebug("event", "%@ %s%s before callout handled: %s%s", self,
 985                      ![mySyncPeerIDs isEmpty] || ![mySyncBackupPeerIDs isEmpty] ? "S" : "s",
 986                      myEnsurePeerRegistration ? "E" : "e",
 987                      ![handledKeys isEmpty] ? "S" : "s",
 988                      handledEnsurePeerRegistration ? "E" : "e");
 989
 990             // In CKDKVSProxy's serial queue
 991             self->_inCallout = NO;
 992
 993             // Update ensurePeerRegistration
 994             self->_ensurePeerRegistration = ((self->_ensurePeerRegistration && !handledEnsurePeerRegistration) || self->_shadowEnsurePeerRegistration);
 995
 996             self->_shadowEnsurePeerRegistration = NO;
 997
 998             if(self->_ensurePeerRegistration && ![self.lockMonitor locked])
 999                 [self doEnsurePeerRegistration];
1000
1001             bool hadShadowPeerIDs = ![self->_shadowPendingSyncPeerIDs isEmpty] || ![self->_shadowPendingSyncBackupPeerIDs isEmpty];
1002
1003             // Update SyncWithPeers stuff.
1004             if (handledSyncs) {
1005                 [self->_pendingSyncPeerIDs minusSet: handledSyncs];
1006                 [self->_pendingSyncBackupPeerIDs minusSet: handledSyncs];
1007
1008                 if (![handledSyncs isEmpty]) {
1009                     secnotice("sync-ids", "handled syncIDs: %@", [handledSyncs logIDs]);
1010                     secnotice("sync-ids", "remaining peerIDs: %@", [self->_pendingSyncPeerIDs logIDs]);
1011                     secnotice("sync-ids", "remaining backupIDs: %@", [self->_pendingSyncBackupPeerIDs logIDs]);
1012
1013                     if (hadShadowPeerIDs) {
1014                         secnotice("sync-ids", "signaled peerIDs: %@", [self->_shadowPendingSyncPeerIDs logIDs]);
1015                         secnotice("sync-ids", "signaled backupIDs: %@", [self->_shadowPendingSyncBackupPeerIDs logIDs]);
1016                     }
1017                 }
1018
1019                 self->_shadowPendingSyncPeerIDs = nil;
1020                 self->_shadowPendingSyncBackupPeerIDs = nil;
1021             }
1022
1023
1024             // Update pendingKeys and handle them
1025             [self->_pendingKeys removeObject: [NSNull null]]; // Don't let NULL hang around
1026
1027             [self->_pendingKeys minusSet: handledKeys];
1028             bool hadShadowPendingKeys = [self->_shadowPendingKeys count];
1029             // Move away shadownPendingKeys first, because pendKeysAndGetPendingForCurrentLockState
1030             // will look at them. See rdar://problem/20733166.
1031             NSSet *oldShadowPendingKeys = self->_shadowPendingKeys;
1032             self->_shadowPendingKeys = nil;
1033
1034             NSSet *filteredKeys = [self pendKeysAndGetPendingForCurrentLockState:oldShadowPendingKeys];
1035
1036             secnoticeq("keytrace", "%@ account handled: %@ pending: %@", self,
1037                        [[handledKeys allObjects] componentsJoinedByString: @" "],
1038                        [[filteredKeys allObjects] componentsJoinedByString: @" "]);
1039
1040             // Write state to disk
1041             [self persistState];
1042
1043             // Handle shadow pended stuff
1044
1045             // We only kick off another sync if we got new stuff during handling
1046             if (hadShadowPeerIDs && ![self.lockMonitor locked])
1047                 [self newPeersToSyncWith];
1048
1049             /* We don't want to call processKeyChangedEvent if we failed to
1050              handle pending keys and the device didn't unlock nor receive
1051              any kvs changes while we were in our callout.
1052              Doing so will lead to securityd and CloudKeychainProxy
1053              talking to each other forever in a tight loop if securityd
1054              repeatedly returns an error processing the same message.
1055              Instead we leave any old pending keys until the next event. */
1056             if (hadShadowPendingKeys || (![self.lockMonitor locked] && wasLocked)){
1057                 [self processKeyChangedEvent:[self copyValues:filteredKeys]];
1058                 if(self->_shadowFlushBlock != NULL)
1059                     secerror("Flush block is not null and sending new keys");
1060             }
1061
1062             if(self->_shadowFlushBlock != NULL){
1063                 dispatch_async(self->_calloutQueue, self->_shadowFlushBlock);
1064                 self->_shadowFlushBlock = NULL;
1065             }
1066
1067             if (failure) {
1068                 [self.lockMonitor recheck];
1069             }
1070
1071             xpc_transaction_end();
1072         });
1073     });
1074 }
1075
1076 - (void) sendKeysCallout: (NSSet *(^)(NSSet* pending, NSError** error)) handleKeys {
1077     [self calloutWith: ^(NSSet *pending, NSSet* pendingSyncIDs, NSSet* pendingBackupSyncIDs, bool ensurePeerRegistration, dispatch_queue_t queue, void(^done)(NSSet *handledKeys, NSSet *handledSyncs, bool handledEnsurePeerRegistration, NSError* error)) {
1078         NSError* error = NULL;
1079
1080         secnotice("CloudKeychainProxy", "send keys: %@", pending);
1081         NSSet * handled = handleKeys(pending, &error);
1082
1083         dispatch_async(queue, ^{
1084             if (!handled) {
1085                 secerror("%@ ensurePeerRegistration failed: %@", self, error);
1086             }
1087
1088             done(handled, nil, NO, error);
1089         });
1090     }];
1091 }
1092
1093 - (void) doEnsurePeerRegistration
1094 {
1095     NSObject<CKDAccount>* accountDelegate = [self account];
1096     [self calloutWith:^(NSSet *pending, NSSet* pendingSyncIDs, NSSet* pendingBackupSyncIDs, bool ensurePeerRegistration, dispatch_queue_t queue, void(^done)(NSSet *handledKeys, NSSet *handledSyncs, bool handledEnsurePeerRegistration, NSError* error)) {
1097         NSError* error = nil;
1098         bool handledEnsurePeerRegistration = [accountDelegate ensurePeerRegistration:&error];
1099         secnotice("EnsurePeerRegistration", "%@ ensurePeerRegistration called, %@ (%@)", self, handledEnsurePeerRegistration ? @"success" : @"failure", error);
1100         if (!handledEnsurePeerRegistration) {
1101             [self.lockMonitor recheck];
1102             handledEnsurePeerRegistration = ![self.lockMonitor locked]; // If we're unlocked we handled it, if we're locked we didn't.
1103                                                               // This means we get to fail once per unlock and then cut that spinning out.
1104         }
1105         dispatch_async(queue, ^{
1106             done(nil, nil, handledEnsurePeerRegistration, error);
1107         });
1108     }];
1109 }
1110
1111 - (void) doSyncWithPendingPeers
1112 {
1113     NSObject<CKDAccount>* accountDelegate = [self account];
1114     [self calloutWith:^(NSSet *pending, NSSet* pendingSyncIDs, NSSet* pendingBackupSyncIDs, bool ensurePeerRegistration, dispatch_queue_t queue, void(^done)(NSSet *handledKeys, NSSet *handledSyncs, bool handledEnsurePeerRegistration, NSError* error)) {
1115         NSError* error = NULL;
1116         secnotice("syncwith", "%@ syncwith peers: %@", self, [[pendingSyncIDs allObjects] componentsJoinedByString:@" "]);
1117         secnotice("syncwith", "%@ syncwith backups: %@", self, [[pendingBackupSyncIDs allObjects] componentsJoinedByString:@" "]);
1118         NSSet<NSString*>* handled = [accountDelegate syncWithPeers:pendingSyncIDs backups:pendingBackupSyncIDs error:&error];
1119         secnotice("syncwith", "%@ syncwith handled: %@", self, [[handled allObjects] componentsJoinedByString:@" "]);
1120         dispatch_async(queue, ^{
1121             if (!handled) {
1122                 // We might be confused about lock state
1123                 [self.lockMonitor recheck];
1124             }
1125
1126             done(nil, handled, false, error);
1127         });
1128     }];
1129 }
1130
1131 - (void) doSyncWithAllPeers
1132 {
1133     NSObject<CKDAccount>* accountDelegate = [self account];
1134     [self calloutWith:^(NSSet *pending, NSSet* pendingSyncIDs, NSSet* pendingBackupSyncIDs, bool ensurePeerRegistration, dispatch_queue_t queue, void(^done)(NSSet *handledKeys, NSSet *handledSyncs, bool handledEnsurePeerRegistration, NSError*error)) {
1135         NSError* error = NULL;
1136         bool handled = [accountDelegate syncWithAllPeers:&error];
1137         if (!handled) {
1138             secerror("Failed to syncWithAllPeers: %@", error);
1139         }
1140         dispatch_async(queue, ^{
1141             done(nil, nil, false, error);
1142         });
1143     }];
1144 }
1145
1146 - (void)newPeersToSyncWith
1147 {
1148     secnotice("event", "%@ syncWithPeersPending: %d inCallout: %d isLocked: %d", self, [self hasPendingSyncIDs], _inCallout, [self.lockMonitor locked]);
1149     if(_ensurePeerRegistration){
1150         [self doEnsurePeerRegistration];
1151     }
1152     if ([self hasPendingSyncIDs] && !_inCallout && ![self.lockMonitor locked]){
1153         [self doSyncWithPendingPeers];
1154     }
1155 }
1156
1157 - (bool)hasPendingNonShadowSyncIDs {
1158     return ![_pendingSyncPeerIDs isEmpty] || ![_pendingSyncBackupPeerIDs isEmpty];
1159 }
1160
1161 - (bool)hasPendingShadowSyncIDs {
1162     return (_shadowPendingSyncPeerIDs && ![_shadowPendingSyncPeerIDs isEmpty]) ||
1163     (_shadowPendingSyncBackupPeerIDs && ![_shadowPendingSyncBackupPeerIDs isEmpty]);
1164 }
1165
1166 - (bool)hasPendingSyncIDs
1167 {
1168     bool pendingIDs = [self hasPendingNonShadowSyncIDs];
1169
1170     if (_inCallout) {
1171         pendingIDs |= [self hasPendingShadowSyncIDs];
1172     }
1173
1174     return pendingIDs;
1175 }
1176
1177 - (void)requestSyncWithPeerIDs: (NSArray<NSString*>*) peerIDs backupPeerIDs: (NSArray<NSString*>*) backupPeerIDs
1178 {
1179     if ([peerIDs count] == 0 && [backupPeerIDs count] == 0)
1180         return; // Nothing to do;
1181
1182     NSSet<NSString*>* peerIDsSet = [NSSet setWithArray: peerIDs];
1183     NSSet<NSString*>* backupPeerIDsSet = [NSSet setWithArray: backupPeerIDs];
1184
1185     [_pendingSyncPeerIDs unionSet: peerIDsSet];
1186     [_pendingSyncBackupPeerIDs unionSet: backupPeerIDsSet];
1187
1188     if (_inCallout) {
1189         [_shadowPendingSyncPeerIDs unionSet: peerIDsSet];
1190         [_shadowPendingSyncBackupPeerIDs unionSet: backupPeerIDsSet];
1191     }
1192
1193     [self persistState];
1194
1195     if(_ensurePeerRegistration){
1196         [self doEnsurePeerRegistration];
1197     }
1198     if ([self hasPendingSyncIDs] && !_inCallout && ![self.lockMonitor locked]){
1199         [self doSyncWithPendingPeers];
1200     }
1201 }
1202
1203 - (BOOL)hasSyncPendingFor: (NSString*) peerID {
1204     return [_pendingSyncPeerIDs containsObject: peerID] ||
1205     (_shadowPendingSyncPeerIDs && [_shadowPendingSyncPeerIDs containsObject: peerID]);
1206 }
1207
1208 - (BOOL)hasPendingKey: (NSString*) keyName {
1209     return [self.pendingKeys containsObject: keyName]
1210         || (_shadowPendingKeys && [self.shadowPendingKeys containsObject: keyName]);
1211 }
1212
1213 - (void)requestEnsurePeerRegistration
1214 {
1215 #if !defined(NDEBUG)
1216     NSString *desc = [self description];
1217 #endif
1218
1219     if (_inCallout) {
1220         _shadowEnsurePeerRegistration = YES;
1221     } else {
1222         _ensurePeerRegistration = YES;
1223         if (![self.lockMonitor locked]){
1224             [self doEnsurePeerRegistration];
1225         }
1226         [self persistState];
1227     }
1228
1229     secdebug("event", "%@ %@", desc, self);
1230 }
1231
1232 - (void)_queue_locked
1233 {
1234     dispatch_assert_queue(_ckdkvsproxy_queue);
1235
1236     secnotice("event", "%@ Locked", self);
1237 }
1238
1239 - (void)_queue_unlocked
1240 {
1241     dispatch_assert_queue(_ckdkvsproxy_queue);
1242
1243     secnotice("event", "%@ Unlocked", self);
1244     if (_ensurePeerRegistration) {
1245         [self doEnsurePeerRegistration];
1246     }
1247
1248     // First send changed keys to securityd so it can proccess updates
1249     [self processPendingKeysForCurrentLockState];
1250
1251     // Then, tickle securityd to perform a sync if needed.
1252     if ([self hasPendingSyncIDs]) {
1253         [self doSyncWithPendingPeers];
1254     }
1255 }
1256
1257 - (void) _queue_kvsStoreChange {
1258     dispatch_assert_queue(_ckdkvsproxy_queue);
1259
1260     os_activity_initiate("kvsStoreChange", OS_ACTIVITY_FLAG_DEFAULT, ^{
1261         if (!self->_seenKVSStoreChange) {
1262             secnotice("event", "%@ received darwin notification before first NSNotification", self);
1263             // TODO This might not be needed if we always get the NSNotification
1264             // deleived even if we were launched due to a kvsStoreChange
1265             // Send all keys for current lock state to securityd so it can proccess them
1266             [self pendKeysAndGetNewlyPended: [self copyAllKeyInterests]];
1267             [self processPendingKeysForCurrentLockState];
1268         } else {
1269             secdebug("event", "%@ ignored, waiting for NSNotification", self);
1270         }
1271     });
1272 }
1273
1274 #pragma mark -
1275 #pragma mark XPCNotificationListener
1276
1277 - (void)handleNotification:(const char *) name
1278 {
1279     // sync because we cannot ensure the lifetime of name
1280     dispatch_sync(_ckdkvsproxy_queue, ^{
1281         [self _queue_handleNotification:name];
1282     });
1283 }
1284
1285 #pragma mark -
1286 #pragma mark Calls from -[CKDKVSStore kvsStoreChanged:]
1287
1288 - (void)storeKeysChanged: (NSSet<NSString*>*) changedKeys initial: (bool) initial
1289 {
1290     // sync, caller must wait to ensure correct state
1291     dispatch_sync(_ckdkvsproxy_queue, ^{
1292         [self _queue_storeKeysChanged:changedKeys initial:initial];
1293     });
1294 }
1295
1296 - (void)storeAccountChanged
1297 {
1298     // sync, caller must wait to ensure correct state
1299     dispatch_sync(_ckdkvsproxy_queue, ^{
1300         [self _queue_storeAccountChanged];
1301     });
1302 }
1303
1304 #pragma mark -
1305 #pragma mark CKDLockListener
1306
1307 - (void) locked
1308 {
1309     // sync, otherwise tests fail
1310     dispatch_sync(_ckdkvsproxy_queue, ^{
1311         [self _queue_locked];
1312     });
1313 }
1314
1315 - (void) unlocked
1316 {
1317     // sync, otherwise tests fail
1318     dispatch_sync(_ckdkvsproxy_queue, ^{
1319         [self _queue_unlocked];
1320     });
1321 }
1322
1323 //
1324 // MARK: ----- Key Filtering -----
1325 //
1326
1327 - (NSSet*) keysForCurrentLockState
1328 {
1329     secdebug("filtering", "%@ Filtering: unlockedSinceBoot: %d\n unlocked: %d\n, keysOfInterest: <%@>", self, (int) [self.lockMonitor unlockedSinceBoot], (int) ![self.lockMonitor locked], [self.persistentData compactDescription]);
1330
1331     NSMutableSet *currentStateKeys = [NSMutableSet setWithSet: _alwaysKeys];
1332     if ([self.lockMonitor unlockedSinceBoot])
1333         [currentStateKeys unionSet: _firstUnlockKeys];
1334
1335     if (![self.lockMonitor locked])
1336         [currentStateKeys unionSet: _unlockedKeys];
1337
1338     return currentStateKeys;
1339 }
1340
1341
1342 - (NSMutableSet*) pendKeysAndGetNewlyPended: (NSSet*) keysToPend
1343 {
1344     NSMutableSet *filteredKeysToPend = [self copyAllKeyInterests];
1345     [filteredKeysToPend intersectSet: keysToPend];
1346
1347     NSMutableSet *newlyPendedKeys = [filteredKeysToPend mutableCopy];
1348     [newlyPendedKeys minusSet: _pendingKeys];
1349     if (_shadowPendingKeys) {
1350         [newlyPendedKeys minusSet: _shadowPendingKeys];
1351     }
1352
1353     if (_shadowPendingKeys) {
1354         [_shadowPendingKeys unionSet:filteredKeysToPend];
1355     }
1356     else{
1357         [_pendingKeys unionSet:filteredKeysToPend];
1358     }
1359
1360     return newlyPendedKeys;
1361 }
1362
1363 - (void) intersectWithCurrentLockState: (NSMutableSet*) set
1364 {
1365     [set intersectSet: [self keysForCurrentLockState]];
1366 }
1367
1368 - (NSMutableSet*) pendingKeysForCurrentLockState
1369 {
1370     NSMutableSet * result = [_pendingKeys mutableCopy];
1371     [self intersectWithCurrentLockState:result];
1372     return result;
1373 }
1374
1375 - (NSMutableSet*) pendKeysAndGetPendingForCurrentLockState: (NSSet*) startingSet
1376 {
1377     [self pendKeysAndGetNewlyPended: startingSet];
1378
1379     return [self pendingKeysForCurrentLockState];
1380 }
1381
1382 - (NSMutableDictionary *)copyValues:(NSSet*)keysOfInterest
1383 {
1384     // Grab values from store.
1385     NSObject<CKDStore> *store = [self store];
1386     NSMutableDictionary *changedValues = [NSMutableDictionary dictionaryWithCapacity:0];
1387     [keysOfInterest enumerateObjectsUsingBlock:^(id obj, BOOL *stop)
1388      {
1389          NSString* key = (NSString*) obj;
1390          id objval = [store objectForKey:key];
1391          if (!objval) objval = [NSNull null];
1392
1393          [changedValues setObject:objval forKey:key];
1394          secdebug(XPROXYSCOPE, "%@ storeChanged updated value for %@", self, key);
1395      }];
1396     return changedValues;
1397 }
1398
1399 /*
1400  During RegisterKeys, separate keys-of-interest into three disjoint sets:
1401  - keys that we always want to be notified about; this means we can get the
1402  value at any time
1403  - keys that require the device to have been unlocked at least once
1404  - keys that require the device to be unlocked now
1405
1406  Typically, the sets of keys will be:
1407
1408  - Dk: alwaysKeys
1409  - Ck: firstUnlock
1410  - Ak: unlocked
1411
1412  The caller is responsible for making sure that the keys in e.g. alwaysKeys are
1413  values that can be handled at any time (that is, not when unlocked)
1414
1415  Each time we get a notification from ubiquity that keys have changed, we need to
1416  see if anything of interest changed. If we don't care, then done.
1417
1418  For each key-of-interest that changed, we either notify the client that things
1419  changed, or add it to a pendingNotifications list. If the notification to the
1420  client fails, also add it to the pendingNotifications list. This pending list
1421  should be written to persistent storage and consulted any time we either get an
1422  item changed notification, or get a stream event signalling a change in lock state.
1423
1424  We can notify the client either through XPC if a connection is set up, or call a
1425  routine in securityd to launch it.
1426
1427  */
1428
1429 - (void)processKeyChangedEvent:(NSDictionary *)changedValues
1430 {
1431     NSMutableDictionary* filtered = [NSMutableDictionary dictionary];
1432
1433     secnotice("processKeyChangedEvent", "changedValues:%@", changedValues);
1434     NSMutableArray* nullKeys = [NSMutableArray array];
1435     // Remove nulls because we don't want them in securityd.
1436     [changedValues enumerateKeysAndObjectsUsingBlock:^(id key, id obj, BOOL *stop) {
1437         if (obj == [NSNull null]){
1438             [nullKeys addObject:key];
1439         }else{
1440             filtered[key] = obj;
1441         }
1442     }];
1443     if ([nullKeys count])
1444         [_pendingKeys minusSet: [NSSet setWithArray: nullKeys]];
1445
1446     if([filtered count] != 0 ) {
1447         [self sendKeysCallout:^NSSet *(NSSet *pending, NSError** error) {
1448             secnotice("processing keys", "pending:%@", pending);
1449             NSError *updateError = nil;
1450             return [[self account] keysChanged: filtered error: &updateError];
1451         }];
1452     } else {
1453         secnoticeq("keytrace", "%@ null: %@ pending: %@", self,
1454                   [nullKeys componentsJoinedByString: @" "],
1455                   [[_pendingKeys allObjects] componentsJoinedByString: @" "]);
1456     }
1457 }
1458
1459 - (void) processPendingKeysForCurrentLockState
1460 {
1461     [self processKeyChangedEvent: [self copyValues: [self pendingKeysForCurrentLockState]]];
1462 }
1463
1464 @end
1465
1466