keychain/ckks/CKKSKeychainView.m

   1 /*
   2  * Copyright (c) 2016 Apple Inc. All Rights Reserved.
   3  *
   4  * @APPLE_LICENSE_HEADER_START@
   5  *
   6  * This file contains Original Code and/or Modifications of Original Code
   7  * as defined in and that are subject to the Apple Public Source License
   8  * Version 2.0 (the 'License'). You may not use this file except in
   9  * compliance with the License. Please obtain a copy of the License at
  10  * http://www.opensource.apple.com/apsl/ and read it before using this
  11  * file.
  12  *
  13  * The Original Code and all software distributed under the License are
  14  * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  15  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  16  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
  17  * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
  18  * Please see the License for the specific language governing rights and
  19  * limitations under the License.
  20  *
  21  * @APPLE_LICENSE_HEADER_END@
  22  */
  23
  24 #import "CKKSKeychainView.h"
  25
  26
  27
  28 #if OCTAGON
  29 #import "CloudKitDependencies.h"
  30 #import <CloudKit/CloudKit.h>
  31 #import <CloudKit/CloudKit_Private.h>
  32 #endif
  33
  34 #import "CKKS.h"
  35 #import "CKKSAPSReceiver.h"
  36 #import "CKKSIncomingQueueEntry.h"
  37 #import "CKKSOutgoingQueueEntry.h"
  38 #import "CKKSCurrentKeyPointer.h"
  39 #import "CKKSKey.h"
  40 #import "CKKSMirrorEntry.h"
  41 #import "CKKSZoneStateEntry.h"
  42 #import "CKKSItemEncrypter.h"
  43 #import "CKKSIncomingQueueOperation.h"
  44 #import "CKKSNewTLKOperation.h"
  45 #import "CKKSProcessReceivedKeysOperation.h"
  46 #import "CKKSZone.h"
  47 #import "CKKSFetchAllRecordZoneChangesOperation.h"
  48 #import "CKKSHealKeyHierarchyOperation.h"
  49 #import "CKKSReencryptOutgoingItemsOperation.h"
  50 #import "CKKSScanLocalItemsOperation.h"
  51 #import "CKKSSynchronizeOperation.h"
  52 #import "CKKSRateLimiter.h"
  53 #import "CKKSManifest.h"
  54 #import "CKKSManifestLeafRecord.h"
  55 #import "CKKSZoneChangeFetcher.h"
  56 #import  "CKKSAnalyticsLogger.h"
  57 #import "keychain/ckks/CKKSDeviceStateEntry.h"
  58 #import "keychain/ckks/CKKSNearFutureScheduler.h"
  59 #import "keychain/ckks/CKKSCurrentItemPointer.h"
  60 #import "keychain/ckks/CKKSUpdateCurrentItemPointerOperation.h"
  61 #import "keychain/ckks/CKKSUpdateDeviceStateOperation.h"
  62 #import "keychain/ckks/CKKSLockStateTracker.h"
  63 #import "keychain/ckks/CKKSNotifier.h"
  64 #import "keychain/ckks/CloudKitCategories.h"
  65 #import "keychain/ckks/CKKSTLKShare.h"
  66 #import "keychain/ckks/CKKSHealTLKSharesOperation.h"
  67 #import "keychain/ckks/CKKSLocalSynchronizeOperation.h"
  68
  69 #include <utilities/SecCFWrappers.h>
  70 #include <utilities/SecDb.h>
  71 #include <securityd/SecDbItem.h>
  72 #include <securityd/SecItemDb.h>
  73 #include <securityd/SecItemSchema.h>
  74 #include <securityd/SecItemServer.h>
  75 #include <utilities/debugging.h>
  76 #include <Security/SecItemPriv.h>
  77 #include <Security/SecureObjectSync/SOSAccountTransaction.h>
  78 #include <utilities/SecADWrapper.h>
  79 #include <utilities/SecPLWrappers.h>
  80
  81 #if OCTAGON
  82 @interface CKKSKeychainView()
  83 @property bool setupSuccessful;
  84 @property bool keyStateFetchRequested;
  85 @property bool keyStateFullRefetchRequested;
  86 @property bool keyStateProcessRequested;
  87 @property (atomic) NSString *activeTLK;
  88
  89 @property (readonly) Class<CKKSNotifier> notifierClass;
  90
  91 @property CKKSNearFutureScheduler* initializeScheduler;
  92
  93 // Slows down all outgoing queue operations
  94 @property CKKSNearFutureScheduler* outgoingQueueOperationScheduler;
  95
  96 @property CKKSResultOperation* processIncomingQueueAfterNextUnlockOperation;
  97
  98 @property NSMutableDictionary<NSString*, SecBoolNSErrorCallback>* pendingSyncCallbacks;
  99
 100 @property id<CKKSPeerProvider> currentPeerProvider;
 101
 102 // An extra queue for semaphore-waiting-based NSOperations
 103 @property NSOperationQueue* waitingQueue;
 104
 105 // Make these readwrite
 106 @property (nonatomic, readwrite) CKKSSelves* currentSelfPeers;
 107 @property (nonatomic, readwrite) NSError* currentSelfPeersError;
 108 @property (nonatomic, readwrite) NSSet<id<CKKSPeer>>* currentTrustedPeers;
 109 @property (nonatomic, readwrite) NSError* currentTrustedPeersError;
 110 @end
 111 #endif
 112
 113 @implementation CKKSKeychainView
 114 #if OCTAGON
 115
 116 - (instancetype)initWithContainer:     (CKContainer*) container
 117                              zoneName: (NSString*) zoneName
 118                        accountTracker:(CKKSCKAccountStateTracker*) accountTracker
 119                      lockStateTracker:(CKKSLockStateTracker*) lockStateTracker
 120                      savedTLKNotifier:(CKKSNearFutureScheduler*) savedTLKNotifier
 121                      peerProvider:(id<CKKSPeerProvider>)peerProvider
 122  fetchRecordZoneChangesOperationClass: (Class<CKKSFetchRecordZoneChangesOperation>) fetchRecordZoneChangesOperationClass
 123            fetchRecordsOperationClass: (Class<CKKSFetchRecordsOperation>)fetchRecordsOperationClass
 124                   queryOperationClass:(Class<CKKSQueryOperation>)queryOperationClass
 125     modifySubscriptionsOperationClass: (Class<CKKSModifySubscriptionsOperation>) modifySubscriptionsOperationClass
 126       modifyRecordZonesOperationClass: (Class<CKKSModifyRecordZonesOperation>) modifyRecordZonesOperationClass
 127                    apsConnectionClass: (Class<CKKSAPSConnection>) apsConnectionClass
 128                         notifierClass: (Class<CKKSNotifier>) notifierClass
 129 {
 130
 131     if(self = [super initWithContainer:container
 132                               zoneName:zoneName
 133                         accountTracker:accountTracker
 134   fetchRecordZoneChangesOperationClass:fetchRecordZoneChangesOperationClass
 135             fetchRecordsOperationClass:fetchRecordsOperationClass
 136                    queryOperationClass:queryOperationClass
 137      modifySubscriptionsOperationClass:modifySubscriptionsOperationClass
 138        modifyRecordZonesOperationClass:modifyRecordZonesOperationClass
 139                     apsConnectionClass:apsConnectionClass]) {
 140         __weak __typeof(self) weakSelf = self;
 141
 142         _loggedIn = [[CKKSCondition alloc] init];
 143         _loggedOut = [[CKKSCondition alloc] init];
 144
 145         _incomingQueueOperations = [NSHashTable weakObjectsHashTable];
 146         _outgoingQueueOperations = [NSHashTable weakObjectsHashTable];
 147         _zoneChangeFetcher = [[CKKSZoneChangeFetcher alloc] initWithCKKSKeychainView: self];
 148
 149         _notifierClass = notifierClass;
 150         _notifyViewChangedScheduler = [[CKKSNearFutureScheduler alloc] initWithName:[NSString stringWithFormat: @"%@-notify-scheduler", self.zoneName]
 151                                                             initialDelay:250*NSEC_PER_MSEC
 152                                                          continuingDelay:1*NSEC_PER_SEC
 153                                                         keepProcessAlive:true
 154                                                                    block:^{
 155                                                                        __strong __typeof(self) strongSelf = weakSelf;
 156                                                                        [strongSelf.notifierClass post:[NSString stringWithFormat:@"com.apple.security.view-change.%@", strongSelf.zoneName]];
 157
 158                                                                        // Ugly, but: the Manatee and Engram views need to send a fake 'PCS' view change.
 159                                                                        // TODO: make this data-driven somehow
 160                                                                        if([strongSelf.zoneName isEqualToString:@"Manatee"] || [strongSelf.zoneName isEqualToString:@"Engram"]) {
 161                                                                            [strongSelf.notifierClass post:@"com.apple.security.view-change.PCS"];
 162                                                                        }
 163                                                                    }];
 164
 165         _pendingSyncCallbacks = [[NSMutableDictionary alloc] init];
 166
 167         _lockStateTracker = lockStateTracker;
 168         _savedTLKNotifier = savedTLKNotifier;
 169         _currentPeerProvider = peerProvider;
 170         [_currentPeerProvider registerForPeerChangeUpdates:self];
 171
 172         _setupSuccessful = false;
 173
 174         _keyHierarchyConditions = [[NSMutableDictionary alloc] init];
 175         [CKKSZoneKeyStateMap() enumerateKeysAndObjectsUsingBlock:^(CKKSZoneKeyState * _Nonnull key, NSNumber * _Nonnull obj, BOOL * _Nonnull stop) {
 176             [self.keyHierarchyConditions setObject: [[CKKSCondition alloc] init] forKey:key];
 177         }];
 178
 179         self.keyHierarchyState = SecCKKSZoneKeyStateInitializing;
 180         _keyHierarchyError = nil;
 181         _keyHierarchyOperationGroup = nil;
 182         _keyStateMachineOperation = nil;
 183         _keyStateFetchRequested = false;
 184         _keyStateProcessRequested = false;
 185
 186         _waitingQueue = [[NSOperationQueue alloc] init];
 187         _waitingQueue.maxConcurrentOperationCount = 5;
 188
 189         _keyStateReadyDependency = [self createKeyStateReadyDependency: @"Key state has become ready for the first time." ckoperationGroup:[CKOperationGroup CKKSGroupWithName:@"initial-key-state-ready-scan"]];
 190
 191         dispatch_time_t initializeDelay = SecCKKSTestsEnabled() ? NSEC_PER_MSEC * 600 : NSEC_PER_SEC * 30;
 192         _initializeScheduler = [[CKKSNearFutureScheduler alloc] initWithName:[NSString stringWithFormat: @"%@-zone-initializer", self.zoneName]
 193                                                                 initialDelay:0
 194                                                              continuingDelay:initializeDelay
 195                                                             keepProcessAlive:false
 196                                                                        block:^{
 197                                                                            __strong __typeof(self) strongSelf = weakSelf;
 198                                                                            ckksnotice("ckks", strongSelf, "initialize-scheduler restarting setup");
 199                                                                            [strongSelf maybeRestartSetup];
 200                                                                        }];
 201
 202         dispatch_time_t initialOutgoingQueueDelay = SecCKKSTestsEnabled() ? NSEC_PER_MSEC * 200 : NSEC_PER_SEC * 1;
 203         dispatch_time_t continuingOutgoingQueueDelay = SecCKKSTestsEnabled() ? NSEC_PER_MSEC * 500 : NSEC_PER_SEC * 30;
 204         _outgoingQueueOperationScheduler = [[CKKSNearFutureScheduler alloc] initWithName:[NSString stringWithFormat: @"%@-outgoing-queue-scheduler", self.zoneName]
 205                                                                             initialDelay:initialOutgoingQueueDelay
 206                                                                          continuingDelay:continuingOutgoingQueueDelay
 207                                                                         keepProcessAlive:false
 208                                                                                    block:^{}];
 209     }
 210     return self;
 211 }
 212
 213 - (NSString*)description {
 214     return [NSString stringWithFormat:@"<%@: %@ (%@)>", NSStringFromClass([self class]), self.zoneName, self.keyHierarchyState];
 215 }
 216
 217 - (NSString*)debugDescription {
 218     return [NSString stringWithFormat:@"<%@: %@ (%@) %p>", NSStringFromClass([self class]), self.zoneName, self.keyHierarchyState, self];
 219 }
 220
 221 - (CKKSZoneKeyState*)keyHierarchyState {
 222     return _keyHierarchyState;
 223 }
 224
 225 - (void)setKeyHierarchyState:(CKKSZoneKeyState *)keyHierarchyState {
 226     if((keyHierarchyState == nil && _keyHierarchyState == nil) || ([keyHierarchyState isEqualToString:_keyHierarchyState])) {
 227         // No change, do nothing.
 228     } else {
 229         // Fixup the condition variables
 230         if(_keyHierarchyState) {
 231             self.keyHierarchyConditions[_keyHierarchyState] = [[CKKSCondition alloc] init];
 232         }
 233         if(keyHierarchyState) {
 234             [self.keyHierarchyConditions[keyHierarchyState] fulfill];
 235         }
 236     }
 237
 238     _keyHierarchyState = keyHierarchyState;
 239 }
 240
 241 - (NSString *)lastActiveTLKUUID
 242 {
 243     return self.activeTLK;
 244 }
 245
 246 - (void)maybeRestartSetup {
 247     [self dispatchSync: ^bool{
 248         if([self.viewSetupOperation isPending] || [self.viewSetupOperation isExecuting]) {
 249             ckksinfo("ckks", self, "setup is in-flight. Ignoring timer fire");
 250             return false;
 251         }
 252
 253         [self resetSetup];
 254         [self restartCurrentAccountStateOperation];
 255         return true;
 256     }];
 257 }
 258
 259 - (void)resetSetup {
 260     [super resetSetup];
 261     self.setupSuccessful = false;
 262
 263     // Key hierarchy state machine resets, too
 264     self.keyHierarchyState = SecCKKSZoneKeyStateInitializing;
 265     _keyHierarchyError = nil;
 266 }
 267
 268  - (void)_onqueueHandleCKLogin {
 269     if(!SecCKKSIsEnabled()) {
 270         ckksnotice("ckks", self, "Skipping CloudKit initialization due to disabled CKKS");
 271         return;
 272     }
 273
 274     dispatch_assert_queue(self.queue);
 275
 276     __weak __typeof(self) weakSelf = self;
 277
 278     CKKSZoneStateEntry* ckse = [CKKSZoneStateEntry state: self.zoneName];
 279     [self handleCKLogin:ckse.ckzonecreated zoneSubscribed:ckse.ckzonesubscribed];
 280
 281     self.viewSetupOperation = [CKKSResultOperation operationWithBlock: ^{
 282         __strong __typeof(weakSelf) strongSelf = weakSelf;
 283         if(!strongSelf) {
 284             ckkserror("ckks", strongSelf, "received callback for released object");
 285             return;
 286         }
 287
 288         __block bool quit = false;
 289
 290         [strongSelf dispatchSync: ^bool {
 291             ckksnotice("ckks", strongSelf, "Zone setup progress: %@ %d %@ %d %@",
 292                        [CKKSCKAccountStateTracker stringFromAccountStatus:strongSelf.accountStatus],
 293                        strongSelf.zoneCreated, strongSelf.zoneCreatedError, strongSelf.zoneSubscribed, strongSelf.zoneSubscribedError);
 294
 295             NSError* error = nil;
 296             CKKSZoneStateEntry* ckse = [CKKSZoneStateEntry state: strongSelf.zoneName];
 297             ckse.ckzonecreated = strongSelf.zoneCreated;
 298             ckse.ckzonesubscribed = strongSelf.zoneSubscribed;
 299
 300             // Although, if the zone subscribed error says there's no zone, mark down that there's no zone
 301             if(strongSelf.zoneSubscribedError &&
 302                [strongSelf.zoneSubscribedError.domain isEqualToString:CKErrorDomain] && strongSelf.zoneSubscribedError.code == CKErrorPartialFailure) {
 303                 NSError* subscriptionError = strongSelf.zoneSubscribedError.userInfo[CKPartialErrorsByItemIDKey][strongSelf.zoneID];
 304                 if(subscriptionError && [subscriptionError.domain isEqualToString:CKErrorDomain] && subscriptionError.code == CKErrorZoneNotFound) {
 305
 306                     ckkserror("ckks", strongSelf, "zone subscription error appears to say the zone doesn't exist, fixing status: %@", strongSelf.zoneSubscribedError);
 307                     ckse.ckzonecreated = false;
 308                 }
 309             }
 310
 311             [ckse saveToDatabase: &error];
 312             if(error) {
 313                 ckkserror("ckks", strongSelf, "couldn't save zone creation status for %@: %@", strongSelf.zoneName, error);
 314             }
 315
 316             if(!strongSelf.zoneCreated || !strongSelf.zoneSubscribed || strongSelf.accountStatus != CKAccountStatusAvailable) {
 317                 // Something has gone very wrong. Error out and maybe retry.
 318                 quit = true;
 319
 320                 // Note that CKKSZone has probably called [handleLogout]; which means we have a key hierarchy reset queued up. Error here anyway.
 321                 NSError* realReason = strongSelf.zoneCreatedError ? strongSelf.zoneCreatedError : strongSelf.zoneSubscribedError;
 322                 strongSelf.viewSetupOperation.error = realReason;
 323                 [strongSelf _onqueueAdvanceKeyStateMachineToState: SecCKKSZoneKeyStateError withError: realReason];
 324
 325                 // We're supposed to be up, but something has gone wrong. Blindly retry until it works.
 326                 if(strongSelf.accountStatus == CKKSAccountStatusAvailable) {
 327                     [strongSelf.initializeScheduler trigger];
 328                     ckksnotice("ckks", strongSelf, "We're logged in, but setup didn't work. Scheduling retry for %@", strongSelf.initializeScheduler.nextFireTime);
 329                 }
 330                 return true;
 331             } else {
 332                 strongSelf.setupSuccessful = true;
 333             }
 334
 335             return true;
 336         }];
 337
 338         if(quit) {
 339             ckkserror("ckks", strongSelf, "Quitting setup.");
 340             return;
 341         }
 342
 343         // We can't enter the account queue until an account exists. Before this point, we don't know if one does.
 344         [strongSelf dispatchSyncWithAccountKeys: ^bool{
 345             // Change our condition variables to reflect that we think we're logged in
 346             strongSelf.loggedOut = [[CKKSCondition alloc] initToChain: strongSelf.loggedOut];
 347             [strongSelf.loggedIn fulfill];
 348
 349             CKKSZoneStateEntry* ckse = [CKKSZoneStateEntry state: strongSelf.zoneName];
 350
 351             // Check if we believe we've synced this zone before.
 352             if(ckse.changeToken == nil) {
 353                 strongSelf.keyHierarchyOperationGroup = [CKOperationGroup CKKSGroupWithName:@"initial-setup"];
 354
 355                 ckksnotice("ckks", strongSelf, "No existing change token; going to try to match local items with CloudKit ones.");
 356
 357                 // Onboard this keychain: there's likely items in it that we haven't synced yet.
 358                 // But, there might be items in The Cloud that correspond to these items, with UUIDs that we don't know yet.
 359                 // First, fetch all remote items.
 360                 CKKSResultOperation* fetch = [strongSelf.zoneChangeFetcher requestSuccessfulFetch:CKKSFetchBecauseInitialStart];
 361                 fetch.name = @"initial-fetch";
 362
 363                 // Next, try to process them (replacing local entries)
 364                 CKKSIncomingQueueOperation* initialProcess = [strongSelf processIncomingQueue: true after: fetch ];
 365                 initialProcess.name = @"initial-process-incoming-queue";
 366
 367                 // If all that succeeds, iterate through all keychain items and find the ones which need to be uploaded
 368                 strongSelf.initialScanOperation = [[CKKSScanLocalItemsOperation alloc] initWithCKKSKeychainView:strongSelf ckoperationGroup:strongSelf.keyHierarchyOperationGroup];
 369                 strongSelf.initialScanOperation.name = @"initial-scan-operation";
 370                 [strongSelf.initialScanOperation addNullableDependency:strongSelf.lockStateTracker.unlockDependency];
 371                 [strongSelf.initialScanOperation addDependency: initialProcess];
 372                 [strongSelf scheduleOperation: strongSelf.initialScanOperation];
 373
 374             } else {
 375                 // Likely a restart of securityd!
 376
 377                 // First off, are there any in-flight queue entries? If so, put them back into New.
 378                 // If they're truly in-flight, we'll "conflict" with ourselves, but that should be fine.
 379                 NSError* error = nil;
 380                 [self _onqueueResetAllInflightOQE:&error];
 381                 if(error) {
 382                     ckkserror("ckks", self, "Couldn't reset in-flight OQEs, bad behavior ahead: %@", error);
 383                 }
 384
 385                 // Are there any fixups to run first?
 386                 strongSelf.lastFixupOperation = [CKKSFixups fixup:ckse.lastFixup for:strongSelf];
 387                 if(strongSelf.lastFixupOperation) {
 388                     ckksnotice("ckksfixup", strongSelf, "We have a fixup to perform: %@", strongSelf.lastFixupOperation);
 389                     [strongSelf scheduleOperation:strongSelf.lastFixupOperation];
 390                 }
 391
 392                 strongSelf.keyHierarchyOperationGroup = [CKOperationGroup CKKSGroupWithName:@"restart-setup"];
 393
 394                 if ([CKKSManifest shouldSyncManifests]) {
 395                     strongSelf.egoManifest = [CKKSEgoManifest tryCurrentEgoManifestForZone:strongSelf.zoneName];
 396                 }
 397
 398                 // If it's been more than 24 hours since the last fetch, fetch and process everything.
 399                 // Otherwise, just kick off the local queue processing.
 400
 401                 NSDate* now = [NSDate date];
 402                 NSDateComponents* offset = [[NSDateComponents alloc] init];
 403                 [offset setHour:-24];
 404                 NSDate* deadline = [[NSCalendar currentCalendar] dateByAddingComponents:offset toDate:now options:0];
 405
 406                 NSOperation* initialProcess = nil;
 407                 if(ckse.lastFetchTime == nil || [ckse.lastFetchTime compare: deadline] == NSOrderedAscending) {
 408                     initialProcess = [strongSelf fetchAndProcessCKChanges:CKKSFetchBecauseSecuritydRestart after:strongSelf.lastFixupOperation];
 409                 } else {
 410                     initialProcess = [strongSelf processIncomingQueue:false after:strongSelf.lastFixupOperation];
 411                 }
 412
 413                 if([CKKSManifest shouldSyncManifests]) {
 414                     if (!strongSelf.egoManifest) {
 415                         ckksnotice("ckksmanifest", strongSelf, "No ego manifest on restart; rescanning");
 416                         strongSelf.initialScanOperation = [[CKKSScanLocalItemsOperation alloc] initWithCKKSKeychainView:strongSelf ckoperationGroup:strongSelf.keyHierarchyOperationGroup];
 417                         strongSelf.initialScanOperation.name = @"initial-scan-operation";
 418                         [strongSelf.initialScanOperation addNullableDependency:strongSelf.lastFixupOperation];
 419                         [strongSelf.initialScanOperation addNullableDependency:strongSelf.lockStateTracker.unlockDependency];
 420                         [strongSelf.initialScanOperation addDependency: initialProcess];
 421                         [strongSelf scheduleOperation: strongSelf.initialScanOperation];
 422                     }
 423                 }
 424
 425                 // Process outgoing queue after re-start
 426                 [strongSelf processOutgoingQueueAfter:strongSelf.lastFixupOperation ckoperationGroup:strongSelf.keyHierarchyOperationGroup];
 427             }
 428
 429             // Tell the key state machine to fire off. It should either:
 430             //  Wait for the fixup operation to occur
 431             //  Be initialized
 432             if(strongSelf.lastFixupOperation) {
 433                 [strongSelf _onqueueAdvanceKeyStateMachineToState: SecCKKSZoneKeyStateWaitForFixupOperation withError: nil];
 434             } else {
 435                 [strongSelf _onqueueAdvanceKeyStateMachineToState: SecCKKSZoneKeyStateInitialized withError: nil];
 436             }
 437             return true;
 438         }];
 439     }];
 440     self.viewSetupOperation.name = @"view-setup";
 441
 442     [self.viewSetupOperation addNullableDependency: self.zoneSetupOperation];
 443     [self scheduleAccountStatusOperation: self.viewSetupOperation];
 444 }
 445
 446 - (bool)_onqueueResetLocalData: (NSError * __autoreleasing *) error {
 447     dispatch_assert_queue(self.queue);
 448
 449     NSError* localerror = nil;
 450     bool setError = false; // Ugly, but this is the only way to return the first error given
 451
 452     CKKSZoneStateEntry* ckse = [CKKSZoneStateEntry state: self.zoneName];
 453     ckse.ckzonecreated = false;
 454     ckse.ckzonesubscribed = false; // I'm actually not sure about this: can you be subscribed to a non-existent zone?
 455     ckse.changeToken = NULL;
 456     [ckse saveToDatabase: &localerror];
 457     if(localerror) {
 458         ckkserror("ckks", self, "couldn't reset zone status for %@: %@", self.zoneName, localerror);
 459         if(error && !setError) {
 460             *error = localerror; setError = true;
 461         }
 462     }
 463
 464     [CKKSMirrorEntry deleteAll:self.zoneID error: &localerror];
 465     if(localerror) {
 466         ckkserror("ckks", self, "couldn't delete all CKKSMirrorEntry: %@", localerror);
 467         if(error && !setError) {
 468             *error = localerror; setError = true;
 469         }
 470     }
 471
 472     [CKKSOutgoingQueueEntry deleteAll:self.zoneID error: &localerror];
 473     if(localerror) {
 474         ckkserror("ckks", self, "couldn't delete all CKKSOutgoingQueueEntry: %@", localerror);
 475         if(error && !setError) {
 476             *error = localerror; setError = true;
 477         }
 478     }
 479
 480     [CKKSIncomingQueueEntry deleteAll:self.zoneID error: &localerror];
 481     if(localerror) {
 482         ckkserror("ckks", self, "couldn't delete all CKKSIncomingQueueEntry: %@", localerror);
 483         if(error && !setError) {
 484             *error = localerror; setError = true;
 485         }
 486     }
 487
 488     [CKKSKey deleteAll:self.zoneID error: &localerror];
 489     if(localerror) {
 490         ckkserror("ckks", self, "couldn't delete all CKKSKey: %@", localerror);
 491         if(error && !setError) {
 492             *error = localerror; setError = true;
 493         }
 494     }
 495
 496     [CKKSTLKShare deleteAll:self.zoneID error: &localerror];
 497     if(localerror) {
 498         ckkserror("ckks", self, "couldn't delete all CKKSTLKShare: %@", localerror);
 499         if(error && !setError) {
 500             *error = localerror; setError = true;
 501         }
 502     }
 503
 504     [CKKSCurrentKeyPointer deleteAll:self.zoneID error: &localerror];
 505     if(localerror) {
 506         ckkserror("ckks", self, "couldn't delete all CKKSCurrentKeyPointer: %@", localerror);
 507         if(error && !setError) {
 508             *error = localerror; setError = true;
 509         }
 510     }
 511
 512     [CKKSCurrentItemPointer deleteAll:self.zoneID error: &localerror];
 513     if(localerror) {
 514         ckkserror("ckks", self, "couldn't delete all CKKSCurrentItemPointer: %@", localerror);
 515         if(error && !setError) {
 516             *error = localerror; setError = true;
 517         }
 518     }
 519
 520     [CKKSDeviceStateEntry deleteAll:self.zoneID error:&localerror];
 521     if(localerror) {
 522         ckkserror("ckks", self, "couldn't delete all CKKSDeviceStateEntry: %@", localerror);
 523         if(error && !setError) {
 524             *error = localerror; setError = true;
 525         }
 526     }
 527
 528     [self _onqueueAdvanceKeyStateMachineToState: SecCKKSZoneKeyStateInitializing withError:nil];
 529
 530     return (localerror == nil && !setError);
 531 }
 532
 533 - (CKKSResultOperation*)resetLocalData {
 534     __weak __typeof(self) weakSelf = self;
 535
 536     CKKSGroupOperation* resetFollowUp = [[CKKSGroupOperation alloc] init];
 537     resetFollowUp.name = @"local-reset-follow-up-group";
 538     __weak __typeof(resetFollowUp) weakResetFollowUp = resetFollowUp;
 539
 540     CKKSResultOperation* op = [[CKKSResultOperation alloc] init];
 541     op.name = @"local-reset";
 542
 543     __weak __typeof(op) weakOp = op;
 544     [op addExecutionBlock:^{
 545         __strong __typeof(self) strongSelf = weakSelf;
 546         __strong __typeof(op) strongOp = weakOp;
 547         __strong __typeof(resetFollowUp) strongResetFollowUp = weakResetFollowUp;
 548         if(!strongSelf || !strongOp || !strongResetFollowUp) {
 549             return;
 550         }
 551
 552         __block NSError* error = nil;
 553
 554         [strongSelf dispatchSync: ^bool{
 555             [strongSelf _onqueueResetLocalData: &error];
 556             return true;
 557         }];
 558
 559         if(error) {
 560             ckksnotice("ckksreset", strongSelf, "Local reset finished with error %@", error);
 561             strongOp.error = error;
 562         } else {
 563             if(strongSelf.accountStatus == CKKSAccountStatusAvailable) {
 564                 // Since we're logged in, we expect a reset to fix up the key hierarchy
 565                 ckksnotice("ckksreset", strongSelf, "logged in; re-initializing zone");
 566                 [self.initializeScheduler trigger];
 567
 568                 ckksnotice("ckksreset", strongSelf, "waiting for key hierarchy to become ready");
 569                 CKKSResultOperation* waitOp = [CKKSResultOperation named:@"waiting-for-key-hierarchy" withBlock:^{}];
 570                 [waitOp timeout: 60*NSEC_PER_SEC];
 571                 [waitOp addNullableDependency:strongSelf.keyStateReadyDependency];
 572
 573                 [strongResetFollowUp runBeforeGroupFinished:waitOp];
 574             } else {
 575                 ckksnotice("ckksreset", strongSelf, "logged out; not initializing zone");
 576             }
 577         }
 578     }];
 579
 580     // On a reset, all other operations should be cancelled
 581     [self cancelAllOperations];
 582     [resetFollowUp runBeforeGroupFinished:op];
 583     [self scheduleOperationWithoutDependencies:resetFollowUp];
 584     return resetFollowUp;
 585 }
 586
 587 - (CKKSResultOperation*)resetCloudKitZone {
 588     // On a reset, we should cancel all existing operations
 589     [self cancelAllOperations];
 590     CKKSResultOperation* reset = [super beginResetCloudKitZoneOperation];
 591
 592     __weak __typeof(self) weakSelf = self;
 593     CKKSGroupOperation* resetFollowUp = [[CKKSGroupOperation alloc] init];
 594     resetFollowUp.name = @"cloudkit-reset-follow-up-group";
 595
 596     __weak __typeof(resetFollowUp) weakResetFollowUp = resetFollowUp;
 597     [resetFollowUp runBeforeGroupFinished: [CKKSResultOperation named:@"cloudkit-reset-follow-up" withBlock: ^{
 598         __strong __typeof(weakSelf) strongSelf = weakSelf;
 599         if(!strongSelf) {
 600             ckkserror("ckks", strongSelf, "received callback for released object");
 601             return;
 602         }
 603         __strong __typeof(resetFollowUp) strongResetFollowUp = weakResetFollowUp;
 604
 605         if(!reset.error) {
 606             ckksnotice("ckks", strongSelf, "Successfully deleted zone %@", strongSelf.zoneName);
 607             __block NSError* error = nil;
 608
 609             [strongSelf dispatchSync: ^bool{
 610                 [strongSelf _onqueueResetLocalData: &error];
 611                 strongSelf.setupSuccessful = false;
 612                 return true;
 613             }];
 614
 615             if(strongSelf.accountStatus == CKKSAccountStatusAvailable) {
 616                 // Since we're logged in, we expect a reset to fix up the key hierarchy
 617                 ckksnotice("ckksreset", strongSelf, "re-initializing zone");
 618                 [self.initializeScheduler trigger];
 619
 620                 ckksnotice("ckksreset", strongSelf, "waiting for key hierarchy to become ready");
 621                 CKKSResultOperation* waitOp = [CKKSResultOperation named:@"waiting-for-reset" withBlock:^{}];
 622                 [waitOp timeout: 60*NSEC_PER_SEC];
 623                 [waitOp addNullableDependency:strongSelf.keyStateReadyDependency];
 624
 625                 [strongResetFollowUp runBeforeGroupFinished:waitOp];
 626             } else {
 627                 ckksnotice("ckksreset", strongSelf, "logged out; not initializing zone");
 628             }
 629         } else {
 630             // Shouldn't ever happen, since reset is a successDependency
 631             ckkserror("ckks", strongSelf, "Couldn't reset zone %@: %@", strongSelf.zoneName, reset.error);
 632         }
 633     }]];
 634
 635     [resetFollowUp addSuccessDependency:reset];
 636     [self scheduleOperationWithoutDependencies:resetFollowUp];
 637     return resetFollowUp;
 638 }
 639
 640 - (void)advanceKeyStateMachine {
 641     __weak __typeof(self) weakSelf = self;
 642
 643     [self dispatchAsync: ^bool{
 644         __strong __typeof(weakSelf) strongSelf = weakSelf;
 645         if(!strongSelf) {
 646             ckkserror("ckks", strongSelf, "received callback for released object");
 647             false;
 648         }
 649
 650         [strongSelf _onqueueAdvanceKeyStateMachineToState: nil withError: nil];
 651         return true;
 652     }];
 653 };
 654
 655 - (void)_onqueueKeyStateMachineRequestFetch {
 656     dispatch_assert_queue(self.queue);
 657
 658     // We're going to set this flag, then nudge the key state machine.
 659     // If it was idle, then it should launch a fetch. If there was an active process, this flag will stay high
 660     // and the fetch will be launched later.
 661
 662     self.keyStateFetchRequested = true;
 663     [self _onqueueAdvanceKeyStateMachineToState: nil withError: nil];
 664 }
 665
 666 - (void)_onqueueKeyStateMachineRequestFullRefetch {
 667     dispatch_assert_queue(self.queue);
 668
 669     self.keyStateFullRefetchRequested = true;
 670     [self _onqueueAdvanceKeyStateMachineToState: nil withError: nil];
 671 }
 672
 673 - (void)keyStateMachineRequestProcess {
 674     __weak __typeof(self) weakSelf = self;
 675     [self dispatchAsync: ^bool{
 676         __strong __typeof(weakSelf) strongSelf = weakSelf;
 677         if(!strongSelf) {
 678             ckkserror("ckks", strongSelf, "received callback for released object");
 679             return false;
 680         }
 681
 682         [strongSelf _onqueueKeyStateMachineRequestProcess];
 683         return true;
 684     }];
 685 }
 686
 687 - (CKKSResultOperation*)createKeyStateReadyDependency:(NSString*)message ckoperationGroup:(CKOperationGroup*)group {
 688     __weak __typeof(self) weakSelf = self;
 689     CKKSResultOperation* keyStateReadyDependency = [CKKSResultOperation operationWithBlock:^{
 690         __strong __typeof(self) strongSelf = weakSelf;
 691         if(!strongSelf) {
 692             return;
 693         }
 694         ckksnotice("ckkskey", strongSelf, "%@", message);
 695
 696         [strongSelf dispatchSync:^bool {
 697             if(strongSelf.droppedItems) {
 698                 // While we weren't in 'ready', keychain modifications might have come in and were dropped on the floor. Find them!
 699                 ckksnotice("ckkskey", strongSelf, "Launching scan operation for missed items");
 700                 CKKSScanLocalItemsOperation* scanOperation = [[CKKSScanLocalItemsOperation alloc] initWithCKKSKeychainView: strongSelf ckoperationGroup:group];
 701                 [strongSelf scheduleOperation: scanOperation];
 702             }
 703             return true;
 704         }];
 705     }];
 706     keyStateReadyDependency.name = [NSString stringWithFormat: @"%@-key-state-ready", self.zoneName];
 707     return keyStateReadyDependency;
 708 }
 709
 710 - (void)_onqueueKeyStateMachineRequestProcess {
 711     dispatch_assert_queue(self.queue);
 712
 713     // Set the request flag, then nudge the key state machine.
 714     // If it was idle, then it should launch a process. If there was an active process, this flag will stay high
 715     // and the process will be launched later.
 716
 717     self.keyStateProcessRequested = true;
 718     [self _onqueueAdvanceKeyStateMachineToState: nil withError: nil];
 719 }
 720
 721 // The operations suggested by this state machine should call _onqueueAdvanceKeyStateMachineToState once they are complete.
 722 // At no other time should keyHierarchyState be modified.
 723
 724 // Note that this function cannot rely on doing any database work; it might get rolled back, especially in an error state
 725 - (void)_onqueueAdvanceKeyStateMachineToState: (CKKSZoneKeyState*) state withError: (NSError*) error {
 726     dispatch_assert_queue(self.queue);
 727     __weak __typeof(self) weakSelf = self;
 728
 729     // Resetting back to 'initializing' takes all precedence.
 730     if([state isEqual: SecCKKSZoneKeyStateInitializing]) {
 731         ckksnotice("ckkskey", self, "Resetting the key hierarchy state machine back to 'initializing'");
 732
 733         [self.keyStateMachineOperation cancel];
 734         self.keyStateMachineOperation = nil;
 735
 736         self.keyHierarchyState = SecCKKSZoneKeyStateInitializing;
 737         self.keyHierarchyError = nil;
 738         self.keyStateFetchRequested = false;
 739         self.keyStateProcessRequested = false;
 740
 741         self.keyHierarchyOperationGroup = [CKOperationGroup CKKSGroupWithName:@"key-state-reset"];
 742         NSOperation* oldKSRD = self.keyStateReadyDependency;
 743         self.keyStateReadyDependency = [self createKeyStateReadyDependency:@"Key state has become ready for the first time (after reset)." ckoperationGroup:self.keyHierarchyOperationGroup];
 744         if(oldKSRD) {
 745             [oldKSRD addDependency:self.keyStateReadyDependency];
 746             [self.waitingQueue addOperation:oldKSRD];
 747         }
 748         return;
 749     }
 750
 751     // Cancels and error states take precedence
 752     if([self.keyHierarchyState isEqualToString: SecCKKSZoneKeyStateError] ||
 753        [self.keyHierarchyState isEqualToString: SecCKKSZoneKeyStateCancelled] ||
 754        self.keyHierarchyError != nil) {
 755         // Error state: nowhere to go. Early-exit.
 756         ckkserror("ckkskey", self, "Asked to advance state machine from non-exit state %@: %@", self.keyHierarchyState, self.keyHierarchyError);
 757         return;
 758     }
 759
 760     if(error != nil || [state isEqual: SecCKKSZoneKeyStateError]) {
 761         // But wait! Is this a "we're locked" error?
 762         if([self.lockStateTracker isLockedError:error]) {
 763             ckkserror("ckkskey", self, "advised of 'keychain locked' error, ignoring: coming from state (%@): %@", self.keyHierarchyState, error);
 764             // After the next unlock, fake that we received the last zone transition
 765             CKKSZoneKeyState* lastState = self.keyHierarchyState;
 766             self.keyStateMachineOperation = [NSBlockOperation named:@"key-state-after-unlock" withBlock:^{
 767                 __strong __typeof(self) strongSelf = weakSelf;
 768                 if(!strongSelf) {
 769                     return;
 770                 }
 771                 [strongSelf dispatchSync:^bool{
 772                     [strongSelf _onqueueAdvanceKeyStateMachineToState:lastState withError:nil];
 773                     return true;
 774                 }];
 775             }];
 776             state = nil;
 777             [self.keyStateMachineOperation addNullableDependency:self.lockStateTracker.unlockDependency];
 778             [self scheduleOperation:self.keyStateMachineOperation];
 779
 780         } else {
 781             // Error state: record the error and exit early
 782             ckkserror("ckkskey", self, "advised of error: coming from state (%@): %@", self.keyHierarchyState, error);
 783
 784             [[CKKSAnalyticsLogger logger] logUnrecoverableError:error
 785                                                        forEvent:CKKSEventStateError
 786                                                          inView:self
 787                                                  withAttributes:@{ @"previousKeyHierarchyState" : self.keyHierarchyState }];
 788
 789
 790             self.keyHierarchyState = SecCKKSZoneKeyStateError;
 791             self.keyHierarchyError = error;
 792
 793             return;
 794         }
 795     }
 796
 797     if([state isEqual: SecCKKSZoneKeyStateCancelled]) {
 798         ckkserror("ckkskey", self, "advised of cancel: coming from state (%@): %@", self.keyHierarchyState, error);
 799         self.keyHierarchyState = SecCKKSZoneKeyStateCancelled;
 800         self.keyHierarchyError = error;
 801
 802         // Cancel the key ready dependency. Strictly Speaking, this will cause errors down the line, but we're in a cancel state: those operations should be canceled anyway.
 803         self.keyHierarchyOperationGroup = nil;
 804         [self.keyStateReadyDependency cancel];
 805         self.keyStateReadyDependency = nil;
 806         return;
 807     }
 808
 809     // Now that the current or new state isn't an error or a cancel, proceed.
 810
 811     if(self.keyStateMachineOperation && ![self.keyStateMachineOperation isFinished]) {
 812         if(state == nil) {
 813             // we started this operation to move the state machine. Since you aren't asking for a state transition, and there's an active operation, no need to do anything
 814             ckksnotice("ckkskey", self, "Not advancing state machine: waiting for %@", self.keyStateMachineOperation);
 815             return;
 816         }
 817     }
 818
 819     if(state) {
 820         ckksnotice("ckkskey", self, "Preparing to advance key hierarchy state machine from %@ to %@", self.keyHierarchyState, state);
 821         self.keyStateMachineOperation = nil;
 822     } else {
 823         ckksnotice("ckkskey", self, "Key hierarchy state machine is being poked; currently %@", self.keyHierarchyState);
 824         state = self.keyHierarchyState;
 825     }
 826
 827     // Many of our decisions below will be based on what keys exist. Help them out.
 828     CKKSCurrentKeySet* keyset = [[CKKSCurrentKeySet alloc] initForZone:self.zoneID];
 829     NSError* localerror = nil;
 830     NSArray<CKKSKey*>* localKeys = [CKKSKey localKeys:self.zoneID error:&localerror];
 831     NSArray<CKKSKey*>* remoteKeys = [CKKSKey remoteKeys:self.zoneID error: &localerror];
 832
 833     // We also are checking for OutgoingQueueEntries in the reencrypt state; this is a sign that our key hierarchy is out of date.
 834     NSArray<CKKSOutgoingQueueEntry*>* outdatedOQEs = [CKKSOutgoingQueueEntry allInState: SecCKKSStateReencrypt zoneID:self.zoneID error: &localerror];
 835
 836     SecADSetValueForScalarKey((__bridge CFStringRef) SecCKKSAggdViewKeyCount, [localKeys count]);
 837
 838     if(localerror) {
 839         ckkserror("ckkskey", self, "couldn't fetch keys and OQEs from local database, entering error state: %@", localerror);
 840         self.keyHierarchyState = SecCKKSZoneKeyStateError;
 841         self.keyHierarchyError = localerror;
 842         return;
 843     }
 844
 845 #if !defined(NDEBUG)
 846     NSArray<CKKSKey*>* allKeys = [CKKSKey allKeys:self.zoneID error:&localerror];
 847     ckksdebug("ckkskey", self, "All keys: %@", allKeys);
 848 #endif
 849
 850     NSError* hierarchyError = nil;
 851
 852     if([state isEqualToString: SecCKKSZoneKeyStateInitializing]) {
 853         if(state != nil) {
 854             // Wait for CKKSZone to finish initialization.
 855             ckkserror("ckkskey", self, "Asked to advance state machine (to %@) while CK zone still initializing.", state);
 856         }
 857         return;
 858
 859     } else if([state isEqualToString: SecCKKSZoneKeyStateReady]) {
 860         if(self.keyStateProcessRequested || [remoteKeys count] > 0) {
 861             // We've either received some remote keys from the last fetch, or someone has requested a reprocess.
 862             ckksnotice("ckkskey", self, "Kicking off a key reprocess based on request:%d and remote key count %lu", self.keyStateProcessRequested, (unsigned long)[remoteKeys count]);
 863             [self _onqueueKeyHierarchyProcess];
 864             // Stay in state 'ready': this reprocess might not change anything. If it does, cleanup code elsewhere will
 865             // reencode items that arrive during this ready
 866
 867         } else if(self.keyStateFullRefetchRequested) {
 868             // In ready, but someone has requested a full fetch. Kick it off.
 869             ckksnotice("ckkskey", self, "Kicking off a key refetch based on request:%d", self.keyStateFetchRequested);
 870             [self _onqueueKeyHierarchyRefetch];
 871             state = SecCKKSZoneKeyStateNeedFullRefetch;
 872
 873         } else if(self.keyStateFetchRequested) {
 874             // In ready, but someone has requested a fetch. Kick it off.
 875             ckksnotice("ckkskey", self, "Kicking off a key refetch based on request:%d", self.keyStateFetchRequested);
 876             [self _onqueueKeyHierarchyFetch];
 877             state = SecCKKSZoneKeyStateInitialized; // Don't go to 'ready', go to 'initialized', since we want to fetch again
 878         }
 879         // TODO: kick off a key roll if one has been requested
 880
 881         if(!self.keyStateMachineOperation) {
 882             // We think we're ready. Double check.
 883             CKKSZoneKeyState* checkedstate = [self _onqueueEnsureKeyHierarchyHealth:keyset error:&hierarchyError];
 884             if(![checkedstate isEqualToString:SecCKKSZoneKeyStateReady] || hierarchyError) {
 885                 // Things is bad. Kick off a heal to fix things up.
 886                 ckksnotice("ckkskey", self, "Thought we were ready, but the key hierarchy is %@: %@", checkedstate, hierarchyError);
 887                 state = checkedstate;
 888
 889             }
 890         }
 891
 892     } else if([state isEqualToString: SecCKKSZoneKeyStateInitialized]) {
 893         // We're initialized and CloudKit is ready. See what needs done...
 894
 895         // Check if we have an existing key hierarchy
 896         CKKSKey* tlk    = [CKKSKey currentKeyForClass:SecCKKSKeyClassTLK zoneID:self.zoneID error:&error];
 897         CKKSKey* classA = [CKKSKey currentKeyForClass:SecCKKSKeyClassA   zoneID:self.zoneID error:&error];
 898         CKKSKey* classC = [CKKSKey currentKeyForClass:SecCKKSKeyClassC   zoneID:self.zoneID error:&error];
 899
 900         if(error && !([error.domain isEqual: @"securityd"] && error.code == errSecItemNotFound)) {
 901             ckkserror("ckkskey", self, "Error examining existing key hierarchy: %@", error);
 902         }
 903
 904         if(tlk && classA && classC && !error) {
 905             // This is likely a restart of securityd, and we think we're ready. Double check.
 906
 907             CKKSZoneKeyState* checkedstate = [self _onqueueEnsureKeyHierarchyHealth:keyset error:&hierarchyError];
 908             if([checkedstate isEqualToString:SecCKKSZoneKeyStateReady] && !hierarchyError) {
 909                 ckksnotice("ckkskey", self, "Already have existing key hierarchy for %@; using it.", self.zoneID.zoneName);
 910             } else {
 911                 ckksnotice("ckkskey", self, "Initial scan shows key hierarchy is %@: %@", checkedstate, hierarchyError);
 912                 state = checkedstate;
 913             }
 914
 915         } else {
 916             // We have no local key hierarchy. One might exist in CloudKit, or it might not.
 917             ckksnotice("ckkskey", self, "No existing key hierarchy for %@. Check if there's one in CloudKit...", self.zoneID.zoneName);
 918
 919             [self _onqueueKeyHierarchyFetch];
 920         }
 921
 922     } else if([state isEqualToString:SecCKKSZoneKeyStateWaitForFixupOperation]) {
 923         // We should enter 'initialized' when the fixup operation completes
 924         ckksnotice("ckkskey", self, "Waiting for the fixup operation: %@", self.lastFixupOperation);
 925
 926         self.keyStateMachineOperation = [NSBlockOperation named:@"key-state-after-fixup" withBlock:^{
 927             __strong __typeof(self) strongSelf = weakSelf;
 928             [strongSelf dispatchSyncWithAccountKeys:^bool{
 929                 ckksnotice("ckkskey", self, "Fixup operation complete! Restarting key hierarchy machinery");
 930                 [strongSelf _onqueueAdvanceKeyStateMachineToState:SecCKKSZoneKeyStateInitialized withError:nil];
 931                 return true;
 932             }];
 933         }];
 934         [self.keyStateMachineOperation addNullableDependency:self.lastFixupOperation];
 935
 936     } else if([state isEqualToString: SecCKKSZoneKeyStateFetchComplete]) {
 937         // We've just completed a fetch of everything. Are there any remote keys?
 938         if(remoteKeys.count > 0u) {
 939             // Process the keys we received.
 940             self.keyStateMachineOperation = [[CKKSProcessReceivedKeysOperation alloc] initWithCKKSKeychainView: self];
 941         } else if( (keyset.currentTLKPointer || keyset.currentClassAPointer || keyset.currentClassCPointer) &&
 942                   !(keyset.tlk && keyset.classA && keyset.classC)) {
 943             // Huh. We appear to have current key pointers, but the keys themselves don't exist. That's weird.
 944             // Transfer to the "unhealthy" state to request a fix
 945             ckksnotice("ckkskey", self, "We appear to have current key pointers but no keys to match them. Moving to 'unhealthy'");
 946             state = SecCKKSZoneKeyStateUnhealthy;
 947         } else {
 948             // No remote keys, and the pointers look sane? Do we have an existing key hierarchy?
 949             CKKSZoneKeyState* checkedstate = [self _onqueueEnsureKeyHierarchyHealth:keyset error:&hierarchyError];
 950             if([checkedstate isEqualToString:SecCKKSZoneKeyStateReady] && !hierarchyError) {
 951                 ckksnotice("ckkskey", self, "After fetch, everything looks good.");
 952             } else if(localKeys.count == 0 && remoteKeys.count == 0) {
 953                 ckksnotice("ckkskey", self, "After fetch, we don't have any key hierarchy. Making a new one: %@", hierarchyError);
 954                 self.keyStateMachineOperation = [[CKKSNewTLKOperation alloc] initWithCKKSKeychainView: self ckoperationGroup:self.keyHierarchyOperationGroup];
 955             } else {
 956                 ckksnotice("ckkskey", self, "After fetch, we have a possibly unhealthy key hierarchy. Moving to %@: %@", checkedstate, hierarchyError);
 957                 state = checkedstate;
 958             }
 959         }
 960
 961     } else if([state isEqualToString: SecCKKSZoneKeyStateWaitForTLK]) {
 962         // We're in a hold state: waiting for the TLK bytes to arrive.
 963
 964         if(self.keyStateProcessRequested) {
 965             // Someone has requsted a reprocess! Run a ProcessReceivedKeysOperation.
 966             ckksnotice("ckkskey", self, "Received a nudge that our TLK might be here! Starting operation to check.");
 967             [self _onqueueKeyHierarchyProcess];
 968         }
 969
 970     } else if([state isEqualToString: SecCKKSZoneKeyStateWaitForUnlock]) {
 971         // will be handled later.
 972         ckksnotice("ckkskey", self, "Requested to enter waitforunlock");
 973
 974     } else if([state isEqualToString: SecCKKSZoneKeyStateReadyPendingUnlock]) {
 975         // will be handled later.
 976         ckksnotice("ckkskey", self, "Believe we're ready, but recheck after unlock");
 977
 978     } else if([state isEqualToString: SecCKKSZoneKeyStateBadCurrentPointers]) {
 979         // The current key pointers are broken, but we're not sure why.
 980         ckksnotice("ckkskey", self, "Our current key pointers are reported broken. Attempting a fix!");
 981         self.keyStateMachineOperation = [[CKKSHealKeyHierarchyOperation alloc] initWithCKKSKeychainView: self ckoperationGroup:self.keyHierarchyOperationGroup];
 982
 983     } else if([state isEqualToString: SecCKKSZoneKeyStateNewTLKsFailed]) {
 984         ckksnotice("ckkskey", self, "Creating new TLKs didn't work. Attempting to refetch!");
 985         [self _onqueueKeyHierarchyFetch];
 986
 987     } else if([state isEqualToString: SecCKKSZoneKeyStateHealTLKSharesFailed]) {
 988         ckksnotice("ckkskey", self, "Creating new TLK shares didn't work. Attempting to refetch!");
 989         [self _onqueueKeyHierarchyFetch];
 990
 991     } else if([state isEqualToString: SecCKKSZoneKeyStateNeedFullRefetch]) {
 992         ckksnotice("ckkskey", self, "Informed of request for full refetch");
 993         [self _onqueueKeyHierarchyRefetch];
 994
 995     } else {
 996         ckkserror("ckks", self, "asked to advance state machine to unknown state: %@", state);
 997         self.keyHierarchyState = state;
 998         return;
 999     }
1000
1001     // Check our other states: did the above code ask for a fix up? Are we in ready?
1002     if([state isEqualToString:SecCKKSZoneKeyStateUnhealthy]) {
1003         ckksnotice("ckkskey", self, "Looks like the key hierarchy is unhealthy. Launching fix.");
1004         self.keyStateMachineOperation = [[CKKSHealKeyHierarchyOperation alloc] initWithCKKSKeychainView:self ckoperationGroup:self.keyHierarchyOperationGroup];
1005
1006     } else if([state isEqualToString:SecCKKSZoneKeyStateHealTLKShares]) {
1007         ckksnotice("ckksshare", self, "Key hierarchy is okay, but not shared appropriately. Launching fix.");
1008         self.keyStateMachineOperation = [[CKKSHealTLKSharesOperation alloc] initWithCKKSKeychainView:self
1009                                                                                     ckoperationGroup:self.keyHierarchyOperationGroup];
1010
1011     } else if([state isEqualToString: SecCKKSZoneKeyStateWaitForUnlock] || [state isEqualToString:SecCKKSZoneKeyStateReadyPendingUnlock]) {
1012         // We're in a hold state: waiting for the keybag to unlock so we can reenter the key hierarchy state machine
1013         // After the next unlock, poke ourselves
1014         self.keyStateMachineOperation = [NSBlockOperation named:@"key-state-after-unlock" withBlock:^{
1015             __strong __typeof(self) strongSelf = weakSelf;
1016             if(!strongSelf) {
1017                 return;
1018             }
1019             [strongSelf dispatchSyncWithAccountKeys:^bool{
1020                 [strongSelf _onqueueAdvanceKeyStateMachineToState:SecCKKSZoneKeyStateInitialized withError:nil];
1021                 return true;
1022             }];
1023         }];
1024         [self.keyStateMachineOperation addNullableDependency: self.lockStateTracker.unlockDependency];
1025
1026     }
1027
1028
1029     // Handle the key state ready dependency
1030     if([state isEqualToString:SecCKKSZoneKeyStateReady] || [state isEqualToString:SecCKKSZoneKeyStateReadyPendingUnlock]) {
1031         // Ready enough!
1032         if(self.keyStateReadyDependency) {
1033             [self scheduleOperation: self.keyStateReadyDependency];
1034             self.keyStateReadyDependency = nil;
1035         }
1036
1037         // If there are any OQEs waiting to be encrypted, launch an op to fix them
1038         if([outdatedOQEs count] > 0u) {
1039             ckksnotice("ckksreencrypt", self, "Reencrypting outgoing items as the key hierarchy is ready");
1040             CKKSReencryptOutgoingItemsOperation* op = [[CKKSReencryptOutgoingItemsOperation alloc] initWithCKKSKeychainView:self ckoperationGroup:self.keyHierarchyOperationGroup];
1041             [self scheduleOperation:op];
1042         }
1043     } else {
1044         // Not in ready: we need a key state ready dependency
1045         if(self.keyStateReadyDependency == nil || [self.keyStateReadyDependency isFinished]) {
1046             self.keyHierarchyOperationGroup = [CKOperationGroup CKKSGroupWithName:@"key-state-broken"];
1047             self.keyStateReadyDependency = [self createKeyStateReadyDependency:@"Key state has become ready again." ckoperationGroup:self.keyHierarchyOperationGroup];
1048         }
1049     }
1050
1051     // Start any operations, or log that we aren't
1052     if(self.keyStateMachineOperation) {
1053         [self scheduleOperation: self.keyStateMachineOperation];
1054
1055     } else if([state isEqualToString:SecCKKSZoneKeyStateWaitForTLK]) {
1056         ckksnotice("ckkskey", self, "Entering key state %@", state);
1057     } else if([state isEqualToString:SecCKKSZoneKeyStateError]) {
1058         ckksnotice("ckkskey", self, "Entering key state 'error'");
1059     } else {
1060         // Nothing to do and not in a waiting state? Awesome; we must be in the ready state.
1061         if(!([state isEqualToString:SecCKKSZoneKeyStateReady] || [state isEqualToString:SecCKKSZoneKeyStateReadyPendingUnlock])) {
1062             ckksnotice("ckkskey", self, "No action to take in state %@; we must be ready.", state);
1063             state = SecCKKSZoneKeyStateReady;
1064
1065             self.keyHierarchyOperationGroup = nil;
1066             if(self.keyStateReadyDependency) {
1067                 [self scheduleOperation: self.keyStateReadyDependency];
1068                 self.keyStateReadyDependency = nil;
1069             }
1070         }
1071     }
1072     ckksnotice("ckkskey", self, "Advancing to key state: %@", state);
1073     self.keyHierarchyState = state;
1074 }
1075
1076 // For this key, who doesn't yet have a CKKSTLKShare for it?
1077 // Note that we really want a record sharing the TLK to ourselves, so this function might return
1078 // a non-empty set even if all peers have the TLK: it wants us to make a record for ourself.
1079 - (NSSet<id<CKKSPeer>>*)_onqueueFindPeersMissingShare:(CKKSKey*)key error:(NSError* __autoreleasing*)error {
1080     dispatch_assert_queue(self.queue);
1081
1082     if(!key) {
1083         ckkserror("ckksshare", self, "Attempting to find missing shares for nil key");
1084         return [NSSet set];
1085     }
1086
1087     if(self.currentTrustedPeersError) {
1088         ckkserror("ckksshare", self, "Couldn't find missing shares because trusted peers aren't available: %@", self.currentTrustedPeersError);
1089         if(error) {
1090             *error = self.currentTrustedPeersError;
1091         }
1092         return [NSSet set];
1093     }
1094     if(self.currentSelfPeersError) {
1095         ckkserror("ckksshare", self, "Couldn't find missing shares because self peers aren't available: %@", self.currentSelfPeersError);
1096         if(error) {
1097             *error = self.currentSelfPeersError;
1098         }
1099         return [NSSet set];
1100     }
1101
1102     NSMutableSet<id<CKKSPeer>>* peersMissingShares = [NSMutableSet set];
1103
1104     NSMutableSet<NSString*>* trustedPeerIDs = [NSMutableSet set];
1105     for(id<CKKSPeer> peer in self.currentTrustedPeers) {
1106         [trustedPeerIDs addObject:peer.peerID];
1107     }
1108
1109     for(id<CKKSPeer> peer in self.currentTrustedPeers) {
1110         NSError* peerError = nil;
1111         // Find all the shares for this peer for this key
1112         NSArray<CKKSTLKShare*>* currentPeerShares = [CKKSTLKShare allFor:peer.peerID
1113                                                                  keyUUID:key.uuid
1114                                                                   zoneID:self.zoneID
1115                                                                    error:&peerError];
1116
1117         if(peerError) {
1118             ckkserror("ckksshare", self, "Couldn't load shares for peer %@: %@", peer, peerError);
1119             if(error) {
1120                 *error = peerError;
1121             }
1122             return nil;
1123         }
1124
1125         // Determine if we think this peer has enough things shared to them
1126         bool alreadyShared = false;
1127         for(CKKSTLKShare* existingPeerShare in currentPeerShares) {
1128             if([existingPeerShare.tlkUUID isEqualToString: key.uuid] && [trustedPeerIDs containsObject:existingPeerShare.senderPeerID]) {
1129
1130                 // Was this shared to us?
1131                 if([peer.peerID isEqualToString: self.currentSelfPeers.currentSelf.peerID]) {
1132                     // We only count this as 'found' if we did the sharing
1133                     if([existingPeerShare.senderPeerID isEqualToString:self.currentSelfPeers.currentSelf.peerID]) {
1134                         ckksnotice("ckksshare", self, "Local peer %@ is shared %@ via self: %@", peer, key, existingPeerShare);
1135                         alreadyShared = true;
1136                     } else {
1137                         ckksnotice("ckksshare", self, "Local peer %@ is shared %@ via trusted %@, but that's not good enough", peer, key, existingPeerShare);
1138                     }
1139
1140                 } else {
1141                     // Some other peer has a trusted share. Cool!
1142                     ckksnotice("ckksshare", self, "Peer %@ is shared %@ via trusted %@", peer, key, existingPeerShare);
1143                     alreadyShared = true;
1144                 }
1145             }
1146         }
1147
1148         if(!alreadyShared) {
1149             // Add this peer to our set
1150             [peersMissingShares addObject:peer];
1151         }
1152     }
1153
1154     if(peersMissingShares.count > 0u) {
1155         // Log each and every one of the things
1156         ckksnotice("ckksshare", self, "Missing TLK shares for %lu peers: %@", (unsigned long)peersMissingShares.count, peersMissingShares);
1157         ckksnotice("ckksshare", self, "Self peers are (%@) %@", self.currentSelfPeersError ?: @"no error", self.currentSelfPeers);
1158         ckksnotice("ckksshare", self, "Trusted peers are (%@) %@", self.currentTrustedPeersError ?: @"no error", self.currentTrustedPeers);
1159     }
1160
1161     return peersMissingShares;
1162 }
1163
1164 - (NSSet<CKKSTLKShare*>*)_onqueueCreateMissingKeyShares:(CKKSKey*)key error:(NSError* __autoreleasing*)error {
1165     dispatch_assert_queue(self.queue);
1166
1167     if(self.currentTrustedPeersError) {
1168         ckkserror("ckksshare", self, "Couldn't create missing shares because trusted peers aren't available: %@", self.currentTrustedPeersError);
1169         if(error) {
1170             *error = self.currentTrustedPeersError;
1171         }
1172         return nil;
1173     }
1174     if(self.currentSelfPeersError) {
1175         ckkserror("ckksshare", self, "Couldn't create missing shares because self peers aren't available: %@", self.currentSelfPeersError);
1176         if(error) {
1177             *error = self.currentSelfPeersError;
1178         }
1179         return nil;
1180     }
1181
1182     NSSet<id<CKKSPeer>>* remainingPeers = [self _onqueueFindPeersMissingShare:key error:error];
1183     NSMutableSet<CKKSTLKShare*>* newShares = [NSMutableSet set];
1184
1185     if(!remainingPeers) {
1186         return nil;
1187     }
1188
1189     NSError* localerror = nil;
1190
1191     if(![key ensureKeyLoaded:error]) {
1192         return nil;
1193     }
1194
1195     for(id<CKKSPeer> peer in remainingPeers) {
1196         // Create a share for this peer.
1197         ckksnotice("ckksshare", self, "Creating share of %@ as %@ for %@", key, self.currentSelfPeers.currentSelf, peer);
1198         CKKSTLKShare* newShare = [CKKSTLKShare share:key
1199                                                   as:self.currentSelfPeers.currentSelf
1200                                                   to:peer
1201                                                epoch:-1
1202                                             poisoned:0
1203                                                error:&localerror];
1204
1205         if(localerror) {
1206             ckkserror("ckksshare", self, "Couldn't create new share for %@: %@", peer, localerror);
1207             if(error) {
1208                 *error = localerror;
1209             }
1210             return nil;
1211         }
1212
1213         [newShares addObject: newShare];
1214     }
1215
1216     return newShares;
1217 }
1218
1219 - (CKKSZoneKeyState*)_onqueueEnsureKeyHierarchyHealth:(CKKSCurrentKeySet*)set error:(NSError* __autoreleasing *)error {
1220     dispatch_assert_queue(self.queue);
1221
1222     // Check keyset
1223     if(!set.tlk || !set.classA || !set.classC) {
1224         ckkserror("ckkskey", self, "Error examining existing key hierarchy: %@", set);
1225         if(error) {
1226             *error = set.error;
1227         }
1228         return SecCKKSZoneKeyStateUnhealthy;
1229     }
1230
1231     NSError* localerror = nil;
1232     bool probablyOkIfUnlocked = false;
1233
1234     // keychain being locked is not a fatal error here
1235     [set.tlk loadKeyMaterialFromKeychain:&localerror];
1236     if(localerror && !([localerror.domain isEqual: @"securityd"] && localerror.code == errSecInteractionNotAllowed)) {
1237         ckkserror("ckkskey", self, "Error loading TLK(%@): %@", set.tlk, localerror);
1238         if(error) {
1239             *error = localerror;
1240         }
1241         return SecCKKSZoneKeyStateUnhealthy;
1242     } else if(localerror) {
1243         ckkserror("ckkskey", self, "Soft error loading TLK(%@), maybe locked: %@", set.tlk, localerror);
1244         probablyOkIfUnlocked = true;
1245     }
1246     localerror = nil;
1247
1248     // keychain being locked is not a fatal error here
1249     [set.classA loadKeyMaterialFromKeychain:&localerror];
1250     if(localerror && !([localerror.domain isEqual: @"securityd"] && localerror.code == errSecInteractionNotAllowed)) {
1251         ckkserror("ckkskey", self, "Error loading classA key(%@): %@", set.classA, localerror);
1252         if(error) {
1253             *error = localerror;
1254         }
1255         return SecCKKSZoneKeyStateUnhealthy;
1256     } else if(localerror) {
1257         ckkserror("ckkskey", self, "Soft error loading classA key(%@), maybe locked: %@", set.classA, localerror);
1258         probablyOkIfUnlocked = true;
1259     }
1260     localerror = nil;
1261
1262     // keychain being locked is a fatal error here, since this is class C
1263     [set.classC loadKeyMaterialFromKeychain:&localerror];
1264     if(localerror) {
1265         ckkserror("ckkskey", self, "Error loading classC(%@): %@", set.classC, localerror);
1266         if(error) {
1267             *error = localerror;
1268         }
1269         return SecCKKSZoneKeyStateUnhealthy;
1270     }
1271
1272     // Check that the classA and classC keys point to the current TLK
1273     if(![set.classA.parentKeyUUID isEqualToString: set.tlk.uuid]) {
1274         localerror = [NSError errorWithDomain:CKKSServerExtensionErrorDomain
1275                                          code:CKKSServerUnexpectedSyncKeyInChain
1276                                      userInfo:@{
1277                                                 NSLocalizedDescriptionKey: @"Current class A key does not wrap to current TLK",
1278                                                }];
1279         ckkserror("ckkskey", self, "Key hierarchy unhealthy: %@", localerror);
1280         if(error) {
1281             *error = localerror;
1282         }
1283         return SecCKKSZoneKeyStateUnhealthy;
1284     }
1285     if(![set.classC.parentKeyUUID isEqualToString: set.tlk.uuid]) {
1286         localerror = [NSError errorWithDomain:CKKSServerExtensionErrorDomain
1287                                          code:CKKSServerUnexpectedSyncKeyInChain
1288                                      userInfo:@{
1289                                                 NSLocalizedDescriptionKey: @"Current class C key does not wrap to current TLK",
1290                                                }];
1291         ckkserror("ckkskey", self, "Key hierarchy unhealthy: %@", localerror);
1292         if(error) {
1293             *error = localerror;
1294         }
1295         return SecCKKSZoneKeyStateUnhealthy;
1296     }
1297
1298     self.activeTLK = [set.tlk uuid];
1299
1300     // Now that we're pretty sure we have the keys, are they shared appropriately?
1301     // Check that every trusted peer has at least one TLK share
1302     NSSet<id<CKKSPeer>>* missingShares = [self _onqueueFindPeersMissingShare:set.tlk error:&localerror];
1303     if(localerror && [self.lockStateTracker isLockedError: localerror]) {
1304         ckkserror("ckkskey", self, "Couldn't find missing TLK shares due to lock state: %@", localerror);
1305         probablyOkIfUnlocked = true;
1306
1307     } else if(localerror) {
1308         if(error) {
1309             *error = localerror;
1310         }
1311         ckkserror("ckkskey", self, "Error finding missing TLK shares: %@", localerror);
1312         return SecCKKSZoneKeyStateError;
1313     }
1314
1315     if(!missingShares || missingShares.count != 0u) {
1316         localerror = [NSError errorWithDomain:CKKSErrorDomain code:CKKSMissingTLKShare
1317                                   description:[NSString stringWithFormat:@"Missing shares for %lu peers", (unsigned long)missingShares.count]];
1318         if(error) {
1319             *error = localerror;
1320         }
1321         return SecCKKSZoneKeyStateHealTLKShares;
1322     } else {
1323         ckksnotice("ckksshare", self, "TLK (%@) is shared correctly", set.tlk);
1324     }
1325
1326     // Got to the bottom? Cool! All keys are present and accounted for.
1327     return probablyOkIfUnlocked ? SecCKKSZoneKeyStateReadyPendingUnlock : SecCKKSZoneKeyStateReady;
1328 }
1329
1330 - (void)_onqueueKeyHierarchyFetch {
1331     dispatch_assert_queue(self.queue);
1332
1333     __weak __typeof(self) weakSelf = self;
1334     self.keyStateMachineOperation = [NSBlockOperation blockOperationWithBlock: ^{
1335         __strong __typeof(weakSelf) strongSelf = weakSelf;
1336         if(!strongSelf) {
1337             ckkserror("ckks", strongSelf, "received callback for released object");
1338             return;
1339         }
1340
1341         [strongSelf dispatchSyncWithAccountKeys: ^bool{
1342             [strongSelf _onqueueAdvanceKeyStateMachineToState: SecCKKSZoneKeyStateFetchComplete withError: nil];
1343             return true;
1344         }];
1345     }];
1346     self.keyStateMachineOperation.name = @"waiting-for-fetch";
1347
1348     NSOperation* fetchOp = [self.zoneChangeFetcher requestSuccessfulFetch: CKKSFetchBecauseKeyHierarchy];
1349     [self.keyStateMachineOperation addDependency: fetchOp];
1350
1351     self.keyStateFetchRequested = false;
1352 }
1353
1354 - (void)_onqueueKeyHierarchyRefetch {
1355     dispatch_assert_queue(self.queue);
1356
1357     __weak __typeof(self) weakSelf = self;
1358     self.keyStateMachineOperation = [NSBlockOperation blockOperationWithBlock: ^{
1359         __strong __typeof(weakSelf) strongSelf = weakSelf;
1360         if(!strongSelf) {
1361             ckkserror("ckks", strongSelf, "received callback for released object");
1362             return;
1363         }
1364
1365         [strongSelf dispatchSyncWithAccountKeys: ^bool{
1366             [strongSelf _onqueueAdvanceKeyStateMachineToState: SecCKKSZoneKeyStateFetchComplete withError: nil];
1367             return true;
1368         }];
1369     }];
1370     self.keyStateMachineOperation.name = @"waiting-for-refetch";
1371
1372     NSOperation* fetchOp = [self.zoneChangeFetcher requestSuccessfulResyncFetch: CKKSFetchBecauseKeyHierarchy];
1373     [self.keyStateMachineOperation addDependency: fetchOp];
1374
1375     self.keyStateMachineRefetched = true;
1376     self.keyStateFullRefetchRequested = false;
1377     self.keyStateFetchRequested = false;
1378 }
1379
1380
1381 - (void)_onqueueKeyHierarchyProcess {
1382     dispatch_assert_queue(self.queue);
1383
1384     self.keyStateMachineOperation = [[CKKSProcessReceivedKeysOperation alloc] initWithCKKSKeychainView: self];
1385
1386     // Since we're starting a reprocess, this is answering all previous requests.
1387     self.keyStateProcessRequested = false;
1388 }
1389
1390 - (void) handleKeychainEventDbConnection: (SecDbConnectionRef) dbconn
1391                                    added: (SecDbItemRef) added
1392                                  deleted: (SecDbItemRef) deleted
1393                              rateLimiter: (CKKSRateLimiter*) rateLimiter
1394                             syncCallback: (SecBoolNSErrorCallback) syncCallback {
1395     if(!SecCKKSIsEnabled()) {
1396         ckksnotice("ckks", self, "Skipping handleKeychainEventDbConnection due to disabled CKKS");
1397         return;
1398     }
1399
1400     __block NSError* error = nil;
1401
1402     if(self.accountStatus == CKKSAccountStatusNoAccount && syncCallback) {
1403         // We're positively not logged into CloudKit, and therefore don't expect this item to be synced anytime particularly soon.
1404         [self callSyncCallbackWithErrorNoAccount: syncCallback];
1405         syncCallback = nil;
1406     }
1407
1408     // Tombstones come in as item modifications or item adds. Handle modifications here.
1409     bool addedTombstone   = added   && SecDbItemIsTombstone(added);
1410     bool deletedTombstone = deleted && SecDbItemIsTombstone(deleted);
1411
1412     bool addedSync   = added   && SecDbItemIsSyncable(added);
1413     bool deletedSync = deleted && SecDbItemIsSyncable(deleted);
1414
1415     bool isAdd    = ( added && !deleted) || (added && deleted && !addedTombstone &&  deletedTombstone) || (added && deleted &&  addedSync && !deletedSync);
1416     bool isDelete = (!added &&  deleted) || (added && deleted &&  addedTombstone && !deletedTombstone) || (added && deleted && !addedSync &&  deletedSync);
1417     bool isModify = ( added &&  deleted) && (!isAdd) && (!isDelete);
1418
1419     // On an update that changes an item's primary key, SecDb modifies the existing item, then adds a new tombstone to replace the old primary key.
1420     // Therefore, we might receive an added tombstone here with no deleted item to accompany it. This should be considered a deletion.
1421     if(addedTombstone && !deleted) {
1422         isAdd = false;
1423         isDelete = true;
1424         isModify = false;
1425
1426         // Passed to withItem: below
1427         deleted = added;
1428     }
1429
1430     // If neither item is syncable, don't proceed further in the syncing system
1431     bool proceed = addedSync || deletedSync;
1432
1433     if(!proceed) {
1434         ckksnotice("ckks", self, "skipping sync of non-sync item (%d, %d)", addedSync, deletedSync);
1435         return;
1436     }
1437
1438     // Only synchronize items which can transfer between devices
1439     NSString* protection = (__bridge NSString*)SecDbItemGetCachedValueWithName(added ? added : deleted, kSecAttrAccessible);
1440     if(! ([protection isEqualToString: (__bridge NSString*)kSecAttrAccessibleWhenUnlocked] ||
1441           [protection isEqualToString: (__bridge NSString*)kSecAttrAccessibleAfterFirstUnlock] ||
1442           [protection isEqualToString: (__bridge NSString*)kSecAttrAccessibleAlwaysPrivate])) {
1443         ckksnotice("ckks", self, "skipping sync of device-bound(%@) item", protection);
1444         return;
1445     }
1446
1447     // Our caller gave us a database connection. We must get on the local queue to ensure atomicity
1448     // Note that we're at the mercy of the surrounding db transaction, so don't try to rollback here
1449     [self dispatchSyncWithConnection: dbconn block: ^bool {
1450
1451         // Always record the callback, even if we can't encrypt the item right now. Maybe we'll get to it soon!
1452         if(syncCallback) {
1453             CFErrorRef cferror = NULL;
1454             NSString* uuid = (__bridge_transfer NSString*) CFRetain(SecDbItemGetValue(added, &v10itemuuid, &cferror));
1455             if(!cferror && uuid) {
1456                 self.pendingSyncCallbacks[uuid] = syncCallback;
1457             }
1458             CFReleaseNull(cferror);
1459         }
1460
1461         CKKSOutgoingQueueEntry* oqe = nil;
1462         if       (isAdd) {
1463             oqe = [CKKSOutgoingQueueEntry withItem: added   action: SecCKKSActionAdd    ckks:self error: &error];
1464         } else if(isDelete) {
1465             oqe = [CKKSOutgoingQueueEntry withItem: deleted action: SecCKKSActionDelete ckks:self error: &error];
1466         } else if(isModify) {
1467             oqe = [CKKSOutgoingQueueEntry withItem: added   action: SecCKKSActionModify ckks:self error: &error];
1468         } else {
1469             ckkserror("ckks", self, "processKeychainEventItemAdded given garbage: %@ %@", added, deleted);
1470             return true;
1471         }
1472
1473         CKOperationGroup* operationGroup = [CKOperationGroup CKKSGroupWithName:@"keychain-api-use"];
1474
1475         if(error) {
1476             ckkserror("ckks", self, "Couldn't create outgoing queue entry: %@", error);
1477             self.droppedItems = true;
1478
1479             // If the problem is 'no UUID', launch a scan operation to find and fix it
1480             // We don't want to fix it up here, in the closing moments of a transaction
1481             if([error.domain isEqualToString:CKKSErrorDomain] && error.code == CKKSNoUUIDOnItem) {
1482                 ckksnotice("ckks", self, "Launching scan operation to find UUID");
1483                 CKKSScanLocalItemsOperation* scanOperation = [[CKKSScanLocalItemsOperation alloc] initWithCKKSKeychainView: self ckoperationGroup:operationGroup];
1484                 [self scheduleOperation: scanOperation];
1485             }
1486
1487             // If the problem is 'couldn't load key', tell the key hierarchy state machine to fix it
1488             if([error.domain isEqualToString:CKKSErrorDomain] && error.code == errSecItemNotFound) {
1489                 [self _onqueueAdvanceKeyStateMachineToState: nil withError: nil];
1490             }
1491
1492             return true;
1493         }
1494
1495         if(rateLimiter) {
1496             NSDate* limit = nil;
1497             NSInteger value = [rateLimiter judge:oqe at:[NSDate date] limitTime:&limit];
1498             if(limit) {
1499                 oqe.waitUntil = limit;
1500                 SecPLLogRegisteredEvent(@"CKKSSyncing", @{ @"ratelimit" : @(value), @"accessgroup" : oqe.accessgroup});
1501             }
1502         }
1503
1504         [oqe saveToDatabaseWithConnection: dbconn error: &error];
1505         if(error) {
1506             ckkserror("ckks", self, "Couldn't save outgoing queue entry to database: %@", error);
1507             return true;
1508         } else {
1509             ckksnotice("ckks", self, "Saved %@ to outgoing queue", oqe);
1510         }
1511
1512         // This update supercedes all other local modifications to this item (_except_ those in-flight).
1513         // Delete all items in reencrypt or error.
1514         CKKSOutgoingQueueEntry* reencryptOQE = [CKKSOutgoingQueueEntry tryFromDatabase:oqe.uuid state:SecCKKSStateReencrypt zoneID:self.zoneID error:&error];
1515         if(error) {
1516             ckkserror("ckks", self, "Couldn't load reencrypt OQE sibling for %@: %@", oqe, error);
1517         }
1518         if(reencryptOQE) {
1519             [reencryptOQE deleteFromDatabase:&error];
1520             if(error) {
1521                 ckkserror("ckks", self, "Couldn't delete reencrypt OQE sibling(%@) for %@: %@", reencryptOQE, oqe, error);
1522             }
1523             error = nil;
1524         }
1525
1526         CKKSOutgoingQueueEntry* errorOQE = [CKKSOutgoingQueueEntry tryFromDatabase:oqe.uuid state:SecCKKSStateError zoneID:self.zoneID error:&error];
1527         if(error) {
1528             ckkserror("ckks", self, "Couldn't load error OQE sibling for %@: %@", oqe, error);
1529         }
1530         if(errorOQE) {
1531             [errorOQE deleteFromDatabase:&error];
1532             if(error) {
1533                 ckkserror("ckks", self, "Couldn't delete error OQE sibling(%@) for %@: %@", reencryptOQE, oqe, error);
1534             }
1535         }
1536
1537         // Schedule a "view changed" notification
1538         [self.notifyViewChangedScheduler trigger];
1539
1540         [self processOutgoingQueue:operationGroup];
1541
1542         return true;
1543     }];
1544 }
1545
1546 -(void)setCurrentItemForAccessGroup:(SecDbItemRef)newItem
1547                                hash:(NSData*)newItemSHA1
1548                         accessGroup:(NSString*)accessGroup
1549                          identifier:(NSString*)identifier
1550                           replacing:(SecDbItemRef)oldItem
1551                                hash:(NSData*)oldItemSHA1
1552                            complete:(void (^) (NSError* operror)) complete
1553 {
1554     if(accessGroup == nil || identifier == nil) {
1555         NSError* error = [NSError errorWithDomain:CKKSErrorDomain
1556                                              code:errSecParam
1557                                       description:@"No access group or identifier given"];
1558         ckkserror("ckkscurrent", self, "Cancelling request: %@", error);
1559         complete(error);
1560         return;
1561     }
1562
1563     __weak __typeof(self) weakSelf = self;
1564
1565     [self dispatchSync:^bool {
1566         NSError* error = nil;
1567         CFErrorRef cferror = NULL;
1568
1569         NSString* newItemUUID = nil;
1570         NSString* oldItemUUID = nil;
1571
1572         // Now that we're on the db queue, ensure that the given hashes for the items match the hashes as they are now.
1573         // That is, the items haven't changed since the caller knew about the item.
1574         NSData* newItemComputedSHA1 = (NSData*) CFBridgingRelease(CFRetainSafe(SecDbItemGetSHA1(newItem, &cferror)));
1575         if(!newItemComputedSHA1 || cferror ||
1576            ![newItemComputedSHA1 isEqual:newItemSHA1]) {
1577             ckksnotice("ckkscurrent", self, "Hash mismatch for new item: %@ vs %@", newItemComputedSHA1, newItemSHA1);
1578             error = [NSError errorWithDomain:CKKSErrorDomain
1579                                         code:CKKSItemChanged
1580                                  description:@"New item has changed; hashes mismatch. Refetch and try again."];
1581             complete(error);
1582             CFReleaseNull(cferror);
1583             return false;
1584         }
1585
1586         newItemUUID = (NSString*) CFBridgingRelease(CFRetainSafe(SecDbItemGetValue(newItem, &v10itemuuid, &cferror)));
1587         if(!newItemUUID || cferror) {
1588             ckkserror("ckkscurrent", self, "Error fetching UUID for new item: %@", cferror);
1589             complete((__bridge NSError*) cferror);
1590             CFReleaseNull(cferror);
1591             return false;
1592         }
1593
1594         // We pass this into the change operation. If it's nil, that's an indicator that the old item doesn't exist in the keychain anymore
1595         NSData* oldItemComputedSHA1 = nil;
1596         if(oldItem) {
1597             oldItemComputedSHA1 = (NSData*) CFBridgingRelease(CFRetainSafe(SecDbItemGetSHA1(oldItem, &cferror)));
1598             if(!oldItemComputedSHA1 || cferror ||
1599                ![oldItemComputedSHA1 isEqual:oldItemSHA1]) {
1600                 ckksnotice("ckkscurrent", self, "Hash mismatch for old item: %@ vs %@", oldItemComputedSHA1, oldItemSHA1);
1601                 error = [NSError errorWithDomain:CKKSErrorDomain
1602                                             code:CKKSItemChanged
1603                                      description:@"Old item has changed; hashes mismatch. Refetch and try again."];
1604                 complete(error);
1605                 CFReleaseNull(cferror);
1606                 return false;
1607             }
1608
1609             oldItemUUID = (NSString*) CFBridgingRelease(CFRetainSafe(SecDbItemGetValue(oldItem, &v10itemuuid, &cferror)));
1610             if(!oldItemUUID || cferror) {
1611                 ckkserror("ckkscurrent", self, "Error fetching UUID for old item: %@", cferror);
1612                 complete((__bridge NSError*) cferror);
1613                 CFReleaseNull(cferror);
1614                 return false;
1615             }
1616         }
1617
1618         // Not being in a CloudKit account is an automatic failure.
1619         if(self.accountStatus != CKKSAccountStatusAvailable) {
1620             ckksnotice("ckkscurrent", self, "Rejecting current item pointer set since we don't have an iCloud account.");
1621             error = [NSError errorWithDomain:CKKSErrorDomain
1622                                         code:CKKSNotLoggedIn
1623                                  description:@"User is not signed into iCloud."];
1624             complete(error);
1625             return false;
1626         }
1627
1628         // At this point, we've completed all the checks we need for the SecDbItems. Try to launch this boat!
1629         NSString* currentIdentifier = [NSString stringWithFormat:@"%@-%@", accessGroup, identifier];
1630         ckksnotice("ckkscurrent", self, "Setting current pointer for %@ to %@ (from %@)", currentIdentifier, newItemUUID, oldItemUUID);
1631         CKKSUpdateCurrentItemPointerOperation* ucipo = [[CKKSUpdateCurrentItemPointerOperation alloc] initWithCKKSKeychainView:self
1632                                                                                                                 currentPointer:(NSString*)currentIdentifier
1633                                                                                                                    oldItemUUID:(NSString*)oldItemUUID
1634                                                                                                                    oldItemHash:oldItemComputedSHA1
1635                                                                                                                    newItemUUID:(NSString*)newItemUUID
1636                                                                                                               ckoperationGroup:[CKOperationGroup CKKSGroupWithName:@"currentitem-api"]];
1637         CKKSResultOperation* returnCallback = [CKKSResultOperation operationWithBlock:^{
1638             __strong __typeof(self) strongSelf = weakSelf;
1639
1640             if(ucipo.error) {
1641                 ckkserror("ckkscurrent", strongSelf, "Failed setting a current item pointer for %@ with %@", currentIdentifier, ucipo.error);
1642             } else {
1643                 ckksnotice("ckkscurrent", strongSelf, "Finished setting a current item pointer for %@", currentIdentifier);
1644             }
1645             complete(ucipo.error);
1646         }];
1647         returnCallback.name = @"setCurrentItem-return-callback";
1648         [returnCallback addDependency: ucipo];
1649         [self scheduleOperation: returnCallback];
1650
1651         // Now, schedule ucipo. It modifies the CloudKit zone, so it should insert itself into the list of OutgoingQueueOperations.
1652         // Then, we won't have simultaneous zone-modifying operations.
1653         [ucipo linearDependencies:self.outgoingQueueOperations];
1654
1655         // If this operation hasn't started within 60 seconds, cancel it and return a "timed out" error.
1656         [ucipo timeout:60*NSEC_PER_SEC];
1657
1658         [self scheduleOperation:ucipo];
1659         return true;
1660     }];
1661     return;
1662 }
1663
1664 -(void)getCurrentItemForAccessGroup:(NSString*)accessGroup
1665                          identifier:(NSString*)identifier
1666                     fetchCloudValue:(bool)fetchCloudValue
1667                            complete:(void (^) (NSString* uuid, NSError* operror)) complete
1668 {
1669     if(accessGroup == nil || identifier == nil) {
1670         ckksnotice("ckkscurrent", self, "Rejecting current item pointer get since no access group(%@) or identifier(%@) given", accessGroup, identifier);
1671         complete(NULL, [NSError errorWithDomain:CKKSErrorDomain
1672                                            code:errSecParam
1673                                     description:@"No access group or identifier given"]);
1674         return;
1675     }
1676
1677     // Not being in a CloudKit account is an automatic failure.
1678     if(self.accountStatus != CKKSAccountStatusAvailable) {
1679         ckksnotice("ckkscurrent", self, "Rejecting current item pointer get since we don't have an iCloud account.");
1680         complete(NULL, [NSError errorWithDomain:CKKSErrorDomain
1681                                            code:CKKSNotLoggedIn
1682                                     description:@"User is not signed into iCloud."]);
1683         return;
1684     }
1685
1686     CKKSResultOperation* fetchAndProcess = nil;
1687     if(fetchCloudValue) {
1688         fetchAndProcess = [self fetchAndProcessCKChanges:CKKSFetchBecauseCurrentItemFetchRequest];
1689     }
1690
1691     __weak __typeof(self) weakSelf = self;
1692     CKKSResultOperation* getCurrentItem = [CKKSResultOperation named:@"get-current-item-pointer" withBlock:^{
1693         if(fetchAndProcess.error) {
1694             ckksnotice("ckkscurrent", self, "Rejecting current item pointer get since fetch failed: %@", fetchAndProcess.error);
1695             complete(NULL, fetchAndProcess.error);
1696             return;
1697         }
1698
1699         __strong __typeof(self) strongSelf = weakSelf;
1700
1701         [strongSelf dispatchSync: ^bool {
1702             NSError* error = nil;
1703             NSString* currentIdentifier = [NSString stringWithFormat:@"%@-%@", accessGroup, identifier];
1704
1705             CKKSCurrentItemPointer* cip = [CKKSCurrentItemPointer fromDatabase:currentIdentifier
1706                                                                          state:SecCKKSProcessedStateLocal
1707                                                                         zoneID:strongSelf.zoneID
1708                                                                          error:&error];
1709             if(!cip || error) {
1710                 ckkserror("ckkscurrent", strongSelf, "No current item pointer for %@", currentIdentifier);
1711                 complete(nil, error);
1712                 return false;
1713             }
1714
1715             if(!cip.currentItemUUID) {
1716                 ckkserror("ckkscurrent", strongSelf, "Current item pointer is empty %@", cip);
1717                 complete(nil, [NSError errorWithDomain:CKKSErrorDomain
1718                                                   code:errSecInternalError
1719                                            description:@"Current item pointer is empty"]);
1720                 return false;
1721             }
1722
1723             ckksinfo("ckkscurrent", strongSelf, "Retrieved current item pointer: %@", cip);
1724             complete(cip.currentItemUUID, NULL);
1725             return true;
1726         }];
1727     }];
1728
1729     [getCurrentItem addNullableDependency:fetchAndProcess];
1730     [self scheduleOperation: getCurrentItem];
1731 }
1732
1733 - (CKKSKey*) keyForItem: (SecDbItemRef) item error: (NSError * __autoreleasing *) error {
1734     CKKSKeyClass* class = nil;
1735
1736     NSString* protection = (__bridge NSString*)SecDbItemGetCachedValueWithName(item, kSecAttrAccessible);
1737     if([protection isEqualToString: (__bridge NSString*)kSecAttrAccessibleWhenUnlocked]) {
1738         class = SecCKKSKeyClassA;
1739     } else if([protection isEqualToString: (__bridge NSString*)kSecAttrAccessibleAlwaysPrivate] ||
1740               [protection isEqualToString: (__bridge NSString*)kSecAttrAccessibleAfterFirstUnlock]) {
1741         class = SecCKKSKeyClassC;
1742     } else {
1743         NSError* localError = [NSError errorWithDomain:CKKSErrorDomain
1744                                                   code:CKKSInvalidKeyClass
1745                                            description:[NSString stringWithFormat:@"can't pick key class for protection %@", protection]];
1746         ckkserror("ckks", self, "can't pick key class: %@ %@", localError, item);
1747         if(error) {
1748             *error = localError;
1749         }
1750
1751         return nil;
1752     }
1753
1754     NSError* currentKeyError = nil;
1755     CKKSKey* key = [CKKSKey currentKeyForClass: class zoneID:self.zoneID error:&currentKeyError];
1756     if(!key || currentKeyError) {
1757         ckkserror("ckks", self, "Couldn't find current key for %@: %@", class, currentKeyError);
1758
1759         if(error) {
1760             *error = currentKeyError;
1761         }
1762         return nil;
1763     }
1764
1765     // and make sure it's unwrapped.
1766     NSError* loadedError = nil;
1767     if(![key ensureKeyLoaded:&loadedError]) {
1768         ckkserror("ckks", self, "Couldn't load key(%@): %@", key, loadedError);
1769         if(error) {
1770             *error = loadedError;
1771         }
1772         return nil;
1773     }
1774
1775     return key;
1776 }
1777
1778 // Use the following method to find the first pending operation in a weak collection
1779 - (NSOperation*)findFirstPendingOperation: (NSHashTable*) table {
1780     return [self findFirstPendingOperation:table ofClass:nil];
1781 }
1782
1783 // Use the following method to find the first pending operation in a weak collection
1784 - (NSOperation*)findFirstPendingOperation: (NSHashTable*) table ofClass:(Class)class {
1785     @synchronized(table) {
1786         for(NSOperation* op in table) {
1787             if(op != nil && [op isPending] && (class == nil || [op isKindOfClass: class])) {
1788                 return op;
1789             }
1790         }
1791         return nil;
1792     }
1793 }
1794
1795 // Use the following method to count the pending operations in a weak collection
1796 - (int64_t)countPendingOperations: (NSHashTable*) table {
1797     @synchronized(table) {
1798         int count = 0;
1799         for(NSOperation* op in table) {
1800             if(op != nil && !([op isExecuting] || [op isFinished])) {
1801                 count++;
1802             }
1803         }
1804         return count;
1805     }
1806 }
1807
1808 - (CKKSOutgoingQueueOperation*)processOutgoingQueue:(CKOperationGroup*)ckoperationGroup {
1809     return [self processOutgoingQueueAfter:nil ckoperationGroup:ckoperationGroup];
1810 }
1811
1812 - (CKKSOutgoingQueueOperation*)processOutgoingQueueAfter:(CKKSResultOperation*)after ckoperationGroup:(CKOperationGroup*)ckoperationGroup {
1813     CKKSOutgoingQueueOperation* outgoingop =
1814             (CKKSOutgoingQueueOperation*) [self findFirstPendingOperation:self.outgoingQueueOperations
1815                                                                   ofClass:[CKKSOutgoingQueueOperation class]];
1816     if(outgoingop) {
1817         if(after) {
1818             [outgoingop addDependency: after];
1819         }
1820         if([outgoingop isPending]) {
1821             if(!outgoingop.ckoperationGroup && ckoperationGroup) {
1822                 outgoingop.ckoperationGroup = ckoperationGroup;
1823             } else if(ckoperationGroup) {
1824                 ckkserror("ckks", self, "Throwing away CKOperationGroup(%@) in favor of (%@)", ckoperationGroup.name, outgoingop.ckoperationGroup.name);
1825             }
1826
1827             // Will log any pending dependencies as well
1828             ckksnotice("ckksoutgoing", self, "Returning existing %@", outgoingop);
1829
1830             // Shouldn't be necessary, but can't hurt
1831             [self.outgoingQueueOperationScheduler trigger];
1832             return outgoingop;
1833         }
1834     }
1835
1836     CKKSOutgoingQueueOperation* op = [[CKKSOutgoingQueueOperation alloc] initWithCKKSKeychainView:self ckoperationGroup:ckoperationGroup];
1837     op.name = @"outgoing-queue-operation";
1838     [op addNullableDependency:after];
1839     [op addNullableDependency:self.outgoingQueueOperationScheduler.operationDependency];
1840     [self.outgoingQueueOperationScheduler trigger];
1841
1842     [op addNullableDependency: self.initialScanOperation];
1843
1844     [self scheduleOperation: op];
1845     ckksnotice("ckksoutgoing", self, "Scheduled %@", op);
1846     return op;
1847 }
1848
1849 - (void)processIncomingQueueAfterNextUnlock {
1850     // Thread races aren't so important here; we might end up with two or three copies of this operation, but that's okay.
1851     if(![self.processIncomingQueueAfterNextUnlockOperation isPending]) {
1852         __weak __typeof(self) weakSelf = self;
1853
1854         CKKSResultOperation* restartIncomingQueueOperation = [CKKSResultOperation operationWithBlock:^{
1855             __strong __typeof(self) strongSelf = weakSelf;
1856             // This IQO shouldn't error if the keybag has locked again. It will simply try again later.
1857             [strongSelf processIncomingQueue:false];
1858         }];
1859
1860         restartIncomingQueueOperation.name = @"reprocess-incoming-queue-after-unlock";
1861         self.processIncomingQueueAfterNextUnlockOperation = restartIncomingQueueOperation;
1862
1863         [restartIncomingQueueOperation addNullableDependency:self.lockStateTracker.unlockDependency];
1864         [self scheduleOperation: restartIncomingQueueOperation];
1865     }
1866 }
1867
1868 - (CKKSIncomingQueueOperation*)processIncomingQueue:(bool)failOnClassA {
1869     return [self processIncomingQueue:failOnClassA after: nil];
1870 }
1871
1872 - (CKKSIncomingQueueOperation*) processIncomingQueue:(bool)failOnClassA after: (CKKSResultOperation*) after {
1873     CKKSIncomingQueueOperation* incomingop = (CKKSIncomingQueueOperation*) [self findFirstPendingOperation:self.incomingQueueOperations];
1874     if(incomingop) {
1875         ckksinfo("ckks", self, "Skipping processIncomingQueue due to at least one pending instance");
1876         if(after) {
1877             [incomingop addNullableDependency: after];
1878         }
1879         // check (again) for race condition; if the op has started we need to add another (for the dependency)
1880         if([incomingop isPending]) {
1881             incomingop.errorOnClassAFailure |= failOnClassA;
1882             return incomingop;
1883         }
1884     }
1885
1886     CKKSIncomingQueueOperation* op  = [[CKKSIncomingQueueOperation alloc] initWithCKKSKeychainView:self errorOnClassAFailure:failOnClassA];
1887     op.name = @"incoming-queue-operation";
1888     if(after != nil) {
1889         [op addSuccessDependency: after];
1890     }
1891
1892     [self scheduleOperation: op];
1893     return op;
1894 }
1895
1896 - (CKKSUpdateDeviceStateOperation*)updateDeviceState:(bool)rateLimit
1897                    waitForKeyHierarchyInitialization:(uint64_t)timeout
1898                                     ckoperationGroup:(CKOperationGroup*)ckoperationGroup {
1899     __weak __typeof(self) weakSelf = self;
1900
1901     // If securityd just started, the key state might be in some transient early state. Wait a bit.
1902     CKKSResultOperation* waitForKeyReady = [CKKSResultOperation named:@"device-state-wait" withBlock:^{
1903         __strong __typeof(self) strongSelf = weakSelf;
1904         __block bool wait = timeout > 0;
1905         if(!wait) {
1906             return;
1907         }
1908
1909         // Determine if we're in an initializing state. Otherwise, don't bother waiting.
1910         [strongSelf dispatchSync:^bool {
1911             wait = [strongSelf.keyHierarchyState isEqualToString:SecCKKSZoneKeyStateInitializing] ||
1912                    [strongSelf.keyHierarchyState isEqualToString:SecCKKSZoneKeyStateInitialized];
1913             return false;
1914         }];
1915
1916         if(wait) {
1917             [strongSelf.keyHierarchyConditions[SecCKKSZoneKeyStateReady] wait:timeout];
1918         }
1919         ckksnotice("ckksdevice", strongSelf, "Finished waiting for key hierarchy state, currently %@", strongSelf.keyHierarchyState);
1920     }];
1921     [self.waitingQueue addOperation:waitForKeyReady];
1922
1923     CKKSUpdateDeviceStateOperation* op = [[CKKSUpdateDeviceStateOperation alloc] initWithCKKSKeychainView:self rateLimit:rateLimit ckoperationGroup:ckoperationGroup];
1924     op.name = @"device-state-operation";
1925
1926     [op addDependency: waitForKeyReady];
1927
1928     // op modifies the CloudKit zone, so it should insert itself into the list of OutgoingQueueOperations.
1929     // Then, we won't have simultaneous zone-modifying operations and confuse ourselves.
1930     // However, since we might have pending OQOs, it should try to insert itself at the beginning of the linearized list
1931     [op linearDependenciesWithSelfFirst:self.outgoingQueueOperations];
1932
1933     // CKKSUpdateDeviceStateOperations are special: they should fire even if we don't believe we're in an iCloud account.
1934     // They also shouldn't block or be blocked by any other operation; our wait operation above will handle that
1935     [self scheduleOperationWithoutDependencies:op];
1936     return op;
1937 }
1938
1939 // There are some errors which won't be reported but will be reflected in the CDSE; any error coming out of here is fatal
1940 - (CKKSDeviceStateEntry*)_onqueueCurrentDeviceStateEntry: (NSError* __autoreleasing*)error {
1941     NSError* localerror = nil;
1942
1943     CKKSCKAccountStateTracker* accountTracker = self.accountTracker;
1944
1945     // We must have an iCloud account (with d2de on) to even create one of these
1946     if(accountTracker.currentCKAccountInfo.accountStatus != CKAccountStatusAvailable || accountTracker.currentCKAccountInfo.supportsDeviceToDeviceEncryption != YES) {
1947         ckkserror("ckksdevice", self, "No iCloud account active: %@", accountTracker.currentCKAccountInfo);
1948         localerror = [NSError errorWithDomain:@"securityd"
1949                                          code:errSecInternalError
1950                                      userInfo:@{NSLocalizedDescriptionKey: [NSString stringWithFormat: @"No active HSA2 iCloud account: %@", accountTracker.currentCKAccountInfo]}];
1951         if(error) {
1952             *error = localerror;
1953         }
1954         return nil;
1955     }
1956
1957     CKKSDeviceStateEntry* oldcdse = [CKKSDeviceStateEntry tryFromDatabase:accountTracker.ckdeviceID zoneID:self.zoneID error:&localerror];
1958     if(localerror) {
1959         ckkserror("ckksdevice", self, "Couldn't read old CKKSDeviceStateEntry from database: %@", localerror);
1960         if(error) {
1961             *error = localerror;
1962         }
1963         return nil;
1964     }
1965
1966     // Find out what we think the current keys are
1967     CKKSCurrentKeyPointer* currentTLKPointer    = [CKKSCurrentKeyPointer tryFromDatabase: SecCKKSKeyClassTLK zoneID:self.zoneID error:&localerror];
1968     CKKSCurrentKeyPointer* currentClassAPointer = [CKKSCurrentKeyPointer tryFromDatabase: SecCKKSKeyClassA   zoneID:self.zoneID error:&localerror];
1969     CKKSCurrentKeyPointer* currentClassCPointer = [CKKSCurrentKeyPointer tryFromDatabase: SecCKKSKeyClassC   zoneID:self.zoneID error:&localerror];
1970     if(localerror) {
1971         // Things is broken, but the whole point of this record is to share the brokenness. Continue.
1972         ckkserror("ckksdevice", self, "Couldn't read current key pointers from database: %@; proceeding", localerror);
1973         localerror = nil;
1974     }
1975
1976     CKKSKey* suggestedTLK       = currentTLKPointer.currentKeyUUID    ? [CKKSKey tryFromDatabase:currentTLKPointer.currentKeyUUID    zoneID:self.zoneID error:&localerror] : nil;
1977     CKKSKey* suggestedClassAKey = currentClassAPointer.currentKeyUUID ? [CKKSKey tryFromDatabase:currentClassAPointer.currentKeyUUID zoneID:self.zoneID error:&localerror] : nil;
1978     CKKSKey* suggestedClassCKey = currentClassCPointer.currentKeyUUID ? [CKKSKey tryFromDatabase:currentClassCPointer.currentKeyUUID zoneID:self.zoneID error:&localerror] : nil;
1979
1980     if(localerror) {
1981         // Things is broken, but the whole point of this record is to share the brokenness. Continue.
1982         ckkserror("ckksdevice", self, "Couldn't read keys from database: %@; proceeding", localerror);
1983         localerror = nil;
1984     }
1985
1986     // Check if we posess the keys in the keychain
1987     [suggestedTLK ensureKeyLoaded:&localerror];
1988     if(localerror && [self.lockStateTracker isLockedError:localerror]) {
1989         ckkserror("ckksdevice", self, "Device is locked; couldn't read TLK from keychain. Assuming it is present and continuing; error was %@", localerror);
1990         localerror = nil;
1991     } else if(localerror) {
1992         ckkserror("ckksdevice", self, "Couldn't read TLK from keychain. We do not have a current TLK. Error was %@", localerror);
1993         suggestedTLK = nil;
1994     }
1995
1996     [suggestedClassAKey ensureKeyLoaded:&localerror];
1997     if(localerror && [self.lockStateTracker isLockedError:localerror]) {
1998         ckkserror("ckksdevice", self, "Device is locked; couldn't read ClassA key from keychain. Assuming it is present and continuing; error was %@", localerror);
1999         localerror = nil;
2000     } else if(localerror) {
2001         ckkserror("ckksdevice", self, "Couldn't read ClassA key from keychain. We do not have a current ClassA key. Error was %@", localerror);
2002         suggestedClassAKey = nil;
2003     }
2004
2005     [suggestedClassCKey ensureKeyLoaded:&localerror];
2006     // class C keys are stored class C, so uh, don't check lock state.
2007     if(localerror) {
2008         ckkserror("ckksdevice", self, "Couldn't read ClassC key from keychain. We do not have a current ClassC key. Error was %@", localerror);
2009         suggestedClassCKey = nil;
2010     }
2011
2012     // We'd like to have the circle peer ID. Give the account state tracker a fighting chance, but not having it is not an error
2013     if([accountTracker.accountCirclePeerIDInitialized wait:500*NSEC_PER_MSEC] != 0 && !accountTracker.accountCirclePeerID) {
2014         ckkserror("ckksdevice", self, "No peer ID available");
2015     }
2016
2017     // We only really want the oldcdse for its encodedCKRecord, so make a new cdse here
2018     CKKSDeviceStateEntry* newcdse = [[CKKSDeviceStateEntry alloc] initForDevice:accountTracker.ckdeviceID
2019                                                                    circlePeerID:accountTracker.accountCirclePeerID
2020                                                                    circleStatus:accountTracker.currentCircleStatus
2021                                                                        keyState:self.keyHierarchyState
2022                                                                  currentTLKUUID:suggestedTLK.uuid
2023                                                               currentClassAUUID:suggestedClassAKey.uuid
2024                                                               currentClassCUUID:suggestedClassCKey.uuid
2025                                                                          zoneID:self.zoneID
2026                                                                 encodedCKRecord:oldcdse.encodedCKRecord];
2027     return newcdse;
2028 }
2029
2030 - (CKKSSynchronizeOperation*) resyncWithCloud {
2031     CKKSSynchronizeOperation* op = [[CKKSSynchronizeOperation alloc] initWithCKKSKeychainView: self];
2032     [self scheduleOperation: op];
2033     return op;
2034 }
2035
2036 - (CKKSLocalSynchronizeOperation*)resyncLocal {
2037     CKKSLocalSynchronizeOperation* op = [[CKKSLocalSynchronizeOperation alloc] initWithCKKSKeychainView:self];
2038     [self scheduleOperation: op];
2039     return op;
2040 }
2041
2042 - (CKKSResultOperation*)fetchAndProcessCKChanges:(CKKSFetchBecause*)because {
2043     return [self fetchAndProcessCKChanges:because after:nil];
2044 }
2045
2046 - (CKKSResultOperation*)fetchAndProcessCKChanges:(CKKSFetchBecause*)because after:(CKKSResultOperation*)after {
2047     if(!SecCKKSIsEnabled()) {
2048         ckksinfo("ckks", self, "Skipping fetchAndProcessCKChanges due to disabled CKKS");
2049         return nil;
2050     }
2051
2052     if(after) {
2053         [self.zoneChangeFetcher holdFetchesUntil:after];
2054     }
2055
2056     // We fetched some changes; try to process them!
2057     return [self processIncomingQueue:false after:[self.zoneChangeFetcher requestSuccessfulFetch:because]];
2058 }
2059
2060 // Lets the view know about a failed CloudKit write. If the error is "already have one of these records", it will
2061 // store the new records and kick off the new processing
2062 //
2063 // Note that you need to tell this function the records you wanted to save, so it can determine what needs deletion
2064 - (bool)_onqueueCKWriteFailed:(NSError*)ckerror attemptedRecordsChanged:(NSDictionary<CKRecordID*, CKRecord*>*)savedRecords {
2065     dispatch_assert_queue(self.queue);
2066
2067     NSDictionary<CKRecordID*,NSError*>* partialErrors = ckerror.userInfo[CKPartialErrorsByItemIDKey];
2068     if([ckerror.domain isEqual:CKErrorDomain] && ckerror.code == CKErrorPartialFailure && partialErrors) {
2069         // Check if this error was "you're out of date"
2070         bool recordChanged = true;
2071
2072         for(NSError* error in partialErrors.allValues) {
2073             if((![error.domain isEqual:CKErrorDomain]) || (error.code != CKErrorBatchRequestFailed && error.code != CKErrorServerRecordChanged && error.code != CKErrorUnknownItem)) {
2074                 // There's an error in there that isn't CKErrorServerRecordChanged, CKErrorBatchRequestFailed, or CKErrorUnknownItem. Don't handle nicely...
2075                 recordChanged = false;
2076             }
2077         }
2078
2079         if(recordChanged) {
2080             ckksnotice("ckks", self, "Received a ServerRecordChanged error, attempting to update new records and delete unknown ones");
2081
2082             bool updatedRecord = false;
2083
2084             for(CKRecordID* recordID in partialErrors.allKeys) {
2085                 NSError* error = partialErrors[recordID];
2086                 if([error.domain isEqual:CKErrorDomain] && error.code == CKErrorServerRecordChanged) {
2087                     CKRecord* newRecord = error.userInfo[CKRecordChangedErrorServerRecordKey];
2088                     ckksnotice("ckks", self, "On error: updating our idea of: %@", newRecord);
2089
2090                     updatedRecord |= [self _onqueueCKRecordChanged:newRecord resync:true];
2091                 } else if([error.domain isEqual:CKErrorDomain] && error.code == CKErrorUnknownItem) {
2092                     CKRecord* record = savedRecords[recordID];
2093                     ckksnotice("ckks", self, "On error: handling an unexpected delete of: %@ %@", recordID, record);
2094
2095                     updatedRecord |= [self _onqueueCKRecordDeleted:recordID recordType:record.recordType resync:true];
2096                 }
2097             }
2098
2099             if(updatedRecord) {
2100                 [self processIncomingQueue:false];
2101                 return true;
2102             }
2103         }
2104
2105         // Check if this error was the CKKS server extension rejecting the write
2106         for(CKRecordID* recordID in partialErrors.allKeys) {
2107             NSError* error = partialErrors[recordID];
2108
2109             NSError* underlyingError = error.userInfo[NSUnderlyingErrorKey];
2110             NSError* thirdLevelError = underlyingError.userInfo[NSUnderlyingErrorKey];
2111             ckksnotice("ckks", self, "Examining 'write failed' error: %@ %@ %@", error, underlyingError, thirdLevelError);
2112
2113             if([error.domain isEqualToString:CKErrorDomain] && error.code == CKErrorServerRejectedRequest &&
2114                underlyingError && [underlyingError.domain isEqualToString:CKInternalErrorDomain] && underlyingError.code == CKErrorInternalPluginError &&
2115                thirdLevelError && [thirdLevelError.domain isEqualToString:@"CloudkitKeychainService"]) {
2116
2117                 if(thirdLevelError.code == CKKSServerUnexpectedSyncKeyInChain) {
2118                     // The server thinks the classA/C synckeys don't wrap directly the to top TLK, but we don't (otherwise, we would have fixed it).
2119                     // Issue a key hierarchy fetch and see what's what.
2120                     ckkserror("ckks", self, "CKKS Server extension has told us about %@ for record %@; requesting refetch and reprocess of key hierarchy", thirdLevelError, recordID);
2121                     [self _onqueueKeyStateMachineRequestFetch];
2122                 } else {
2123                     ckkserror("ckks", self, "CKKS Server extension has told us about %@ for record %@, but we don't currently handle this error", thirdLevelError, recordID);
2124                 }
2125             }
2126         }
2127     }
2128
2129     return false;
2130 }
2131
2132 - (bool)_onqueueCKRecordDeleted:(CKRecordID*)recordID recordType:(NSString*)recordType resync:(bool)resync {
2133     dispatch_assert_queue(self.queue);
2134
2135     // TODO: resync doesn't really mean much here; what does it mean for a record to be 'deleted' if you're fetching from scratch?
2136
2137     if([recordType isEqual: SecCKRecordItemType]) {
2138         ckksinfo("ckks", self, "CloudKit notification: deleted record(%@): %@", recordType, recordID);
2139         NSError* error = nil;
2140         NSError* iqeerror = nil;
2141         CKKSMirrorEntry* ckme = [CKKSMirrorEntry fromDatabase: [recordID recordName] zoneID:self.zoneID error: &error];
2142
2143         // Deletes always succeed, not matter the generation count
2144         if(ckme) {
2145             [ckme deleteFromDatabase:&error];
2146
2147             CKKSIncomingQueueEntry* iqe = [[CKKSIncomingQueueEntry alloc] initWithCKKSItem:ckme.item action:SecCKKSActionDelete state:SecCKKSStateNew];
2148             [iqe saveToDatabase:&iqeerror];
2149             if(iqeerror) {
2150                 ckkserror("ckks", self, "Couldn't save incoming queue entry: %@", iqeerror);
2151             }
2152         }
2153         ckksinfo("ckks", self, "CKKSMirrorEntry was deleted: %@ %@ error: %@", recordID, ckme, error);
2154         // TODO: actually pass error back up
2155         return (error == nil);
2156
2157     } else if([recordType isEqual: SecCKRecordCurrentItemType]) {
2158         ckksinfo("ckks", self, "CloudKit notification: deleted current item pointer(%@): %@", recordType, recordID);
2159         NSError* error = nil;
2160
2161         [[CKKSCurrentItemPointer tryFromDatabase:[recordID recordName] state:SecCKKSProcessedStateRemote zoneID:self.zoneID error:&error] deleteFromDatabase:&error];
2162         [[CKKSCurrentItemPointer fromDatabase:[recordID recordName]    state:SecCKKSProcessedStateLocal  zoneID:self.zoneID error:&error] deleteFromDatabase:&error];
2163
2164         ckksinfo("ckks", self, "CKKSCurrentItemPointer was deleted: %@ error: %@", recordID, error);
2165         return (error == nil);
2166
2167     } else if([recordType isEqual: SecCKRecordIntermediateKeyType]) {
2168         // TODO: handle in some interesting way
2169         return true;
2170     } else if([recordType isEqual: SecCKRecordTLKShareType]) {
2171         NSError* error = nil;
2172         ckksinfo("ckks", self, "CloudKit notification: deleted tlk share record(%@): %@", recordType, recordID);
2173         CKKSTLKShare* share = [CKKSTLKShare tryFromDatabaseFromCKRecordID:recordID error:&error];
2174         [share deleteFromDatabase:&error];
2175
2176         if(error) {
2177             ckkserror("ckks", self, "CK notification: Couldn't delete deleted TLKShare: %@ %@", recordID,  error);
2178         }
2179         return (error == nil);
2180
2181     } else if([recordType isEqual: SecCKRecordDeviceStateType]) {
2182         NSError* error = nil;
2183         ckksinfo("ckks", self, "CloudKit notification: deleted device state record(%@): %@", recordType, recordID);
2184
2185         CKKSDeviceStateEntry* cdse = [CKKSDeviceStateEntry tryFromDatabaseFromCKRecordID:recordID error:&error];
2186         [cdse deleteFromDatabase: &error];
2187         ckksinfo("ckks", self, "CKKSCurrentItemPointer(%@) was deleted: %@ error: %@", cdse, recordID, error);
2188
2189         return (error == nil);
2190
2191     } else if ([recordType isEqualToString:SecCKRecordManifestType]) {
2192         ckksinfo("ckks", self, "CloudKit notification: deleted manifest record (%@): %@", recordType, recordID);
2193
2194         NSError* error = nil;
2195         CKKSManifest* manifest = [CKKSManifest manifestForRecordName:recordID.recordName error:&error];
2196         if (manifest) {
2197             [manifest deleteFromDatabase:&error];
2198         }
2199
2200         ckksinfo("ckks", self, "CKKSManifest was deleted: %@ %@ error: %@", recordID, manifest, error);
2201         // TODO: actually pass error back up
2202         return error == nil;
2203     }
2204
2205     else {
2206         ckkserror("ckksfetch", self, "unknown record type: %@ %@", recordType, recordID);
2207         return false;
2208     }
2209 }
2210
2211 - (bool)_onqueueCKRecordChanged:(CKRecord*)record resync:(bool)resync {
2212     dispatch_assert_queue(self.queue);
2213
2214     ckksinfo("ckksfetch", self, "Processing record modification(%@): %@", record.recordType, record);
2215
2216     if([[record recordType] isEqual: SecCKRecordItemType]) {
2217         [self _onqueueCKRecordItemChanged:record resync:resync];
2218         return true;
2219     } else if([[record recordType] isEqual: SecCKRecordCurrentItemType]) {
2220         [self _onqueueCKRecordCurrentItemPointerChanged:record resync:resync];
2221         return true;
2222     } else if([[record recordType] isEqual: SecCKRecordIntermediateKeyType]) {
2223         [self _onqueueCKRecordKeyChanged:record resync:resync];
2224         return true;
2225     } else if ([[record recordType] isEqual: SecCKRecordTLKShareType]) {
2226         [self _onqueueCKRecordTLKShareChanged:record resync:resync];
2227         return true;
2228     } else if([[record recordType] isEqualToString: SecCKRecordCurrentKeyType]) {
2229         [self _onqueueCKRecordCurrentKeyPointerChanged:record resync:resync];
2230         return true;
2231     } else if ([[record recordType] isEqualToString:SecCKRecordManifestType]) {
2232         [self _onqueueCKRecordManifestChanged:record resync:resync];
2233         return true;
2234     } else if ([[record recordType] isEqualToString:SecCKRecordManifestLeafType]) {
2235         [self _onqueueCKRecordManifestLeafChanged:record resync:resync];
2236         return true;
2237     } else if ([[record recordType] isEqualToString:SecCKRecordDeviceStateType]) {
2238         [self _onqueueCKRecordDeviceStateChanged:record resync:resync];
2239         return true;
2240     } else {
2241         ckkserror("ckksfetch", self, "unknown record type: %@ %@", [record recordType], record);
2242         return false;
2243     }
2244 }
2245
2246 - (void)_onqueueCKRecordItemChanged:(CKRecord*)record resync:(bool)resync {
2247     dispatch_assert_queue(self.queue);
2248
2249     NSError* error = nil;
2250     // Find if we knew about this record in the past
2251     bool update = false;
2252     CKKSMirrorEntry* ckme = [CKKSMirrorEntry tryFromDatabase: [[record recordID] recordName] zoneID:self.zoneID error:&error];
2253
2254     if(error) {
2255         ckkserror("ckks", self, "error loading a CKKSMirrorEntry from database: %@", error);
2256         // TODO: quit?
2257     }
2258
2259     if(resync) {
2260         if(!ckme) {
2261             ckkserror("ckksresync", self, "BUG: No local item matching resynced CloudKit record: %@", record);
2262         } else if(![ckme matchesCKRecord:record]) {
2263             ckkserror("ckksresync", self, "BUG: Local item doesn't match resynced CloudKit record: %@ %@", ckme, record);
2264         } else {
2265             ckksnotice("ckksresync", self, "Already know about this item record, updating anyway: %@", record.recordID);
2266         }
2267     }
2268
2269     if(ckme && ckme.item && ckme.item.generationCount > [record[SecCKRecordGenerationCountKey] unsignedLongLongValue]) {
2270         ckkserror("ckks", self, "received a record from CloudKit with a bad generation count: %@ (%ld > %@)", ckme.uuid,
2271                  (long) ckme.item.generationCount,
2272                  record[SecCKRecordGenerationCountKey]);
2273         // Abort processing this record.
2274         return;
2275     }
2276
2277     // If we found an old version in the database; this might be an update
2278     if(ckme) {
2279         if([ckme matchesCKRecord:record] && !resync) {
2280             // This is almost certainly a record we uploaded; CKFetchChanges sends them back as new records
2281             ckksnotice("ckks", self, "CloudKit has told us of record we already know about; skipping update");
2282             return;
2283         }
2284
2285         update = true;
2286         // Set the CKKSMirrorEntry's fields to be whatever this record holds
2287         [ckme setFromCKRecord: record];
2288     } else {
2289         // Have to make a new CKKSMirrorEntry
2290         ckme = [[CKKSMirrorEntry alloc] initWithCKRecord: record];
2291     }
2292
2293     [ckme saveToDatabase: &error];
2294
2295     if(error) {
2296         ckkserror("ckks", self, "couldn't save new CKRecord to database: %@ %@", record, error);
2297     } else {
2298         ckksdebug("ckks", self, "CKKSMirrorEntry was created: %@", ckme);
2299     }
2300
2301     NSError* iqeerror = nil;
2302     CKKSIncomingQueueEntry* iqe = [[CKKSIncomingQueueEntry alloc] initWithCKKSItem:ckme.item
2303                                                                             action:(update ? SecCKKSActionModify : SecCKKSActionAdd)
2304                                                                              state:SecCKKSStateNew];
2305     [iqe saveToDatabase:&iqeerror];
2306     if(iqeerror) {
2307         ckkserror("ckks", self, "Couldn't save modified incoming queue entry: %@", iqeerror);
2308     } else {
2309         ckksdebug("ckks", self, "CKKSIncomingQueueEntry was created: %@", iqe);
2310     }
2311
2312     // A remote change has occured for this record. Delete any pending local changes; they will be overwritten.
2313     CKKSOutgoingQueueEntry* oqe = [CKKSOutgoingQueueEntry tryFromDatabase:ckme.uuid state: SecCKKSStateNew zoneID:self.zoneID error: &error];
2314     if(error) {
2315         ckkserror("ckks", self, "Couldn't load OutgoingQueueEntry: %@", error);
2316     }
2317     if(oqe) {
2318         [self _onqueueChangeOutgoingQueueEntry:oqe toState:SecCKKSStateDeleted error:&error];
2319     }
2320
2321     // Reencryptions are pending changes too
2322     oqe = [CKKSOutgoingQueueEntry tryFromDatabase:ckme.uuid state: SecCKKSStateReencrypt zoneID:self.zoneID error: &error];
2323     if(error) {
2324         ckkserror("ckks", self, "Couldn't load reencrypted OutgoingQueueEntry: %@", error);
2325     }
2326     if(oqe) {
2327         [oqe deleteFromDatabase:&error];
2328         if(error) {
2329             ckkserror("ckks", self, "Couldn't delete reencrypted oqe(%@): %@", oqe, error);
2330         }
2331     }
2332 }
2333
2334 - (void)_onqueueCKRecordKeyChanged:(CKRecord*)record resync:(bool)resync {
2335     dispatch_assert_queue(self.queue);
2336
2337     NSError* error = nil;
2338
2339     if(resync) {
2340         NSError* resyncerror = nil;
2341
2342         CKKSKey* key = [CKKSKey tryFromDatabaseAnyState:record.recordID.recordName zoneID:self.zoneID error:&resyncerror];
2343         if(resyncerror) {
2344             ckkserror("ckksresync", self, "error loading key: %@", resyncerror);
2345         }
2346         if(!key) {
2347             ckkserror("ckksresync", self, "BUG: No sync key matching resynced CloudKit record: %@", record);
2348         } else if(![key matchesCKRecord:record]) {
2349             ckkserror("ckksresync", self, "BUG: Local sync key doesn't match resynced CloudKit record(s): %@ %@", key, record);
2350         } else {
2351             ckksnotice("ckksresync", self, "Already know about this sync key, skipping update: %@", record);
2352             return;
2353         }
2354     }
2355
2356     CKKSKey* remotekey = [[CKKSKey alloc] initWithCKRecord: record];
2357
2358     // Do we already know about this key?
2359     CKKSKey* possibleLocalKey = [CKKSKey tryFromDatabase:remotekey.uuid zoneID:self.zoneID error:&error];
2360     if(error) {
2361         ckkserror("ckkskey", self, "Error findibg exsiting local key for %@: %@", remotekey, error);
2362         // Go on, assuming there isn't a local key
2363     } else if(possibleLocalKey && [possibleLocalKey matchesCKRecord:record]) {
2364         // Okay, nothing new here. Update the CKRecord and move on.
2365         // Note: If the new record doesn't match the local copy, we have to go through the whole dance below
2366         possibleLocalKey.storedCKRecord = record;
2367         [possibleLocalKey saveToDatabase:&error];
2368
2369         if(error) {
2370             ckkserror("ckkskey", self, "Couldn't update existing key: %@: %@", possibleLocalKey, error);
2371         }
2372         return;
2373     }
2374
2375     // Drop into the synckeys table as a 'remote' key, then ask for a rekey operation.
2376     remotekey.state = SecCKKSProcessedStateRemote;
2377     remotekey.currentkey = false;
2378
2379     [remotekey saveToDatabase:&error];
2380     if(error) {
2381         ckkserror("ckkskey", self, "Couldn't save key record to database: %@: %@", remotekey, error);
2382         ckksinfo("ckkskey", self, "CKRecord was %@", record);
2383     }
2384
2385     // We've saved a new key in the database; trigger a rekey operation.
2386     [self _onqueueKeyStateMachineRequestProcess];
2387 }
2388
2389 - (void)_onqueueCKRecordTLKShareChanged:(CKRecord*)record resync:(bool)resync {
2390     dispatch_assert_queue(self.queue);
2391
2392     NSError* error = nil;
2393     if(resync) {
2394         // TODO fill in
2395     }
2396
2397     // CKKSTLKShares get saved with no modification
2398     CKKSTLKShare* share = [[CKKSTLKShare alloc] initWithCKRecord:record];
2399     [share saveToDatabase:&error];
2400     if(error) {
2401         ckkserror("ckksshare", self, "Couldn't save new TLK share to database: %@ %@", share, error);
2402     }
2403
2404     [self _onqueueKeyStateMachineRequestProcess];
2405 }
2406
2407 - (void)_onqueueCKRecordCurrentKeyPointerChanged:(CKRecord*)record resync:(bool)resync {
2408     dispatch_assert_queue(self.queue);
2409
2410     // Pull out the old CKP, if it exists
2411     NSError* ckperror = nil;
2412     CKKSCurrentKeyPointer* oldckp = [CKKSCurrentKeyPointer tryFromDatabase:((CKKSKeyClass*) record.recordID.recordName) zoneID:self.zoneID error:&ckperror];
2413     if(ckperror) {
2414         ckkserror("ckkskey", self, "error loading ckp: %@", ckperror);
2415     }
2416
2417     if(resync) {
2418         if(!oldckp) {
2419             ckkserror("ckksresync", self, "BUG: No current key pointer matching resynced CloudKit record: %@", record);
2420         } else if(![oldckp matchesCKRecord:record]) {
2421             ckkserror("ckksresync", self, "BUG: Local current key pointer doesn't match resynced CloudKit record: %@ %@", oldckp, record);
2422         } else {
2423             ckksnotice("ckksresync", self, "Current key pointer has 'changed', but it matches our local copy: %@", record);
2424         }
2425     }
2426
2427     NSError* error = nil;
2428     CKKSCurrentKeyPointer* currentkey = [[CKKSCurrentKeyPointer alloc] initWithCKRecord: record];
2429
2430     [currentkey saveToDatabase: &error];
2431     if(error) {
2432         ckkserror("ckkskey", self, "Couldn't save current key pointer to database: %@: %@", currentkey, error);
2433         ckksinfo("ckkskey", self, "CKRecord was %@", record);
2434     }
2435
2436     if([oldckp matchesCKRecord:record]) {
2437         ckksnotice("ckkskey", self, "Current key pointer modification doesn't change anything interesting; skipping reprocess: %@", record);
2438     } else {
2439         // We've saved a new key in the database; trigger a rekey operation.
2440         [self _onqueueKeyStateMachineRequestProcess];
2441     }
2442 }
2443
2444 - (void)_onqueueCKRecordCurrentItemPointerChanged:(CKRecord*)record resync:(bool)resync {
2445     dispatch_assert_queue(self.queue);
2446
2447     if(resync) {
2448         NSError* ciperror = nil;
2449         CKKSCurrentItemPointer* localcip  = [CKKSCurrentItemPointer tryFromDatabase:record.recordID.recordName state:SecCKKSProcessedStateLocal  zoneID:self.zoneID error:&ciperror];
2450         CKKSCurrentItemPointer* remotecip = [CKKSCurrentItemPointer tryFromDatabase:record.recordID.recordName state:SecCKKSProcessedStateRemote zoneID:self.zoneID error:&ciperror];
2451         if(ciperror) {
2452             ckkserror("ckksresync", self, "error loading cip: %@", ciperror);
2453         }
2454         if(!(localcip || remotecip)) {
2455             ckkserror("ckksresync", self, "BUG: No current item pointer matching resynced CloudKit record: %@", record);
2456         } else if(! ([localcip matchesCKRecord:record] || [remotecip matchesCKRecord:record]) ) {
2457             ckkserror("ckksresync", self, "BUG: Local current item pointer doesn't match resynced CloudKit record(s): %@ %@ %@", localcip, remotecip, record);
2458         } else {
2459             ckksnotice("ckksresync", self, "Already know about this current item pointer, skipping update: %@", record);
2460             return;
2461         }
2462     }
2463
2464     NSError* error = nil;
2465     CKKSCurrentItemPointer* cip = [[CKKSCurrentItemPointer alloc] initWithCKRecord: record];
2466     cip.state = SecCKKSProcessedStateRemote;
2467
2468     [cip saveToDatabase: &error];
2469     if(error) {
2470         ckkserror("currentitem", self, "Couldn't save current item pointer to database: %@: %@ %@", cip, error, record);
2471     }
2472 }
2473
2474 - (void)_onqueueCKRecordManifestChanged:(CKRecord*)record resync:(bool)resync
2475 {
2476     NSError* error = nil;
2477     CKKSPendingManifest* manifest = [[CKKSPendingManifest alloc] initWithCKRecord:record];
2478     [manifest saveToDatabase:&error];
2479     if (error) {
2480         ckkserror("CKKS", self, "Failed to save fetched manifest record to database: %@: %@", manifest, error);
2481         ckksinfo("CKKS", self, "manifest CKRecord was %@", record);
2482     }
2483 }
2484
2485 - (void)_onqueueCKRecordManifestLeafChanged:(CKRecord*)record resync:(bool)resync
2486 {
2487     NSError* error = nil;
2488     CKKSManifestLeafRecord* manifestLeaf = [[CKKSManifestPendingLeafRecord alloc] initWithCKRecord:record];
2489     [manifestLeaf saveToDatabase:&error];
2490     if (error) {
2491         ckkserror("CKKS", self, "Failed to save fetched manifest leaf record to database: %@: %@", manifestLeaf, error);
2492         ckksinfo("CKKS", self, "manifest leaf CKRecord was %@", record);
2493     }
2494 }
2495
2496 - (void)_onqueueCKRecordDeviceStateChanged:(CKRecord*)record resync:(bool)resync {
2497     if(resync) {
2498         NSError* dserror = nil;
2499         CKKSDeviceStateEntry* cdse  = [CKKSDeviceStateEntry tryFromDatabase:record.recordID.recordName zoneID:self.zoneID error:&dserror];
2500         if(dserror) {
2501             ckkserror("ckksresync", self, "error loading cdse: %@", dserror);
2502         }
2503         if(!cdse) {
2504             ckkserror("ckksresync", self, "BUG: No current device state entry matching resynced CloudKit record: %@", record);
2505         } else if(![cdse matchesCKRecord:record]) {
2506             ckkserror("ckksresync", self, "BUG: Local current device state entry doesn't match resynced CloudKit record(s): %@ %@", cdse, record);
2507         } else {
2508             ckksnotice("ckksresync", self, "Already know about this current item pointer, skipping update: %@", record);
2509             return;
2510         }
2511     }
2512
2513     NSError* error = nil;
2514     CKKSDeviceStateEntry* cdse = [[CKKSDeviceStateEntry alloc] initWithCKRecord:record];
2515     [cdse saveToDatabase:&error];
2516     if (error) {
2517         ckkserror("ckksdevice", self, "Failed to save device record to database: %@: %@ %@", cdse, error, record);
2518     }
2519 }
2520
2521 - (bool)_onqueueResetAllInflightOQE:(NSError**)error {
2522     NSError* localError = nil;
2523
2524     while(true) {
2525         NSArray<CKKSOutgoingQueueEntry*> * inflightQueueEntries = [CKKSOutgoingQueueEntry fetch:SecCKKSOutgoingQueueItemsAtOnce
2526                                                                                           state:SecCKKSStateInFlight
2527                                                                                          zoneID:self.zoneID
2528                                                                                           error:&localError];
2529
2530         if(localError != nil) {
2531             ckkserror("ckks", self, "Error finding inflight outgoing queue records: %@", localError);
2532             if(error) {
2533                 *error = localError;
2534             }
2535             return false;
2536         }
2537
2538         if([inflightQueueEntries count] == 0u) {
2539             break;
2540         }
2541
2542         for(CKKSOutgoingQueueEntry* oqe in inflightQueueEntries) {
2543             [self _onqueueChangeOutgoingQueueEntry:oqe toState:SecCKKSStateNew error:&localError];
2544
2545             if(localError) {
2546                 ckkserror("ckks", self, "Error fixing up inflight OQE(%@): %@", oqe, localError);
2547                 if(error) {
2548                     *error = localError;
2549                 }
2550                 return false;
2551             }
2552         }
2553     }
2554
2555     return true;
2556 }
2557
2558 - (bool)_onqueueChangeOutgoingQueueEntry: (CKKSOutgoingQueueEntry*) oqe toState: (NSString*) state error: (NSError* __autoreleasing*) error {
2559     dispatch_assert_queue(self.queue);
2560
2561     NSError* localerror = nil;
2562
2563     if([state isEqualToString: SecCKKSStateDeleted]) {
2564         // Hurray, this must be a success
2565         SecBoolNSErrorCallback callback = self.pendingSyncCallbacks[oqe.uuid];
2566         if(callback) {
2567             callback(true, nil);
2568             self.pendingSyncCallbacks[oqe.uuid] = nil;
2569         }
2570
2571         [oqe deleteFromDatabase: &localerror];
2572         if(localerror) {
2573             ckkserror("ckks", self, "Couldn't delete %@: %@", oqe, localerror);
2574         }
2575
2576     } else if([oqe.state isEqualToString:SecCKKSStateInFlight] && [state isEqualToString:SecCKKSStateNew]) {
2577         // An in-flight OQE is moving to new? See if it's been superceded
2578         CKKSOutgoingQueueEntry* newOQE = [CKKSOutgoingQueueEntry tryFromDatabase:oqe.uuid state:SecCKKSStateNew zoneID:self.zoneID error:&localerror];
2579         if(localerror) {
2580             ckkserror("ckksoutgoing", self, "Couldn't fetch an overwriting OQE, assuming one doesn't exist: %@", localerror);
2581             newOQE = nil;
2582         }
2583
2584         if(newOQE) {
2585             ckksnotice("ckksoutgoing", self, "New modification has come in behind inflight %@; dropping failed change", oqe);
2586             // recurse for that lovely code reuse
2587             [self _onqueueChangeOutgoingQueueEntry:oqe toState:SecCKKSStateDeleted error:&localerror];
2588             if(localerror) {
2589                 ckkserror("ckksoutgoing", self, "Couldn't delete in-flight OQE: %@", localerror);
2590                 if(error) {
2591                     *error = localerror;
2592                 }
2593             }
2594         } else {
2595             oqe.state = state;
2596             [oqe saveToDatabase: &localerror];
2597             if(localerror) {
2598                 ckkserror("ckks", self, "Couldn't save %@ as %@: %@", oqe, state, localerror);
2599             }
2600         }
2601
2602     } else {
2603         oqe.state = state;
2604         [oqe saveToDatabase: &localerror];
2605         if(localerror) {
2606             ckkserror("ckks", self, "Couldn't save %@ as %@: %@", oqe, state, localerror);
2607         }
2608     }
2609
2610     if(error && localerror) {
2611         *error = localerror;
2612     }
2613     return localerror == nil;
2614 }
2615
2616 - (bool)_onqueueErrorOutgoingQueueEntry: (CKKSOutgoingQueueEntry*) oqe itemError: (NSError*) itemError error: (NSError* __autoreleasing*) error {
2617     dispatch_assert_queue(self.queue);
2618
2619     SecBoolNSErrorCallback callback = self.pendingSyncCallbacks[oqe.uuid];
2620     if(callback) {
2621         callback(false, itemError);
2622         self.pendingSyncCallbacks[oqe.uuid] = nil;
2623     }
2624     NSError* localerror = nil;
2625
2626     // Now, delete the OQE: it's never coming back
2627     [oqe deleteFromDatabase:&localerror];
2628     if(localerror) {
2629         ckkserror("ckks", self, "Couldn't delete %@ (due to error %@): %@", oqe, itemError, localerror);
2630     }
2631
2632     if(error && localerror) {
2633         *error = localerror;
2634     }
2635     return localerror == nil;
2636 }
2637
2638 - (bool)_onqueueUpdateLatestManifestWithError:(NSError**)error
2639 {
2640     dispatch_assert_queue(self.queue);
2641     CKKSManifest* manifest = [CKKSManifest latestTrustedManifestForZone:self.zoneName error:error];
2642     if (manifest) {
2643         self.latestManifest = manifest;
2644         return true;
2645     }
2646     else {
2647         return false;
2648     }
2649 }
2650
2651 - (bool)_onqueueWithAccountKeysCheckTLK:(CKKSKey*)proposedTLK error:(NSError* __autoreleasing *)error {
2652     dispatch_assert_queue(self.queue);
2653     // First, if we have a local identity, check for any TLK shares
2654     NSError* localerror = nil;
2655
2656     if(![proposedTLK wrapsSelf]) {
2657         ckkserror("ckksshare", self, "Potential TLK %@ does not wrap self; skipping TLK share checking", proposedTLK);
2658     } else {
2659         if(!self.currentSelfPeers.currentSelf || self.currentSelfPeersError) {
2660             ckkserror("ckksshare", self, "Couldn't fetch self peers: %@", self.currentSelfPeersError);
2661             if(error) {
2662                 *error = self.currentSelfPeersError;
2663             }
2664             return false;
2665         }
2666
2667         if(!self.currentTrustedPeers || self.currentTrustedPeersError) {
2668             ckkserror("ckksshare", self, "Couldn't fetch trusted peers: %@", self.currentTrustedPeersError);
2669             if(error) {
2670                 *error = self.currentTrustedPeersError;
2671             }
2672             return false;
2673         }
2674
2675         NSArray<CKKSTLKShare*>* possibleShares = [CKKSTLKShare allFor:self.currentSelfPeers.currentSelf.peerID
2676                                                               keyUUID:proposedTLK.uuid
2677                                                                zoneID:self.zoneID
2678                                                                 error:&localerror];
2679         if(localerror) {
2680             ckkserror("ckksshare", self, "Error fetching CKKSTLKShares: %@", localerror);
2681         }
2682
2683         if(possibleShares.count == 0) {
2684             ckksnotice("ckksshare", self, "No CKKSTLKShares for %@", proposedTLK);
2685         }
2686
2687         for(CKKSTLKShare* possibleShare in possibleShares) {
2688             NSError* possibleShareError = nil;
2689             ckksnotice("ckksshare", self, "Checking possible TLK share %@ as %@",
2690                        possibleShare, self.currentSelfPeers.currentSelf);
2691
2692             CKKSKey* possibleKey = [possibleShare recoverTLK:self.currentSelfPeers.currentSelf
2693                                                 trustedPeers:self.currentTrustedPeers
2694                                                        error:&possibleShareError];
2695
2696             if(possibleShareError) {
2697                 ckkserror("ckksshare", self, "Unable to unwrap TLKShare(%@) as %@: %@",
2698                           possibleShare, self.currentSelfPeers.currentSelf, possibleShareError);
2699                 ckkserror("ckksshare", self, "Current trust set: %@", self.currentTrustedPeers);
2700                 // TODO: save error
2701                 continue;
2702             }
2703
2704             bool result = [proposedTLK trySelfWrappedKeyCandidate:possibleKey.aessivkey error:&possibleShareError];
2705             if(possibleShareError) {
2706                 ckkserror("ckksshare", self, "Unwrapped TLKShare(%@) does not unwrap proposed TLK(%@) as %@: %@",
2707                           possibleShare, proposedTLK, self.currentSelfPeers.currentSelf, possibleShareError);
2708                 // TODO save error
2709                 continue;
2710             }
2711
2712             if(result) {
2713                 ckksnotice("ckksshare", self, "TLKShare(%@) unlocked TLK(%@) as %@",
2714                            possibleShare, proposedTLK, self.currentSelfPeers.currentSelf);
2715
2716                 // The proposed TLK is trusted key material. Persist it as a "trusted" key.
2717                 [proposedTLK saveKeyMaterialToKeychain:true error: &possibleShareError];
2718                 if(possibleShareError) {
2719                     ckkserror("ckksshare", self, "Couldn't store the new TLK(%@) to the keychain: %@", proposedTLK, possibleShareError);
2720                     if(error) {
2721                         *error = possibleShareError;
2722                     }
2723                     return false;
2724                 }
2725
2726                 return true;
2727             }
2728         }
2729     }
2730
2731     if([proposedTLK loadKeyMaterialFromKeychain:error]) {
2732         // Hurray!
2733         return true;
2734     } else {
2735         return false;
2736     }
2737 }
2738
2739 - (void) dispatchAsync: (bool (^)(void)) block {
2740     // We need to call kc_with_dbt, which blocks. Route up through a global queue...
2741     __weak __typeof(self) weakSelf = self;
2742
2743     dispatch_async(dispatch_get_global_queue(QOS_CLASS_DEFAULT, 0), ^{
2744         [weakSelf dispatchSync:block];
2745     });
2746 }
2747
2748 - (bool)dispatchSyncWithConnection:(SecDbConnectionRef _Nonnull)dbconn block:(bool (^)(void))block {
2749     CFErrorRef cferror = NULL;
2750
2751     // Take the DB transaction, then get on the local queue.
2752     // In the case of exclusive DB transactions, we don't really _need_ the local queue, but, it's here for future use.
2753     bool ret = kc_transaction_type(dbconn, kSecDbExclusiveRemoteCKKSTransactionType, &cferror, ^bool{
2754         __block bool ok = false;
2755
2756         dispatch_sync(self.queue, ^{
2757             ok = block();
2758         });
2759
2760         return ok;
2761     });
2762
2763     if(cferror) {
2764         ckkserror("ckks", self, "error doing database transaction, major problems ahead: %@", cferror);
2765     }
2766     return ret;
2767 }
2768
2769 - (void)dispatchSync: (bool (^)(void)) block {
2770     // important enough to block this thread. Must get a connection first, though!
2771     CFErrorRef cferror = NULL;
2772     kc_with_dbt(true, &cferror, ^bool (SecDbConnectionRef dbt) {
2773         return [self dispatchSyncWithConnection:dbt block:block];
2774     });
2775     if(cferror) {
2776         ckkserror("ckks", self, "error getting database connection, major problems ahead: %@", cferror);
2777     }
2778 }
2779
2780 - (void)dispatchSyncWithAccountKeys:(bool (^)(void))block
2781 {
2782     [SOSAccount performOnAccountQueue: ^{
2783         NSError* selfPeersError = nil;
2784         CKKSSelves* currentSelfPeers = [self.currentPeerProvider fetchSelfPeers:&selfPeersError];
2785
2786         NSError* trustedPeersError = nil;
2787         NSSet<id<CKKSPeer>>* currentTrustedPeers = [self.currentPeerProvider fetchTrustedPeers:&trustedPeersError];
2788
2789         [self dispatchSync:^bool{
2790             self.currentSelfPeers = currentSelfPeers;
2791             self.currentSelfPeersError = selfPeersError;
2792
2793             self.currentTrustedPeers = currentTrustedPeers;
2794             self.currentTrustedPeersError = trustedPeersError;
2795
2796             __block bool result = false;
2797             [SOSAccount performWhileHoldingAccountQueue:^{ // so any calls through SOS account will know they can perform their work without dispatching to the account queue, which we already hold
2798                 result = block();
2799             }];
2800
2801             // Forget the peers; they might have class A key material
2802             self.currentSelfPeers = nil;
2803             self.currentSelfPeersError = [NSError errorWithDomain:CKKSErrorDomain code:CKKSNoPeersAvailable description:@"No current self peer available"];
2804             self.currentTrustedPeers = nil;
2805             self.currentTrustedPeersError = [NSError errorWithDomain:CKKSErrorDomain code:CKKSNoPeersAvailable description:@"No current trusted peers available"];
2806
2807             return result;
2808         }];
2809     }];
2810 }
2811
2812 #pragma mark - CKKSZoneUpdateReceiver
2813
2814 - (void)notifyZoneChange: (CKRecordZoneNotification*) notification {
2815     ckksnotice("ckks", self, "received a zone change notification for %@ %@", self, notification);
2816
2817     [self fetchAndProcessCKChanges:CKKSFetchBecauseAPNS];
2818 }
2819
2820 - (void)handleCKLogin {
2821     ckksnotice("ckks", self, "received a notification of CK login");
2822
2823     __weak __typeof(self) weakSelf = self;
2824     CKKSResultOperation* login = [CKKSResultOperation named:@"ckks-login" withBlock:^{
2825         __strong __typeof(self) strongSelf = weakSelf;
2826
2827         [strongSelf dispatchSync:^bool{
2828             strongSelf.accountStatus = CKKSAccountStatusAvailable;
2829             [strongSelf _onqueueHandleCKLogin];
2830             return true;
2831         }];
2832     }];
2833
2834     [self scheduleAccountStatusOperation:login];
2835 }
2836
2837 - (void)superHandleCKLogout {
2838     [super handleCKLogout];
2839 }
2840
2841 - (void)handleCKLogout {
2842     __weak __typeof(self) weakSelf = self;
2843     CKKSResultOperation* logout = [CKKSResultOperation named:@"ckks-logout" withBlock: ^{
2844         __strong __typeof(self) strongSelf = weakSelf;
2845         if(!strongSelf) {
2846             return;
2847         }
2848         [strongSelf dispatchSync:^bool {
2849             ckksnotice("ckks", strongSelf, "received a notification of CK logout");
2850             [strongSelf superHandleCKLogout];
2851
2852             NSError* error = nil;
2853             [strongSelf _onqueueResetLocalData: &error];
2854             if(error) {
2855                 ckkserror("ckks", strongSelf, "error while resetting local data: %@", error);
2856             }
2857
2858             // Tell all pending sync clients that we don't expect to ever sync
2859             for(NSString* callbackUUID in strongSelf.pendingSyncCallbacks.allKeys) {
2860                 [strongSelf callSyncCallbackWithErrorNoAccount:strongSelf.pendingSyncCallbacks[callbackUUID]];
2861                 strongSelf.pendingSyncCallbacks[callbackUUID] = nil;
2862             }
2863
2864             strongSelf.loggedIn = [[CKKSCondition alloc] initToChain: strongSelf.loggedIn];
2865             [strongSelf.loggedOut fulfill];
2866
2867             return true;
2868         }];
2869     }];
2870
2871     [self scheduleAccountStatusOperation: logout];
2872 }
2873
2874 - (void)callSyncCallbackWithErrorNoAccount:(SecBoolNSErrorCallback)syncCallback {
2875     CKKSAccountStatus accountStatus = self.accountStatus;
2876     dispatch_async(self.queue, ^{
2877         syncCallback(false, [NSError errorWithDomain:@"securityd"
2878                                                 code:errSecNotLoggedIn
2879                                             userInfo:@{NSLocalizedDescriptionKey:
2880                                                            [NSString stringWithFormat: @"No iCloud account available(%d); item is not expected to sync", (int)accountStatus]}]);
2881     });
2882 }
2883
2884 #pragma mark - CKKSChangeFetcherErrorOracle
2885
2886 - (bool) isFatalCKFetchError: (NSError*) error {
2887     __weak __typeof(self) weakSelf = self;
2888
2889     // Again, note that this handles exactly one zone. Mutli-zone errors are not supported.
2890     bool isChangeTokenExpiredError = false;
2891     if([error.domain isEqualToString:CKErrorDomain] && (error.code == CKErrorChangeTokenExpired)) {
2892         isChangeTokenExpiredError = true;
2893     } else if([error.domain isEqualToString:CKErrorDomain] && (error.code == CKErrorPartialFailure)) {
2894         NSDictionary* partialErrors = error.userInfo[CKPartialErrorsByItemIDKey];
2895         for(NSError* partialError in partialErrors.allValues) {
2896             if([partialError.domain isEqualToString:CKErrorDomain] && (partialError.code == CKErrorChangeTokenExpired)) {
2897                 isChangeTokenExpiredError = true;
2898             }
2899         }
2900     }
2901
2902     if(isChangeTokenExpiredError) {
2903         ckkserror("ckks", self, "Received notice that our change token is out of date. Resetting local data...");
2904         [self cancelAllOperations];
2905         CKKSResultOperation* resetOp = [self resetLocalData];
2906         CKKSResultOperation* resetHandler = [CKKSResultOperation named:@"local-reset-handler" withBlock:^{
2907             __strong __typeof(self) strongSelf = weakSelf;
2908             if(!strongSelf) {
2909                 ckkserror("ckks", strongSelf, "received callback for released object");
2910                 return;
2911             }
2912
2913             if(resetOp.error) {
2914                 ckksnotice("ckks", strongSelf, "CloudKit-inspired local reset of %@ ended with error: %@", strongSelf.zoneID, error);
2915             } else {
2916                 ckksnotice("ckksreset", strongSelf, "re-initializing zone %@", strongSelf.zoneID);
2917                 [self.initializeScheduler trigger];
2918             }
2919         }];
2920
2921         [resetHandler addDependency:resetOp];
2922         [self scheduleOperation:resetHandler];
2923         return true;
2924     }
2925
2926     bool isDeletedZoneError = false;
2927     if([error.domain isEqualToString:CKErrorDomain] && ((error.code == CKErrorUserDeletedZone) || (error.code == CKErrorZoneNotFound))) {
2928         isDeletedZoneError = true;
2929     } else if([error.domain isEqualToString:CKErrorDomain] && (error.code == CKErrorPartialFailure)) {
2930         NSDictionary* partialErrors = error.userInfo[CKPartialErrorsByItemIDKey];
2931         for(NSError* partialError in partialErrors.allValues) {
2932             if([partialError.domain isEqualToString:CKErrorDomain] && ((partialError.code == CKErrorUserDeletedZone) || (partialError.code == CKErrorZoneNotFound))) {
2933                 isDeletedZoneError = true;
2934             }
2935         }
2936     }
2937
2938     if(isDeletedZoneError) {
2939         ckkserror("ckks", self, "Received notice that our zone does not exist. Resetting all data.");
2940         [self cancelAllOperations];
2941         CKKSResultOperation* resetOp = [self resetCloudKitZone];
2942         CKKSResultOperation* resetHandler = [CKKSResultOperation named:@"reset-handler" withBlock:^{
2943             __strong __typeof(self) strongSelf = weakSelf;
2944             if(!strongSelf) {
2945                 ckkserror("ckksreset", strongSelf, "received callback for released object");
2946                 return;
2947             }
2948
2949             if(resetOp.error) {
2950                 ckksnotice("ckksreset", strongSelf, "CloudKit-inspired zone reset of %@ ended with error: %@", strongSelf.zoneID, resetOp.error);
2951             }
2952         }];
2953
2954         [resetHandler addDependency:resetOp];
2955         [self scheduleOperation:resetHandler];
2956         return true;
2957     }
2958
2959     if([error.domain isEqualToString:CKErrorDomain] && (error.code == CKErrorBadContainer)) {
2960         ckkserror("ckks", self, "Received notice that our container does not exist. Nothing to do.");
2961         return true;
2962     }
2963
2964     return false;
2965 }
2966
2967 #pragma mark CKKSPeerUpdateListener
2968
2969 - (void)selfPeerChanged {
2970     // Currently, we have no idea what to do with this. Kick off a key reprocess?
2971     ckkserror("ckks", self, "Received update that our self identity has changed");
2972     [self keyStateMachineRequestProcess];
2973 }
2974
2975 - (void)trustedPeerSetChanged {
2976     // We might need to share the TLK to some new people, or we might now trust the TLKs we have.
2977     // The key state machine should handle that, so poke it.
2978     ckkserror("ckks", self, "Received update that the trust set has changed");
2979     [self keyStateMachineRequestProcess];
2980 }
2981
2982 #pragma mark - Test Support
2983
2984 - (bool) outgoingQueueEmpty: (NSError * __autoreleasing *) error {
2985     __block bool ret = false;
2986     [self dispatchSync: ^bool{
2987         NSArray* queueEntries = [CKKSOutgoingQueueEntry all: error];
2988         ret = queueEntries && ([queueEntries count] == 0);
2989         return true;
2990     }];
2991
2992     return ret;
2993 }
2994
2995 - (CKKSResultOperation*)waitForFetchAndIncomingQueueProcessing {
2996     CKKSResultOperation* op = [self fetchAndProcessCKChanges:CKKSFetchBecauseTesting];
2997     [op waitUntilFinished];
2998     return op;
2999 }
3000
3001 - (void)waitForKeyHierarchyReadiness {
3002     if(self.keyStateReadyDependency) {
3003         [self.keyStateReadyDependency waitUntilFinished];
3004     }
3005 }
3006
3007 - (void)cancelAllOperations {
3008     [self.zoneSetupOperation cancel];
3009     [self.keyStateMachineOperation cancel];
3010     [self.keyStateReadyDependency cancel];
3011     [self.zoneChangeFetcher cancel];
3012     [self.notifyViewChangedScheduler cancel];
3013
3014     for(NSOperation* op in self.outgoingQueueOperations) {
3015         [op cancel];
3016     }
3017     [self.outgoingQueueOperations removeAllObjects];
3018
3019     for(NSOperation* op in self.incomingQueueOperations) {
3020         [op cancel];
3021     }
3022     [self.incomingQueueOperations removeAllObjects];
3023
3024     [super cancelAllOperations];
3025
3026     [self dispatchSync:^bool{
3027         [self _onqueueAdvanceKeyStateMachineToState: SecCKKSZoneKeyStateCancelled withError: nil];
3028         return true;
3029     }];
3030 }
3031
3032 - (void)halt {
3033     [super halt];
3034
3035     // Don't send any more notifications, either
3036     _notifierClass = nil;
3037 }
3038
3039 - (NSDictionary*)status {
3040 #define stringify(obj) CKKSNilToNSNull([obj description])
3041 #define boolstr(obj) (!!(obj) ? @"yes" : @"no")
3042     __block NSDictionary* ret = nil;
3043     __block NSError* error = nil;
3044     CKKSManifest* manifest = [CKKSManifest latestTrustedManifestForZone:self.zoneName error:&error];
3045     [self dispatchSync: ^bool {
3046
3047         NSString* uuidTLK    = [CKKSKey currentKeyForClass:SecCKKSKeyClassTLK zoneID:self.zoneID error:&error].uuid;
3048         NSString* uuidClassA = [CKKSKey currentKeyForClass:SecCKKSKeyClassA   zoneID:self.zoneID error:&error].uuid;
3049         NSString* uuidClassC = [CKKSKey currentKeyForClass:SecCKKSKeyClassC   zoneID:self.zoneID error:&error].uuid;
3050
3051         NSString* manifestGeneration = manifest ? [NSString stringWithFormat:@"%lu", (unsigned long)manifest.generationCount] : nil;
3052
3053         if(error) {
3054             ckkserror("ckks", self, "error during status: %@", error);
3055         }
3056         // We actually don't care about this error, especially if it's "no current key pointers"...
3057         error = nil;
3058
3059         // Map deviceStates to strings to avoid NSXPC issues. Obj-c, why is this so hard?
3060         NSArray* deviceStates = [CKKSDeviceStateEntry allInZone:self.zoneID error:&error];
3061         NSMutableArray<NSString*>* mutDeviceStates = [[NSMutableArray alloc] init];
3062         [deviceStates enumerateObjectsUsingBlock:^(id  _Nonnull obj, NSUInteger idx, BOOL * _Nonnull stop) {
3063             [mutDeviceStates addObject: [obj description]];
3064         }];
3065
3066         NSArray* tlkShares = [CKKSTLKShare allForUUID:uuidTLK zoneID:self.zoneID error:&error];
3067         NSMutableArray<NSString*>* mutTLKShares = [[NSMutableArray alloc] init];
3068         [tlkShares enumerateObjectsUsingBlock:^(id  _Nonnull obj, NSUInteger idx, BOOL * _Nonnull stop) {
3069             [mutTLKShares addObject: [obj description]];
3070         }];
3071
3072
3073         ret = @{
3074                  @"view":                CKKSNilToNSNull(self.zoneName),
3075                  @"ckaccountstatus":     self.accountStatus == CKAccountStatusCouldNotDetermine ? @"could not determine" :
3076                                          self.accountStatus == CKAccountStatusAvailable         ? @"logged in" :
3077                                          self.accountStatus == CKAccountStatusRestricted        ? @"restricted" :
3078                                          self.accountStatus == CKAccountStatusNoAccount         ? @"logged out" : @"unknown",
3079                  @"lockstatetracker":    stringify(self.lockStateTracker),
3080                  @"accounttracker":      stringify(self.accountTracker),
3081                  @"fetcher":             stringify(self.zoneChangeFetcher),
3082                  @"setup":               boolstr([self.viewSetupOperation isFinished]),
3083                  @"zoneCreated":         boolstr(self.zoneCreated),
3084                  @"zoneCreatedError":    stringify(self.zoneCreatedError),
3085                  @"zoneSubscribed":      boolstr(self.zoneSubscribed),
3086                  @"zoneSubscribedError": stringify(self.zoneSubscribedError),
3087                  @"zoneInitializeScheduler": stringify(self.initializeScheduler),
3088                  @"keystate":            CKKSNilToNSNull(self.keyHierarchyState),
3089                  @"keyStateError":       stringify(self.keyHierarchyError),
3090                  @"statusError":         stringify(error),
3091                  @"oqe":                 CKKSNilToNSNull([CKKSOutgoingQueueEntry countsByState:self.zoneID error:&error]),
3092                  @"iqe":                 CKKSNilToNSNull([CKKSIncomingQueueEntry countsByState:self.zoneID error:&error]),
3093                  @"ckmirror":            CKKSNilToNSNull([CKKSMirrorEntry        countsByParentKey:self.zoneID error:&error]),
3094                  @"devicestates":        CKKSNilToNSNull(mutDeviceStates),
3095                  @"tlkshares":           CKKSNilToNSNull(mutTLKShares),
3096                  @"keys":                CKKSNilToNSNull([CKKSKey countsByClass:self.zoneID error:&error]),
3097                  @"currentTLK":          CKKSNilToNSNull(uuidTLK),
3098                  @"currentClassA":       CKKSNilToNSNull(uuidClassA),
3099                  @"currentClassC":       CKKSNilToNSNull(uuidClassC),
3100                  @"currentManifestGen":   CKKSNilToNSNull(manifestGeneration),
3101
3102
3103                  @"zoneSetupOperation":                 stringify(self.zoneSetupOperation),
3104                  @"viewSetupOperation":                 stringify(self.viewSetupOperation),
3105                  @"keyStateOperation":                  stringify(self.keyStateMachineOperation),
3106                  @"lastIncomingQueueOperation":         stringify(self.lastIncomingQueueOperation),
3107                  @"lastNewTLKOperation":                stringify(self.lastNewTLKOperation),
3108                  @"lastOutgoingQueueOperation":         stringify(self.lastOutgoingQueueOperation),
3109                  @"lastRecordZoneChangesOperation":     stringify(self.lastRecordZoneChangesOperation),
3110                  @"lastProcessReceivedKeysOperation":   stringify(self.lastProcessReceivedKeysOperation),
3111                  @"lastReencryptOutgoingItemsOperation":stringify(self.lastReencryptOutgoingItemsOperation),
3112                  @"lastScanLocalItemsOperation":        stringify(self.lastScanLocalItemsOperation),
3113                  };
3114         return false;
3115     }];
3116     return ret;
3117 }
3118
3119
3120
3121 #endif /* OCTAGON */
3122 @end