keychain/ckks/CKKSKeychainView.m

   1 /*
   2  * Copyright (c) 2016 Apple Inc. All Rights Reserved.
   3  *
   4  * @APPLE_LICENSE_HEADER_START@
   5  *
   6  * This file contains Original Code and/or Modifications of Original Code
   7  * as defined in and that are subject to the Apple Public Source License
   8  * Version 2.0 (the 'License'). You may not use this file except in
   9  * compliance with the License. Please obtain a copy of the License at
  10  * http://www.opensource.apple.com/apsl/ and read it before using this
  11  * file.
  12  *
  13  * The Original Code and all software distributed under the License are
  14  * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  15  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  16  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
  17  * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
  18  * Please see the License for the specific language governing rights and
  19  * limitations under the License.
  20  *
  21  * @APPLE_LICENSE_HEADER_END@
  22  */
  23
  24 #import "CKKSKeychainView.h"
  25
  26
  27
  28 #if OCTAGON
  29 #import "CloudKitDependencies.h"
  30 #import <CloudKit/CloudKit.h>
  31 #import <CloudKit/CloudKit_Private.h>
  32 #endif
  33
  34 #import "CKKS.h"
  35 #import "OctagonAPSReceiver.h"
  36 #import "CKKSIncomingQueueEntry.h"
  37 #import "CKKSOutgoingQueueEntry.h"
  38 #import "CKKSCurrentKeyPointer.h"
  39 #import "CKKSKey.h"
  40 #import "CKKSMirrorEntry.h"
  41 #import "CKKSZoneStateEntry.h"
  42 #import "CKKSItemEncrypter.h"
  43 #import "CKKSIncomingQueueOperation.h"
  44 #import "CKKSNewTLKOperation.h"
  45 #import "CKKSProcessReceivedKeysOperation.h"
  46 #import "CKKSZone.h"
  47 #import "CKKSFetchAllRecordZoneChangesOperation.h"
  48 #import "CKKSHealKeyHierarchyOperation.h"
  49 #import "CKKSReencryptOutgoingItemsOperation.h"
  50 #import "CKKSScanLocalItemsOperation.h"
  51 #import "CKKSSynchronizeOperation.h"
  52 #import "CKKSRateLimiter.h"
  53 #import "CKKSManifest.h"
  54 #import "CKKSManifestLeafRecord.h"
  55 #import "CKKSZoneChangeFetcher.h"
  56 #import "CKKSAnalytics.h"
  57 #import "keychain/analytics/CKKSLaunchSequence.h"
  58 #import "keychain/ckks/CKKSCloudKitClassDependencies.h"
  59 #import "keychain/ckks/CKKSDeviceStateEntry.h"
  60 #import "keychain/ckks/CKKSNearFutureScheduler.h"
  61 #import "keychain/ckks/CKKSCurrentItemPointer.h"
  62 #import "keychain/ckks/CKKSUpdateCurrentItemPointerOperation.h"
  63 #import "keychain/ckks/CKKSUpdateDeviceStateOperation.h"
  64 #import "keychain/ckks/CKKSNotifier.h"
  65 #import "keychain/ckks/CloudKitCategories.h"
  66 #import "keychain/ckks/CKKSTLKShareRecord.h"
  67 #import "keychain/ckks/CKKSHealTLKSharesOperation.h"
  68 #import "keychain/ckks/CKKSLocalSynchronizeOperation.h"
  69 #import "keychain/ckks/CKKSPeerProvider.h"
  70 #import "keychain/categories/NSError+UsefulConstructors.h"
  71
  72 #import "keychain/ot/OTConstants.h"
  73 #import "keychain/ot/OTDefines.h"
  74 #import "keychain/ot/OctagonCKKSPeerAdapter.h"
  75 #import "keychain/ot/ObjCImprovements.h"
  76
  77 #include <utilities/SecCFWrappers.h>
  78 #include <utilities/SecTrace.h>
  79 #include <utilities/SecDb.h>
  80 #include "keychain/securityd/SecDbItem.h"
  81 #include "keychain/securityd/SecItemDb.h"
  82 #include "keychain/securityd/SecItemSchema.h"
  83 #include "keychain/securityd/SecItemServer.h"
  84 #include <utilities/debugging.h>
  85 #include <Security/SecItemPriv.h>
  86 #include "keychain/SecureObjectSync/SOSAccountTransaction.h"
  87 #include <utilities/SecADWrapper.h>
  88 #include <utilities/SecPLWrappers.h>
  89 #include <os/transaction_private.h>
  90
  91 #if OCTAGON
  92
  93 @interface CKKSKeychainView()
  94 @property bool keyStateFetchRequested;
  95 @property bool keyStateProcessRequested;
  96 @property bool trustedPeersSetChanged;
  97
  98 @property bool keyStateCloudKitDeleteRequested;
  99 @property NSHashTable<CKKSResultOperation*>* cloudkitDeleteZoneOperations;
 100
 101 @property bool keyStateLocalResetRequested;
 102 @property NSHashTable<CKKSResultOperation*>* localResetOperations;
 103
 104 @property bool tlkCreationRequested;
 105 @property NSHashTable<CKKSResultOperation<CKKSKeySetProviderOperationProtocol>*>* keysetProviderOperations;
 106
 107
 108 @property (atomic) NSString *activeTLK;
 109
 110 @property (readonly) Class<CKKSNotifier> notifierClass;
 111
 112 // Slows down all outgoing queue operations
 113 @property CKKSNearFutureScheduler* outgoingQueueOperationScheduler;
 114
 115 @property CKKSResultOperation* processIncomingQueueAfterNextUnlockOperation;
 116 @property CKKSResultOperation* resultsOfNextIncomingQueueOperationOperation;
 117
 118 @property NSMutableDictionary<NSString*, SecBoolNSErrorCallback>* pendingSyncCallbacks;
 119
 120 // An extra queue for semaphore-waiting-based NSOperations
 121 @property NSOperationQueue* waitingQueue;
 122
 123 // Scratch space for resyncs
 124 @property (nullable) NSMutableSet<NSString*>* resyncRecordsSeen;
 125
 126 // Make these readwrite
 127 @property NSArray<id<CKKSPeerProvider>>* currentPeerProviders;
 128 @property NSArray<CKKSPeerProviderState*>* currentTrustStates;
 129
 130 @end
 131 #endif
 132
 133 @implementation CKKSKeychainView
 134 #if OCTAGON
 135
 136 - (instancetype)initWithContainer:(CKContainer*)container
 137                          zoneName:(NSString*)zoneName
 138                    accountTracker:(CKKSAccountStateTracker*)accountTracker
 139                  lockStateTracker:(CKKSLockStateTracker*)lockStateTracker
 140               reachabilityTracker:(CKKSReachabilityTracker*)reachabilityTracker
 141                     changeFetcher:(CKKSZoneChangeFetcher*)fetcher
 142                      zoneModifier:(CKKSZoneModifier*)zoneModifier
 143                  savedTLKNotifier:(CKKSNearFutureScheduler*)savedTLKNotifier
 144         cloudKitClassDependencies:(CKKSCloudKitClassDependencies*)cloudKitClassDependencies
 145 {
 146
 147     if(self = [super initWithContainer:container
 148                               zoneName:zoneName
 149                         accountTracker:accountTracker
 150                    reachabilityTracker:reachabilityTracker
 151                           zoneModifier:zoneModifier
 152              cloudKitClassDependencies:cloudKitClassDependencies]) {
 153         WEAKIFY(self);
 154
 155         _loggedIn = [[CKKSCondition alloc] init];
 156         _loggedOut = [[CKKSCondition alloc] init];
 157         _accountStateKnown = [[CKKSCondition alloc] init];
 158
 159         _trustStatus = CKKSAccountStatusUnknown;
 160         _trustDependency = [CKKSResultOperation named:@"wait-for-trust" withBlock:^{}];
 161
 162         _incomingQueueOperations = [NSHashTable weakObjectsHashTable];
 163         _outgoingQueueOperations = [NSHashTable weakObjectsHashTable];
 164         _cloudkitDeleteZoneOperations = [NSHashTable weakObjectsHashTable];
 165         _localResetOperations = [NSHashTable weakObjectsHashTable];
 166         _keysetProviderOperations = [NSHashTable weakObjectsHashTable];
 167
 168         _currentPeerProviders = @[];
 169         _currentTrustStates = @[];
 170
 171         _launch = [[CKKSLaunchSequence alloc] initWithRocketName:@"com.apple.security.ckks.launch"];
 172         [_launch addAttribute:@"view" value:zoneName];
 173
 174         _zoneChangeFetcher = fetcher;
 175         [fetcher registerClient:self];
 176
 177         _resyncRecordsSeen = nil;
 178
 179         _notifierClass = cloudKitClassDependencies.notifierClass;
 180         _notifyViewChangedScheduler = [[CKKSNearFutureScheduler alloc] initWithName:[NSString stringWithFormat: @"%@-notify-scheduler", self.zoneName]
 181                                                             initialDelay:250*NSEC_PER_MSEC
 182                                                          continuingDelay:1*NSEC_PER_SEC
 183                                                         keepProcessAlive:true
 184                                                           dependencyDescriptionCode:CKKSResultDescriptionPendingViewChangedScheduling
 185                                                                    block:^{
 186                                                                        STRONGIFY(self);
 187                                                                        [self.notifierClass post:[NSString stringWithFormat:@"com.apple.security.view-change.%@", self.zoneName]];
 188
 189                                                                        // Ugly, but: the Manatee and Engram views need to send a fake 'PCS' view change.
 190                                                                        // TODO: make this data-driven somehow
 191                                                                        if([self.zoneName isEqualToString:@"Manatee"] ||
 192                                                                           [self.zoneName isEqualToString:@"Engram"] ||
 193                                                                           [self.zoneName isEqualToString:@"ApplePay"] ||
 194                                                                           [self.zoneName isEqualToString:@"Home"] ||
 195                                                                           [self.zoneName isEqualToString:@"LimitedPeersAllowed"]) {
 196                                                                            [self.notifierClass post:@"com.apple.security.view-change.PCS"];
 197                                                                        }
 198                                                                    }];
 199
 200         _notifyViewReadyScheduler = [[CKKSNearFutureScheduler alloc] initWithName:[NSString stringWithFormat: @"%@-ready-scheduler", self.zoneName]
 201                                                                        initialDelay:250*NSEC_PER_MSEC
 202                                                                     continuingDelay:120*NSEC_PER_SEC
 203                                                                    keepProcessAlive:true
 204                                                           dependencyDescriptionCode:CKKSResultDescriptionPendingViewChangedScheduling
 205                                                                               block:^{
 206                                                                                   STRONGIFY(self);
 207                                                                                   NSDistributedNotificationCenter *center = [self.cloudKitClassDependencies.nsdistributednotificationCenterClass defaultCenter];
 208
 209                                                                                   [center postNotificationName:@"com.apple.security.view-become-ready"
 210                                                                                                         object:nil
 211                                                                                                       userInfo:@{ @"view" : self.zoneName ?: @"unknown" }
 212                                                                                                        options:0];
 213                                                                               }];
 214
 215
 216         _pendingSyncCallbacks = [[NSMutableDictionary alloc] init];
 217
 218         _lockStateTracker = lockStateTracker;
 219         _savedTLKNotifier = savedTLKNotifier;
 220
 221         _keyHierarchyConditions = [[NSMutableDictionary alloc] init];
 222         [CKKSZoneKeyStateMap() enumerateKeysAndObjectsUsingBlock:^(CKKSZoneKeyState * _Nonnull key, NSNumber * _Nonnull obj, BOOL * _Nonnull stop) {
 223             [self.keyHierarchyConditions setObject: [[CKKSCondition alloc] init] forKey:key];
 224         }];
 225
 226         // Use the keyHierarchyState setter to modify the zone key state map
 227         self.keyHierarchyState = SecCKKSZoneKeyStateLoggedOut;
 228
 229         _keyHierarchyError = nil;
 230         _keyHierarchyOperationGroup = nil;
 231         _keyStateMachineOperation = nil;
 232         _keyStateFetchRequested = false;
 233         _keyStateProcessRequested = false;
 234         _tlkCreationRequested = false;
 235
 236         _waitingQueue = [[NSOperationQueue alloc] init];
 237         _waitingQueue.maxConcurrentOperationCount = 5;
 238
 239         _keyStateReadyDependency = [self createKeyStateReadyDependency: @"Key state has become ready for the first time." ckoperationGroup:[CKOperationGroup CKKSGroupWithName:@"initial-key-state-ready-scan"]];
 240
 241         _keyStateNonTransientDependency = [self createKeyStateNontransientDependency];
 242
 243         dispatch_time_t initialOutgoingQueueDelay = SecCKKSReduceRateLimiting() ? NSEC_PER_MSEC * 200 : NSEC_PER_SEC * 1;
 244         dispatch_time_t continuingOutgoingQueueDelay = SecCKKSReduceRateLimiting() ? NSEC_PER_MSEC * 200 : NSEC_PER_SEC * 30;
 245         _outgoingQueueOperationScheduler = [[CKKSNearFutureScheduler alloc] initWithName:[NSString stringWithFormat: @"%@-outgoing-queue-scheduler", self.zoneName]
 246                                                                             initialDelay:initialOutgoingQueueDelay
 247                                                                          continuingDelay:continuingOutgoingQueueDelay
 248                                                                         keepProcessAlive:false
 249                                                                dependencyDescriptionCode:CKKSResultDescriptionPendingOutgoingQueueScheduling
 250                                                                                    block:^{}];
 251
 252
 253         dispatch_time_t initialKeyHierachyPokeDelay = SecCKKSReduceRateLimiting() ? NSEC_PER_MSEC * 100 : NSEC_PER_MSEC * 500;
 254         dispatch_time_t continuingKeyHierachyPokeDelay = SecCKKSReduceRateLimiting() ? NSEC_PER_MSEC * 200 : NSEC_PER_SEC * 5;
 255         _pokeKeyStateMachineScheduler = [[CKKSNearFutureScheduler alloc] initWithName:[NSString stringWithFormat: @"%@-reprocess-scheduler", self.zoneName]
 256                                                                          initialDelay:initialKeyHierachyPokeDelay
 257                                                                       continuingDelay:continuingKeyHierachyPokeDelay
 258                                                                      keepProcessAlive:true
 259                                                             dependencyDescriptionCode:CKKSResultDescriptionPendingKeyHierachyPokeScheduling
 260                                                                                      block:^{
 261                                                                                          STRONGIFY(self);
 262                                                                                          [self dispatchSyncWithAccountKeys: ^bool{
 263                                                                                              STRONGIFY(self);
 264
 265                                                                                              [self _onqueueAdvanceKeyStateMachineToState:nil withError:nil];
 266                                                                                              return true;
 267                                                                                          }];
 268                                                                                      }];
 269     }
 270     return self;
 271 }
 272
 273 - (NSString*)description {
 274     return [NSString stringWithFormat:@"<%@: %@ (%@)>", NSStringFromClass([self class]), self.zoneName, self.keyHierarchyState];
 275 }
 276
 277 - (NSString*)debugDescription {
 278     return [NSString stringWithFormat:@"<%@: %@ (%@) %p>", NSStringFromClass([self class]), self.zoneName, self.keyHierarchyState, self];
 279 }
 280
 281 - (CKKSZoneKeyState*)keyHierarchyState {
 282     return _keyHierarchyState;
 283 }
 284
 285 - (void)setKeyHierarchyState:(CKKSZoneKeyState *)keyHierarchyState {
 286     if((keyHierarchyState == nil && _keyHierarchyState == nil) || ([keyHierarchyState isEqualToString:_keyHierarchyState])) {
 287         // No change, do nothing.
 288     } else {
 289         // Fixup the condition variables as part of setting this state
 290         if(_keyHierarchyState) {
 291             self.keyHierarchyConditions[_keyHierarchyState] = [[CKKSCondition alloc] init];
 292         }
 293
 294         _keyHierarchyState = keyHierarchyState;
 295
 296         if(keyHierarchyState) {
 297             [self.keyHierarchyConditions[keyHierarchyState] fulfill];
 298         }
 299     }
 300 }
 301
 302 - (NSString *)lastActiveTLKUUID
 303 {
 304     return self.activeTLK;
 305 }
 306
 307 - (void)_onqueueResetSetup:(CKKSZoneKeyState*)newState resetMessage:(NSString*)resetMessage ckoperationGroup:(CKOperationGroup*)group {
 308     [super resetSetup];
 309
 310     self.keyHierarchyState = newState;
 311     self.keyHierarchyError = nil;
 312
 313     [self.keyStateMachineOperation cancel];
 314     self.keyStateMachineOperation = nil;
 315
 316     self.keyStateFetchRequested = false;
 317     self.keyStateProcessRequested = false;
 318
 319     self.keyHierarchyOperationGroup = group;
 320
 321     [self ensureKeyStateReadyDependency:resetMessage];
 322
 323     NSOperation* oldKSNTD = self.keyStateNonTransientDependency;
 324     self.keyStateNonTransientDependency = [self createKeyStateNontransientDependency];
 325     if(oldKSNTD) {
 326         [oldKSNTD addDependency:self.keyStateNonTransientDependency];
 327         [self.waitingQueue addOperation:oldKSNTD];
 328     }
 329 }
 330
 331 - (void)ensureKeyStateReadyDependency:(NSString*)resetMessage {
 332     NSOperation* oldKSRD = self.keyStateReadyDependency;
 333     self.keyStateReadyDependency = [self createKeyStateReadyDependency:resetMessage ckoperationGroup:self.keyHierarchyOperationGroup];
 334     if(oldKSRD) {
 335         [oldKSRD addDependency:self.keyStateReadyDependency];
 336         [self.waitingQueue addOperation:oldKSRD];
 337     }
 338 }
 339
 340 - (CKKSResultOperation*)createPendingInitializationOperation {
 341
 342     WEAKIFY(self);
 343     CKKSResultOperation* initializationOp = [CKKSGroupOperation named:@"view-initialization" withBlockTakingSelf:^(CKKSGroupOperation * _Nonnull strongOp) {
 344         STRONGIFY(self);
 345
 346         __block CKKSResultOperation* zoneCreationOperation = nil;
 347         [self dispatchSync:^bool {
 348             CKKSZoneStateEntry* ckse = [CKKSZoneStateEntry state: self.zoneName];
 349             zoneCreationOperation = [self handleCKLogin:ckse.ckzonecreated zoneSubscribed:ckse.ckzonesubscribed];
 350             return true;
 351         }];
 352
 353         CKKSResultOperation* viewInitializationOperation = [CKKSResultOperation named:@"view-initialization" withBlockTakingSelf:^(CKKSResultOperation * _Nonnull strongInternalOp) {
 354             STRONGIFY(self);
 355             if(!self) {
 356                 ckkserror("ckks", self, "received callback for released object");
 357                 return;
 358             }
 359
 360             [self dispatchSyncWithAccountKeys: ^bool {
 361                 ckksnotice("ckks", self, "Zone setup progress: %@ %d %@ %d %@",
 362                            [CKKSAccountStateTracker stringFromAccountStatus:self.accountStatus],
 363                            self.zoneCreated, self.zoneCreatedError, self.zoneSubscribed, self.zoneSubscribedError);
 364
 365                 NSError* error = nil;
 366                 CKKSZoneStateEntry* ckse = [CKKSZoneStateEntry state: self.zoneName];
 367                 ckse.ckzonecreated = self.zoneCreated;
 368                 ckse.ckzonesubscribed = self.zoneSubscribed;
 369
 370                 // Although, if the zone subscribed error says there's no zone, mark down that there's no zone
 371                 if(self.zoneSubscribedError &&
 372                    [self.zoneSubscribedError.domain isEqualToString:CKErrorDomain] && self.zoneSubscribedError.code == CKErrorPartialFailure) {
 373                     NSError* subscriptionError = self.zoneSubscribedError.userInfo[CKPartialErrorsByItemIDKey][self.zoneID];
 374                     if(subscriptionError && [subscriptionError.domain isEqualToString:CKErrorDomain] && subscriptionError.code == CKErrorZoneNotFound) {
 375
 376                         ckkserror("ckks", self, "zone subscription error appears to say the zone doesn't exist, fixing status: %@", self.zoneSubscribedError);
 377                         ckse.ckzonecreated = false;
 378                     }
 379                 }
 380
 381                 [ckse saveToDatabase: &error];
 382                 if(error) {
 383                     ckkserror("ckks", self, "couldn't save zone creation status for %@: %@", self.zoneName, error);
 384                 }
 385
 386                 if(!self.zoneCreated || !self.zoneSubscribed) {
 387                     // Go into 'zonecreationfailed'
 388                     strongInternalOp.error = self.zoneCreatedError ? self.zoneCreatedError : self.zoneSubscribedError;
 389                     [self _onqueueAdvanceKeyStateMachineToState:SecCKKSZoneKeyStateZoneCreationFailed withError:strongInternalOp.error];
 390
 391                     return true;
 392                 } else {
 393                     [self _onqueueAdvanceKeyStateMachineToState:SecCKKSZoneKeyStateInitialized withError:nil];
 394                 }
 395
 396                 return true;
 397             }];
 398         }];
 399
 400         [viewInitializationOperation addDependency:zoneCreationOperation];
 401         [strongOp runBeforeGroupFinished:viewInitializationOperation];
 402     }];
 403
 404     return initializationOp;
 405 }
 406
 407 - (void)_onqueuePerformKeyStateInitialized:(CKKSZoneStateEntry*)ckse {
 408     CKKSOutgoingQueueOperation* outgoingOperation = nil;
 409     NSOperation* initialProcess = nil;
 410
 411     // Check if we believe we've synced this zone before.
 412     if(ckse.changeToken == nil) {
 413         self.keyHierarchyOperationGroup = [CKOperationGroup CKKSGroupWithName:@"initial-setup"];
 414
 415         ckksnotice("ckks", self, "No existing change token; going to try to match local items with CloudKit ones.");
 416
 417         // Onboard this keychain: there's likely items in it that we haven't synced yet.
 418         // But, there might be items in The Cloud that correspond to these items, with UUIDs that we don't know yet.
 419         // First, fetch all remote items.
 420         CKKSResultOperation* fetch = [self.zoneChangeFetcher requestSuccessfulFetch:CKKSFetchBecauseInitialStart];
 421         fetch.name = @"initial-fetch";
 422
 423         // Next, try to process them (replacing local entries)
 424         initialProcess = [self processIncomingQueue:true after:fetch];
 425         initialProcess.name = @"initial-process-incoming-queue";
 426
 427         // If all that succeeds, iterate through all keychain items and find the ones which need to be uploaded
 428         self.initialScanOperation = [self scanLocalItems:@"initial-scan-operation"
 429                                         ckoperationGroup:self.keyHierarchyOperationGroup
 430                                                    after:initialProcess];
 431
 432     } else {
 433         // Likely a restart of securityd!
 434
 435         // First off, are there any in-flight queue entries? If so, put them back into New.
 436         // If they're truly in-flight, we'll "conflict" with ourselves, but that should be fine.
 437         NSError* error = nil;
 438         [self _onqueueResetAllInflightOQE:&error];
 439         if(error) {
 440             ckkserror("ckks", self, "Couldn't reset in-flight OQEs, bad behavior ahead: %@", error);
 441         }
 442
 443         // Are there any fixups to run first?
 444         self.lastFixupOperation = [CKKSFixups fixup:ckse.lastFixup for:self];
 445         if(self.lastFixupOperation) {
 446             ckksnotice("ckksfixup", self, "We have a fixup to perform: %@", self.lastFixupOperation);
 447             [self scheduleOperation:self.lastFixupOperation];
 448         }
 449
 450         self.keyHierarchyOperationGroup = [CKOperationGroup CKKSGroupWithName:@"restart-setup"];
 451
 452         if ([CKKSManifest shouldSyncManifests]) {
 453             self.egoManifest = [CKKSEgoManifest tryCurrentEgoManifestForZone:self.zoneName];
 454         }
 455
 456         // If it's been more than 24 hours since the last fetch, fetch and process everything.
 457         // Or, if we think we were interrupted in the middle of fetching, fetch some more.
 458         // Otherwise, just kick off the local queue processing.
 459
 460         NSDate* now = [NSDate date];
 461         NSDateComponents* offset = [[NSDateComponents alloc] init];
 462         [offset setHour:-24];
 463         NSDate* deadline = [[NSCalendar currentCalendar] dateByAddingComponents:offset toDate:now options:0];
 464
 465         if(ckse.lastFetchTime == nil ||
 466            [ckse.lastFetchTime compare: deadline] == NSOrderedAscending ||
 467            ckse.moreRecordsInCloudKit) {
 468             initialProcess = [self fetchAndProcessCKChanges:CKKSFetchBecauseSecuritydRestart after:self.lastFixupOperation];
 469
 470             // Also, kick off a scan local items: it'll find any out-of-sync issues in the local keychain
 471             self.initialScanOperation = [self scanLocalItems:@"24-hr-scan-operation"
 472                                                 ckoperationGroup:self.keyHierarchyOperationGroup
 473                                                        after:initialProcess];
 474         } else {
 475             initialProcess = [self processIncomingQueue:false after:self.lastFixupOperation];
 476         }
 477
 478         if([CKKSManifest shouldSyncManifests]) {
 479             if (!self.egoManifest && !self.initialScanOperation) {
 480                 ckksnotice("ckksmanifest", self, "No ego manifest on restart; rescanning");
 481                 self.initialScanOperation = [self scanLocalItems:@"initial-scan-operation"
 482                                                 ckoperationGroup:self.keyHierarchyOperationGroup
 483                                                            after:initialProcess];
 484             }
 485         }
 486
 487         // Process outgoing queue after re-start
 488         outgoingOperation = [self processOutgoingQueueAfter:self.lastFixupOperation ckoperationGroup:self.keyHierarchyOperationGroup];
 489     }
 490
 491     /*
 492      * Launch time is determined by when the zone have:
 493      *  1. keystate have become ready
 494      *  2. scan local items (if needed)
 495      *  3. processed all outgoing item (if needed)
 496      */
 497
 498     WEAKIFY(self);
 499     NSBlockOperation *seemReady = [NSBlockOperation named:[NSString stringWithFormat:@"seemsReadyForSyncing-%@", self.zoneName] withBlock:^void{
 500         STRONGIFY(self);
 501         NSError *error = nil;
 502         ckksnotice("launch", self, "Launch complete");
 503         NSNumber *zoneSize = [CKKSMirrorEntry counts:self.zoneID error:&error];
 504         if (zoneSize) {
 505             zoneSize = @(SecBucket1Significant([zoneSize longValue]));
 506             [self.launch addAttribute:@"zonesize" value:zoneSize];
 507         }
 508         [self.launch launch];
 509
 510         /*
 511          * Since we think we are ready, signal to CK that its to check for PCS identities again, and create the
 512          * since before we completed this operation, we would probably have failed with a timeout because
 513          * we where busy downloading items from CloudKit and then processing them.
 514          */
 515         [self.notifyViewReadyScheduler trigger];
 516     }];
 517
 518     [seemReady addNullableDependency:self.keyStateReadyDependency];
 519     [seemReady addNullableDependency:outgoingOperation];
 520     [seemReady addNullableDependency:self.initialScanOperation];
 521     [seemReady addNullableDependency:initialProcess];
 522
 523     [self scheduleOperation: seemReady];
 524 }
 525
 526 - (bool)_onqueueResetLocalData: (NSError * __autoreleasing *) error {
 527     dispatch_assert_queue(self.queue);
 528
 529     NSError* localerror = nil;
 530     bool setError = false; // Ugly, but this is the only way to return the first error given
 531
 532     CKKSZoneStateEntry* ckse = [CKKSZoneStateEntry state: self.zoneName];
 533     ckse.ckzonecreated = false;
 534     ckse.ckzonesubscribed = false; // I'm actually not sure about this: can you be subscribed to a non-existent zone?
 535     ckse.changeToken = NULL;
 536     [ckse saveToDatabase: &localerror];
 537     if(localerror) {
 538         ckkserror("ckks", self, "couldn't reset zone status for %@: %@", self.zoneName, localerror);
 539         if(error && !setError) {
 540             *error = localerror; setError = true;
 541         }
 542     }
 543
 544     [CKKSMirrorEntry deleteAll:self.zoneID error: &localerror];
 545     if(localerror) {
 546         ckkserror("ckks", self, "couldn't delete all CKKSMirrorEntry: %@", localerror);
 547         if(error && !setError) {
 548             *error = localerror; setError = true;
 549         }
 550     }
 551
 552     [CKKSOutgoingQueueEntry deleteAll:self.zoneID error: &localerror];
 553     if(localerror) {
 554         ckkserror("ckks", self, "couldn't delete all CKKSOutgoingQueueEntry: %@", localerror);
 555         if(error && !setError) {
 556             *error = localerror; setError = true;
 557         }
 558     }
 559
 560     [CKKSIncomingQueueEntry deleteAll:self.zoneID error: &localerror];
 561     if(localerror) {
 562         ckkserror("ckks", self, "couldn't delete all CKKSIncomingQueueEntry: %@", localerror);
 563         if(error && !setError) {
 564             *error = localerror; setError = true;
 565         }
 566     }
 567
 568     [CKKSKey deleteAll:self.zoneID error: &localerror];
 569     if(localerror) {
 570         ckkserror("ckks", self, "couldn't delete all CKKSKey: %@", localerror);
 571         if(error && !setError) {
 572             *error = localerror; setError = true;
 573         }
 574     }
 575
 576     [CKKSTLKShareRecord deleteAll:self.zoneID error: &localerror];
 577     if(localerror) {
 578         ckkserror("ckks", self, "couldn't delete all CKKSTLKShare: %@", localerror);
 579         if(error && !setError) {
 580             *error = localerror; setError = true;
 581         }
 582     }
 583
 584     [CKKSCurrentKeyPointer deleteAll:self.zoneID error: &localerror];
 585     if(localerror) {
 586         ckkserror("ckks", self, "couldn't delete all CKKSCurrentKeyPointer: %@", localerror);
 587         if(error && !setError) {
 588             *error = localerror; setError = true;
 589         }
 590     }
 591
 592     [CKKSCurrentItemPointer deleteAll:self.zoneID error: &localerror];
 593     if(localerror) {
 594         ckkserror("ckks", self, "couldn't delete all CKKSCurrentItemPointer: %@", localerror);
 595         if(error && !setError) {
 596             *error = localerror; setError = true;
 597         }
 598     }
 599
 600     [CKKSDeviceStateEntry deleteAll:self.zoneID error:&localerror];
 601     if(localerror) {
 602         ckkserror("ckks", self, "couldn't delete all CKKSDeviceStateEntry: %@", localerror);
 603         if(error && !setError) {
 604             *error = localerror; setError = true;
 605         }
 606     }
 607
 608     return (localerror == nil && !setError);
 609 }
 610
 611 - (CKKSResultOperation*)createPendingResetLocalDataOperation {
 612     @synchronized(self.localResetOperations) {
 613         CKKSResultOperation* pendingResetLocalOperation = (CKKSResultOperation*) [self findFirstPendingOperation:self.localResetOperations];
 614         if(!pendingResetLocalOperation) {
 615             WEAKIFY(self);
 616             pendingResetLocalOperation = [CKKSResultOperation named:@"reset-local" withBlockTakingSelf:^(CKKSResultOperation * _Nonnull strongOp) {
 617                 STRONGIFY(self);
 618                 __block NSError* error = nil;
 619
 620                 [self dispatchSync: ^bool{
 621                     [self _onqueueResetLocalData: &error];
 622                     return true;
 623                 }];
 624
 625                 strongOp.error = error;
 626             }];
 627             [pendingResetLocalOperation linearDependencies:self.localResetOperations];
 628         }
 629         return pendingResetLocalOperation;
 630     }
 631 }
 632
 633 - (CKKSResultOperation*)resetLocalData {
 634     // Not overly thread-safe, but a single read is okay
 635     CKKSAccountStatus accountStatus = self.accountStatus;
 636     ckksnotice("ckksreset", self, "Requesting local data reset");
 637
 638     // If we're currently signed in, the reset operation will be handled by the CKKS key state machine, and a reset should end up in 'ready'
 639     if(accountStatus == CKKSAccountStatusAvailable) {
 640         WEAKIFY(self);
 641         CKKSGroupOperation* resetOperationGroup = [CKKSGroupOperation named:@"local-reset" withBlockTakingSelf:^(CKKSGroupOperation *strongOp) {
 642             STRONGIFY(self);
 643
 644             __block CKKSResultOperation* resetOperation = nil;
 645
 646             [self dispatchSyncWithAccountKeys:^bool {
 647                 self.keyStateLocalResetRequested = true;
 648                 resetOperation = [self createPendingResetLocalDataOperation];
 649                 [self _onqueueAdvanceKeyStateMachineToState:nil withError:nil];
 650                 return true;
 651             }];
 652
 653             [strongOp dependOnBeforeGroupFinished:resetOperation];
 654         }];
 655         [self scheduleOperationWithoutDependencies:resetOperationGroup];
 656
 657         CKKSGroupOperation* viewReset = [CKKSGroupOperation named:@"local-data-reset" withBlockTakingSelf:^(CKKSGroupOperation *strongOp) {
 658             STRONGIFY(self);
 659             // Now that the local reset finished, wait for the key hierarchy state machine to churn
 660             ckksnotice("ckksreset", self, "waiting for key hierarchy to become nontransient (after local reset)");
 661             CKKSResultOperation* waitOp = [CKKSResultOperation named:@"waiting-for-local-reset" withBlock:^{}];
 662             [waitOp timeout: 60*NSEC_PER_SEC];
 663             [waitOp addNullableDependency:self.keyStateNonTransientDependency];
 664
 665             [strongOp runBeforeGroupFinished:waitOp];
 666         }];
 667         [viewReset addSuccessDependency:resetOperationGroup];
 668
 669         [self scheduleOperationWithoutDependencies:viewReset];
 670         return viewReset;
 671     } else {
 672         // Since we're logged out, we must run the reset ourselves
 673         WEAKIFY(self);
 674         CKKSResultOperation* pendingResetLocalOperation = [CKKSResultOperation named:@"reset-local"
 675                                                                  withBlockTakingSelf:^(CKKSResultOperation * _Nonnull strongOp) {
 676             STRONGIFY(self);
 677             __block NSError* error = nil;
 678
 679             [self dispatchSync: ^bool{
 680                 [self _onqueueResetLocalData: &error];
 681                 return true;
 682             }];
 683
 684             strongOp.error = error;
 685         }];
 686         [self scheduleOperationWithoutDependencies:pendingResetLocalOperation];
 687         return pendingResetLocalOperation;
 688     }
 689 }
 690
 691 - (CKKSResultOperation*)createPendingDeleteZoneOperation:(CKOperationGroup*)operationGroup {
 692     @synchronized(self.cloudkitDeleteZoneOperations) {
 693         CKKSResultOperation* pendingDeleteOperation = (CKKSResultOperation*) [self findFirstPendingOperation:self.cloudkitDeleteZoneOperations];
 694         if(!pendingDeleteOperation) {
 695             pendingDeleteOperation = [self deleteCloudKitZoneOperation:operationGroup];
 696             [pendingDeleteOperation linearDependencies:self.cloudkitDeleteZoneOperations];
 697         }
 698         return pendingDeleteOperation;
 699     }
 700 }
 701
 702 - (CKKSResultOperation*)resetCloudKitZone:(CKOperationGroup*)operationGroup {
 703     [self.accountStateKnown wait:(SecCKKSTestsEnabled() ? 1*NSEC_PER_SEC : 10*NSEC_PER_SEC)];
 704
 705     // Not overly thread-safe, but a single read is okay
 706     if(self.accountStatus != CKKSAccountStatusAvailable) {
 707         // No CK account? goodbye!
 708         ckksnotice("ckksreset", self, "Requesting reset of CK zone, but no CK account exists");
 709         CKKSResultOperation* errorOp = [CKKSResultOperation named:@"fail" withBlockTakingSelf:^(CKKSResultOperation * _Nonnull op) {
 710             op.error = [NSError errorWithDomain:CKKSErrorDomain
 711                                           code:CKKSNotLoggedIn
 712                                    description:@"User is not signed into iCloud."];
 713         }];
 714
 715         [self scheduleOperationWithoutDependencies:errorOp];
 716         return errorOp;
 717     }
 718
 719     // Actually running the delete operation will be handled by the CKKS key state machine
 720     ckksnotice("ckksreset", self, "Requesting reset of CK zone (logged in)");
 721
 722     __block CKKSResultOperation* deleteOperation = nil;
 723     [self dispatchSyncWithAccountKeys:^bool {
 724         self.keyStateCloudKitDeleteRequested = true;
 725         deleteOperation = [self createPendingDeleteZoneOperation:operationGroup];
 726         [self _onqueueAdvanceKeyStateMachineToState:nil withError:nil];
 727         return true;
 728     }];
 729
 730     WEAKIFY(self);
 731     CKKSGroupOperation* viewReset = [CKKSGroupOperation named:[NSString stringWithFormat:@"cloudkit-view-reset-%@", self.zoneName]
 732                                           withBlockTakingSelf:^(CKKSGroupOperation *strongOp) {
 733         STRONGIFY(self);
 734         // Now that the delete finished, wait for the key hierarchy state machine
 735         ckksnotice("ckksreset", self, "waiting for key hierarchy to become nontransient (after cloudkit reset)");
 736         CKKSResultOperation* waitOp = [CKKSResultOperation named:@"waiting-for-reset" withBlock:^{}];
 737         [waitOp timeout: 60*NSEC_PER_SEC];
 738         [waitOp addNullableDependency:self.keyStateNonTransientDependency];
 739
 740         [strongOp runBeforeGroupFinished:waitOp];
 741     }];
 742
 743     [viewReset timeout:30*NSEC_PER_SEC];
 744     [viewReset addDependency:deleteOperation];
 745     [self.waitingQueue addOperation:viewReset];
 746
 747     return viewReset;
 748 }
 749
 750 - (void)_onqueueKeyStateMachineRequestFetch {
 751     dispatch_assert_queue(self.queue);
 752
 753     // We're going to set this flag, then nudge the key state machine.
 754     // If it was idle, then it should launch a fetch. If there was an active process, this flag will stay high
 755     // and the fetch will be launched later.
 756
 757     self.keyStateFetchRequested = true;
 758     [self _onqueueAdvanceKeyStateMachineToState: nil withError: nil];
 759 }
 760
 761 - (void)keyStateMachineRequestProcess {
 762     // Since bools are atomic, we don't need to get on-queue here
 763     // Just set the flag high and hope
 764     self.keyStateProcessRequested = true;
 765     [self.pokeKeyStateMachineScheduler trigger];
 766 }
 767
 768 - (void)_onqueueKeyStateMachineRequestProcess {
 769     dispatch_assert_queue(self.queue);
 770
 771     // Set the request flag, then nudge the key state machine.
 772     // If it was idle, then it should launch a process. If there was an active process, this flag will stay high
 773     // and the process will be launched later.
 774
 775     self.keyStateProcessRequested = true;
 776     [self _onqueueAdvanceKeyStateMachineToState: nil withError: nil];
 777 }
 778
 779 - (CKKSResultOperation*)createKeyStateReadyDependency:(NSString*)message ckoperationGroup:(CKOperationGroup*)group {
 780     WEAKIFY(self);
 781     CKKSResultOperation* keyStateReadyDependency = [CKKSResultOperation operationWithBlock:^{
 782         STRONGIFY(self);
 783         if(!self) {
 784             return;
 785         }
 786         ckksnotice("ckkskey", self, "%@", message);
 787
 788         [self dispatchSync:^bool {
 789             if(self.droppedItems) {
 790                 // While we weren't in 'ready', keychain modifications might have come in and were dropped on the floor. Find them!
 791                 ckksnotice("ckkskey", self, "Launching scan operation for missed items");
 792                 [self scanLocalItems:@"ready-again-scan" ckoperationGroup:group after:nil];
 793             }
 794             return true;
 795         }];
 796     }];
 797     keyStateReadyDependency.name = [NSString stringWithFormat: @"%@-key-state-ready", self.zoneName];
 798     keyStateReadyDependency.descriptionErrorCode = CKKSResultDescriptionPendingKeyReady;
 799     return keyStateReadyDependency;
 800 }
 801
 802 - (CKKSResultOperation*)createKeyStateNontransientDependency {
 803     WEAKIFY(self);
 804     return [CKKSResultOperation named:[NSString stringWithFormat: @"%@-key-state-nontransient", self.zoneName] withBlock:^{
 805         STRONGIFY(self);
 806         ckksnotice("ckkskey", self, "Key state is now non-transient");
 807     }];
 808 }
 809
 810 // The operations suggested by this state machine should call _onqueueAdvanceKeyStateMachineToState once they are complete.
 811 // At no other time should keyHierarchyState be modified.
 812
 813 // Note that this function cannot rely on doing any database work; it might get rolled back, especially in an error state
 814 - (void)_onqueueAdvanceKeyStateMachineToState: (CKKSZoneKeyState*) state withError: (NSError*) error {
 815     dispatch_assert_queue(self.queue);
 816     WEAKIFY(self);
 817
 818     // Resetting back to 'loggedout' takes all precedence.
 819     if([state isEqual:SecCKKSZoneKeyStateLoggedOut]) {
 820         ckksnotice("ckkskey", self, "Resetting the key hierarchy state machine back to '%@'", state);
 821
 822         [self _onqueueResetSetup:SecCKKSZoneKeyStateLoggedOut
 823                     resetMessage:@"Key state has become ready for the first time (after reset)."
 824                 ckoperationGroup:[CKOperationGroup CKKSGroupWithName:@"key-state-after-logout"]];
 825
 826         [self _onqueueHandleKeyStateNonTransientDependency:nil];
 827         self.launch = nil;
 828         return;
 829     }
 830
 831     [self.launch addEvent:state];
 832
 833     // Resetting back to 'initialized' also takes precedence
 834     if([state isEqual:SecCKKSZoneKeyStateInitializing]) {
 835         ckksnotice("ckkskey", self, "Resetting the key hierarchy state machine back to '%@'", state);
 836
 837         [self _onqueueResetSetup:SecCKKSZoneKeyStateInitializing
 838                     resetMessage:@"Key state has become ready for the first time (after re-initializing)."
 839                 ckoperationGroup:[CKOperationGroup CKKSGroupWithName:@"key-state-reset-to-initializing"]];
 840
 841         // Begin initialization, but rate-limit it
 842         self.keyStateMachineOperation = [self createPendingInitializationOperation];
 843         [self.keyStateMachineOperation addNullableDependency:self.zoneModifier.cloudkitRetryAfter.operationDependency];
 844         [self.zoneModifier.cloudkitRetryAfter trigger];
 845         [self scheduleOperation:self.keyStateMachineOperation];
 846
 847         [self _onqueueHandleKeyStateNonTransientDependency:nil];
 848         return;
 849     }
 850
 851     // Resetting to 'waitfortrust' also takes precedence
 852     if([state isEqualToString:SecCKKSZoneKeyStateWaitForTrust]) {
 853         if([self.keyHierarchyState isEqualToString:SecCKKSZoneKeyStateLoggedOut]) {
 854             ckksnotice("ckks", self, "Asked to waitfortrust, but we're already in loggedout. Ignoring...");
 855             return;
 856         }
 857
 858         ckksnotice("ckks", self, "Entering waitfortrust");
 859         self.keyHierarchyState = SecCKKSZoneKeyStateWaitForTrust;
 860         self.keyHierarchyError = nil;
 861         self.keyStateMachineOperation = nil;
 862
 863         [self ensureKeyStateReadyDependency:@"Key state has become ready for the first time (after lacking trust)."];
 864
 865         if(self.trustStatus == CKKSAccountStatusAvailable) {
 866             // Note: we go to initialized here, since to enter waitfortrust CKKS has already gone through initializing
 867             // initialized should refetch only if needed.
 868             ckksnotice("ckks", self, "CKKS is trusted, moving to initialized");
 869             self.keyStateMachineOperation = [self operationToEnterState:SecCKKSZoneKeyStateInitialized
 870                                                           keyStateError:nil
 871                                                                   named:@"re-enter initialized"];
 872             [self scheduleOperation:self.keyStateMachineOperation];
 873         }
 874
 875         // In wait for trust, we might have a keyset. Who knows!
 876         CKKSCurrentKeySet* keyset = [CKKSCurrentKeySet loadForZone:self.zoneID];
 877         [self _onqueueHandleKeyStateNonTransientDependency:keyset];
 878
 879         return;
 880     }
 881
 882     // Cancels and error states take precedence
 883     if([self.keyHierarchyState isEqualToString: SecCKKSZoneKeyStateError] ||
 884        [self.keyHierarchyState isEqualToString: SecCKKSZoneKeyStateCancelled] ||
 885        self.keyHierarchyError != nil) {
 886         // Error state: nowhere to go. Early-exit.
 887         ckkserror("ckkskey", self, "Asked to advance state machine from non-exit state %@ (to %@): %@", self.keyHierarchyState, state, self.keyHierarchyError);
 888         return;
 889     }
 890
 891     if([state isEqual: SecCKKSZoneKeyStateError]) {
 892         // But wait! Is this a "we're locked" error?
 893         if(error && [self.lockStateTracker isLockedError:error]) {
 894             ckkserror("ckkskey", self, "advised of 'keychain locked' error, ignoring: coming from state (%@): %@", self.keyHierarchyState, error);
 895             // After the next unlock, fake that we received the last zone transition
 896             CKKSZoneKeyState* lastState = self.keyHierarchyState;
 897             self.keyStateMachineOperation = [NSBlockOperation named:@"key-state-after-unlock" withBlock:^{
 898                 STRONGIFY(self);
 899                 if(!self) {
 900                     return;
 901                 }
 902                 [self dispatchSyncWithAccountKeys:^bool{
 903                     [self _onqueueAdvanceKeyStateMachineToState:lastState withError:nil];
 904                     return true;
 905                 }];
 906             }];
 907             state = nil;
 908
 909             self.keyHierarchyState = SecCKKSZoneKeyStateWaitForUnlock;
 910
 911             [self.keyStateMachineOperation addNullableDependency:self.lockStateTracker.unlockDependency];
 912             [self scheduleOperation:self.keyStateMachineOperation];
 913
 914             [self _onqueueHandleKeyStateNonTransientDependency:nil];
 915             return;
 916
 917         } else {
 918             // Error state: record the error and exit early
 919             ckkserror("ckkskey", self, "advised of error: coming from state (%@): %@", self.keyHierarchyState, error);
 920
 921             [[CKKSAnalytics logger] logUnrecoverableError:error
 922                                                  forEvent:CKKSEventStateError
 923                                                    inView:self
 924                                            withAttributes:@{ @"previousKeyHierarchyState" : self.keyHierarchyState }];
 925
 926
 927             self.keyHierarchyState = SecCKKSZoneKeyStateError;
 928             self.keyHierarchyError = error;
 929
 930             [self _onqueueHandleKeyStateNonTransientDependency:nil];
 931             return;
 932         }
 933     }
 934
 935     if([state isEqual: SecCKKSZoneKeyStateCancelled]) {
 936         ckkserror("ckkskey", self, "advised of cancel: coming from state (%@): %@", self.keyHierarchyState, error);
 937         self.keyHierarchyState = SecCKKSZoneKeyStateCancelled;
 938         self.keyHierarchyError = error;
 939
 940         // Cancel the key ready dependency. Strictly Speaking, this will cause errors down the line, but we're in a cancel state: those operations should be canceled anyway.
 941         self.keyHierarchyOperationGroup = nil;
 942         [self.keyStateReadyDependency cancel];
 943         self.keyStateReadyDependency = nil;
 944
 945         [self.keyStateNonTransientDependency cancel];
 946         self.keyStateNonTransientDependency = nil;
 947         return;
 948     }
 949
 950     // Now that the current or new state isn't an error or a cancel, proceed.
 951     if(self.keyStateMachineOperation && ![self.keyStateMachineOperation isFinished]) {
 952         if(state == nil) {
 953             // we started this operation to move the state machine. Since you aren't asking for a state transition, and there's an active operation, no need to do anything
 954             ckksnotice("ckkskey", self, "Not advancing state machine: waiting for %@", self.keyStateMachineOperation);
 955             return;
 956         }
 957     }
 958
 959     if(state) {
 960         ckksnotice("ckkskey", self, "Preparing to advance key hierarchy state machine from %@ to %@", self.keyHierarchyState, state);
 961         self.keyStateMachineOperation = nil;
 962     } else {
 963         ckksnotice("ckkskey", self, "Key hierarchy state machine is being poked; currently %@", self.keyHierarchyState);
 964         state = self.keyHierarchyState;
 965     }
 966
 967 #if DEBUG
 968     // During testing, keep the developer honest: this function should always have the self identities, unless the account has lost trust
 969     if(self.trustStatus == CKKSAccountStatusAvailable && ![state isEqualToString:SecCKKSZoneKeyStateLoggedOut]) {
 970         bool hasSelfIdentities = false;
 971         NSAssert(self.currentTrustStates.count > 0, @"Should have at least one trust state");
 972         for(CKKSPeerProviderState* state in self.currentTrustStates) {
 973             if(state.currentSelfPeersError == nil || state.currentSelfPeersError.code != CKKSNoPeersAvailable) {
 974                 hasSelfIdentities = true;
 975             }
 976         }
 977
 978         NSAssert(hasSelfIdentities, @"Must have viable (or errored) self peers to advance key state");
 979     }
 980 #endif
 981
 982     // Do any of these state transitions below want to change which state we're in?
 983     CKKSZoneKeyState* nextState = nil;
 984     NSError* nextError = nil;
 985
 986     // Any state that wants should fill this in; it'll be used at the end of this function as well
 987     CKKSCurrentKeySet* keyset = nil;
 988
 989 #if !defined(NDEBUG)
 990     {
 991         NSError* localerror = nil;
 992         NSError* allKeysError = nil;
 993         NSArray<CKKSKey*>* allKeys = [CKKSKey allKeys:self.zoneID error:&allKeysError];
 994
 995         if(localerror) {
 996             ckkserror("ckkskey", self, "couldn't fetch all keys from local database, entering error state: %@", allKeysError);
 997         }
 998         ckksdebug("ckkskey", self, "All keys: %@", allKeys);
 999     }
1000 #endif
1001
1002     NSError* hierarchyError = nil;
1003
1004     if(self.keyStateCloudKitDeleteRequested || [state isEqualToString:SecCKKSZoneKeyStateResettingZone]) {
1005         // CloudKit reset requests take precedence over all other state transitions
1006         ckksnotice("ckkskey", self, "Deleting the CloudKit Zone");
1007         CKKSGroupOperation* op = [[CKKSGroupOperation alloc] init];
1008
1009         CKKSResultOperation* deleteOp = [self createPendingDeleteZoneOperation:self.keyHierarchyOperationGroup];
1010         [op runBeforeGroupFinished: deleteOp];
1011
1012         NSOperation* nextStateOp = [CKKSResultOperation named:@"inspect-zone-delete" withBlockTakingSelf:^(CKKSResultOperation * _Nonnull op) {
1013             STRONGIFY(self);
1014             [self dispatchSyncWithAccountKeys:^bool {
1015                 // Did the delete op succeed?
1016                 if(deleteOp.error == nil) {
1017                     ckksnotice("ckkskey", self, "Zone deletion operation complete! Proceeding to reset local data");
1018                     [self _onqueueAdvanceKeyStateMachineToState:SecCKKSZoneKeyStateResettingLocalData withError:nil];
1019                     return true;
1020                 }
1021
1022                 ckksnotice("ckkskey", self, "Zone deletion operation failed, will retry: %@", deleteOp.error);
1023                 [self _onqueueAdvanceKeyStateMachineToState:SecCKKSZoneKeyStateResettingZone withError:nil];
1024
1025                 return true;
1026             }];
1027         }];
1028
1029         [nextStateOp addDependency:deleteOp];
1030         [op runBeforeGroupFinished:nextStateOp];
1031
1032         self.keyStateMachineOperation = op;
1033         self.keyStateCloudKitDeleteRequested = false;
1034
1035         // Also, pending operations should be cancelled
1036         [self cancelPendingOperations];
1037
1038     } else if(self.keyStateLocalResetRequested || [state isEqualToString:SecCKKSZoneKeyStateResettingLocalData]) {
1039         // Local reset requests take precedence over all other state transitions
1040         ckksnotice("ckkskey", self, "Resetting local data");
1041         CKKSGroupOperation* op = [[CKKSGroupOperation alloc] init];
1042
1043         CKKSResultOperation* resetOp = [self createPendingResetLocalDataOperation];
1044         [op runBeforeGroupFinished: resetOp];
1045
1046         NSOperation* nextStateOp = [self operationToEnterState:SecCKKSZoneKeyStateInitializing keyStateError:nil named:@"state-resetting-initialize"];
1047         [nextStateOp addDependency:resetOp];
1048         [op runBeforeGroupFinished:nextStateOp];
1049
1050         self.keyStateMachineOperation = op;
1051         self.keyStateLocalResetRequested = false;
1052
1053     } else if([state isEqualToString:SecCKKSZoneKeyStateZoneCreationFailed]) {
1054         //Prepare to go back into initializing, as soon as the cloudkitRetryAfter is happy
1055         self.keyStateMachineOperation = [self operationToEnterState:SecCKKSZoneKeyStateInitializing keyStateError:nil named:@"recover-from-cloudkit-failure"];
1056         [self.keyStateMachineOperation addNullableDependency:self.zoneModifier.cloudkitRetryAfter.operationDependency];
1057         [self.zoneModifier.cloudkitRetryAfter trigger];
1058
1059     } else if([state isEqualToString:SecCKKSZoneKeyStateWaitForTrust]) {
1060         // Actually entering this state should have been handled above, so let's check if we can exit it here...
1061         if(self.trustStatus == CKKSAccountStatusAvailable) {
1062             ckksnotice("ckkskey", self, "Beginning trusted state machine operation");
1063             nextState = SecCKKSZoneKeyStateInitialized;
1064
1065         } else if (self.tlkCreationRequested) {
1066             ckksnotice("ckkskey", self, "No trust, but TLK creation is requested. Moving to fetchcomplete.");
1067             nextState = SecCKKSZoneKeyStateFetchComplete;
1068
1069         } else {
1070             ckksnotice("ckkskey", self, "Remaining in 'waitfortrust'");
1071         }
1072
1073     } else if([state isEqualToString: SecCKKSZoneKeyStateReady]) {
1074         NSError* localerror = nil;
1075         NSArray<CKKSKey*>* remoteKeys = [CKKSKey remoteKeys:self.zoneID error: &localerror];
1076
1077         if(remoteKeys == nil || localerror) {
1078             ckkserror("ckkskey", self, "couldn't fetch keys from local database, entering error state: %@", localerror);
1079             self.keyHierarchyState = SecCKKSZoneKeyStateError;
1080             self.keyHierarchyError = localerror;
1081             [self _onqueueHandleKeyStateNonTransientDependency:nil];
1082             return;
1083         }
1084
1085         if(self.keyStateProcessRequested || [remoteKeys count] > 0) {
1086             // We've either received some remote keys from the last fetch, or someone has requested a reprocess.
1087             ckksnotice("ckkskey", self, "Kicking off a key reprocess based on request:%d and remote key count %lu", self.keyStateProcessRequested, (unsigned long)[remoteKeys count]);
1088             nextState = SecCKKSZoneKeyStateProcess;
1089
1090         } else if(self.keyStateFullRefetchRequested) {
1091             // In ready, but someone has requested a full fetch. Kick it off.
1092             ckksnotice("ckkskey", self, "Kicking off a full key refetch based on request:%d", self.keyStateFullRefetchRequested);
1093             nextState = SecCKKSZoneKeyStateNeedFullRefetch;
1094
1095         } else if(self.keyStateFetchRequested) {
1096             // In ready, but someone has requested a fetch. Kick it off.
1097             ckksnotice("ckkskey", self, "Kicking off a key refetch based on request:%d", self.keyStateFetchRequested);
1098             nextState = SecCKKSZoneKeyStateFetch; // Don't go to 'ready', go to 'initialized', since we want to fetch again
1099         } else if (self.trustStatus != CKKSAccountStatusAvailable) {
1100             ckksnotice("ckkskey", self, "Asked to go into ready, but there's no trust; going into waitfortrust");
1101             nextState = SecCKKSZoneKeyStateWaitForTrust;
1102         } else if (self.trustedPeersSetChanged) {
1103             ckksnotice("ckkskey", self, "Received a nudge that the trusted peers set might have changed! Reprocessing.");
1104             nextState = SecCKKSZoneKeyStateProcess;
1105             self.trustedPeersSetChanged = false;
1106         }
1107
1108         // TODO: kick off a key roll if one has been requested
1109
1110         if(!self.keyStateMachineOperation && !nextState) {
1111             // We think we're ready. Double check.
1112             keyset = [CKKSCurrentKeySet loadForZone:self.zoneID];
1113             CKKSZoneKeyState* checkedstate = [self _onqueueEnsureKeyHierarchyHealth:keyset error:&hierarchyError];
1114             if(![checkedstate isEqualToString:SecCKKSZoneKeyStateReady] || hierarchyError) {
1115                 // Things is bad. Kick off a heal to fix things up.
1116                 ckksnotice("ckkskey", self, "Thought we were ready, but the key hierarchy is %@: %@", checkedstate, hierarchyError);
1117                 nextState = checkedstate;
1118                 if([nextState isEqualToString:SecCKKSZoneKeyStateError]) {
1119                     nextError = hierarchyError;
1120                 }
1121             }
1122         }
1123
1124     } else if([state isEqualToString: SecCKKSZoneKeyStateInitialized]) {
1125         // We're initialized and CloudKit is ready. If we're trusted, see what needs done. Otherwise, wait.
1126
1127         // Note: we might be still 'untrusted' at this point. The state machine is responsible for not entering 'ready' until
1128         // we are trusted.
1129         // This is acceptable only if the key state machine does not make new TLKs without being trusted!
1130
1131         // Set this state, for test use
1132         self.keyHierarchyState = SecCKKSZoneKeyStateInitialized;
1133
1134         CKKSZoneStateEntry* ckse = [CKKSZoneStateEntry state:self.zoneName];
1135         [self _onqueuePerformKeyStateInitialized:ckse];
1136
1137         // We need to either:
1138         //  Wait for the fixup operation to occur
1139         //  Go into 'ready'
1140         //  Or start a key state fetch
1141         if(self.lastFixupOperation && ![self.lastFixupOperation isFinished]) {
1142             nextState = SecCKKSZoneKeyStateWaitForFixupOperation;
1143         } else {
1144             // Check if we have an existing key hierarchy in keyset
1145             keyset = [CKKSCurrentKeySet loadForZone:self.zoneID];
1146             if(keyset.error && !([keyset.error.domain isEqual: @"securityd"] && keyset.error.code == errSecItemNotFound)) {
1147                 ckkserror("ckkskey", self, "Error examining existing key hierarchy: %@", error);
1148             }
1149
1150             if(keyset.tlk && keyset.classA && keyset.classC && !keyset.error) {
1151                 // This is likely a restart of securityd, and we think we're ready. Double check.
1152
1153                 CKKSZoneKeyState* checkedstate = [self _onqueueEnsureKeyHierarchyHealth:keyset error:&hierarchyError];
1154                 if([checkedstate isEqualToString:SecCKKSZoneKeyStateReady] && !hierarchyError) {
1155                     ckksnotice("ckkskey", self, "Already have existing key hierarchy for %@; using it.", self.zoneID.zoneName);
1156                 } else {
1157                     ckksnotice("ckkskey", self, "Initial scan shows key hierarchy is %@: %@", checkedstate, hierarchyError);
1158                 }
1159                 nextState = checkedstate;
1160
1161             } else {
1162                 // We have no local key hierarchy. One might exist in CloudKit, or it might not.
1163                 ckksnotice("ckkskey", self, "No existing key hierarchy for %@. Check if there's one in CloudKit...", self.zoneID.zoneName);
1164                 nextState = SecCKKSZoneKeyStateFetch;
1165             }
1166         }
1167
1168     } else if([state isEqualToString:SecCKKSZoneKeyStateFetch]) {
1169         ckksnotice("ckkskey", self, "Starting a key hierarchy fetch");
1170         [self _onqueueKeyHierarchyFetch];
1171
1172     } else if([state isEqualToString: SecCKKSZoneKeyStateNeedFullRefetch]) {
1173         ckksnotice("ckkskey", self, "Starting a key hierarchy full refetch");
1174         [self _onqueueKeyHierarchyFetchForReasons:[NSSet setWithObjects:CKKSFetchBecauseKeyHierarchy, CKKSFetchBecauseResync, nil]];
1175         self.keyStateMachineRefetched = true;
1176         self.keyStateFullRefetchRequested = false;
1177
1178     } else if([state isEqualToString:SecCKKSZoneKeyStateWaitForFixupOperation]) {
1179         // We should enter 'initialized' when the fixup operation completes
1180         ckksnotice("ckkskey", self, "Waiting for the fixup operation: %@", self.lastFixupOperation);
1181
1182         self.keyStateMachineOperation = [NSBlockOperation named:@"key-state-after-fixup" withBlock:^{
1183             STRONGIFY(self);
1184             [self dispatchSyncWithAccountKeys:^bool{
1185                 ckksnotice("ckkskey", self, "Fixup operation complete! Restarting key hierarchy machinery");
1186                 [self _onqueueAdvanceKeyStateMachineToState:SecCKKSZoneKeyStateInitialized withError:nil];
1187                 return true;
1188             }];
1189         }];
1190         [self.keyStateMachineOperation addNullableDependency:self.lastFixupOperation];
1191
1192     } else if([state isEqualToString: SecCKKSZoneKeyStateFetchComplete]) {
1193         // We've just completed a fetch of everything. Are there any remote keys?
1194         keyset = [CKKSCurrentKeySet loadForZone:self.zoneID];
1195
1196         NSError* localerror = nil;
1197
1198         NSArray<CKKSKey*>* localKeys = [CKKSKey localKeys:self.zoneID error:&localerror];
1199         NSArray<CKKSKey*>* remoteKeys = [CKKSKey remoteKeys:self.zoneID error: &localerror];
1200
1201         if(localKeys == nil || remoteKeys == nil || localerror) {
1202             ckkserror("ckkskey", self, "couldn't fetch keys from local database, entering error state: %@", localerror);
1203             self.keyHierarchyState = SecCKKSZoneKeyStateError;
1204             self.keyHierarchyError = localerror;
1205             [self _onqueueHandleKeyStateNonTransientDependency:nil];
1206             return;
1207         }
1208
1209         if(remoteKeys.count > 0u) {
1210             // Process the keys we received.
1211             self.keyStateMachineOperation = [[CKKSProcessReceivedKeysStateMachineOperation alloc] initWithCKKSKeychainView: self];
1212         } else if( (keyset.currentTLKPointer || keyset.currentClassAPointer || keyset.currentClassCPointer) &&
1213                   !(keyset.tlk && keyset.classA && keyset.classC)) {
1214             // Huh. We appear to have current key pointers, but the keys themselves don't exist. That's weird.
1215             // Transfer to the "unhealthy" state to request a fix
1216             ckksnotice("ckkskey", self, "We appear to have current key pointers but no keys to match them: %@ Moving to 'unhealthy'", keyset);
1217             nextState = SecCKKSZoneKeyStateUnhealthy;
1218         } else {
1219             // No remote keys, and the pointers look sane? Do we have an existing key hierarchy?
1220             CKKSZoneKeyState* checkedstate = [self _onqueueEnsureKeyHierarchyHealth:keyset error:&hierarchyError];
1221             if([checkedstate isEqualToString:SecCKKSZoneKeyStateReady] && !hierarchyError) {
1222                 ckksnotice("ckkskey", self, "After fetch, everything looks good.");
1223                 nextState = checkedstate;
1224
1225             } else if(localKeys.count == 0 && remoteKeys.count == 0) {
1226                 ckksnotice("ckkskey", self, "After fetch, we don't have any key hierarchy. Entering a waiting state: %@", hierarchyError ?: @"no error");
1227                 nextState = SecCKKSZoneKeyStateWaitForTLKCreation;
1228             } else {
1229                 ckksnotice("ckkskey", self, "After fetch, we have a possibly unhealthy key hierarchy. Moving to %@: %@", checkedstate, hierarchyError ?: @"no error");
1230                 nextState = checkedstate;
1231             }
1232         }
1233
1234     } else if([state isEqualToString:SecCKKSZoneKeyStateWaitForTLKCreation]) {
1235
1236         if(self.tlkCreationRequested) {
1237             self.tlkCreationRequested = false;
1238             ckksnotice("ckkskey", self, "TLK creation requested; kicking off operation");
1239             self.keyStateMachineOperation = [[CKKSNewTLKOperation alloc] initWithCKKSKeychainView: self ckoperationGroup:self.keyHierarchyOperationGroup];
1240
1241         } else if(self.keyStateProcessRequested) {
1242             ckksnotice("ckkskey", self, "We believe we need to create TLKs but we also received a key nudge; moving to key state Process.");
1243             nextState = SecCKKSZoneKeyStateProcess;
1244
1245         } else {
1246             ckksnotice("ckkskey", self, "We believe we need to create TLKs; waiting for Octagon (via %@)", self.suggestTLKUpload);
1247             [self.suggestTLKUpload trigger];
1248         }
1249
1250
1251     } else if([state isEqualToString:SecCKKSZoneKeyStateWaitForTLKUpload]) {
1252         ckksnotice("ckkskey", self, "We believe we have TLKs that need uploading");
1253
1254
1255         if(self.keyStateProcessRequested) {
1256             keyset = [CKKSCurrentKeySet loadForZone:self.zoneID];
1257             if(keyset.currentTLKPointer.currentKeyUUID) {
1258                 ckksnotice("ckkskey", self, "Received a nudge that our TLK records might be here (and there's some current TLK pointer)");
1259                 nextState = SecCKKSZoneKeyStateProcess;
1260             } else {
1261                 ckksnotice("ckkskey", self, "Received a nudge that our TLK records might be here, but there's no TLK pointer. Staying in WaitForTLKUpload.");
1262                 self.keyStateProcessRequested = false;
1263             }
1264         }
1265
1266         if(nextState == nil) {
1267             ckksnotice("ckkskey", self, "Alerting any listener of our proposed keyset: %@", self.lastNewTLKOperation.keyset);
1268             [self _onqueueRunKeysetProviderOperations:self.lastNewTLKOperation.keyset];
1269
1270             ckksnotice("ckkskey", self, "Notifying Octagon again, just in case");
1271             [self.suggestTLKUpload trigger];
1272         }
1273
1274     } else if([state isEqualToString: SecCKKSZoneKeyStateWaitForTLK]) {
1275         // We're in a hold state: waiting for the TLK bytes to arrive.
1276
1277         if(self.keyStateProcessRequested) {
1278             // Someone has requsted a reprocess! Go to the correct state.
1279             ckksnotice("ckkskey", self, "Received a nudge that our TLK might be here! Reprocessing.");
1280             nextState = SecCKKSZoneKeyStateProcess;
1281
1282         } else if(self.trustedPeersSetChanged) {
1283             // Hmm, maybe this trust set change will cause us to recover this TLK (due to a previously-untrusted share becoming trusted). Worth a shot!
1284             ckksnotice("ckkskey", self, "Received a nudge that the trusted peers set might have changed! Reprocessing.");
1285             nextState = SecCKKSZoneKeyStateProcess;
1286             self.trustedPeersSetChanged = false;
1287
1288         } else {
1289             keyset = [CKKSCurrentKeySet loadForZone:self.zoneID];
1290
1291             // Should we nuke this zone?
1292             if(self.trustStatus == CKKSAccountStatusAvailable) {
1293                 if([self _onqueueOtherDevicesReportHavingTLKs:keyset]) {
1294                     ckksnotice("ckkskey", self, "Other devices report having TLK(%@). Entering a waiting state", keyset.currentTLKPointer);
1295                 } else {
1296                     ckksnotice("ckkskey", self, "No other devices have TLK(%@). Beginning zone reset...", keyset.currentTLKPointer);
1297                     self.keyHierarchyOperationGroup = [CKOperationGroup CKKSGroupWithName:@"tlk-missing"];
1298                     nextState = SecCKKSZoneKeyStateResettingZone;
1299                 }
1300             } else {
1301                 ckksnotice("ckkskey", self, "This device isn't trusted, so don't modify the existing TLK(%@)", keyset.currentTLKPointer);
1302                 nextState = SecCKKSZoneKeyStateWaitForTrust;
1303             }
1304         }
1305
1306     } else if([state isEqualToString: SecCKKSZoneKeyStateWaitForUnlock]) {
1307         ckksnotice("ckkskey", self, "Requested to enter waitforunlock");
1308         self.keyStateMachineOperation = [self operationToEnterState:SecCKKSZoneKeyStateInitialized keyStateError:nil named:@"key-state-after-unlock"];
1309         [self.keyStateMachineOperation addNullableDependency: self.lockStateTracker.unlockDependency];
1310
1311     } else if([state isEqualToString: SecCKKSZoneKeyStateReadyPendingUnlock]) {
1312         ckksnotice("ckkskey", self, "Believe we're ready, but rechecking after unlock");
1313         self.keyStateMachineOperation = [self operationToEnterState:SecCKKSZoneKeyStateInitialized keyStateError:nil named:@"key-state-after-unlock"];
1314         [self.keyStateMachineOperation addNullableDependency: self.lockStateTracker.unlockDependency];
1315
1316     } else if([state isEqualToString: SecCKKSZoneKeyStateBadCurrentPointers]) {
1317         // The current key pointers are broken, but we're not sure why.
1318         ckksnotice("ckkskey", self, "Our current key pointers are reported broken. Attempting a fix!");
1319         self.keyStateMachineOperation = [[CKKSHealKeyHierarchyOperation alloc] initWithCKKSKeychainView: self ckoperationGroup:self.keyHierarchyOperationGroup];
1320
1321     } else if([state isEqualToString: SecCKKSZoneKeyStateNewTLKsFailed]) {
1322         ckksnotice("ckkskey", self, "Creating new TLKs didn't work. Attempting to refetch!");
1323         [self _onqueueKeyHierarchyFetch];
1324
1325     } else if([state isEqualToString: SecCKKSZoneKeyStateHealTLKSharesFailed]) {
1326         ckksnotice("ckkskey", self, "Creating new TLK shares didn't work. Attempting to refetch!");
1327         [self _onqueueKeyHierarchyFetch];
1328
1329     } else if([state isEqualToString:SecCKKSZoneKeyStateUnhealthy]) {
1330         if(self.trustStatus != CKKSAccountStatusAvailable) {
1331             ckksnotice("ckkskey", self, "Looks like the key hierarchy is unhealthy, but we're untrusted.");
1332             nextState = SecCKKSZoneKeyStateWaitForTrust;
1333
1334         } else {
1335             ckksnotice("ckkskey", self, "Looks like the key hierarchy is unhealthy. Launching fix.");
1336             self.keyStateMachineOperation = [[CKKSHealKeyHierarchyOperation alloc] initWithCKKSKeychainView:self ckoperationGroup:self.keyHierarchyOperationGroup];
1337         }
1338
1339     } else if([state isEqualToString:SecCKKSZoneKeyStateHealTLKShares]) {
1340         ckksnotice("ckksshare", self, "Key hierarchy is okay, but not shared appropriately. Launching fix.");
1341         self.keyStateMachineOperation = [[CKKSHealTLKSharesOperation alloc] initWithCKKSKeychainView:self
1342                                                                                     ckoperationGroup:self.keyHierarchyOperationGroup];
1343
1344     } else if([state isEqualToString:SecCKKSZoneKeyStateProcess]) {
1345         ckksnotice("ckksshare", self, "Launching key state process");
1346         self.keyStateMachineOperation = [[CKKSProcessReceivedKeysStateMachineOperation alloc] initWithCKKSKeychainView: self];
1347
1348         // Since we're starting a reprocess, this is answering all previous requests.
1349         self.keyStateProcessRequested = false;
1350
1351     } else {
1352         ckkserror("ckks", self, "asked to advance state machine to unknown state: %@", state);
1353         self.keyHierarchyState = state;
1354
1355         keyset = [CKKSCurrentKeySet loadForZone:self.zoneID];
1356         [self _onqueueHandleKeyStateNonTransientDependency:keyset];
1357         return;
1358     }
1359
1360     // Handle the key state ready dependency
1361     // If we're in ready and not entering a non-ready state, we should activate the ready dependency. Otherwise, we should create it.
1362     if(([state isEqualToString:SecCKKSZoneKeyStateReady] || [state isEqualToString:SecCKKSZoneKeyStateReadyPendingUnlock]) &&
1363        (nextState == nil || [nextState isEqualToString:SecCKKSZoneKeyStateReady] || [nextState isEqualToString:SecCKKSZoneKeyStateReadyPendingUnlock])) {
1364
1365         // Ready enough!
1366         [[CKKSAnalytics logger] setDateProperty:[NSDate date] forKey:CKKSAnalyticsLastKeystateReady inView:self];
1367
1368         if(self.keyStateReadyDependency) {
1369             [self scheduleOperation: self.keyStateReadyDependency];
1370             self.keyStateReadyDependency = nil;
1371         }
1372
1373         // If there are any OQEs waiting to be encrypted, launch an op to fix them
1374         NSError* localerror = nil;
1375         NSInteger outdatedOQEs = [CKKSOutgoingQueueEntry countByState:SecCKKSStateReencrypt zone:self.zoneID error:&localerror];
1376
1377         if(localerror) {
1378            ckkserror("ckkskey", self, "couldn't fetch OQEs from local database, entering error state: %@", localerror);
1379             self.keyHierarchyState = SecCKKSZoneKeyStateError;
1380             self.keyHierarchyError = localerror;
1381             [self _onqueueHandleKeyStateNonTransientDependency:nil];
1382             return;
1383         }
1384
1385         if(outdatedOQEs > 0) {
1386             ckksnotice("ckksreencrypt", self, "Reencrypting outgoing items as the key hierarchy is ready");
1387             CKKSReencryptOutgoingItemsOperation* op = [[CKKSReencryptOutgoingItemsOperation alloc] initWithCKKSKeychainView:self ckoperationGroup:self.keyHierarchyOperationGroup];
1388             [self scheduleOperation:op];
1389         }
1390     } else {
1391         // Not in ready: we need a key state ready dependency
1392         if(self.keyStateReadyDependency == nil || [self.keyStateReadyDependency isFinished]) {
1393             self.keyHierarchyOperationGroup = [CKOperationGroup CKKSGroupWithName:@"key-state-broken"];
1394             self.keyStateReadyDependency = [self createKeyStateReadyDependency:@"Key state has become ready again." ckoperationGroup:self.keyHierarchyOperationGroup];
1395         }
1396     }
1397
1398     NSAssert(!((self.keyStateMachineOperation != nil) &&
1399                (nextState != nil)),
1400              @"Should have a machine operation or a next state, not both");
1401
1402     // Start any operations, or log that we aren't
1403     if(self.keyStateMachineOperation) {
1404         [self scheduleOperation: self.keyStateMachineOperation];
1405         ckksnotice("ckkskey", self, "Now in key state: %@", state);
1406         self.keyHierarchyState = state;
1407
1408     } else if([state isEqualToString:SecCKKSZoneKeyStateError]) {
1409         ckksnotice("ckkskey", self, "Entering key state 'error'");
1410         self.keyHierarchyState = state;
1411
1412     } else if(nextState == nil) {
1413         ckksnotice("ckkskey", self, "Entering key state: %@", state);
1414         self.keyHierarchyState = state;
1415
1416     } else if(![state isEqualToString: nextState]) {
1417         ckksnotice("ckkskey", self, "Staying in state %@, but proceeding to %@ as soon as possible", self.keyHierarchyState, nextState);
1418         self.keyStateMachineOperation = [self operationToEnterState:nextState keyStateError:nextError named:[NSString stringWithFormat:@"next-key-state-%@", nextState]];
1419         [self scheduleOperation: self.keyStateMachineOperation];
1420
1421     } else {
1422         // Nothing to do and not in a waiting state? This is likely a bug, but, hey: pretend to be in ready!
1423         if(!([state isEqualToString:SecCKKSZoneKeyStateReady] || [state isEqualToString:SecCKKSZoneKeyStateReadyPendingUnlock])) {
1424             ckkserror("ckkskey", self, "No action to take in state %@; BUG, but: maybe we're ready?", state);
1425             nextState = SecCKKSZoneKeyStateReady;
1426             self.keyStateMachineOperation = [self operationToEnterState:nextState keyStateError:nil named:@"next-key-state"];
1427             [self scheduleOperation: self.keyStateMachineOperation];
1428         }
1429     }
1430
1431     // If the keystate is non-transient, ensure we've loaded the keyset, and provide it to any waiters
1432     // If it is transient, just call the handler anyway: it needs to set up the dependency
1433     if(!CKKSKeyStateTransient(self.keyHierarchyState) && keyset == nil) {
1434         keyset = [CKKSCurrentKeySet loadForZone:self.zoneID];
1435     }
1436     [self _onqueueHandleKeyStateNonTransientDependency:keyset];
1437 }
1438
1439 - (void)_onqueueHandleKeyStateNonTransientDependency:(CKKSCurrentKeySet* _Nullable)keyset {
1440     dispatch_assert_queue(self.queue);
1441
1442     if(CKKSKeyStateTransient(self.keyHierarchyState)) {
1443         if(self.keyStateNonTransientDependency == nil || [self.keyStateNonTransientDependency isFinished]) {
1444             self.keyStateNonTransientDependency = [self createKeyStateNontransientDependency];
1445         }
1446     } else {
1447         // Nontransient: go for it
1448         if(self.keyStateNonTransientDependency) {
1449             [self scheduleOperation: self.keyStateNonTransientDependency];
1450             self.keyStateNonTransientDependency = nil;
1451         }
1452
1453         if(keyset && keyset.currentTLKPointer.currentKeyUUID) {
1454             [self _onqueueRunKeysetProviderOperations:keyset];
1455         } else {
1456             ckksnotice("ckkskey", self, "State machine is nontransient, but no keyset...");
1457         }
1458     }
1459 }
1460
1461 - (NSOperation*)operationToEnterState:(CKKSZoneKeyState*)state keyStateError:(NSError* _Nullable)keyStateError named:(NSString*)name {
1462     WEAKIFY(self);
1463
1464     return [NSBlockOperation named:name withBlock:^{
1465         STRONGIFY(self);
1466         if(!self) {
1467             return;
1468         }
1469         [self dispatchSyncWithAccountKeys:^bool{
1470             [self _onqueueAdvanceKeyStateMachineToState:state withError:keyStateError];
1471             return true;
1472         }];
1473     }];
1474 }
1475
1476 - (BOOL)otherDevicesReportHavingTLKs:(CKKSCurrentKeySet*)keyset
1477 {
1478     __block BOOL report = false;
1479     [self dispatchSync:^bool{
1480         report = [self _onqueueOtherDevicesReportHavingTLKs:keyset];
1481         return true;
1482     }];
1483     return report ? YES : NO;
1484 }
1485
1486 - (bool)_onqueueOtherDevicesReportHavingTLKs:(CKKSCurrentKeySet*)keyset
1487 {
1488     dispatch_assert_queue(self.queue);
1489
1490     //Has there been any activity indicating that other trusted devices have keys in the past 45 days, or untrusted devices in the past 4?
1491     // (We chose 4 as devices attempt to upload their device state every 3 days. If a device is unceremoniously kicked out of circle, we normally won't immediately reset.)
1492     NSDate* now = [NSDate date];
1493     NSDateComponents* trustedOffset = [[NSDateComponents alloc] init];
1494     [trustedOffset setDay:-45];
1495     NSDate* trustedDeadline = [[NSCalendar currentCalendar] dateByAddingComponents:trustedOffset toDate:now options:0];
1496
1497     NSDateComponents* untrustedOffset = [[NSDateComponents alloc] init];
1498     [untrustedOffset setDay:-4];
1499     NSDate* untrustedDeadline = [[NSCalendar currentCalendar] dateByAddingComponents:untrustedOffset toDate:now options:0];
1500
1501
1502     NSMutableSet<NSString*>* trustedPeerIDs = [NSMutableSet set];
1503     for(CKKSPeerProviderState* trustState in self.currentTrustStates) {
1504         for(id<CKKSPeer> peer in trustState.currentTrustedPeers) {
1505             [trustedPeerIDs addObject:peer.peerID];
1506         }
1507     }
1508
1509     NSError* localerror = nil;
1510
1511     NSArray<CKKSDeviceStateEntry*>* allDeviceStates = [CKKSDeviceStateEntry allInZone:self.zoneID error:&localerror];
1512     if(localerror) {
1513         ckkserror("ckkskey", self, "Error fetching device states: %@", localerror);
1514         localerror = nil;
1515         return true;
1516     }
1517     for(CKKSDeviceStateEntry* device in allDeviceStates) {
1518         // The peerIDs in CDSEs aren't written with the peer prefix. Make sure we match both.
1519         NSString* sosPeerID = device.circlePeerID ? [CKKSSOSPeerPrefix stringByAppendingString:device.circlePeerID] : nil;
1520
1521         if([trustedPeerIDs containsObject:device.circlePeerID] ||
1522            [trustedPeerIDs containsObject:sosPeerID] ||
1523            [trustedPeerIDs containsObject:device.octagonPeerID]) {
1524             // Is this a recent DSE? If it's older than the deadline, skip it
1525             if([device.storedCKRecord.modificationDate compare:trustedDeadline] == NSOrderedAscending) {
1526                 ckksnotice("ckkskey", self, "Trusted device state (%@) is too old; ignoring", device);
1527                 continue;
1528             }
1529         } else {
1530             // Device is untrusted. How does it fare with the untrustedDeadline?
1531             if([device.storedCKRecord.modificationDate compare:untrustedDeadline] == NSOrderedAscending) {
1532                 ckksnotice("ckkskey", self, "Device (%@) is not trusted and from too long ago; ignoring device state (%@)", device.circlePeerID, device);
1533                 continue;
1534             } else {
1535                 ckksnotice("ckkskey", self, "Device (%@) is not trusted, but very recent. Including in heuristic: %@", device.circlePeerID, device);
1536             }
1537         }
1538
1539         if([device.keyState isEqualToString:SecCKKSZoneKeyStateReady] ||
1540            [device.keyState isEqualToString:SecCKKSZoneKeyStateReadyPendingUnlock]) {
1541             ckksnotice("ckkskey", self, "Other device (%@) has keys; it should send them to us", device);
1542             return true;
1543         }
1544     }
1545
1546     NSArray<CKKSTLKShareRecord*>* tlkShares = [CKKSTLKShareRecord allForUUID:keyset.currentTLKPointer.currentKeyUUID
1547                                                           zoneID:self.zoneID
1548                                                            error:&localerror];
1549     if(localerror) {
1550         ckkserror("ckkskey", self, "Error fetching device states: %@", localerror);
1551         localerror = nil;
1552         return false;
1553     }
1554
1555     for(CKKSTLKShareRecord* tlkShare in tlkShares) {
1556         if([trustedPeerIDs containsObject:tlkShare.senderPeerID] &&
1557            [tlkShare.storedCKRecord.modificationDate compare:trustedDeadline] == NSOrderedDescending) {
1558             ckksnotice("ckkskey", self, "Trusted TLK Share (%@) created recently; other devices have keys and should send them to us", tlkShare);
1559             return true;
1560         }
1561     }
1562
1563     // Okay, how about the untrusted deadline?
1564     for(CKKSTLKShareRecord* tlkShare in tlkShares) {
1565         if([tlkShare.storedCKRecord.modificationDate compare:untrustedDeadline] == NSOrderedDescending) {
1566             ckksnotice("ckkskey", self, "Untrusted TLK Share (%@) created very recently; other devices might have keys and should rejoin the circle (and send them to us)", tlkShare);
1567             return true;
1568         }
1569     }
1570
1571     return false;
1572 }
1573
1574 // For this key, who doesn't yet have a valid CKKSTLKShare for it?
1575 // Note that we really want a record sharing the TLK to ourselves, so this function might return
1576 // a non-empty set even if all peers have the TLK: it wants us to make a record for ourself.
1577 - (NSSet<id<CKKSPeer>>*)_onqueueFindPeers:(CKKSPeerProviderState*)trustState
1578                              missingShare:(CKKSKey*)key
1579                            afterUploading:(NSSet<CKKSTLKShareRecord*>* _Nullable)newShares
1580                                     error:(NSError* __autoreleasing*)error
1581 {
1582     dispatch_assert_queue(self.queue);
1583
1584     if(!key) {
1585         ckkserror("ckksshare", self, "Attempting to find missing shares for nil key");
1586         return [NSSet set];
1587     }
1588
1589     if(trustState.currentTrustedPeersError) {
1590         ckkserror("ckksshare", self, "Couldn't find missing shares because trusted peers aren't available: %@", trustState.currentTrustedPeersError);
1591         if(error) {
1592             *error = trustState.currentTrustedPeersError;
1593         }
1594         return [NSSet set];
1595     }
1596     if(trustState.currentSelfPeersError) {
1597         ckkserror("ckksshare", self, "Couldn't find missing shares because self peers aren't available: %@", trustState.currentSelfPeersError);
1598         if(error) {
1599             *error = trustState.currentSelfPeersError;
1600         }
1601         return [NSSet set];
1602     }
1603
1604     NSMutableSet<id<CKKSPeer>>* peersMissingShares = [NSMutableSet set];
1605
1606     // Ensure that the 'self peer' is one of the current trusted peers. Otherwise, any TLKShare we create
1607     // won't be considered trusted the next time through...
1608     if(![trustState.currentTrustedPeerIDs containsObject:trustState.currentSelfPeers.currentSelf.peerID]) {
1609         ckkserror("ckksshare", self, "current self peer (%@) is not in the set of trusted peers: %@",
1610                   trustState.currentSelfPeers.currentSelf.peerID,
1611                   trustState.currentTrustedPeerIDs);
1612
1613         if(error) {
1614             *error = [NSError errorWithDomain:CKKSErrorDomain
1615                                          code:CKKSLackingTrust
1616                                   description:[NSString stringWithFormat:@"current self peer (%@) is not in the set of trusted peers",
1617                                                trustState.currentSelfPeers.currentSelf.peerID]];
1618         }
1619
1620         return nil;
1621     }
1622
1623     for(id<CKKSRemotePeerProtocol> peer in trustState.currentTrustedPeers) {
1624         if(![peer shouldHaveView:self.zoneName]) {
1625             ckkserror("ckksshare", self, "Peer (%@) is not supposed to have view, skipping", peer);
1626             continue;
1627         }
1628
1629         NSError* peerError = nil;
1630         // Find all the shares for this peer for this key
1631         NSArray<CKKSTLKShareRecord*>* currentPeerShares = [CKKSTLKShareRecord allFor:peer.peerID
1632                                                                  keyUUID:key.uuid
1633                                                                   zoneID:self.zoneID
1634                                                                    error:&peerError];
1635
1636         if(peerError) {
1637             ckkserror("ckksshare", self, "Couldn't load shares for peer %@: %@", peer, peerError);
1638             if(error) {
1639                 *error = peerError;
1640             }
1641             return nil;
1642         }
1643
1644         // Include the new shares, too....
1645         NSArray<CKKSTLKShareRecord*>* possiblePeerShares = newShares ? [currentPeerShares arrayByAddingObjectsFromArray:[newShares allObjects]] : currentPeerShares;
1646
1647         // Determine if we think this peer has enough things shared to them
1648         bool alreadyShared = false;
1649         for(CKKSTLKShareRecord* existingPeerShare in possiblePeerShares) {
1650             // Ensure this share is to this peer...
1651             if(![existingPeerShare.share.receiverPeerID isEqualToString:peer.peerID]) {
1652                 continue;
1653             }
1654
1655             // If an SOS Peer sent this share, is its signature still valid? Or did the signing key change?
1656             if([existingPeerShare.senderPeerID hasPrefix:CKKSSOSPeerPrefix]) {
1657                 NSError* signatureError = nil;
1658                 if(![existingPeerShare signatureVerifiesWithPeerSet:trustState.currentTrustedPeers error:&signatureError]) {
1659                     ckksnotice("ckksshare", self, "Existing TLKShare's signature doesn't verify with current peer set: %@ %@", signatureError, existingPeerShare);
1660                     continue;
1661                 }
1662             }
1663
1664             if([existingPeerShare.tlkUUID isEqualToString:key.uuid] && [trustState.currentTrustedPeerIDs containsObject:existingPeerShare.senderPeerID]) {
1665                 // Was this shared to us?
1666                 if([peer.peerID isEqualToString: trustState.currentSelfPeers.currentSelf.peerID]) {
1667                     // We only count this as 'found' if we did the sharing and it's to our current keys
1668                     NSData* currentKey = trustState.currentSelfPeers.currentSelf.publicEncryptionKey.keyData;
1669
1670                     if([existingPeerShare.senderPeerID isEqualToString:trustState.currentSelfPeers.currentSelf.peerID] &&
1671                        [existingPeerShare.share.receiverPublicEncryptionKeySPKI isEqual:currentKey]) {
1672                         ckksnotice("ckksshare", self, "Local peer %@ is shared %@ via self: %@", peer, key, existingPeerShare);
1673                         alreadyShared = true;
1674                         break;
1675                     } else {
1676                         ckksnotice("ckksshare", self, "Local peer %@ is shared %@ via trusted %@, but that's not good enough", peer, key, existingPeerShare);
1677                     }
1678
1679                 } else {
1680                     // Was this shared to the remote peer's current keys?
1681                     NSData* currentKeySPKI = peer.publicEncryptionKey.keyData;
1682
1683                     if([existingPeerShare.share.receiverPublicEncryptionKeySPKI isEqual:currentKeySPKI]) {
1684                         // Some other peer has a trusted share. Cool!
1685                         ckksnotice("ckksshare", self, "Peer %@ is shared %@ via trusted %@", peer, key, existingPeerShare);
1686                         alreadyShared = true;
1687                         break;
1688                     } else {
1689                         ckksnotice("ckksshare", self, "Peer %@ has a share for %@, but to old keys: %@", peer, key, existingPeerShare);
1690                     }
1691                 }
1692             }
1693         }
1694
1695         if(!alreadyShared) {
1696             // Add this peer to our set, if it has an encryption key to receive the share
1697             if(peer.publicEncryptionKey) {
1698                 [peersMissingShares addObject:peer];
1699             }
1700         }
1701     }
1702
1703     if(peersMissingShares.count > 0u) {
1704         // Log each and every one of the things
1705         ckksnotice("ckksshare", self, "Missing TLK shares for %lu peers: %@", (unsigned long)peersMissingShares.count, peersMissingShares);
1706         ckksnotice("ckksshare", self, "Self peers are (%@) %@", trustState.currentSelfPeersError ?: @"no error", trustState.currentSelfPeers);
1707         ckksnotice("ckksshare", self, "Trusted peers are (%@) %@", trustState.currentTrustedPeersError ?: @"no error", trustState.currentTrustedPeers);
1708     }
1709
1710     return peersMissingShares;
1711 }
1712
1713 - (BOOL)_onqueueAreNewSharesSufficient:(NSSet<CKKSTLKShareRecord*>*)newShares
1714                             currentTLK:(CKKSKey*)key
1715                                  error:(NSError* __autoreleasing*)error
1716 {
1717     dispatch_assert_queue(self.queue);
1718
1719     for(CKKSPeerProviderState* trustState in self.currentTrustStates) {
1720         NSError* localError = nil;
1721         NSSet<id<CKKSPeer>>* peersMissingShares = [self _onqueueFindPeers:trustState
1722                                                              missingShare:key
1723                                                            afterUploading:newShares
1724                                                                     error:&localError];
1725         if(peersMissingShares == nil || localError) {
1726             if(trustState.essential) {
1727                 if(error) {
1728                     *error = localError;
1729                 }
1730                 return NO;
1731             } else {
1732                 ckksnotice("ckksshare", self, "Failed to find peers for nonessential system: %@", trustState);
1733                 // Not a hard failure.
1734             }
1735         }
1736
1737         if(peersMissingShares.count > 0) {
1738             ckksnotice("ckksshare", self, "New share set is missing shares for peers: %@", peersMissingShares);
1739             return NO;
1740         }
1741     }
1742
1743     return YES;
1744 }
1745
1746 - (NSSet<CKKSTLKShareRecord*>*)_onqueueCreateMissingKeyShares:(CKKSKey*)key
1747                                                         error:(NSError* __autoreleasing*)error
1748 {
1749     NSError* localerror = nil;
1750     NSSet<CKKSTLKShareRecord*>* newShares = nil;
1751
1752     // If any one of our trust states succeed, this function doesn't have an error
1753     for(CKKSPeerProviderState* trustState in self.currentTrustStates) {
1754         NSError* stateError = nil;
1755
1756         NSSet<CKKSTLKShareRecord*>* newTrustShares = [self _onqueueCreateMissingKeyShares:key
1757                                                                                     peers:trustState
1758                                                                                     error:&stateError];
1759
1760
1761         if(newTrustShares && !stateError) {
1762             newShares = newShares ? [newShares setByAddingObjectsFromSet:newTrustShares] : newTrustShares;
1763         } else {
1764             ckksnotice("ckksshare", self, "Unable to create shares for trust set %@: %@", trustState, stateError);
1765             if(localerror == nil) {
1766                 localerror = stateError;
1767             }
1768         }
1769     }
1770
1771     // Only report an error if none of the trust states were able to succeed
1772     if(newShares) {
1773         return newShares;
1774     } else {
1775         if(error && localerror) {
1776             *error = localerror;
1777         }
1778         return nil;
1779     }
1780 }
1781
1782 - (NSSet<CKKSTLKShareRecord*>*)_onqueueCreateMissingKeyShares:(CKKSKey*)key
1783                                                         peers:(CKKSPeerProviderState*)trustState
1784                                                         error:(NSError* __autoreleasing*)error
1785 {
1786     dispatch_assert_queue(self.queue);
1787
1788     if(trustState.currentTrustedPeersError) {
1789         ckkserror("ckksshare", self, "Couldn't create missing shares because trusted peers aren't available: %@", trustState.currentTrustedPeersError);
1790         if(error) {
1791             *error = trustState.currentTrustedPeersError;
1792         }
1793         return nil;
1794     }
1795     if(trustState.currentSelfPeersError) {
1796         ckkserror("ckksshare", self, "Couldn't create missing shares because self peers aren't available: %@", trustState.currentSelfPeersError);
1797         if(error) {
1798             *error = trustState.currentSelfPeersError;
1799         }
1800         return nil;
1801     }
1802
1803     NSSet<id<CKKSPeer>>* remainingPeers = [self _onqueueFindPeers:trustState missingShare:key afterUploading:nil error:error];
1804     NSMutableSet<CKKSTLKShareRecord*>* newShares = [NSMutableSet set];
1805
1806     if(!remainingPeers) {
1807         return nil;
1808     }
1809
1810     NSError* localerror = nil;
1811
1812     if(![key ensureKeyLoaded:error]) {
1813         return nil;
1814     }
1815
1816     for(id<CKKSPeer> peer in remainingPeers) {
1817         if(!peer.publicEncryptionKey) {
1818             ckksnotice("ckksshare", self, "No need to make TLK for %@; they don't have any encryption keys", peer);
1819             continue;
1820         }
1821
1822         // Create a share for this peer.
1823         ckksnotice("ckksshare", self, "Creating share of %@ as %@ for %@", key, trustState.currentSelfPeers.currentSelf, peer);
1824         CKKSTLKShareRecord* newShare = [CKKSTLKShareRecord share:key
1825                                                   as:trustState.currentSelfPeers.currentSelf
1826                                                   to:peer
1827                                                epoch:-1
1828                                             poisoned:0
1829                                                error:&localerror];
1830
1831         if(localerror) {
1832             ckkserror("ckksshare", self, "Couldn't create new share for %@: %@", peer, localerror);
1833             if(error) {
1834                 *error = localerror;
1835             }
1836             return nil;
1837         }
1838
1839         [newShares addObject: newShare];
1840     }
1841
1842     return newShares;
1843 }
1844
1845 - (CKKSZoneKeyState*)_onqueueEnsureKeyHierarchyHealth:(CKKSCurrentKeySet*)set error:(NSError* __autoreleasing *)error {
1846     dispatch_assert_queue(self.queue);
1847
1848     if(!set.currentTLKPointer && !set.currentClassAPointer && !set.currentClassCPointer) {
1849         ckkserror("ckkskey", self, "Error examining existing key hierarchy (missing all CKPs, likely no hierarchy exists): %@", set);
1850         return SecCKKSZoneKeyStateWaitForTLKCreation;
1851     }
1852
1853     // Check keyset
1854     if(!set.tlk || !set.classA || !set.classC) {
1855         ckkserror("ckkskey", self, "Error examining existing key hierarchy (missing at least one key): %@", set);
1856         if(error) {
1857             *error = set.error;
1858         }
1859         return SecCKKSZoneKeyStateUnhealthy;
1860     }
1861
1862     NSError* localerror = nil;
1863     bool probablyOkIfUnlocked = false;
1864
1865     // keychain being locked is not a fatal error here
1866     [set.tlk loadKeyMaterialFromKeychain:&localerror];
1867     if(localerror && !([localerror.domain isEqual: @"securityd"] && localerror.code == errSecInteractionNotAllowed)) {
1868         ckkserror("ckkskey", self, "Error loading TLK(%@): %@", set.tlk, localerror);
1869         if(error) {
1870             *error = localerror;
1871         }
1872         return SecCKKSZoneKeyStateUnhealthy;
1873     } else if(localerror) {
1874         ckkserror("ckkskey", self, "Soft error loading TLK(%@), maybe locked: %@", set.tlk, localerror);
1875         probablyOkIfUnlocked = true;
1876     }
1877     localerror = nil;
1878
1879     // keychain being locked is not a fatal error here
1880     [set.classA loadKeyMaterialFromKeychain:&localerror];
1881     if(localerror && !([localerror.domain isEqual: @"securityd"] && localerror.code == errSecInteractionNotAllowed)) {
1882         ckkserror("ckkskey", self, "Error loading classA key(%@): %@", set.classA, localerror);
1883         if(error) {
1884             *error = localerror;
1885         }
1886         return SecCKKSZoneKeyStateUnhealthy;
1887     } else if(localerror) {
1888         ckkserror("ckkskey", self, "Soft error loading classA key(%@), maybe locked: %@", set.classA, localerror);
1889         probablyOkIfUnlocked = true;
1890     }
1891     localerror = nil;
1892
1893     // keychain being locked is a fatal error here, since this is class C
1894     [set.classC loadKeyMaterialFromKeychain:&localerror];
1895     if(localerror) {
1896         ckkserror("ckkskey", self, "Error loading classC(%@): %@", set.classC, localerror);
1897         if(error) {
1898             *error = localerror;
1899         }
1900         return SecCKKSZoneKeyStateUnhealthy;
1901     }
1902
1903     // Check that the classA and classC keys point to the current TLK
1904     if(![set.classA.parentKeyUUID isEqualToString: set.tlk.uuid]) {
1905         localerror = [NSError errorWithDomain:CKKSServerExtensionErrorDomain
1906                                          code:CKKSServerUnexpectedSyncKeyInChain
1907                                      userInfo:@{
1908                                                 NSLocalizedDescriptionKey: @"Current class A key does not wrap to current TLK",
1909                                                }];
1910         ckkserror("ckkskey", self, "Key hierarchy unhealthy: %@", localerror);
1911         if(error) {
1912             *error = localerror;
1913         }
1914         return SecCKKSZoneKeyStateUnhealthy;
1915     }
1916     if(![set.classC.parentKeyUUID isEqualToString: set.tlk.uuid]) {
1917         localerror = [NSError errorWithDomain:CKKSServerExtensionErrorDomain
1918                                          code:CKKSServerUnexpectedSyncKeyInChain
1919                                      userInfo:@{
1920                                                 NSLocalizedDescriptionKey: @"Current class C key does not wrap to current TLK",
1921                                                }];
1922         ckkserror("ckkskey", self, "Key hierarchy unhealthy: %@", localerror);
1923         if(error) {
1924             *error = localerror;
1925         }
1926         return SecCKKSZoneKeyStateUnhealthy;
1927     }
1928
1929     self.activeTLK = [set.tlk uuid];
1930
1931     // Now that we're pretty sure we have the keys, are they shared appropriately?
1932     // We need trust in order to proceed here
1933     if(self.currentTrustStates.count == 0u) {
1934         ckkserror("ckkskey", self, "Can't check TLKShares due to missing trust states");
1935         return SecCKKSZoneKeyStateWaitForTrust;
1936     }
1937
1938     // Check that every trusted peer has at least one TLK share
1939     // If any trust state check works, don't error out
1940     bool anyTrustStateSucceeded = false;
1941     for(CKKSPeerProviderState* trustState in self.currentTrustStates) {
1942         NSSet<id<CKKSPeer>>* missingShares = [self _onqueueFindPeers:trustState missingShare:set.tlk afterUploading:nil error:&localerror];
1943         if(localerror && [self.lockStateTracker isLockedError: localerror]) {
1944             ckkserror("ckkskey", self, "Couldn't find missing TLK shares due to lock state: %@", localerror);
1945             probablyOkIfUnlocked = true;
1946
1947         } else if(([localerror.domain isEqualToString:TrustedPeersHelperErrorDomain] && localerror.code == TrustedPeersHelperErrorNoPreparedIdentity) ||
1948                   ([localerror.domain isEqualToString:CKKSErrorDomain] && localerror.code == CKKSLackingTrust) ||
1949                   ([localerror.domain isEqualToString:CKKSErrorDomain] && localerror.code == CKKSNoPeersAvailable)) {
1950             ckkserror("ckkskey", self, "Couldn't find missing TLK shares due some trust issue: %@", localerror);
1951
1952             if(trustState.essential) {
1953                 ckkserror("ckkskey", self, "Trust state is considered essential; entering waitfortrust: %@", trustState);
1954
1955                 // Octagon can reinform us when it thinks we should start again
1956                 self.trustStatus = CKKSAccountStatusUnknown;
1957                 return SecCKKSZoneKeyStateWaitForTrust;
1958             } else {
1959                 ckkserror("ckkskey", self, "Peer provider is considered nonessential; ignoring error: %@", trustState);
1960                 continue;
1961             }
1962
1963         } else if(localerror) {
1964             ckkserror("ckkskey", self, "Error finding missing TLK shares: %@", localerror);
1965             continue;
1966         }
1967
1968         if(!missingShares || missingShares.count != 0u) {
1969             localerror = [NSError errorWithDomain:CKKSErrorDomain code:CKKSMissingTLKShare
1970                                       description:[NSString stringWithFormat:@"Missing shares for %lu peers", (unsigned long)missingShares.count]];
1971             if(error) {
1972                 *error = localerror;
1973             }
1974             return SecCKKSZoneKeyStateHealTLKShares;
1975         } else {
1976             ckksnotice("ckksshare", self, "TLK (%@) is shared correctly for trust state %@", set.tlk, trustState.peerProviderID);
1977         }
1978
1979         anyTrustStateSucceeded |= true;
1980     }
1981
1982     if(!anyTrustStateSucceeded) {
1983         if(error) {
1984             *error = localerror;
1985         }
1986
1987         return SecCKKSZoneKeyStateError;
1988     }
1989
1990     // Got to the bottom? Cool! All keys are present and accounted for.
1991     return probablyOkIfUnlocked ? SecCKKSZoneKeyStateReadyPendingUnlock : SecCKKSZoneKeyStateReady;
1992 }
1993
1994 - (void)_onqueueKeyHierarchyFetch {
1995     [self _onqueueKeyHierarchyFetchForReasons:[NSSet setWithArray:@[CKKSFetchBecauseKeyHierarchy]]];
1996 }
1997
1998 - (void)_onqueueKeyHierarchyFetchForReasons:(NSSet<CKKSFetchBecause*>*)reasons
1999 {
2000     dispatch_assert_queue(self.queue);
2001
2002     WEAKIFY(self);
2003     self.keyStateMachineOperation = [NSBlockOperation blockOperationWithBlock: ^{
2004         STRONGIFY(self);
2005         if(!self) {
2006             ckkserror("ckks", self, "received callback for released object");
2007             return;
2008         }
2009         [self.launch addEvent:@"fetch-complete"];
2010
2011         [self dispatchSyncWithAccountKeys: ^bool{
2012             [self _onqueueAdvanceKeyStateMachineToState: SecCKKSZoneKeyStateFetchComplete withError: nil];
2013             return true;
2014         }];
2015     }];
2016     self.keyStateMachineOperation.name = @"waiting-for-fetch";
2017
2018     NSOperation* fetchOp = [self.zoneChangeFetcher requestSuccessfulFetchForManyReasons:reasons];
2019     [self.keyStateMachineOperation addDependency: fetchOp];
2020
2021     self.keyStateFetchRequested = false;
2022 }
2023
2024 - (void) handleKeychainEventDbConnection: (SecDbConnectionRef) dbconn
2025                                    added: (SecDbItemRef) added
2026                                  deleted: (SecDbItemRef) deleted
2027                              rateLimiter: (CKKSRateLimiter*) rateLimiter
2028                             syncCallback: (SecBoolNSErrorCallback) syncCallback {
2029     if(!SecCKKSIsEnabled()) {
2030         ckksnotice("ckks", self, "Skipping handleKeychainEventDbConnection due to disabled CKKS");
2031         return;
2032     }
2033
2034     __block NSError* error = nil;
2035
2036     // Tombstones come in as item modifications or item adds. Handle modifications here.
2037     bool addedTombstone   = added   && SecDbItemIsTombstone(added);
2038     bool deletedTombstone = deleted && SecDbItemIsTombstone(deleted);
2039
2040     bool addedSync   = added   && SecDbItemIsSyncable(added);
2041     bool deletedSync = deleted && SecDbItemIsSyncable(deleted);
2042
2043     bool isAdd    = ( added && !deleted) || (added && deleted && !addedTombstone &&  deletedTombstone) || (added && deleted &&  addedSync && !deletedSync);
2044     bool isDelete = (!added &&  deleted) || (added && deleted &&  addedTombstone && !deletedTombstone) || (added && deleted && !addedSync &&  deletedSync);
2045     bool isModify = ( added &&  deleted) && (!isAdd) && (!isDelete);
2046
2047     // On an update that changes an item's primary key, SecDb modifies the existing item, then adds a new tombstone to replace the old primary key.
2048     // Therefore, we might receive an added tombstone here with no deleted item to accompany it. This should be considered a deletion.
2049     if(addedTombstone && !deleted) {
2050         isAdd = false;
2051         isDelete = true;
2052         isModify = false;
2053
2054         // Passed to withItem: below
2055         deleted = added;
2056     }
2057
2058     // If neither item is syncable, don't proceed further in the syncing system
2059     bool proceed = addedSync || deletedSync;
2060
2061     if(!proceed) {
2062         ckksnotice("ckks", self, "skipping sync of non-sync item (%d, %d)", addedSync, deletedSync);
2063         return;
2064     }
2065
2066     // It's possible to ask for an item to be deleted without adding a corresponding tombstone.
2067     // This is arguably a bug, as it generates an out-of-sync state, but it is in the API contract.
2068     // CKKS should ignore these, but log very upset messages.
2069     if(isDelete && !addedTombstone) {
2070         ckksnotice("ckks", self, "Client has asked for an item deletion to not sync. Keychain is now out of sync with account");
2071         return;
2072     }
2073
2074     // Only synchronize items which can transfer between devices
2075     NSString* protection = (__bridge NSString*)SecDbItemGetCachedValueWithName(added ? added : deleted, kSecAttrAccessible);
2076     if(! ([protection isEqualToString: (__bridge NSString*)kSecAttrAccessibleWhenUnlocked] ||
2077           [protection isEqualToString: (__bridge NSString*)kSecAttrAccessibleAfterFirstUnlock] ||
2078           [protection isEqualToString: (__bridge NSString*)kSecAttrAccessibleAlwaysPrivate])) {
2079         ckksnotice("ckks", self, "skipping sync of device-bound(%@) item", protection);
2080         return;
2081     }
2082
2083     // Our caller gave us a database connection. We must get on the local queue to ensure atomicity
2084     // Note that we're at the mercy of the surrounding db transaction, so don't try to rollback here
2085     [self dispatchSyncWithConnection: dbconn block: ^bool {
2086         // Schedule a "view changed" notification
2087         [self.notifyViewChangedScheduler trigger];
2088
2089         if(self.accountStatus == CKKSAccountStatusNoAccount) {
2090             // No account; CKKS shouldn't attempt anything.
2091             self.droppedItems = true;
2092             ckksnotice("ckks", self, "Dropping sync item modification due to CK account state; will scan to find changes later");
2093
2094             if(syncCallback) {
2095                 // We're positively not logged into CloudKit, and therefore don't expect this item to be synced anytime particularly soon.
2096                 [self callSyncCallbackWithErrorNoAccount: syncCallback];
2097             }
2098             return true;
2099         }
2100
2101         // Always record the callback, even if we can't encrypt the item right now. Maybe we'll get to it soon!
2102         if(syncCallback) {
2103             CFErrorRef cferror = NULL;
2104             NSString* uuid = (__bridge_transfer NSString*) CFRetain(SecDbItemGetValue(added, &v10itemuuid, &cferror));
2105             if(!cferror && uuid) {
2106                 self.pendingSyncCallbacks[uuid] = syncCallback;
2107             }
2108             CFReleaseNull(cferror);
2109         }
2110
2111         CKKSOutgoingQueueEntry* oqe = nil;
2112         if       (isAdd) {
2113             oqe = [CKKSOutgoingQueueEntry withItem: added   action: SecCKKSActionAdd    ckks:self error: &error];
2114         } else if(isDelete) {
2115             oqe = [CKKSOutgoingQueueEntry withItem: deleted action: SecCKKSActionDelete ckks:self error: &error];
2116         } else if(isModify) {
2117             oqe = [CKKSOutgoingQueueEntry withItem: added   action: SecCKKSActionModify ckks:self error: &error];
2118         } else {
2119             ckkserror("ckks", self, "processKeychainEventItemAdded given garbage: %@ %@", added, deleted);
2120             return true;
2121         }
2122
2123         CKOperationGroup* operationGroup = [CKOperationGroup CKKSGroupWithName:@"keychain-api-use"];
2124
2125         if(error) {
2126             ckkserror("ckks", self, "Couldn't create outgoing queue entry: %@", error);
2127             self.droppedItems = true;
2128
2129             // If the problem is 'no UUID', launch a scan operation to find and fix it
2130             // We don't want to fix it up here, in the closing moments of a transaction
2131             if([error.domain isEqualToString:CKKSErrorDomain] && error.code == CKKSNoUUIDOnItem) {
2132                 ckksnotice("ckks", self, "Launching scan operation to find UUID");
2133                 [self scanLocalItems:@"uuid-find-scan" ckoperationGroup:operationGroup after:nil];
2134             }
2135
2136             // If the problem is 'couldn't load key', tell the key hierarchy state machine to fix it
2137             if([error.domain isEqualToString:CKKSErrorDomain] && error.code == errSecItemNotFound) {
2138                 [self.pokeKeyStateMachineScheduler trigger];
2139             }
2140
2141             return true;
2142         }
2143
2144         if(rateLimiter) {
2145             NSDate* limit = nil;
2146             NSInteger value = [rateLimiter judge:oqe at:[NSDate date] limitTime:&limit];
2147             if(limit) {
2148                 oqe.waitUntil = limit;
2149                 SecPLLogRegisteredEvent(@"CKKSSyncing", @{ @"ratelimit" : @(value), @"accessgroup" : oqe.accessgroup});
2150             }
2151         }
2152
2153         [oqe saveToDatabaseWithConnection: dbconn error: &error];
2154         if(error) {
2155             ckkserror("ckks", self, "Couldn't save outgoing queue entry to database: %@", error);
2156             return true;
2157         } else {
2158             ckksnotice("ckks", self, "Saved %@ to outgoing queue", oqe);
2159         }
2160
2161         // This update supercedes all other local modifications to this item (_except_ those in-flight).
2162         // Delete all items in reencrypt or error.
2163         CKKSOutgoingQueueEntry* reencryptOQE = [CKKSOutgoingQueueEntry tryFromDatabase:oqe.uuid state:SecCKKSStateReencrypt zoneID:self.zoneID error:&error];
2164         if(error) {
2165             ckkserror("ckks", self, "Couldn't load reencrypt OQE sibling for %@: %@", oqe, error);
2166         }
2167         if(reencryptOQE) {
2168             [reencryptOQE deleteFromDatabase:&error];
2169             if(error) {
2170                 ckkserror("ckks", self, "Couldn't delete reencrypt OQE sibling(%@) for %@: %@", reencryptOQE, oqe, error);
2171             }
2172             error = nil;
2173         }
2174
2175         CKKSOutgoingQueueEntry* errorOQE = [CKKSOutgoingQueueEntry tryFromDatabase:oqe.uuid state:SecCKKSStateError zoneID:self.zoneID error:&error];
2176         if(error) {
2177             ckkserror("ckks", self, "Couldn't load error OQE sibling for %@: %@", oqe, error);
2178         }
2179         if(errorOQE) {
2180             [errorOQE deleteFromDatabase:&error];
2181             if(error) {
2182                 ckkserror("ckks", self, "Couldn't delete error OQE sibling(%@) for %@: %@", reencryptOQE, oqe, error);
2183             }
2184         }
2185
2186         [self processOutgoingQueue:operationGroup];
2187
2188         return true;
2189     }];
2190 }
2191
2192 -(void)setCurrentItemForAccessGroup:(NSData* _Nonnull)newItemPersistentRef
2193                                hash:(NSData*)newItemSHA1
2194                         accessGroup:(NSString*)accessGroup
2195                          identifier:(NSString*)identifier
2196                           replacing:(NSData* _Nullable)oldCurrentItemPersistentRef
2197                                hash:(NSData*)oldItemSHA1
2198                            complete:(void (^) (NSError* operror)) complete
2199 {
2200     if(accessGroup == nil || identifier == nil) {
2201         NSError* error = [NSError errorWithDomain:CKKSErrorDomain
2202                                              code:errSecParam
2203                                       description:@"No access group or identifier given"];
2204         ckkserror("ckkscurrent", self, "Cancelling request: %@", error);
2205         complete(error);
2206         return;
2207     }
2208
2209     // Not being in a CloudKit account is an automatic failure.
2210     // But, wait a good long while for the CloudKit account state to be known (in the case of daemon startup)
2211     [self.accountStateKnown wait:(SecCKKSTestsEnabled() ? 1*NSEC_PER_SEC : 30*NSEC_PER_SEC)];
2212
2213     if(self.accountStatus != CKKSAccountStatusAvailable) {
2214         NSError* error = [NSError errorWithDomain:CKKSErrorDomain
2215                                              code:CKKSNotLoggedIn
2216                                       description:@"User is not signed into iCloud."];
2217         ckksnotice("ckkscurrent", self, "Rejecting current item pointer set since we don't have an iCloud account.");
2218         complete(error);
2219         return;
2220     }
2221
2222     ckksnotice("ckkscurrent", self, "Starting change current pointer operation for %@-%@", accessGroup, identifier);
2223     CKKSUpdateCurrentItemPointerOperation* ucipo = [[CKKSUpdateCurrentItemPointerOperation alloc] initWithCKKSKeychainView:self
2224                                                                                                                    newItem:newItemPersistentRef
2225                                                                                                                       hash:newItemSHA1
2226                                                                                                                accessGroup:accessGroup
2227                                                                                                                 identifier:identifier
2228                                                                                                                  replacing:oldCurrentItemPersistentRef
2229                                                                                                                       hash:oldItemSHA1
2230                                                                                                           ckoperationGroup:[CKOperationGroup CKKSGroupWithName:@"currentitem-api"]];
2231
2232     WEAKIFY(self);
2233     CKKSResultOperation* returnCallback = [CKKSResultOperation operationWithBlock:^{
2234         STRONGIFY(self);
2235
2236         if(ucipo.error) {
2237             ckkserror("ckkscurrent", self, "Failed setting a current item pointer for %@ with %@", ucipo.currentPointerIdentifier, ucipo.error);
2238         } else {
2239             ckksnotice("ckkscurrent", self, "Finished setting a current item pointer for %@", ucipo.currentPointerIdentifier);
2240         }
2241         complete(ucipo.error);
2242     }];
2243     returnCallback.name = @"setCurrentItem-return-callback";
2244     [returnCallback addDependency: ucipo];
2245     [self scheduleOperation: returnCallback];
2246
2247     // Now, schedule ucipo. It modifies the CloudKit zone, so it should insert itself into the list of OutgoingQueueOperations.
2248     // Then, we won't have simultaneous zone-modifying operations.
2249     [ucipo linearDependencies:self.outgoingQueueOperations];
2250
2251     // If this operation hasn't started within 60 seconds, cancel it and return a "timed out" error.
2252     [ucipo timeout:60*NSEC_PER_SEC];
2253
2254     [self scheduleOperation:ucipo];
2255     return;
2256 }
2257
2258 -(void)getCurrentItemForAccessGroup:(NSString*)accessGroup
2259                          identifier:(NSString*)identifier
2260                     fetchCloudValue:(bool)fetchCloudValue
2261                            complete:(void (^) (NSString* uuid, NSError* operror)) complete
2262 {
2263     if(accessGroup == nil || identifier == nil) {
2264         ckksnotice("ckkscurrent", self, "Rejecting current item pointer get since no access group(%@) or identifier(%@) given", accessGroup, identifier);
2265         complete(NULL, [NSError errorWithDomain:CKKSErrorDomain
2266                                            code:errSecParam
2267                                     description:@"No access group or identifier given"]);
2268         return;
2269     }
2270
2271     // Not being in a CloudKit account is an automatic failure.
2272     // But, wait a good long while for the CloudKit account state to be known (in the case of daemon startup)
2273     [self.accountStateKnown wait:(SecCKKSTestsEnabled() ? 1*NSEC_PER_SEC : 30*NSEC_PER_SEC)];
2274
2275     if(self.accountStatus != CKKSAccountStatusAvailable) {
2276         ckksnotice("ckkscurrent", self, "Rejecting current item pointer get since we don't have an iCloud account.");
2277         complete(NULL, [NSError errorWithDomain:CKKSErrorDomain
2278                                            code:CKKSNotLoggedIn
2279                                     description:@"User is not signed into iCloud."]);
2280         return;
2281     }
2282
2283     CKKSResultOperation* fetchAndProcess = nil;
2284     if(fetchCloudValue) {
2285         fetchAndProcess = [self fetchAndProcessCKChanges:CKKSFetchBecauseCurrentItemFetchRequest];
2286     }
2287
2288     WEAKIFY(self);
2289     CKKSResultOperation* getCurrentItem = [CKKSResultOperation named:@"get-current-item-pointer" withBlock:^{
2290         if(fetchAndProcess.error) {
2291             ckksnotice("ckkscurrent", self, "Rejecting current item pointer get since fetch failed: %@", fetchAndProcess.error);
2292             complete(NULL, fetchAndProcess.error);
2293             return;
2294         }
2295
2296         STRONGIFY(self);
2297
2298         [self dispatchSync: ^bool {
2299             NSError* error = nil;
2300             NSString* currentIdentifier = [NSString stringWithFormat:@"%@-%@", accessGroup, identifier];
2301
2302             CKKSCurrentItemPointer* cip = [CKKSCurrentItemPointer fromDatabase:currentIdentifier
2303                                                                          state:SecCKKSProcessedStateLocal
2304                                                                         zoneID:self.zoneID
2305                                                                          error:&error];
2306             if(!cip || error) {
2307                 ckkserror("ckkscurrent", self, "No current item pointer for %@", currentIdentifier);
2308                 complete(nil, error);
2309                 return false;
2310             }
2311
2312             if(!cip.currentItemUUID) {
2313                 ckkserror("ckkscurrent", self, "Current item pointer is empty %@", cip);
2314                 complete(nil, [NSError errorWithDomain:CKKSErrorDomain
2315                                                   code:errSecInternalError
2316                                            description:@"Current item pointer is empty"]);
2317                 return false;
2318             }
2319
2320             ckksinfo("ckkscurrent", self, "Retrieved current item pointer: %@", cip);
2321             complete(cip.currentItemUUID, NULL);
2322             return true;
2323         }];
2324     }];
2325
2326     [getCurrentItem addNullableDependency:fetchAndProcess];
2327     [self scheduleOperation: getCurrentItem];
2328 }
2329
2330 - (CKKSKey*) keyForItem: (SecDbItemRef) item error: (NSError * __autoreleasing *) error {
2331     CKKSKeyClass* class = nil;
2332
2333     NSString* protection = (__bridge NSString*)SecDbItemGetCachedValueWithName(item, kSecAttrAccessible);
2334     if([protection isEqualToString: (__bridge NSString*)kSecAttrAccessibleWhenUnlocked]) {
2335         class = SecCKKSKeyClassA;
2336     } else if([protection isEqualToString: (__bridge NSString*)kSecAttrAccessibleAlwaysPrivate] ||
2337               [protection isEqualToString: (__bridge NSString*)kSecAttrAccessibleAfterFirstUnlock]) {
2338         class = SecCKKSKeyClassC;
2339     } else {
2340         NSError* localError = [NSError errorWithDomain:CKKSErrorDomain
2341                                                   code:CKKSInvalidKeyClass
2342                                            description:[NSString stringWithFormat:@"can't pick key class for protection %@", protection]];
2343         ckkserror("ckks", self, "can't pick key class: %@ %@", localError, item);
2344         if(error) {
2345             *error = localError;
2346         }
2347
2348         return nil;
2349     }
2350
2351     NSError* currentKeyError = nil;
2352     CKKSKey* key = [CKKSKey currentKeyForClass: class zoneID:self.zoneID error:&currentKeyError];
2353     if(!key || currentKeyError) {
2354         ckkserror("ckks", self, "Couldn't find current key for %@: %@", class, currentKeyError);
2355
2356         if(error) {
2357             *error = currentKeyError;
2358         }
2359         return nil;
2360     }
2361
2362     // and make sure it's unwrapped.
2363     NSError* loadedError = nil;
2364     if(![key ensureKeyLoaded:&loadedError]) {
2365         ckkserror("ckks", self, "Couldn't load key(%@): %@", key, loadedError);
2366         if(error) {
2367             *error = loadedError;
2368         }
2369         return nil;
2370     }
2371
2372     return key;
2373 }
2374
2375 - (CKKSResultOperation<CKKSKeySetProviderOperationProtocol>*)findKeySet
2376 {
2377     __block CKKSResultOperation<CKKSKeySetProviderOperationProtocol>* keysetOp = nil;
2378
2379     [self dispatchSyncWithAccountKeys:^bool {
2380         CKKSCurrentKeySet* keyset = [CKKSCurrentKeySet loadForZone:self.zoneID];
2381         if(keyset.currentTLKPointer.currentKeyUUID && keyset.tlk.uuid) {
2382             ckksnotice("ckks", self, "Already have keyset %@", keyset);
2383
2384             keysetOp = [[CKKSProvideKeySetOperation alloc] initWithZoneName:self.zoneName keySet:keyset];
2385             [self scheduleOperationWithoutDependencies:keysetOp];
2386             return true;
2387         } else if([self.keyHierarchyState isEqualToString:SecCKKSZoneKeyStateWaitForTLKUpload]) {
2388             CKKSCurrentKeySet* proposedKeySet = self.lastNewTLKOperation.keyset;
2389             ckksnotice("ckks", self, "Already have proposed keyset %@", proposedKeySet);
2390
2391             keysetOp = [[CKKSProvideKeySetOperation alloc] initWithZoneName:self.zoneName keySet:proposedKeySet];
2392             [self scheduleOperationWithoutDependencies:keysetOp];
2393             return true;
2394         } else {
2395             // No existing keyset (including keys) exists.
2396             // The state machine will know what to do!
2397             self.tlkCreationRequested = true;
2398
2399             ckksnotice("ckks", self, "Received a keyset request; forwarding to state machine");
2400
2401             keysetOp = (CKKSProvideKeySetOperation*) [self findFirstPendingOperation:self.keysetProviderOperations];
2402             if(!keysetOp) {
2403                 keysetOp = [[CKKSProvideKeySetOperation alloc] initWithZoneName:self.zoneName];
2404                 [self.keysetProviderOperations addObject:keysetOp];
2405
2406                 // This is an abuse of operations: they should generally run when added to a queue, not wait, but this allows recipients to set timeouts
2407                 [self scheduleOperationWithoutDependencies:keysetOp];
2408             }
2409
2410             [self _onqueueAdvanceKeyStateMachineToState:nil withError:nil];
2411         }
2412
2413         return true;
2414     }];
2415
2416     return keysetOp;
2417 }
2418
2419 - (void)_onqueueRunKeysetProviderOperations:(CKKSCurrentKeySet*)keyset
2420 {
2421     ckksnotice("ckkskey", self, "Providing keyset (%@) to listeners", keyset);
2422
2423     // We have some keyset; they can ask again if they want a new one
2424     self.tlkCreationRequested = false;
2425
2426     for(CKKSResultOperation<CKKSKeySetProviderOperationProtocol>* op in self.keysetProviderOperations) {
2427         if([op isPending]) {
2428             [op provideKeySet:keyset];
2429         }
2430     }
2431 }
2432
2433 - (void)receiveTLKUploadRecords:(NSArray<CKRecord*>*)records
2434 {
2435     // First, filter for records matching this zone
2436     NSMutableArray<CKRecord*>* zoneRecords = [NSMutableArray array];
2437     for(CKRecord* record in records) {
2438         if([record.recordID.zoneID isEqual:self.zoneID]) {
2439             [zoneRecords addObject:record];
2440         }
2441     }
2442
2443     ckksnotice("ckkskey", self, "Received a set of %lu TLK upload records", (unsigned long)zoneRecords.count);
2444
2445     if(!zoneRecords || zoneRecords.count == 0) {
2446         return;
2447     }
2448
2449     [self dispatchSyncWithAccountKeys:^bool {
2450
2451         for(CKRecord* record in zoneRecords) {
2452             [self _onqueueCKRecordChanged:record resync:false];
2453         }
2454
2455         return true;
2456     }];
2457 }
2458
2459 // Use the following method to find the first pending operation in a weak collection
2460 - (NSOperation*)findFirstPendingOperation: (NSHashTable*) table {
2461     return [self findFirstPendingOperation:table ofClass:nil];
2462 }
2463
2464 // Use the following method to find the first pending operation in a weak collection
2465 - (NSOperation*)findFirstPendingOperation: (NSHashTable*) table ofClass:(Class)class {
2466     @synchronized(table) {
2467         for(NSOperation* op in table) {
2468             if(op != nil && [op isPending] && (class == nil || [op isKindOfClass: class])) {
2469                 return op;
2470             }
2471         }
2472         return nil;
2473     }
2474 }
2475
2476 // Use the following method to count the pending operations in a weak collection
2477 - (int64_t)countPendingOperations: (NSHashTable*) table {
2478     @synchronized(table) {
2479         int count = 0;
2480         for(NSOperation* op in table) {
2481             if(op != nil && !([op isExecuting] || [op isFinished])) {
2482                 count++;
2483             }
2484         }
2485         return count;
2486     }
2487 }
2488
2489 - (NSSet<NSString*>*)_onqueuePriorityOutgoingQueueUUIDs
2490 {
2491     return [self.pendingSyncCallbacks.allKeys copy];
2492 }
2493
2494 - (CKKSOutgoingQueueOperation*)processOutgoingQueue:(CKOperationGroup*)ckoperationGroup {
2495     return [self processOutgoingQueueAfter:nil ckoperationGroup:ckoperationGroup];
2496 }
2497
2498 - (CKKSOutgoingQueueOperation*)processOutgoingQueueAfter:(CKKSResultOperation*)after ckoperationGroup:(CKOperationGroup*)ckoperationGroup {
2499     return [self processOutgoingQueueAfter:after requiredDelay:DISPATCH_TIME_FOREVER ckoperationGroup:ckoperationGroup];
2500 }
2501
2502 - (CKKSOutgoingQueueOperation*)processOutgoingQueueAfter:(CKKSResultOperation*)after
2503                                            requiredDelay:(uint64_t)requiredDelay
2504                                         ckoperationGroup:(CKOperationGroup*)ckoperationGroup
2505 {
2506     CKKSOutgoingQueueOperation* outgoingop =
2507             (CKKSOutgoingQueueOperation*) [self findFirstPendingOperation:self.outgoingQueueOperations
2508                                                                   ofClass:[CKKSOutgoingQueueOperation class]];
2509     if(outgoingop) {
2510         if(after) {
2511             [outgoingop addDependency: after];
2512         }
2513         if([outgoingop isPending]) {
2514             if(!outgoingop.ckoperationGroup && ckoperationGroup) {
2515                 outgoingop.ckoperationGroup = ckoperationGroup;
2516             } else if(ckoperationGroup) {
2517                 ckkserror("ckks", self, "Throwing away CKOperationGroup(%@) in favor of (%@)", ckoperationGroup.name, outgoingop.ckoperationGroup.name);
2518             }
2519
2520             // Will log any pending dependencies as well
2521             ckksnotice("ckksoutgoing", self, "Returning existing %@", outgoingop);
2522
2523             // Shouldn't be necessary, but can't hurt
2524             [self.outgoingQueueOperationScheduler triggerAt:requiredDelay];
2525             return outgoingop;
2526         }
2527     }
2528
2529     CKKSOutgoingQueueOperation* op = [[CKKSOutgoingQueueOperation alloc] initWithCKKSKeychainView:self ckoperationGroup:ckoperationGroup];
2530     op.name = @"outgoing-queue-operation";
2531     [op addNullableDependency:after];
2532     [op addNullableDependency:self.outgoingQueueOperationScheduler.operationDependency];
2533
2534     [self.outgoingQueueOperationScheduler triggerAt:requiredDelay];
2535
2536     [self scheduleOperation: op];
2537     ckksnotice("ckksoutgoing", self, "Scheduled %@", op);
2538     return op;
2539 }
2540
2541 - (void)processIncomingQueueAfterNextUnlock {
2542     // Thread races aren't so important here; we might end up with two or three copies of this operation, but that's okay.
2543     if(![self.processIncomingQueueAfterNextUnlockOperation isPending]) {
2544         WEAKIFY(self);
2545
2546         CKKSResultOperation* restartIncomingQueueOperation = [CKKSResultOperation operationWithBlock:^{
2547             STRONGIFY(self);
2548             // This IQO shouldn't error if the keybag has locked again. It will simply try again later.
2549             [self processIncomingQueue:false];
2550         }];
2551
2552         restartIncomingQueueOperation.name = @"reprocess-incoming-queue-after-unlock";
2553         self.processIncomingQueueAfterNextUnlockOperation = restartIncomingQueueOperation;
2554
2555         [restartIncomingQueueOperation addNullableDependency:self.lockStateTracker.unlockDependency];
2556         [self scheduleOperation: restartIncomingQueueOperation];
2557     }
2558 }
2559
2560 - (CKKSResultOperation*)resultsOfNextProcessIncomingQueueOperation {
2561     if(self.resultsOfNextIncomingQueueOperationOperation && [self.resultsOfNextIncomingQueueOperationOperation isPending]) {
2562         return self.resultsOfNextIncomingQueueOperationOperation;
2563     }
2564
2565     // Else, make a new one.
2566     self.resultsOfNextIncomingQueueOperationOperation = [CKKSResultOperation named:[NSString stringWithFormat:@"wait-for-next-incoming-queue-operation-%@", self.zoneName] withBlock:^{}];
2567     return self.resultsOfNextIncomingQueueOperationOperation;
2568 }
2569
2570 - (CKKSIncomingQueueOperation*)processIncomingQueue:(bool)failOnClassA {
2571     return [self processIncomingQueue:failOnClassA after: nil];
2572 }
2573
2574 - (CKKSIncomingQueueOperation*) processIncomingQueue:(bool)failOnClassA after: (CKKSResultOperation*) after {
2575     CKKSIncomingQueueOperation* incomingop = (CKKSIncomingQueueOperation*) [self findFirstPendingOperation:self.incomingQueueOperations];
2576     if(incomingop) {
2577         ckksinfo("ckks", self, "Skipping processIncomingQueue due to at least one pending instance");
2578         if(after) {
2579             [incomingop addNullableDependency: after];
2580         }
2581
2582         // check (again) for race condition; if the op has started we need to add another (for the dependency)
2583         if([incomingop isPending]) {
2584             incomingop.errorOnClassAFailure |= failOnClassA;
2585             return incomingop;
2586         }
2587     }
2588
2589     CKKSIncomingQueueOperation* op = [[CKKSIncomingQueueOperation alloc] initWithCKKSKeychainView:self errorOnClassAFailure:failOnClassA];
2590     op.name = @"incoming-queue-operation";
2591     if(after != nil) {
2592         [op addSuccessDependency: after];
2593     }
2594
2595     if(self.resultsOfNextIncomingQueueOperationOperation) {
2596         [self.resultsOfNextIncomingQueueOperationOperation addSuccessDependency:op];
2597         [self scheduleOperation:self.resultsOfNextIncomingQueueOperationOperation];
2598     }
2599
2600     [self scheduleOperation: op];
2601     return op;
2602 }
2603
2604 - (CKKSScanLocalItemsOperation*)scanLocalItems:(NSString*)operationName {
2605     return [self scanLocalItems:operationName ckoperationGroup:nil after:nil];
2606 }
2607
2608 - (CKKSScanLocalItemsOperation*)scanLocalItems:(NSString*)operationName ckoperationGroup:(CKOperationGroup*)operationGroup after:(NSOperation*)after {
2609     CKKSScanLocalItemsOperation* scanOperation = [[CKKSScanLocalItemsOperation alloc] initWithCKKSKeychainView:self ckoperationGroup:operationGroup];
2610     scanOperation.name = operationName;
2611
2612     [scanOperation addNullableDependency:self.lastFixupOperation];
2613     [scanOperation addNullableDependency:self.lockStateTracker.unlockDependency];
2614     [scanOperation addNullableDependency:self.keyStateReadyDependency];
2615     [scanOperation addNullableDependency:after];
2616
2617     [self scheduleOperation: scanOperation];
2618     return scanOperation;
2619 }
2620
2621 - (CKKSUpdateDeviceStateOperation*)updateDeviceState:(bool)rateLimit
2622                    waitForKeyHierarchyInitialization:(uint64_t)timeout
2623                                     ckoperationGroup:(CKOperationGroup*)ckoperationGroup {
2624
2625     WEAKIFY(self);
2626
2627     // If securityd just started, the key state might be in some transient early state. Wait a bit.
2628     CKKSResultOperation* waitForKeyReady = [CKKSResultOperation named:@"device-state-wait" withBlock:^{
2629         STRONGIFY(self);
2630         ckksnotice("ckksdevice", self, "Finished waiting for key hierarchy transient state, currently %@", self.keyHierarchyState);
2631     }];
2632
2633     [waitForKeyReady addNullableDependency:self.keyStateNonTransientDependency];
2634     [waitForKeyReady timeout:timeout];
2635     [self.waitingQueue addOperation:waitForKeyReady];
2636
2637     CKKSUpdateDeviceStateOperation* op = [[CKKSUpdateDeviceStateOperation alloc] initWithCKKSKeychainView:self rateLimit:rateLimit ckoperationGroup:ckoperationGroup];
2638     op.name = @"device-state-operation";
2639
2640     [op addDependency: waitForKeyReady];
2641
2642     // op modifies the CloudKit zone, so it should insert itself into the list of OutgoingQueueOperations.
2643     // Then, we won't have simultaneous zone-modifying operations and confuse ourselves.
2644     // However, since we might have pending OQOs, it should try to insert itself at the beginning of the linearized list
2645     [op linearDependenciesWithSelfFirst:self.outgoingQueueOperations];
2646
2647     // CKKSUpdateDeviceStateOperations are special: they should fire even if we don't believe we're in an iCloud account.
2648     // They also shouldn't block or be blocked by any other operation; our wait operation above will handle that
2649     [self scheduleOperationWithoutDependencies:op];
2650     return op;
2651 }
2652
2653 // There are some errors which won't be reported but will be reflected in the CDSE; any error coming out of here is fatal
2654 - (CKKSDeviceStateEntry*)_onqueueCurrentDeviceStateEntry: (NSError* __autoreleasing*)error {
2655     NSError* localerror = nil;
2656
2657     CKKSAccountStateTracker* accountTracker = self.accountTracker;
2658     CKKSAccountStatus hsa2Status = accountTracker.hsa2iCloudAccountStatus;
2659
2660     // We must have an HSA2 iCloud account and a CloudKit account to even create one of these
2661     if(hsa2Status != CKKSAccountStatusAvailable ||
2662        accountTracker.currentCKAccountInfo.accountStatus != CKAccountStatusAvailable) {
2663         ckkserror("ckksdevice", self, "No iCloud account active: %@ hsa2 account:%@",
2664                   accountTracker.currentCKAccountInfo,
2665                   CKKSAccountStatusToString(hsa2Status));
2666         localerror = [NSError errorWithDomain:@"securityd"
2667                                          code:errSecInternalError
2668                                      userInfo:@{NSLocalizedDescriptionKey: [NSString stringWithFormat: @"No active HSA2 iCloud account: %@", accountTracker.currentCKAccountInfo]}];
2669         if(error) {
2670             *error = localerror;
2671         }
2672         return nil;
2673     }
2674
2675     NSString* ckdeviceID = accountTracker.ckdeviceID;
2676     if(ckdeviceID == nil) {
2677         ckkserror("ckksdevice", self, "No CK device ID available; cannot make device state entry");
2678         localerror = [NSError errorWithDomain:CKKSErrorDomain
2679                                          code:CKKSNotLoggedIn
2680                                      userInfo:@{NSLocalizedDescriptionKey: [NSString stringWithFormat: @"No CK device ID: %@", accountTracker.currentCKAccountInfo]}];
2681         if(error) {
2682             *error = localerror;
2683         }
2684         return nil;
2685     }
2686
2687     CKKSDeviceStateEntry* oldcdse = [CKKSDeviceStateEntry tryFromDatabase:ckdeviceID zoneID:self.zoneID error:&localerror];
2688     if(localerror) {
2689         ckkserror("ckksdevice", self, "Couldn't read old CKKSDeviceStateEntry from database: %@", localerror);
2690         if(error) {
2691             *error = localerror;
2692         }
2693         return nil;
2694     }
2695
2696     // Find out what we think the current keys are
2697     CKKSCurrentKeyPointer* currentTLKPointer    = [CKKSCurrentKeyPointer tryFromDatabase: SecCKKSKeyClassTLK zoneID:self.zoneID error:&localerror];
2698     CKKSCurrentKeyPointer* currentClassAPointer = [CKKSCurrentKeyPointer tryFromDatabase: SecCKKSKeyClassA   zoneID:self.zoneID error:&localerror];
2699     CKKSCurrentKeyPointer* currentClassCPointer = [CKKSCurrentKeyPointer tryFromDatabase: SecCKKSKeyClassC   zoneID:self.zoneID error:&localerror];
2700     if(localerror) {
2701         // Things is broken, but the whole point of this record is to share the brokenness. Continue.
2702         ckkserror("ckksdevice", self, "Couldn't read current key pointers from database: %@; proceeding", localerror);
2703         localerror = nil;
2704     }
2705
2706     CKKSKey* suggestedTLK       = currentTLKPointer.currentKeyUUID    ? [CKKSKey tryFromDatabase:currentTLKPointer.currentKeyUUID    zoneID:self.zoneID error:&localerror] : nil;
2707     CKKSKey* suggestedClassAKey = currentClassAPointer.currentKeyUUID ? [CKKSKey tryFromDatabase:currentClassAPointer.currentKeyUUID zoneID:self.zoneID error:&localerror] : nil;
2708     CKKSKey* suggestedClassCKey = currentClassCPointer.currentKeyUUID ? [CKKSKey tryFromDatabase:currentClassCPointer.currentKeyUUID zoneID:self.zoneID error:&localerror] : nil;
2709
2710     if(localerror) {
2711         // Things is broken, but the whole point of this record is to share the brokenness. Continue.
2712         ckkserror("ckksdevice", self, "Couldn't read keys from database: %@; proceeding", localerror);
2713         localerror = nil;
2714     }
2715
2716     // Check if we posess the keys in the keychain
2717     [suggestedTLK ensureKeyLoaded:&localerror];
2718     if(localerror && [self.lockStateTracker isLockedError:localerror]) {
2719         ckkserror("ckksdevice", self, "Device is locked; couldn't read TLK from keychain. Assuming it is present and continuing; error was %@", localerror);
2720         localerror = nil;
2721     } else if(localerror) {
2722         ckkserror("ckksdevice", self, "Couldn't read TLK from keychain. We do not have a current TLK. Error was %@", localerror);
2723         suggestedTLK = nil;
2724     }
2725
2726     [suggestedClassAKey ensureKeyLoaded:&localerror];
2727     if(localerror && [self.lockStateTracker isLockedError:localerror]) {
2728         ckkserror("ckksdevice", self, "Device is locked; couldn't read ClassA key from keychain. Assuming it is present and continuing; error was %@", localerror);
2729         localerror = nil;
2730     } else if(localerror) {
2731         ckkserror("ckksdevice", self, "Couldn't read ClassA key from keychain. We do not have a current ClassA key. Error was %@", localerror);
2732         suggestedClassAKey = nil;
2733     }
2734
2735     [suggestedClassCKey ensureKeyLoaded:&localerror];
2736     // class C keys are stored class C, so uh, don't check lock state.
2737     if(localerror) {
2738         ckkserror("ckksdevice", self, "Couldn't read ClassC key from keychain. We do not have a current ClassC key. Error was %@", localerror);
2739         suggestedClassCKey = nil;
2740     }
2741
2742     // We'd like to have the circle peer ID. Give the account state tracker a fighting chance, but not having it is not an error
2743     // But, if the platform doesn't have SOS, don't bother
2744     if(OctagonPlatformSupportsSOS() && [accountTracker.accountCirclePeerIDInitialized wait:500*NSEC_PER_MSEC] != 0 && !accountTracker.accountCirclePeerID) {
2745         ckkserror("ckksdevice", self, "No SOS peer ID available");
2746     }
2747
2748     // We'd also like the Octagon status
2749     if([accountTracker.octagonInformationInitialized wait:500*NSEC_PER_MSEC] != 0 && !accountTracker.octagonPeerID) {
2750         ckkserror("ckksdevice", self, "No octagon peer ID available");
2751     }
2752
2753     // Reset the last unlock time to 'day' granularity in UTC
2754     NSCalendar* calendar = [NSCalendar calendarWithIdentifier:NSCalendarIdentifierISO8601];
2755     calendar.timeZone = [NSTimeZone timeZoneWithAbbreviation:@"UTC"];
2756     NSDate* lastUnlockDay = self.lockStateTracker.lastUnlockTime;
2757     lastUnlockDay = lastUnlockDay ? [calendar startOfDayForDate:lastUnlockDay] : nil;
2758
2759     // We only really want the oldcdse for its encodedCKRecord, so make a new cdse here
2760     CKKSDeviceStateEntry* newcdse = [[CKKSDeviceStateEntry alloc] initForDevice:ckdeviceID
2761                                                                       osVersion:SecCKKSHostOSVersion()
2762                                                                  lastUnlockTime:lastUnlockDay
2763                                                                   octagonPeerID:accountTracker.octagonPeerID
2764                                                                   octagonStatus:accountTracker.octagonStatus
2765                                                                    circlePeerID:accountTracker.accountCirclePeerID
2766                                                                    circleStatus:accountTracker.currentCircleStatus.status
2767                                                                        keyState:self.keyHierarchyState
2768                                                                  currentTLKUUID:suggestedTLK.uuid
2769                                                               currentClassAUUID:suggestedClassAKey.uuid
2770                                                               currentClassCUUID:suggestedClassCKey.uuid
2771                                                                          zoneID:self.zoneID
2772                                                                 encodedCKRecord:oldcdse.encodedCKRecord];
2773     return newcdse;
2774 }
2775
2776 - (CKKSSynchronizeOperation*) resyncWithCloud {
2777     CKKSSynchronizeOperation* op = [[CKKSSynchronizeOperation alloc] initWithCKKSKeychainView: self];
2778     [self scheduleOperation: op];
2779     return op;
2780 }
2781
2782 - (CKKSLocalSynchronizeOperation*)resyncLocal {
2783     CKKSLocalSynchronizeOperation* op = [[CKKSLocalSynchronizeOperation alloc] initWithCKKSKeychainView:self];
2784     [self scheduleOperation: op];
2785     return op;
2786 }
2787
2788 - (CKKSResultOperation*)fetchAndProcessCKChanges:(CKKSFetchBecause*)because {
2789     return [self fetchAndProcessCKChanges:because after:nil];
2790 }
2791
2792 - (CKKSResultOperation*)fetchAndProcessCKChanges:(CKKSFetchBecause*)because after:(CKKSResultOperation*)after {
2793     if(!SecCKKSIsEnabled()) {
2794         ckksinfo("ckks", self, "Skipping fetchAndProcessCKChanges due to disabled CKKS");
2795         return nil;
2796     }
2797
2798     if(after) {
2799         [self.zoneChangeFetcher holdFetchesUntil:after];
2800     }
2801
2802     // We fetched some changes; try to process them!
2803     return [self processIncomingQueue:false after:[self.zoneChangeFetcher requestSuccessfulFetch:because]];
2804 }
2805
2806 - (CKKSResultOperation*)fetchAndProcessCKChangesDueToAPNS:(CKRecordZoneNotification*)notification {
2807     if(!SecCKKSIsEnabled()) {
2808         ckksinfo("ckks", self, "Skipping fetchAndProcessCKChanges due to disabled CKKS");
2809         return nil;
2810     }
2811
2812     // We fetched some changes; try to process them!
2813     return [self processIncomingQueue:false after:[self.zoneChangeFetcher requestSuccessfulFetchDueToAPNS:notification]];
2814 }
2815
2816 // Lets the view know about a failed CloudKit write. If the error is "already have one of these records", it will
2817 // store the new records and kick off the new processing
2818 //
2819 // Note that you need to tell this function the records you wanted to save, so it can determine what needs deletion
2820 - (bool)_onqueueCKWriteFailed:(NSError*)ckerror attemptedRecordsChanged:(NSDictionary<CKRecordID*, CKRecord*>*)savedRecords {
2821     dispatch_assert_queue(self.queue);
2822
2823     NSDictionary<CKRecordID*,NSError*>* partialErrors = ckerror.userInfo[CKPartialErrorsByItemIDKey];
2824     if([ckerror.domain isEqual:CKErrorDomain] && ckerror.code == CKErrorPartialFailure && partialErrors) {
2825         // Check if this error was "you're out of date"
2826         bool recordChanged = true;
2827
2828         for(NSError* error in partialErrors.allValues) {
2829             if((![error.domain isEqual:CKErrorDomain]) || (error.code != CKErrorBatchRequestFailed && error.code != CKErrorServerRecordChanged && error.code != CKErrorUnknownItem)) {
2830                 // There's an error in there that isn't CKErrorServerRecordChanged, CKErrorBatchRequestFailed, or CKErrorUnknownItem. Don't handle nicely...
2831                 recordChanged = false;
2832             }
2833         }
2834
2835         if(recordChanged) {
2836             ckksnotice("ckks", self, "Received a ServerRecordChanged error, attempting to update new records and delete unknown ones");
2837
2838             bool updatedRecord = false;
2839
2840             for(CKRecordID* recordID in partialErrors.allKeys) {
2841                 NSError* error = partialErrors[recordID];
2842                 if([error.domain isEqual:CKErrorDomain] && error.code == CKErrorServerRecordChanged) {
2843                     CKRecord* newRecord = error.userInfo[CKRecordChangedErrorServerRecordKey];
2844                     ckksnotice("ckks", self, "On error: updating our idea of: %@", newRecord);
2845
2846                     updatedRecord |= [self _onqueueCKRecordChanged:newRecord resync:true];
2847                 } else if([error.domain isEqual:CKErrorDomain] && error.code == CKErrorUnknownItem) {
2848                     CKRecord* record = savedRecords[recordID];
2849                     ckksnotice("ckks", self, "On error: handling an unexpected delete of: %@ %@", recordID, record);
2850
2851                     updatedRecord |= [self _onqueueCKRecordDeleted:recordID recordType:record.recordType resync:true];
2852                 }
2853             }
2854
2855             if(updatedRecord) {
2856                 [self processIncomingQueue:false];
2857                 return true;
2858             }
2859         }
2860
2861         // Check if this error was the CKKS server extension rejecting the write
2862         for(CKRecordID* recordID in partialErrors.allKeys) {
2863             NSError* error = partialErrors[recordID];
2864
2865             NSError* underlyingError = error.userInfo[NSUnderlyingErrorKey];
2866             NSError* thirdLevelError = underlyingError.userInfo[NSUnderlyingErrorKey];
2867             ckksnotice("ckks", self, "Examining 'write failed' error: %@ %@ %@", error, underlyingError, thirdLevelError);
2868
2869             if([error.domain isEqualToString:CKErrorDomain] && error.code == CKErrorServerRejectedRequest &&
2870                underlyingError && [underlyingError.domain isEqualToString:CKInternalErrorDomain] && underlyingError.code == CKErrorInternalPluginError &&
2871                thirdLevelError && [thirdLevelError.domain isEqualToString:@"CloudkitKeychainService"]) {
2872
2873                 if(thirdLevelError.code == CKKSServerUnexpectedSyncKeyInChain) {
2874                     // The server thinks the classA/C synckeys don't wrap directly the to top TLK, but we don't (otherwise, we would have fixed it).
2875                     // Issue a key hierarchy fetch and see what's what.
2876                     ckkserror("ckks", self, "CKKS Server extension has told us about %@ for record %@; requesting refetch and reprocess of key hierarchy", thirdLevelError, recordID);
2877                     [self _onqueueKeyStateMachineRequestFetch];
2878                 } else {
2879                     ckkserror("ckks", self, "CKKS Server extension has told us about %@ for record %@, but we don't currently handle this error", thirdLevelError, recordID);
2880                 }
2881             }
2882         }
2883     }
2884
2885     return false;
2886 }
2887
2888 - (bool)_onqueueCKRecordDeleted:(CKRecordID*)recordID recordType:(NSString*)recordType resync:(bool)resync {
2889     dispatch_assert_queue(self.queue);
2890
2891     // TODO: resync doesn't really mean much here; what does it mean for a record to be 'deleted' if you're fetching from scratch?
2892
2893     if([recordType isEqual: SecCKRecordItemType]) {
2894         ckksinfo("ckks", self, "CloudKit notification: deleted record(%@): %@", recordType, recordID);
2895         NSError* error = nil;
2896         NSError* iqeerror = nil;
2897         CKKSMirrorEntry* ckme = [CKKSMirrorEntry fromDatabase: [recordID recordName] zoneID:self.zoneID error: &error];
2898
2899         // Deletes always succeed, not matter the generation count
2900         if(ckme) {
2901             [ckme deleteFromDatabase:&error];
2902
2903             CKKSIncomingQueueEntry* iqe = [[CKKSIncomingQueueEntry alloc] initWithCKKSItem:ckme.item action:SecCKKSActionDelete state:SecCKKSStateNew];
2904             [iqe saveToDatabase:&iqeerror];
2905             if(iqeerror) {
2906                 ckkserror("ckks", self, "Couldn't save incoming queue entry: %@", iqeerror);
2907             }
2908         }
2909         ckksinfo("ckks", self, "CKKSMirrorEntry was deleted: %@ %@ error: %@", recordID, ckme, error);
2910         // TODO: actually pass error back up
2911         return (error == nil);
2912
2913     } else if([recordType isEqual: SecCKRecordCurrentItemType]) {
2914         ckksinfo("ckks", self, "CloudKit notification: deleted current item pointer(%@): %@", recordType, recordID);
2915         NSError* error = nil;
2916
2917         [[CKKSCurrentItemPointer tryFromDatabase:[recordID recordName] state:SecCKKSProcessedStateRemote zoneID:self.zoneID error:&error] deleteFromDatabase:&error];
2918         [[CKKSCurrentItemPointer fromDatabase:[recordID recordName]    state:SecCKKSProcessedStateLocal  zoneID:self.zoneID error:&error] deleteFromDatabase:&error];
2919
2920         ckksinfo("ckks", self, "CKKSCurrentItemPointer was deleted: %@ error: %@", recordID, error);
2921         return (error == nil);
2922
2923     } else if([recordType isEqual: SecCKRecordIntermediateKeyType]) {
2924         // TODO: handle in some interesting way
2925         return true;
2926     } else if([recordType isEqual: SecCKRecordTLKShareType]) {
2927         NSError* error = nil;
2928         ckksinfo("ckks", self, "CloudKit notification: deleted tlk share record(%@): %@", recordType, recordID);
2929         CKKSTLKShareRecord* share = [CKKSTLKShareRecord tryFromDatabaseFromCKRecordID:recordID error:&error];
2930         [share deleteFromDatabase:&error];
2931
2932         if(error) {
2933             ckkserror("ckks", self, "CK notification: Couldn't delete deleted TLKShare: %@ %@", recordID,  error);
2934         }
2935         return (error == nil);
2936
2937     } else if([recordType isEqual: SecCKRecordDeviceStateType]) {
2938         NSError* error = nil;
2939         ckksinfo("ckks", self, "CloudKit notification: deleted device state record(%@): %@", recordType, recordID);
2940
2941         CKKSDeviceStateEntry* cdse = [CKKSDeviceStateEntry tryFromDatabaseFromCKRecordID:recordID error:&error];
2942         [cdse deleteFromDatabase: &error];
2943         ckksinfo("ckks", self, "CKKSCurrentItemPointer(%@) was deleted: %@ error: %@", cdse, recordID, error);
2944
2945         return (error == nil);
2946
2947     } else if ([recordType isEqualToString:SecCKRecordManifestType]) {
2948         ckksinfo("ckks", self, "CloudKit notification: deleted manifest record (%@): %@", recordType, recordID);
2949
2950         NSError* error = nil;
2951         CKKSManifest* manifest = [CKKSManifest manifestForRecordName:recordID.recordName error:&error];
2952         if (manifest) {
2953             [manifest deleteFromDatabase:&error];
2954         }
2955
2956         ckksinfo("ckks", self, "CKKSManifest was deleted: %@ %@ error: %@", recordID, manifest, error);
2957         // TODO: actually pass error back up
2958         return error == nil;
2959     }
2960
2961     else {
2962         ckkserror("ckksfetch", self, "unknown record type: %@ %@", recordType, recordID);
2963         return false;
2964     }
2965 }
2966
2967 - (bool)_onqueueCKRecordChanged:(CKRecord*)record resync:(bool)resync {
2968     dispatch_assert_queue(self.queue);
2969
2970     @autoreleasepool {
2971         ckksnotice("ckksfetch", self, "Processing record modification(%@): %@", record.recordType, record);
2972
2973         if([[record recordType] isEqual: SecCKRecordItemType]) {
2974             [self _onqueueCKRecordItemChanged:record resync:resync];
2975             return true;
2976         } else if([[record recordType] isEqual: SecCKRecordCurrentItemType]) {
2977             [self _onqueueCKRecordCurrentItemPointerChanged:record resync:resync];
2978             return true;
2979         } else if([[record recordType] isEqual: SecCKRecordIntermediateKeyType]) {
2980             [self _onqueueCKRecordKeyChanged:record resync:resync];
2981             return true;
2982         } else if ([[record recordType] isEqual: SecCKRecordTLKShareType]) {
2983             [self _onqueueCKRecordTLKShareChanged:record resync:resync];
2984             return true;
2985         } else if([[record recordType] isEqualToString: SecCKRecordCurrentKeyType]) {
2986             [self _onqueueCKRecordCurrentKeyPointerChanged:record resync:resync];
2987             return true;
2988         } else if ([[record recordType] isEqualToString:SecCKRecordManifestType]) {
2989             [self _onqueueCKRecordManifestChanged:record resync:resync];
2990             return true;
2991         } else if ([[record recordType] isEqualToString:SecCKRecordManifestLeafType]) {
2992             [self _onqueueCKRecordManifestLeafChanged:record resync:resync];
2993             return true;
2994         } else if ([[record recordType] isEqualToString:SecCKRecordDeviceStateType]) {
2995             [self _onqueueCKRecordDeviceStateChanged:record resync:resync];
2996             return true;
2997         } else {
2998             ckkserror("ckksfetch", self, "unknown record type: %@ %@", [record recordType], record);
2999             return false;
3000         }
3001     }
3002 }
3003
3004 - (void)_onqueueCKRecordItemChanged:(CKRecord*)record resync:(bool)resync {
3005     dispatch_assert_queue(self.queue);
3006
3007     NSError* error = nil;
3008     // Find if we knew about this record in the past
3009     bool update = false;
3010     CKKSMirrorEntry* ckme = [CKKSMirrorEntry tryFromDatabase: [[record recordID] recordName] zoneID:self.zoneID error:&error];
3011
3012     if(error) {
3013         ckkserror("ckks", self, "error loading a CKKSMirrorEntry from database: %@", error);
3014         // TODO: quit?
3015     }
3016
3017     if(resync) {
3018         if(!ckme) {
3019             ckkserror("ckksresync", self, "BUG: No local item matching resynced CloudKit record: %@", record);
3020         } else if(![ckme matchesCKRecord:record]) {
3021             ckkserror("ckksresync", self, "BUG: Local item doesn't match resynced CloudKit record: %@ %@", ckme, record);
3022         } else {
3023             ckksnotice("ckksresync", self, "Already know about this item record, updating anyway: %@", record.recordID);
3024         }
3025     }
3026
3027     if(ckme && ckme.item && ckme.item.generationCount > [record[SecCKRecordGenerationCountKey] unsignedLongLongValue]) {
3028         ckkserror("ckks", self, "received a record from CloudKit with a bad generation count: %@ (%ld > %@)", ckme.uuid,
3029                  (long) ckme.item.generationCount,
3030                  record[SecCKRecordGenerationCountKey]);
3031         // Abort processing this record.
3032         return;
3033     }
3034
3035     // If we found an old version in the database; this might be an update
3036     if(ckme) {
3037         if([ckme matchesCKRecord:record] && !resync) {
3038             // This is almost certainly a record we uploaded; CKFetchChanges sends them back as new records
3039             ckksnotice("ckks", self, "CloudKit has told us of record we already know about; skipping update");
3040             return;
3041         }
3042
3043         update = true;
3044         // Set the CKKSMirrorEntry's fields to be whatever this record holds
3045         [ckme setFromCKRecord: record];
3046     } else {
3047         // Have to make a new CKKSMirrorEntry
3048         ckme = [[CKKSMirrorEntry alloc] initWithCKRecord: record];
3049     }
3050
3051     [ckme saveToDatabase: &error];
3052
3053     if(error) {
3054         ckkserror("ckks", self, "couldn't save new CKRecord to database: %@ %@", record, error);
3055     } else {
3056         ckksdebug("ckks", self, "CKKSMirrorEntry was created: %@", ckme);
3057     }
3058
3059     NSError* iqeerror = nil;
3060     CKKSIncomingQueueEntry* iqe = [[CKKSIncomingQueueEntry alloc] initWithCKKSItem:ckme.item
3061                                                                             action:(update ? SecCKKSActionModify : SecCKKSActionAdd)
3062                                                                              state:SecCKKSStateNew];
3063     [iqe saveToDatabase:&iqeerror];
3064     if(iqeerror) {
3065         ckkserror("ckks", self, "Couldn't save modified incoming queue entry: %@", iqeerror);
3066     } else {
3067         ckksdebug("ckks", self, "CKKSIncomingQueueEntry was created: %@", iqe);
3068     }
3069
3070     // A remote change has occured for this record. Delete any pending local changes; they will be overwritten.
3071     CKKSOutgoingQueueEntry* oqe = [CKKSOutgoingQueueEntry tryFromDatabase:ckme.uuid state: SecCKKSStateNew zoneID:self.zoneID error: &error];
3072     if(error) {
3073         ckkserror("ckks", self, "Couldn't load OutgoingQueueEntry: %@", error);
3074     }
3075     if(oqe) {
3076         [self _onqueueChangeOutgoingQueueEntry:oqe toState:SecCKKSStateDeleted error:&error];
3077     }
3078
3079     // Reencryptions are pending changes too
3080     oqe = [CKKSOutgoingQueueEntry tryFromDatabase:ckme.uuid state: SecCKKSStateReencrypt zoneID:self.zoneID error: &error];
3081     if(error) {
3082         ckkserror("ckks", self, "Couldn't load reencrypted OutgoingQueueEntry: %@", error);
3083     }
3084     if(oqe) {
3085         [oqe deleteFromDatabase:&error];
3086         if(error) {
3087             ckkserror("ckks", self, "Couldn't delete reencrypted oqe(%@): %@", oqe, error);
3088         }
3089     }
3090 }
3091
3092 - (void)_onqueueCKRecordKeyChanged:(CKRecord*)record resync:(bool)resync {
3093     dispatch_assert_queue(self.queue);
3094
3095     NSError* error = nil;
3096
3097     if(resync) {
3098         NSError* resyncerror = nil;
3099
3100         CKKSKey* key = [CKKSKey tryFromDatabaseAnyState:record.recordID.recordName zoneID:self.zoneID error:&resyncerror];
3101         if(resyncerror) {
3102             ckkserror("ckksresync", self, "error loading key: %@", resyncerror);
3103         }
3104         if(!key) {
3105             ckkserror("ckksresync", self, "BUG: No sync key matching resynced CloudKit record: %@", record);
3106         } else if(![key matchesCKRecord:record]) {
3107             ckkserror("ckksresync", self, "BUG: Local sync key doesn't match resynced CloudKit record(s): %@ %@", key, record);
3108         } else {
3109             ckksnotice("ckksresync", self, "Already know about this sync key, skipping update: %@", record);
3110             return;
3111         }
3112     }
3113
3114     CKKSKey* remotekey = [[CKKSKey alloc] initWithCKRecord: record];
3115
3116     // Do we already know about this key?
3117     CKKSKey* possibleLocalKey = [CKKSKey tryFromDatabase:remotekey.uuid zoneID:self.zoneID error:&error];
3118     if(error) {
3119         ckkserror("ckkskey", self, "Error findibg exsiting local key for %@: %@", remotekey, error);
3120         // Go on, assuming there isn't a local key
3121     } else if(possibleLocalKey && [possibleLocalKey matchesCKRecord:record]) {
3122         // Okay, nothing new here. Update the CKRecord and move on.
3123         // Note: If the new record doesn't match the local copy, we have to go through the whole dance below
3124         possibleLocalKey.storedCKRecord = record;
3125         [possibleLocalKey saveToDatabase:&error];
3126
3127         if(error) {
3128             ckkserror("ckkskey", self, "Couldn't update existing key: %@: %@", possibleLocalKey, error);
3129         }
3130         return;
3131     }
3132
3133     // Drop into the synckeys table as a 'remote' key, then ask for a rekey operation.
3134     remotekey.state = SecCKKSProcessedStateRemote;
3135     remotekey.currentkey = false;
3136
3137     [remotekey saveToDatabase:&error];
3138     if(error) {
3139         ckkserror("ckkskey", self, "Couldn't save key record to database: %@: %@", remotekey, error);
3140         ckksinfo("ckkskey", self, "CKRecord was %@", record);
3141     }
3142
3143     // We've saved a new key in the database; trigger a rekey operation.
3144     [self _onqueueKeyStateMachineRequestProcess];
3145 }
3146
3147 - (void)_onqueueCKRecordTLKShareChanged:(CKRecord*)record resync:(bool)resync {
3148     dispatch_assert_queue(self.queue);
3149
3150     NSError* error = nil;
3151     if(resync) {
3152         // TODO fill in
3153     }
3154
3155     // CKKSTLKShares get saved with no modification
3156     CKKSTLKShareRecord* share = [[CKKSTLKShareRecord alloc] initWithCKRecord:record];
3157     [share saveToDatabase:&error];
3158     if(error) {
3159         ckkserror("ckksshare", self, "Couldn't save new TLK share to database: %@ %@", share, error);
3160     }
3161
3162     [self _onqueueKeyStateMachineRequestProcess];
3163 }
3164
3165 - (void)_onqueueCKRecordCurrentKeyPointerChanged:(CKRecord*)record resync:(bool)resync {
3166     dispatch_assert_queue(self.queue);
3167
3168     // Pull out the old CKP, if it exists
3169     NSError* ckperror = nil;
3170     CKKSCurrentKeyPointer* oldckp = [CKKSCurrentKeyPointer tryFromDatabase:((CKKSKeyClass*) record.recordID.recordName) zoneID:self.zoneID error:&ckperror];
3171     if(ckperror) {
3172         ckkserror("ckkskey", self, "error loading ckp: %@", ckperror);
3173     }
3174
3175     if(resync) {
3176         if(!oldckp) {
3177             ckkserror("ckksresync", self, "BUG: No current key pointer matching resynced CloudKit record: %@", record);
3178         } else if(![oldckp matchesCKRecord:record]) {
3179             ckkserror("ckksresync", self, "BUG: Local current key pointer doesn't match resynced CloudKit record: %@ %@", oldckp, record);
3180         } else {
3181             ckksnotice("ckksresync", self, "Current key pointer has 'changed', but it matches our local copy: %@", record);
3182         }
3183     }
3184
3185     NSError* error = nil;
3186     CKKSCurrentKeyPointer* currentkey = [[CKKSCurrentKeyPointer alloc] initWithCKRecord: record];
3187
3188     [currentkey saveToDatabase: &error];
3189     if(error) {
3190         ckkserror("ckkskey", self, "Couldn't save current key pointer to database: %@: %@", currentkey, error);
3191         ckksinfo("ckkskey", self, "CKRecord was %@", record);
3192     }
3193
3194     if([oldckp matchesCKRecord:record]) {
3195         ckksnotice("ckkskey", self, "Current key pointer modification doesn't change anything interesting; skipping reprocess: %@", record);
3196     } else {
3197         // We've saved a new key in the database; trigger a rekey operation.
3198         [self _onqueueKeyStateMachineRequestProcess];
3199     }
3200 }
3201
3202 - (void)_onqueueCKRecordCurrentItemPointerChanged:(CKRecord*)record resync:(bool)resync {
3203     dispatch_assert_queue(self.queue);
3204
3205     if(resync) {
3206         NSError* ciperror = nil;
3207         CKKSCurrentItemPointer* localcip  = [CKKSCurrentItemPointer tryFromDatabase:record.recordID.recordName state:SecCKKSProcessedStateLocal  zoneID:self.zoneID error:&ciperror];
3208         CKKSCurrentItemPointer* remotecip = [CKKSCurrentItemPointer tryFromDatabase:record.recordID.recordName state:SecCKKSProcessedStateRemote zoneID:self.zoneID error:&ciperror];
3209         if(ciperror) {
3210             ckkserror("ckksresync", self, "error loading cip: %@", ciperror);
3211         }
3212         if(!(localcip || remotecip)) {
3213             ckkserror("ckksresync", self, "BUG: No current item pointer matching resynced CloudKit record: %@", record);
3214         } else if(! ([localcip matchesCKRecord:record] || [remotecip matchesCKRecord:record]) ) {
3215             ckkserror("ckksresync", self, "BUG: Local current item pointer doesn't match resynced CloudKit record(s): %@ %@ %@", localcip, remotecip, record);
3216         } else {
3217             ckksnotice("ckksresync", self, "Already know about this current item pointer, skipping update: %@", record);
3218             return;
3219         }
3220     }
3221
3222     NSError* error = nil;
3223     CKKSCurrentItemPointer* cip = [[CKKSCurrentItemPointer alloc] initWithCKRecord: record];
3224     cip.state = SecCKKSProcessedStateRemote;
3225
3226     [cip saveToDatabase: &error];
3227     if(error) {
3228         ckkserror("currentitem", self, "Couldn't save current item pointer to database: %@: %@ %@", cip, error, record);
3229     }
3230 }
3231
3232 - (void)_onqueueCKRecordManifestChanged:(CKRecord*)record resync:(bool)resync
3233 {
3234     NSError* error = nil;
3235     CKKSPendingManifest* manifest = [[CKKSPendingManifest alloc] initWithCKRecord:record];
3236     [manifest saveToDatabase:&error];
3237     if (error) {
3238         ckkserror("CKKS", self, "Failed to save fetched manifest record to database: %@: %@", manifest, error);
3239         ckksinfo("CKKS", self, "manifest CKRecord was %@", record);
3240     }
3241 }
3242
3243 - (void)_onqueueCKRecordManifestLeafChanged:(CKRecord*)record resync:(bool)resync
3244 {
3245     NSError* error = nil;
3246     CKKSManifestLeafRecord* manifestLeaf = [[CKKSManifestPendingLeafRecord alloc] initWithCKRecord:record];
3247     [manifestLeaf saveToDatabase:&error];
3248     if (error) {
3249         ckkserror("CKKS", self, "Failed to save fetched manifest leaf record to database: %@: %@", manifestLeaf, error);
3250         ckksinfo("CKKS", self, "manifest leaf CKRecord was %@", record);
3251     }
3252 }
3253
3254 - (void)_onqueueCKRecordDeviceStateChanged:(CKRecord*)record resync:(bool)resync {
3255     if(resync) {
3256         NSError* dserror = nil;
3257         CKKSDeviceStateEntry* cdse  = [CKKSDeviceStateEntry tryFromDatabase:record.recordID.recordName zoneID:self.zoneID error:&dserror];
3258         if(dserror) {
3259             ckkserror("ckksresync", self, "error loading cdse: %@", dserror);
3260         }
3261         if(!cdse) {
3262             ckkserror("ckksresync", self, "BUG: No current device state entry matching resynced CloudKit record: %@", record);
3263         } else if(![cdse matchesCKRecord:record]) {
3264             ckkserror("ckksresync", self, "BUG: Local current device state entry doesn't match resynced CloudKit record(s): %@ %@", cdse, record);
3265         } else {
3266             ckksnotice("ckksresync", self, "Already know about this current item pointer, skipping update: %@", record);
3267             return;
3268         }
3269     }
3270
3271     NSError* error = nil;
3272     CKKSDeviceStateEntry* cdse = [[CKKSDeviceStateEntry alloc] initWithCKRecord:record];
3273     [cdse saveToDatabase:&error];
3274     if (error) {
3275         ckkserror("ckksdevice", self, "Failed to save device record to database: %@: %@ %@", cdse, error, record);
3276     }
3277 }
3278
3279 - (bool)_onqueueResetAllInflightOQE:(NSError**)error {
3280     NSError* localError = nil;
3281
3282     while(true) {
3283         NSArray<CKKSOutgoingQueueEntry*> * inflightQueueEntries = [CKKSOutgoingQueueEntry fetch:SecCKKSOutgoingQueueItemsAtOnce
3284                                                                                           state:SecCKKSStateInFlight
3285                                                                                          zoneID:self.zoneID
3286                                                                                           error:&localError];
3287
3288         if(localError != nil) {
3289             ckkserror("ckks", self, "Error finding inflight outgoing queue records: %@", localError);
3290             if(error) {
3291                 *error = localError;
3292             }
3293             return false;
3294         }
3295
3296         if([inflightQueueEntries count] == 0u) {
3297             break;
3298         }
3299
3300         for(CKKSOutgoingQueueEntry* oqe in inflightQueueEntries) {
3301             [self _onqueueChangeOutgoingQueueEntry:oqe toState:SecCKKSStateNew error:&localError];
3302
3303             if(localError) {
3304                 ckkserror("ckks", self, "Error fixing up inflight OQE(%@): %@", oqe, localError);
3305                 if(error) {
3306                     *error = localError;
3307                 }
3308                 return false;
3309             }
3310         }
3311     }
3312
3313     return true;
3314 }
3315
3316 - (bool)_onqueueChangeOutgoingQueueEntry: (CKKSOutgoingQueueEntry*) oqe toState: (NSString*) state error: (NSError* __autoreleasing*) error {
3317     dispatch_assert_queue(self.queue);
3318
3319     NSError* localerror = nil;
3320
3321     if([state isEqualToString: SecCKKSStateDeleted]) {
3322         // Hurray, this must be a success
3323         SecBoolNSErrorCallback callback = self.pendingSyncCallbacks[oqe.uuid];
3324         if(callback) {
3325             callback(true, nil);
3326             self.pendingSyncCallbacks[oqe.uuid] = nil;
3327         }
3328
3329         [oqe deleteFromDatabase: &localerror];
3330         if(localerror) {
3331             ckkserror("ckks", self, "Couldn't delete %@: %@", oqe, localerror);
3332         }
3333
3334     } else if([oqe.state isEqualToString:SecCKKSStateInFlight] && [state isEqualToString:SecCKKSStateNew]) {
3335         // An in-flight OQE is moving to new? See if it's been superceded
3336         CKKSOutgoingQueueEntry* newOQE = [CKKSOutgoingQueueEntry tryFromDatabase:oqe.uuid state:SecCKKSStateNew zoneID:self.zoneID error:&localerror];
3337         if(localerror) {
3338             ckkserror("ckksoutgoing", self, "Couldn't fetch an overwriting OQE, assuming one doesn't exist: %@", localerror);
3339             newOQE = nil;
3340         }
3341
3342         if(newOQE) {
3343             ckksnotice("ckksoutgoing", self, "New modification has come in behind inflight %@; dropping failed change", oqe);
3344             // recurse for that lovely code reuse
3345             [self _onqueueChangeOutgoingQueueEntry:oqe toState:SecCKKSStateDeleted error:&localerror];
3346             if(localerror) {
3347                 ckkserror("ckksoutgoing", self, "Couldn't delete in-flight OQE: %@", localerror);
3348                 if(error) {
3349                     *error = localerror;
3350                 }
3351             }
3352         } else {
3353             oqe.state = state;
3354             [oqe saveToDatabase: &localerror];
3355             if(localerror) {
3356                 ckkserror("ckks", self, "Couldn't save %@ as %@: %@", oqe, state, localerror);
3357             }
3358         }
3359
3360     } else {
3361         oqe.state = state;
3362         [oqe saveToDatabase: &localerror];
3363         if(localerror) {
3364             ckkserror("ckks", self, "Couldn't save %@ as %@: %@", oqe, state, localerror);
3365         }
3366     }
3367
3368     if(error && localerror) {
3369         *error = localerror;
3370     }
3371     return localerror == nil;
3372 }
3373
3374 - (bool)_onqueueErrorOutgoingQueueEntry: (CKKSOutgoingQueueEntry*) oqe itemError: (NSError*) itemError error: (NSError* __autoreleasing*) error {
3375     dispatch_assert_queue(self.queue);
3376
3377     SecBoolNSErrorCallback callback = self.pendingSyncCallbacks[oqe.uuid];
3378     if(callback) {
3379         callback(false, itemError);
3380         self.pendingSyncCallbacks[oqe.uuid] = nil;
3381     }
3382     NSError* localerror = nil;
3383
3384     // Now, delete the OQE: it's never coming back
3385     [oqe deleteFromDatabase:&localerror];
3386     if(localerror) {
3387         ckkserror("ckks", self, "Couldn't delete %@ (due to error %@): %@", oqe, itemError, localerror);
3388     }
3389
3390     if(error && localerror) {
3391         *error = localerror;
3392     }
3393     return localerror == nil;
3394 }
3395
3396 - (bool)_onqueueUpdateLatestManifestWithError:(NSError**)error
3397 {
3398     dispatch_assert_queue(self.queue);
3399     CKKSManifest* manifest = [CKKSManifest latestTrustedManifestForZone:self.zoneName error:error];
3400     if (manifest) {
3401         self.latestManifest = manifest;
3402         return true;
3403     }
3404     else {
3405         return false;
3406     }
3407 }
3408
3409 - (bool)_onqueueWithAccountKeysCheckTLK:(CKKSKey*)proposedTLK error:(NSError* __autoreleasing *)error {
3410     dispatch_assert_queue(self.queue);
3411     // First, if we have a local identity, check for any TLK shares
3412     NSError* localerror = nil;
3413
3414     if(![proposedTLK wrapsSelf]) {
3415         localerror = [NSError errorWithDomain:CKKSErrorDomain code:CKKSKeyNotSelfWrapped description:[NSString stringWithFormat:@"Potential TLK %@ doesn't wrap itself: %@", proposedTLK, proposedTLK.parentKeyUUID] underlying:NULL];
3416         ckkserror("ckksshare", self, "%@", localerror);
3417         if (error) {
3418             *error = localerror;
3419         }
3420     } else {
3421         bool tlkShares = [self _onqueueWithAccountKeysCheckTLKFromShares:proposedTLK error:&localerror];
3422         // We only want to error out if a positive error occurred. "No shares" is okay.
3423         if(!tlkShares || localerror) {
3424             bool noTrustedTLKShares = [localerror.domain isEqualToString:CKKSErrorDomain] && localerror.code == CKKSNoTrustedTLKShares;
3425             bool noSelfPeer = [localerror.domain isEqualToString:CKKSErrorDomain] && localerror.code == CKKSNoEncryptionKey;
3426             bool noTrust = [localerror.domain isEqualToString:CKKSErrorDomain] && localerror.code == CKKSLackingTrust;
3427
3428             // If this error was something worse than 'couldn't unwrap for reasons including there not being data', report it
3429             if(!(noTrustedTLKShares || noSelfPeer || noTrust)) {
3430                 if(error) {
3431                     *error = localerror;
3432                 }
3433                 ckkserror("ckksshare", self, "Errored unwrapping TLK with TLKShares: %@", localerror);
3434                 return false;
3435             } else {
3436                 ckkserror("ckksshare", self, "Non-fatal error unwrapping TLK with TLKShares: %@", localerror);
3437             }
3438         }
3439     }
3440
3441     if([proposedTLK loadKeyMaterialFromKeychain:error]) {
3442         // Hurray!
3443         return true;
3444     } else {
3445         return false;
3446     }
3447 }
3448
3449 // This version only examines if this TLK is recoverable from TLK shares
3450 - (bool)_onqueueWithAccountKeysCheckTLKFromShares:(CKKSKey*)proposedTLK error:(NSError* __autoreleasing *)error {
3451     // But being recoverable from any trust set is okay
3452     NSError* localerror = nil;
3453
3454     if(self.currentTrustStates.count == 0u) {
3455         if(error) {
3456             *error = [NSError errorWithDomain:CKKSErrorDomain
3457                                          code:CKKSLackingTrust
3458                                   description:@"No current trust states; can't check TLK"];
3459         }
3460         return false;
3461     }
3462
3463     for(CKKSPeerProviderState* trustState in self.currentTrustStates) {
3464         ckkserror("ckksshare", self, "Checking TLK from trust state %@", trustState);
3465         bool recovered = [self _onqueueWithAccountKeysWithPeers:trustState
3466                                                        checkTLK:proposedTLK
3467                                                           error:&localerror];
3468
3469         if(recovered) {
3470             ckkserror("ckksshare", self, "Recovered the TLK");
3471             return true;
3472         }
3473
3474         ckkserror("ckksshare", self, "Unable to recover TLK from trust set: %@", localerror);
3475     }
3476
3477     // Only report the last error
3478     if(error && localerror) {
3479         *error = localerror;
3480     }
3481     return false;
3482 }
3483
3484 - (bool)_onqueueWithAccountKeysWithPeers:(CKKSPeerProviderState*)trustState
3485                                 checkTLK:(CKKSKey*)proposedTLK
3486                                   error:(NSError* __autoreleasing *)error
3487 {
3488     NSError* localerror = NULL;
3489     if(!trustState.currentSelfPeers.currentSelf || trustState.currentSelfPeersError) {
3490         ckkserror("ckksshare", self, "Don't have self peers for %@: %@", trustState.peerProviderID, trustState.currentSelfPeersError);
3491         if(error) {
3492             if([self.lockStateTracker isLockedError:trustState.currentSelfPeersError]) {
3493                 // Locked error should propagate
3494                 *error = trustState.currentSelfPeersError;
3495             } else {
3496                 *error = [NSError errorWithDomain:CKKSErrorDomain
3497                                              code:CKKSNoEncryptionKey
3498                                       description:@"No current self peer"
3499                                        underlying:trustState.currentSelfPeersError];
3500             }
3501         }
3502         return false;
3503     }
3504
3505     if(!trustState.currentTrustedPeers || trustState.currentTrustedPeersError) {
3506         ckkserror("ckksshare", self, "Don't have trusted peers: %@", trustState.currentTrustedPeersError);
3507         if(error) {
3508             *error = [NSError errorWithDomain:CKKSErrorDomain
3509                                          code:CKKSNoPeersAvailable
3510                                   description:@"No trusted peers"
3511                                    underlying:trustState.currentTrustedPeersError];
3512         }
3513         return false;
3514     }
3515
3516     NSError* lastShareError = nil;
3517
3518     for(id<CKKSSelfPeer> selfPeer in trustState.currentSelfPeers.allSelves) {
3519         NSArray<CKKSTLKShareRecord*>* possibleShares = [CKKSTLKShareRecord allFor:selfPeer.peerID
3520                                                               keyUUID:proposedTLK.uuid
3521                                                                zoneID:self.zoneID
3522                                                                 error:&localerror];
3523         if(localerror) {
3524             ckkserror("ckksshare", self, "Error fetching CKKSTLKShares for %@: %@", selfPeer, localerror);
3525         }
3526
3527         if(possibleShares.count == 0) {
3528             ckksnotice("ckksshare", self, "No CKKSTLKShares to %@ for %@", selfPeer, proposedTLK);
3529             continue;
3530         }
3531
3532         for(CKKSTLKShareRecord* possibleShare in possibleShares) {
3533             NSError* possibleShareError = nil;
3534             ckksnotice("ckksshare", self, "Checking possible TLK share %@ as %@", possibleShare, selfPeer);
3535
3536             CKKSKey* possibleKey = [possibleShare recoverTLK:selfPeer
3537                                                 trustedPeers:trustState.currentTrustedPeers
3538                                                        error:&possibleShareError];
3539
3540             if(possibleShareError) {
3541                 ckkserror("ckksshare", self, "Unable to unwrap TLKShare(%@) as %@: %@",
3542                           possibleShare, selfPeer, possibleShareError);
3543                 ckkserror("ckksshare", self, "Current trust set: %@", trustState.currentTrustedPeers);
3544                 lastShareError = possibleShareError;
3545                 continue;
3546             }
3547
3548             bool result = [proposedTLK trySelfWrappedKeyCandidate:possibleKey.aessivkey error:&possibleShareError];
3549             if(possibleShareError) {
3550                 ckkserror("ckksshare", self, "Unwrapped TLKShare(%@) does not unwrap proposed TLK(%@) as %@: %@",
3551                           possibleShare, proposedTLK, trustState.currentSelfPeers.currentSelf, possibleShareError);
3552                 lastShareError = possibleShareError;
3553                 continue;
3554             }
3555
3556             if(result) {
3557                 ckksnotice("ckksshare", self, "TLKShare(%@) unlocked TLK(%@) as %@",
3558                            possibleShare, proposedTLK, selfPeer);
3559
3560                 // The proposed TLK is trusted key material. Persist it as a "trusted" key.
3561                 [proposedTLK saveKeyMaterialToKeychain:true error:&possibleShareError];
3562                 if(possibleShareError) {
3563                     ckkserror("ckksshare", self, "Couldn't store the new TLK(%@) to the keychain: %@", proposedTLK, possibleShareError);
3564                     if(error) {
3565                         *error = possibleShareError;
3566                     }
3567                     return false;
3568                 }
3569
3570                 return true;
3571             }
3572         }
3573     }
3574
3575     if(error) {
3576         *error = [NSError errorWithDomain:CKKSErrorDomain
3577                                      code:CKKSNoTrustedTLKShares
3578                               description:[NSString stringWithFormat:@"No trusted TLKShares for %@", proposedTLK]
3579                                underlying:lastShareError];
3580     }
3581     return false;
3582 }
3583
3584 - (bool)dispatchSyncWithConnection:(SecDbConnectionRef _Nonnull)dbconn block:(bool (^)(void))block {
3585     CFErrorRef cferror = NULL;
3586
3587     // Take the DB transaction, then get on the local queue.
3588     // In the case of exclusive DB transactions, we don't really _need_ the local queue, but, it's here for future use.
3589     bool ret = kc_transaction_type(dbconn, kSecDbExclusiveRemoteCKKSTransactionType, &cferror, ^bool{
3590         __block bool ok = false;
3591
3592         dispatch_sync(self.queue, ^{
3593             ok = block();
3594         });
3595
3596         return ok;
3597     });
3598
3599     if(cferror) {
3600         ckkserror("ckks", self, "error doing database transaction, major problems ahead: %@", cferror);
3601     }
3602     return ret;
3603 }
3604
3605 - (void)dispatchSync: (bool (^)(void)) block {
3606     // important enough to block this thread. Must get a connection first, though!
3607
3608     // Please don't jetsam us...
3609     os_transaction_t transaction = os_transaction_create([[NSString stringWithFormat:@"com.apple.securityd.ckks.%@", self.zoneName] UTF8String]);
3610
3611     CFErrorRef cferror = NULL;
3612     kc_with_dbt(true, &cferror, ^bool (SecDbConnectionRef dbt) {
3613         return [self dispatchSyncWithConnection:dbt block:block];
3614     });
3615     if(cferror) {
3616         ckkserror("ckks", self, "error getting database connection, major problems ahead: %@", cferror);
3617     }
3618
3619     (void)transaction;
3620 }
3621
3622 - (void)dispatchSyncWithAccountKeys:(bool (^)(void))block
3623 {
3624     [self dispatchSyncWithPeerProviders:self.currentPeerProviders override:false block:block];
3625 }
3626
3627 - (void)dispatchSyncWithPeerProviders:(NSArray<id<CKKSPeerProvider>>*)peerProviders
3628                              override:(bool)overridePeerProviders
3629                                 block:(bool (^)(void))block
3630 {
3631     NSArray<id<CKKSPeerProvider>>* actualPeerProviders = overridePeerProviders ? peerProviders : self.currentPeerProviders;
3632     NSMutableArray<CKKSPeerProviderState*>* trustStates = [NSMutableArray array];
3633
3634     for(id<CKKSPeerProvider> provider in actualPeerProviders) {
3635         ckksnotice("ckks", self, "Fetching account keys for provider %@", provider);
3636         [trustStates addObject:provider.currentState];
3637     }
3638
3639     [self dispatchSync:^bool{
3640         if(overridePeerProviders) {
3641             self.currentPeerProviders = peerProviders;
3642         }
3643         self.currentTrustStates = trustStates;
3644
3645         bool result = block();
3646
3647         // Forget the peers; they might have class A key material
3648         NSMutableArray<CKKSPeerProviderState*>* noTrustStates = [NSMutableArray array];
3649         for(id<CKKSPeerProvider> provider in peerProviders) {
3650             (void)provider;
3651             [noTrustStates addObject:[CKKSPeerProviderState noPeersState:provider]];
3652         }
3653         self.currentTrustStates = noTrustStates;
3654
3655         return result;
3656     }];
3657 }
3658
3659 #pragma mark - CKKSZoneUpdateReceiver
3660
3661 - (void)notifyZoneChange: (CKRecordZoneNotification*) notification {
3662     ckksnotice("ckks", self, "received a zone change notification for %@ %@", self, notification);
3663
3664     [self fetchAndProcessCKChangesDueToAPNS:notification];
3665 }
3666
3667 - (void)superHandleCKLogin {
3668     [super handleCKLogin];
3669 }
3670
3671 - (void)handleCKLogin {
3672     ckksnotice("ckks", self, "received a notification of CK login");
3673     if(!SecCKKSIsEnabled()) {
3674         ckksnotice("ckks", self, "Skipping CloudKit initialization due to disabled CKKS");
3675         return;
3676     }
3677
3678     WEAKIFY(self);
3679     CKKSResultOperation* login = [CKKSResultOperation named:@"ckks-login" withBlock:^{
3680         STRONGIFY(self);
3681
3682         [self dispatchSyncWithAccountKeys:^bool{
3683             [self superHandleCKLogin];
3684
3685             // Reset key hierarchy state machine to initializing
3686             [self _onqueueAdvanceKeyStateMachineToState:SecCKKSZoneKeyStateInitializing withError:nil];
3687             return true;
3688         }];
3689
3690         // Change our condition variables to reflect that we think we're logged in
3691         self.loggedOut = [[CKKSCondition alloc] initToChain:self.loggedOut];
3692         [self.loggedIn fulfill];
3693         [self.accountStateKnown fulfill];
3694     }];
3695
3696     [self scheduleAccountStatusOperation:login];
3697 }
3698
3699 - (void)superHandleCKLogout {
3700     [super handleCKLogout];
3701 }
3702
3703 - (void)handleCKLogout {
3704     WEAKIFY(self);
3705     CKKSResultOperation* logout = [CKKSResultOperation named:@"ckks-logout" withBlock: ^{
3706         STRONGIFY(self);
3707         if(!self) {
3708             return;
3709         }
3710         [self dispatchSync:^bool {
3711             ckksnotice("ckks", self, "received a notification of CK logout");
3712             [self superHandleCKLogout];
3713
3714             NSError* error = nil;
3715             [self _onqueueResetLocalData: &error];
3716             if(error) {
3717                 ckkserror("ckks", self, "error while resetting local data: %@", error);
3718             }
3719
3720             [self _onqueueAdvanceKeyStateMachineToState:SecCKKSZoneKeyStateLoggedOut withError:nil];
3721
3722             self.loggedIn = [[CKKSCondition alloc] initToChain: self.loggedIn];
3723             [self.loggedOut fulfill];
3724             [self.accountStateKnown fulfill];
3725
3726             // Tell all pending sync clients that we don't expect to ever sync
3727             for(NSString* callbackUUID in self.pendingSyncCallbacks.allKeys) {
3728                 [self callSyncCallbackWithErrorNoAccount:self.pendingSyncCallbacks[callbackUUID]];
3729                 self.pendingSyncCallbacks[callbackUUID] = nil;
3730             }
3731
3732             return true;
3733         }];
3734     }];
3735
3736     [self scheduleAccountStatusOperation: logout];
3737 }
3738
3739 - (void)callSyncCallbackWithErrorNoAccount:(SecBoolNSErrorCallback)syncCallback {
3740     CKKSAccountStatus accountStatus = self.accountStatus;
3741     dispatch_async(self.queue, ^{
3742         syncCallback(false, [NSError errorWithDomain:@"securityd"
3743                                                 code:errSecNotLoggedIn
3744                                             userInfo:@{NSLocalizedDescriptionKey:
3745                                                            [NSString stringWithFormat: @"No iCloud account available(%d); item is not expected to sync", (int)accountStatus]}]);
3746     });
3747 }
3748
3749 #pragma mark - Trust operations
3750
3751 - (void)beginTrustedOperation:(NSArray<id<CKKSPeerProvider>>*)peerProviders
3752              suggestTLKUpload:(CKKSNearFutureScheduler*)suggestTLKUpload
3753 {
3754     for(id<CKKSPeerProvider> peerProvider in peerProviders) {
3755         [peerProvider registerForPeerChangeUpdates:self];
3756     }
3757
3758     [self.launch addEvent:@"beginTrusted"];
3759
3760     [self dispatchSyncWithPeerProviders:peerProviders override:true block:^bool {
3761         ckksnotice("ckkstrust", self, "Beginning trusted operation");
3762         CKKSAccountStatus oldTrustStatus = self.trustStatus;
3763
3764         self.suggestTLKUpload = suggestTLKUpload;
3765
3766         self.trustStatus = CKKSAccountStatusAvailable;
3767         if(self.trustDependency) {
3768             [self scheduleOperation: self.trustDependency];
3769             self.trustDependency = nil;
3770         }
3771         [self _onqueueAdvanceKeyStateMachineToState:nil withError:nil];
3772
3773         if(oldTrustStatus == CKKSAccountStatusNoAccount) {
3774             ckksnotice("ckkstrust", self, "Moving from an untrusted status; we need to process incoming queue and scan for any new items");
3775
3776             // Next, try to process them (replacing local entries)
3777             CKKSIncomingQueueOperation* initialProcess = [self processIncomingQueue:true after:nil];
3778             initialProcess.name = @"initial-process-incoming-queue";
3779
3780             // If all that succeeds, iterate through all keychain items and find the ones which need to be uploaded
3781             self.initialScanOperation = [self scanLocalItems:@"newly-trusted-scan"
3782                                             ckoperationGroup:nil
3783                                                        after:initialProcess];
3784         }
3785
3786         return true;
3787     }];
3788 }
3789
3790 - (void)endTrustedOperation
3791 {
3792     [self.launch addEvent:@"endTrusted"];
3793
3794     [self dispatchSyncWithPeerProviders:nil override:true block:^bool {
3795         ckksnotice("ckkstrust", self, "Ending trusted operation");
3796
3797         self.suggestTLKUpload = nil;
3798
3799         self.trustStatus = CKKSAccountStatusNoAccount;
3800         if(!self.trustDependency) {
3801             self.trustDependency = [CKKSResultOperation named:@"wait-for-trust" withBlock:^{}];
3802         }
3803         [self _onqueueAdvanceKeyStateMachineToState:nil withError:nil];
3804         return true;
3805     }];
3806 }
3807
3808 #pragma mark - CKKSChangeFetcherClient
3809
3810 - (CKKSCloudKitFetchRequest*)participateInFetch
3811 {
3812     __block CKKSCloudKitFetchRequest* request = [[CKKSCloudKitFetchRequest alloc] init];
3813     [self dispatchSync: ^bool {
3814         if(self.accountStatus != CKKSAccountStatusAvailable) {
3815             ckksnotice("ckksfetch", self, "Not participating in fetch: not logged in");
3816             request.participateInFetch = false;
3817             return false;
3818         }
3819
3820         if(!self.zoneCreated) {
3821             ckksnotice("ckksfetch", self, "Not participating in fetch: zone not created yet");
3822             request.participateInFetch = false;
3823             return false;
3824         }
3825
3826         request.participateInFetch = true;
3827         [self.launch addEvent:@"fetch"];
3828
3829         if([self.keyHierarchyState isEqualToString:SecCKKSZoneKeyStateNeedFullRefetch]) {
3830             // We want to return a nil change tag (to force a resync)
3831             ckksnotice("ckksfetch", self, "Beginning refetch");
3832             request.changeToken = nil;
3833             request.resync = true;
3834         } else {
3835             CKKSZoneStateEntry* ckse = [CKKSZoneStateEntry state:self.zoneName];
3836             if(!ckse) {
3837                 ckkserror("ckksfetch", self, "couldn't fetch zone change token for %@", self.zoneName);
3838                 return false;
3839             }
3840             request.changeToken = ckse.changeToken;
3841         }
3842         return true;
3843     }];
3844
3845     if (request.changeToken == nil) {
3846         self.launch.firstLaunch = true;
3847     }
3848
3849     return request;
3850 }
3851
3852 - (void)changesFetched:(NSArray<CKRecord*>*)changedRecords
3853       deletedRecordIDs:(NSArray<CKKSCloudKitDeletion*>*)deletedRecords
3854         newChangeToken:(CKServerChangeToken*)newChangeToken
3855             moreComing:(BOOL)moreComing
3856                 resync:(BOOL)resync
3857 {
3858     [self.launch addEvent:@"changes-fetched"];
3859
3860     [self dispatchSyncWithAccountKeys:^bool{
3861         for (CKRecord* record in changedRecords) {
3862             [self _onqueueCKRecordChanged:record resync:resync];
3863         }
3864
3865         for (CKKSCloudKitDeletion* deletion in deletedRecords) {
3866             [self _onqueueCKRecordDeleted:deletion.recordID recordType:deletion.recordType resync:resync];
3867         }
3868
3869         NSError* error = nil;
3870         if(resync) {
3871             // If we're performing a resync, we need to keep track of everything that's actively in
3872             // CloudKit during the fetch, (so that we can find anything that's on-disk and not in CloudKit).
3873             // Please note that if, during a resync, the fetch errors, we won't be notified. If a record is in
3874             // the first refetch but not the second, it'll be added to our set, and the second resync will not
3875             // delete the record (which is a consistency violation, but only with actively changing records).
3876             // A third resync should correctly delete that record.
3877
3878             if(self.resyncRecordsSeen == nil) {
3879                 self.resyncRecordsSeen = [NSMutableSet set];
3880             }
3881             for(CKRecord* r in changedRecords) {
3882                 [self.resyncRecordsSeen addObject:r.recordID.recordName];
3883             }
3884
3885             // Is there More Coming? If not, self.resyncRecordsSeen contains everything in CloudKit. Inspect for anything extra!
3886             if(moreComing) {
3887                 ckksnotice("ckksresync", self, "In a resync, but there's More Coming. Waiting to scan for extra items.");
3888
3889             } else {
3890                 // Scan through all CKMirrorEntries and determine if any exist that CloudKit didn't tell us about
3891                 ckksnotice("ckksresync", self, "Comparing local UUIDs against the CloudKit list");
3892                 NSMutableArray<NSString*>* uuids = [[CKKSMirrorEntry allUUIDs:self.zoneID error:&error] mutableCopy];
3893
3894                 for(NSString* uuid in uuids) {
3895                     if([self.resyncRecordsSeen containsObject:uuid]) {
3896                         ckksnotice("ckksresync", self, "UUID %@ is still in CloudKit; carry on.", uuid);
3897                     } else {
3898                         CKKSMirrorEntry* ckme = [CKKSMirrorEntry tryFromDatabase:uuid zoneID:self.zoneID error:&error];
3899                         if(error != nil) {
3900                             ckkserror("ckksresync", self, "Couldn't read an item from the database, but it used to be there: %@ %@", uuid, error);
3901                             continue;
3902                         }
3903                         if(!ckme) {
3904                             ckkserror("ckksresync", self, "Couldn't read ckme(%@) from database; continuing", uuid);
3905                             continue;
3906                         }
3907
3908                         ckkserror("ckksresync", self, "BUG: Local item %@ not found in CloudKit, deleting", uuid);
3909                         [self _onqueueCKRecordDeleted:ckme.item.storedCKRecord.recordID recordType:ckme.item.storedCKRecord.recordType resync:resync];
3910                     }
3911                 }
3912
3913                 // Now that we've inspected resyncRecordsSeen, reset it for the next time through
3914                 self.resyncRecordsSeen = nil;
3915             }
3916         }
3917
3918         CKKSZoneStateEntry* state = [CKKSZoneStateEntry state:self.zoneName];
3919         state.lastFetchTime = [NSDate date]; // The last fetch happened right now!
3920         state.changeToken = newChangeToken;
3921         state.moreRecordsInCloudKit = moreComing;
3922         [state saveToDatabase:&error];
3923         if(error) {
3924             ckkserror("ckksfetch", self, "Couldn't save new server change token: %@", error);
3925         }
3926
3927         if(!moreComing) {
3928             // Might as well kick off a IQO!
3929             [self processIncomingQueue:false];
3930             ckksnotice("ckksfetch", self, "Beginning incoming processing for %@", self.zoneID);
3931         }
3932
3933         ckksnotice("ckksfetch", self, "Finished processing changes for %@", self.zoneID);
3934
3935         return true;
3936     }];
3937 }
3938
3939 - (bool)ckErrorOrPartialError:(NSError *)error isError:(CKErrorCode)errorCode
3940 {
3941     if((error.code == errorCode) && [error.domain isEqualToString:CKErrorDomain]) {
3942         return true;
3943     } else if((error.code == CKErrorPartialFailure) && [error.domain isEqualToString:CKErrorDomain]) {
3944         NSDictionary* partialErrors = error.userInfo[CKPartialErrorsByItemIDKey];
3945
3946         NSError* partialError = partialErrors[self.zoneID];
3947         if ((partialError.code == errorCode) && [partialError.domain isEqualToString:CKErrorDomain]) {
3948             return true;
3949         }
3950     }
3951     return false;
3952 }
3953
3954 - (bool)shouldRetryAfterFetchError:(NSError*)error {
3955
3956     bool isChangeTokenExpiredError = [self ckErrorOrPartialError:error isError:CKErrorChangeTokenExpired];
3957     if(isChangeTokenExpiredError) {
3958         ckkserror("ckks", self, "Received notice that our change token is out of date (for %@). Resetting local data...", self.zoneID);
3959
3960         // This is a bit scary: we might confuse some poor key hierarchy state machine operation. But, if a key state machine
3961         // operation is waiting for a successful fetch, we need to do this reset
3962         [self dispatchSyncWithAccountKeys:^bool{
3963             NSError* error = nil;
3964             [self _onqueueResetLocalData:&error];
3965
3966             if(error) {
3967                 ckksnotice("ckksreset", self, "CloudKit-inspired local reset of %@ ended with error: %@", self.zoneID, error);
3968             } else {
3969                 ckksnotice("ckksreset", self, "CloudKit-inspired local reset of %@ ended successfully", self.zoneID);
3970             }
3971
3972             // If we're in the middle of a fetch for the key state, then the retried fetch (which should succeed) will be sufficient to progress
3973             // Otherwise, we need to poke the key hierarchy state machine: all of its data is gone
3974             if(![self.keyHierarchyState isEqualToString:SecCKKSZoneKeyStateFetch]) {
3975                 [self _onqueueKeyStateMachineRequestFetch];
3976             }
3977
3978             return true;
3979         }];
3980
3981         return true;
3982     }
3983
3984     bool isDeletedZoneError = [self ckErrorOrPartialError:error isError:CKErrorZoneNotFound];
3985     if(isDeletedZoneError) {
3986         ckkserror("ckks", self, "Received notice that our zone(%@) does not exist. Resetting local data.", self.zoneID);
3987
3988         /*
3989          * If someone delete our zone, lets just start over from the begining
3990          */
3991         [self dispatchSync: ^bool{
3992             NSError* resetError = nil;
3993
3994             [self _onqueueResetLocalData: &resetError];
3995             if(resetError) {
3996                 ckksnotice("ckksreset", self, "CloudKit-inspired local reset of %@ ended with error: %@", self.zoneID, resetError);
3997             } else {
3998                 ckksnotice("ckksreset", self, "CloudKit-inspired local reset of %@ ended successfully", self.zoneID);
3999             }
4000
4001             [self _onqueueAdvanceKeyStateMachineToState:SecCKKSZoneKeyStateInitializing withError:nil];
4002             return true;
4003         }];
4004
4005         return false;
4006     }
4007
4008     if([error.domain isEqualToString:CKErrorDomain] && (error.code == CKErrorBadContainer)) {
4009         ckkserror("ckks", self, "Received notice that our container does not exist. Nothing to do.");
4010         return false;
4011     }
4012
4013     return true;
4014 }
4015
4016 #pragma mark CKKSPeerUpdateListener
4017
4018 - (void)selfPeerChanged:(id<CKKSPeerProvider>)provider
4019 {
4020     // Currently, we have no idea what to do with this. Kick off a key reprocess?
4021     ckkserror("ckks", self, "Received update that our self identity has changed");
4022     [self keyStateMachineRequestProcess];
4023 }
4024
4025 - (void)trustedPeerSetChanged:(id<CKKSPeerProvider>)provider
4026 {
4027     // We might need to share the TLK to some new people, or we might now trust the TLKs we have.
4028     // The key state machine should handle that, so poke it.
4029     ckkserror("ckks", self, "Received update that the trust set has changed");
4030
4031     self.trustedPeersSetChanged = true;
4032     [self.pokeKeyStateMachineScheduler trigger];
4033 }
4034
4035 #pragma mark - Test Support
4036
4037 - (bool) outgoingQueueEmpty: (NSError * __autoreleasing *) error {
4038     __block bool ret = false;
4039     [self dispatchSync: ^bool{
4040         NSArray* queueEntries = [CKKSOutgoingQueueEntry all: error];
4041         ret = queueEntries && ([queueEntries count] == 0);
4042         return true;
4043     }];
4044
4045     return ret;
4046 }
4047
4048 - (CKKSResultOperation*)waitForFetchAndIncomingQueueProcessing {
4049     CKKSResultOperation* op = [self fetchAndProcessCKChanges:CKKSFetchBecauseTesting];
4050     [op waitUntilFinished];
4051     return op;
4052 }
4053
4054 - (void)waitForKeyHierarchyReadiness {
4055     if(self.keyStateReadyDependency) {
4056         [self.keyStateReadyDependency waitUntilFinished];
4057     }
4058 }
4059
4060 - (void)cancelPendingOperations {
4061     @synchronized(self.outgoingQueueOperations) {
4062         for(NSOperation* op in self.outgoingQueueOperations) {
4063             [op cancel];
4064         }
4065         [self.outgoingQueueOperations removeAllObjects];
4066     }
4067
4068     @synchronized(self.incomingQueueOperations) {
4069         for(NSOperation* op in self.incomingQueueOperations) {
4070             [op cancel];
4071         }
4072         [self.incomingQueueOperations removeAllObjects];
4073     }
4074
4075     [super cancelAllOperations];
4076 }
4077
4078 - (void)cancelAllOperations {
4079     [self.zoneSetupOperation cancel];
4080     [self.keyStateMachineOperation cancel];
4081     [self.keyStateReadyDependency cancel];
4082     [self.keyStateNonTransientDependency cancel];
4083     [self.zoneChangeFetcher cancel];
4084     [self.notifyViewChangedScheduler cancel];
4085
4086     [self cancelPendingOperations];
4087
4088     [self dispatchSync:^bool{
4089         [self _onqueueAdvanceKeyStateMachineToState: SecCKKSZoneKeyStateCancelled withError: nil];
4090         return true;
4091     }];
4092 }
4093
4094 - (void)halt {
4095     [super halt];
4096
4097     // Don't send any more notifications, either
4098     _notifierClass = nil;
4099 }
4100
4101 - (NSDictionary*)status {
4102 #define stringify(obj) CKKSNilToNSNull([obj description])
4103 #define boolstr(obj) (!!(obj) ? @"yes" : @"no")
4104     __block NSMutableDictionary* ret = nil;
4105     __block NSError* error = nil;
4106     CKKSManifest* manifest = nil;
4107
4108     ret = [[self fastStatus] mutableCopy];
4109
4110     manifest = [CKKSManifest latestTrustedManifestForZone:self.zoneName error:&error];
4111     [self dispatchSync: ^bool {
4112
4113         CKKSCurrentKeySet* keyset = [CKKSCurrentKeySet loadForZone:self.zoneID];
4114         if(keyset.error) {
4115             error = keyset.error;
4116         }
4117
4118         NSString* manifestGeneration = manifest ? [NSString stringWithFormat:@"%lu", (unsigned long)manifest.generationCount] : nil;
4119
4120         if(error) {
4121             ckkserror("ckks", self, "error during status: %@", error);
4122         }
4123         // We actually don't care about this error, especially if it's "no current key pointers"...
4124         error = nil;
4125
4126         // Map deviceStates to strings to avoid NSXPC issues. Obj-c, why is this so hard?
4127         NSArray* deviceStates = [CKKSDeviceStateEntry allInZone:self.zoneID error:&error];
4128         NSMutableArray<NSString*>* mutDeviceStates = [[NSMutableArray alloc] init];
4129         [deviceStates enumerateObjectsUsingBlock:^(id  _Nonnull obj, NSUInteger idx, BOOL * _Nonnull stop) {
4130             [mutDeviceStates addObject: [obj description]];
4131         }];
4132
4133         NSArray* tlkShares = [CKKSTLKShareRecord allForUUID:keyset.currentTLKPointer.currentKeyUUID zoneID:self.zoneID error:&error];
4134         NSMutableArray<NSString*>* mutTLKShares = [[NSMutableArray alloc] init];
4135         [tlkShares enumerateObjectsUsingBlock:^(id  _Nonnull obj, NSUInteger idx, BOOL * _Nonnull stop) {
4136             [mutTLKShares addObject: [obj description]];
4137         }];
4138
4139         [ret addEntriesFromDictionary:@{
4140                  @"statusError":         stringify(error),
4141                  @"oqe":                 CKKSNilToNSNull([CKKSOutgoingQueueEntry countsByStateInZone:self.zoneID error:&error]),
4142                  @"iqe":                 CKKSNilToNSNull([CKKSIncomingQueueEntry countsByStateInZone:self.zoneID error:&error]),
4143                  @"ckmirror":            CKKSNilToNSNull([CKKSMirrorEntry        countsByParentKey:self.zoneID error:&error]),
4144                  @"devicestates":        CKKSNilToNSNull(mutDeviceStates),
4145                  @"tlkshares":           CKKSNilToNSNull(mutTLKShares),
4146                  @"keys":                CKKSNilToNSNull([CKKSKey countsByClass:self.zoneID error:&error]),
4147                  @"currentTLK":          CKKSNilToNSNull(keyset.tlk.uuid),
4148                  @"currentClassA":       CKKSNilToNSNull(keyset.classA.uuid),
4149                  @"currentClassC":       CKKSNilToNSNull(keyset.classC.uuid),
4150                  @"currentTLKPtr":       CKKSNilToNSNull(keyset.currentTLKPointer.currentKeyUUID),
4151                  @"currentClassAPtr":    CKKSNilToNSNull(keyset.currentClassAPointer.currentKeyUUID),
4152                  @"currentClassCPtr":    CKKSNilToNSNull(keyset.currentClassCPointer.currentKeyUUID),
4153                  @"currentManifestGen":  CKKSNilToNSNull(manifestGeneration),
4154             }];
4155         return false;
4156     }];
4157     return ret;
4158 }
4159
4160 - (NSDictionary*)fastStatus {
4161
4162     __block NSDictionary* ret = nil;
4163
4164     [self dispatchSync: ^bool {
4165
4166         ret = @{
4167             @"view":                CKKSNilToNSNull(self.zoneName),
4168             @"ckaccountstatus":     self.accountStatus == CKAccountStatusCouldNotDetermine ? @"could not determine" :
4169                 self.accountStatus == CKAccountStatusAvailable         ? @"logged in" :
4170                 self.accountStatus == CKAccountStatusRestricted        ? @"restricted" :
4171                 self.accountStatus == CKAccountStatusNoAccount         ? @"logged out" : @"unknown",
4172             @"accounttracker":      stringify(self.accountTracker),
4173             @"fetcher":             stringify(self.zoneChangeFetcher),
4174             @"zoneCreated":         boolstr(self.zoneCreated),
4175             @"zoneCreatedError":    stringify(self.zoneCreatedError),
4176             @"zoneSubscribed":      boolstr(self.zoneSubscribed),
4177             @"zoneSubscribedError": stringify(self.zoneSubscribedError),
4178             @"keystate":            CKKSNilToNSNull(self.keyHierarchyState),
4179             @"keyStateError":       stringify(self.keyHierarchyError),
4180             @"statusError":         [NSNull null],
4181             @"launchSequence":      CKKSNilToNSNull([self.launch eventsByTime]),
4182
4183             @"zoneSetupOperation":                 stringify(self.zoneSetupOperation),
4184             @"keyStateOperation":                  stringify(self.keyStateMachineOperation),
4185             @"lastIncomingQueueOperation":         stringify(self.lastIncomingQueueOperation),
4186             @"lastNewTLKOperation":                stringify(self.lastNewTLKOperation),
4187             @"lastOutgoingQueueOperation":         stringify(self.lastOutgoingQueueOperation),
4188             @"lastProcessReceivedKeysOperation":   stringify(self.lastProcessReceivedKeysOperation),
4189             @"lastReencryptOutgoingItemsOperation":stringify(self.lastReencryptOutgoingItemsOperation),
4190             @"lastScanLocalItemsOperation":        stringify(self.lastScanLocalItemsOperation),
4191         };
4192         return false;
4193     }];
4194
4195     return ret;
4196 }
4197
4198 #endif /* OCTAGON */
4199 @end