keychain/ckks/CKKSKeychainView.m

   1 /*
   2  * Copyright (c) 2016 Apple Inc. All Rights Reserved.
   3  *
   4  * @APPLE_LICENSE_HEADER_START@
   5  *
   6  * This file contains Original Code and/or Modifications of Original Code
   7  * as defined in and that are subject to the Apple Public Source License
   8  * Version 2.0 (the 'License'). You may not use this file except in
   9  * compliance with the License. Please obtain a copy of the License at
  10  * http://www.opensource.apple.com/apsl/ and read it before using this
  11  * file.
  12  *
  13  * The Original Code and all software distributed under the License are
  14  * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  15  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  16  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
  17  * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
  18  * Please see the License for the specific language governing rights and
  19  * limitations under the License.
  20  *
  21  * @APPLE_LICENSE_HEADER_END@
  22  */
  23
  24 #import "CKKSKeychainView.h"
  25
  26
  27
  28 #if OCTAGON
  29 #import "CloudKitDependencies.h"
  30 #import <CloudKit/CloudKit.h>
  31 #import <CloudKit/CloudKit_Private.h>
  32 #endif
  33
  34 #import "CKKS.h"
  35 #import "OctagonAPSReceiver.h"
  36 #import "CKKSIncomingQueueEntry.h"
  37 #import "CKKSOutgoingQueueEntry.h"
  38 #import "CKKSCurrentKeyPointer.h"
  39 #import "CKKSKey.h"
  40 #import "CKKSMirrorEntry.h"
  41 #import "CKKSZoneStateEntry.h"
  42 #import "CKKSItemEncrypter.h"
  43 #import "CKKSIncomingQueueOperation.h"
  44 #import "CKKSNewTLKOperation.h"
  45 #import "CKKSProcessReceivedKeysOperation.h"
  46 #import "CKKSZone.h"
  47 #import "CKKSFetchAllRecordZoneChangesOperation.h"
  48 #import "CKKSHealKeyHierarchyOperation.h"
  49 #import "CKKSReencryptOutgoingItemsOperation.h"
  50 #import "CKKSScanLocalItemsOperation.h"
  51 #import "CKKSSynchronizeOperation.h"
  52 #import "CKKSRateLimiter.h"
  53 #import "CKKSManifest.h"
  54 #import "CKKSManifestLeafRecord.h"
  55 #import "CKKSZoneChangeFetcher.h"
  56 #import "CKKSAnalytics.h"
  57 #import "keychain/analytics/CKKSLaunchSequence.h"
  58 #import "keychain/ckks/CKKSCloudKitClassDependencies.h"
  59 #import "keychain/ckks/CKKSDeviceStateEntry.h"
  60 #import "keychain/ckks/CKKSNearFutureScheduler.h"
  61 #import "keychain/ckks/CKKSCurrentItemPointer.h"
  62 #import "keychain/ckks/CKKSUpdateCurrentItemPointerOperation.h"
  63 #import "keychain/ckks/CKKSUpdateDeviceStateOperation.h"
  64 #import "keychain/ckks/CKKSNotifier.h"
  65 #import "keychain/ckks/CloudKitCategories.h"
  66 #import "keychain/ckks/CKKSTLKShareRecord.h"
  67 #import "keychain/ckks/CKKSHealTLKSharesOperation.h"
  68 #import "keychain/ckks/CKKSLocalSynchronizeOperation.h"
  69 #import "keychain/categories/NSError+UsefulConstructors.h"
  70
  71 #import "keychain/ot/OTConstants.h"
  72 #import "keychain/ot/OTDefines.h"
  73 #import "keychain/ot/OctagonCKKSPeerAdapter.h"
  74 #import "keychain/ot/ObjCImprovements.h"
  75
  76 #include <utilities/SecCFWrappers.h>
  77 #include <utilities/SecTrace.h>
  78 #include <utilities/SecDb.h>
  79 #include <securityd/SecDbItem.h>
  80 #include <securityd/SecItemDb.h>
  81 #include <securityd/SecItemSchema.h>
  82 #include <securityd/SecItemServer.h>
  83 #include <utilities/debugging.h>
  84 #include <Security/SecItemPriv.h>
  85 #include "keychain/SecureObjectSync/SOSAccountTransaction.h"
  86 #include <utilities/SecADWrapper.h>
  87 #include <utilities/SecPLWrappers.h>
  88 #include <os/transaction_private.h>
  89
  90 #if OCTAGON
  91 @implementation CKKSPeerProviderState
  92 - (instancetype)initWithPeerProviderID:(NSString*)providerID
  93                              essential:(BOOL)essential
  94                              selfPeers:(CKKSSelves* _Nullable)selfPeers
  95                         selfPeersError:(NSError* _Nullable)selfPeersError
  96                           trustedPeers:(NSSet<id<CKKSRemotePeerProtocol>>* _Nullable)currentTrustedPeers
  97                      trustedPeersError:(NSError* _Nullable)trustedPeersError
  98 {
  99     if((self = [super init])) {
 100         _peerProviderID = providerID;
 101         _essential = essential;
 102         _currentSelfPeers = selfPeers;
 103         _currentSelfPeersError = selfPeersError;
 104         _currentTrustedPeers = currentTrustedPeers;
 105         _currentTrustedPeersError = trustedPeersError;
 106
 107         if(_currentTrustedPeers) {
 108             NSMutableSet<NSString*>* trustedPeerIDs = [NSMutableSet set];
 109             for(id<CKKSPeer> peer in _currentTrustedPeers) {
 110                 [trustedPeerIDs addObject:peer.peerID];
 111             }
 112             _currentTrustedPeerIDs = trustedPeerIDs;
 113         }
 114     }
 115     return self;
 116 }
 117
 118 - (NSString*)description
 119 {
 120     return [NSString stringWithFormat:@"<CKKSPeerProviderState(%@): %@%@ %@%@>",
 121             self.peerProviderID,
 122             self.currentSelfPeers,
 123             self.currentSelfPeersError ?: @"",
 124             self.currentTrustedPeers,
 125             self.currentTrustedPeersError ?: @""];
 126 }
 127
 128 + (CKKSPeerProviderState*)noPeersState:(id<CKKSPeerProvider>)provider
 129 {
 130     return [[CKKSPeerProviderState alloc] initWithPeerProviderID:provider.providerID
 131                                                        essential:provider.essential
 132                                                        selfPeers:nil
 133                                              selfPeersError:[NSError errorWithDomain:CKKSErrorDomain
 134                                                                                  code:CKKSNoPeersAvailable
 135                                                                           description:@"No current self peer available"]
 136                                                trustedPeers:nil
 137                                           trustedPeersError:[NSError errorWithDomain:CKKSErrorDomain
 138                                                                                  code:CKKSNoPeersAvailable
 139                                                                           description:@"No current trusted peers available"]];
 140 }
 141 @end
 142
 143 @interface CKKSKeychainView()
 144 @property bool keyStateFetchRequested;
 145 @property bool keyStateFullRefetchRequested;
 146 @property bool keyStateProcessRequested;
 147 @property bool trustedPeersSetChanged;
 148
 149 @property bool keyStateCloudKitDeleteRequested;
 150 @property NSHashTable<CKKSResultOperation*>* cloudkitDeleteZoneOperations;
 151
 152 @property bool keyStateLocalResetRequested;
 153 @property NSHashTable<CKKSResultOperation*>* localResetOperations;
 154
 155 @property bool tlkCreationRequested;
 156 @property NSHashTable<CKKSResultOperation<CKKSKeySetProviderOperationProtocol>*>* keysetProviderOperations;
 157
 158
 159 @property (atomic) NSString *activeTLK;
 160
 161 @property (readonly) Class<CKKSNotifier> notifierClass;
 162
 163 // Slows down all outgoing queue operations
 164 @property CKKSNearFutureScheduler* outgoingQueueOperationScheduler;
 165
 166 @property CKKSResultOperation* processIncomingQueueAfterNextUnlockOperation;
 167 @property CKKSResultOperation* resultsOfNextIncomingQueueOperationOperation;
 168
 169 @property NSMutableDictionary<NSString*, SecBoolNSErrorCallback>* pendingSyncCallbacks;
 170
 171 // An extra queue for semaphore-waiting-based NSOperations
 172 @property NSOperationQueue* waitingQueue;
 173
 174 // Make these readwrite
 175 @property NSArray<id<CKKSPeerProvider>>* currentPeerProviders;
 176 @property NSArray<CKKSPeerProviderState*>* currentTrustStates;
 177
 178 @end
 179 #endif
 180
 181 @implementation CKKSKeychainView
 182 #if OCTAGON
 183
 184 - (instancetype)initWithContainer:(CKContainer*)container
 185                          zoneName:(NSString*)zoneName
 186                    accountTracker:(CKKSAccountStateTracker*)accountTracker
 187                  lockStateTracker:(CKKSLockStateTracker*)lockStateTracker
 188               reachabilityTracker:(CKKSReachabilityTracker*)reachabilityTracker
 189                     changeFetcher:(CKKSZoneChangeFetcher*)fetcher
 190                      zoneModifier:(CKKSZoneModifier*)zoneModifier
 191                  savedTLKNotifier:(CKKSNearFutureScheduler*)savedTLKNotifier
 192         cloudKitClassDependencies:(CKKSCloudKitClassDependencies*)cloudKitClassDependencies
 193 {
 194
 195     if(self = [super initWithContainer:container
 196                               zoneName:zoneName
 197                         accountTracker:accountTracker
 198                    reachabilityTracker:reachabilityTracker
 199                           zoneModifier:zoneModifier
 200              cloudKitClassDependencies:cloudKitClassDependencies]) {
 201         WEAKIFY(self);
 202
 203         _loggedIn = [[CKKSCondition alloc] init];
 204         _loggedOut = [[CKKSCondition alloc] init];
 205         _accountStateKnown = [[CKKSCondition alloc] init];
 206
 207         _trustStatus = CKKSAccountStatusUnknown;
 208         _trustDependency = [CKKSResultOperation named:@"wait-for-trust" withBlock:^{}];
 209
 210         _incomingQueueOperations = [NSHashTable weakObjectsHashTable];
 211         _outgoingQueueOperations = [NSHashTable weakObjectsHashTable];
 212         _cloudkitDeleteZoneOperations = [NSHashTable weakObjectsHashTable];
 213         _localResetOperations = [NSHashTable weakObjectsHashTable];
 214         _keysetProviderOperations = [NSHashTable weakObjectsHashTable];
 215
 216         _currentPeerProviders = @[];
 217         _currentTrustStates = @[];
 218
 219         _launch = [[CKKSLaunchSequence alloc] initWithRocketName:@"com.apple.security.ckks.launch"];
 220         [_launch addAttribute:@"view" value:zoneName];
 221
 222         _zoneChangeFetcher = fetcher;
 223         [fetcher registerClient:self];
 224
 225         _notifierClass = cloudKitClassDependencies.notifierClass;
 226         _notifyViewChangedScheduler = [[CKKSNearFutureScheduler alloc] initWithName:[NSString stringWithFormat: @"%@-notify-scheduler", self.zoneName]
 227                                                             initialDelay:250*NSEC_PER_MSEC
 228                                                          continuingDelay:1*NSEC_PER_SEC
 229                                                         keepProcessAlive:true
 230                                                           dependencyDescriptionCode:CKKSResultDescriptionPendingViewChangedScheduling
 231                                                                    block:^{
 232                                                                        STRONGIFY(self);
 233                                                                        [self.notifierClass post:[NSString stringWithFormat:@"com.apple.security.view-change.%@", self.zoneName]];
 234
 235                                                                        // Ugly, but: the Manatee and Engram views need to send a fake 'PCS' view change.
 236                                                                        // TODO: make this data-driven somehow
 237                                                                        if([self.zoneName isEqualToString:@"Manatee"] ||
 238                                                                           [self.zoneName isEqualToString:@"Engram"] ||
 239                                                                           [self.zoneName isEqualToString:@"ApplePay"] ||
 240                                                                           [self.zoneName isEqualToString:@"Home"] ||
 241                                                                           [self.zoneName isEqualToString:@"LimitedPeersAllowed"]) {
 242                                                                            [self.notifierClass post:@"com.apple.security.view-change.PCS"];
 243                                                                        }
 244                                                                    }];
 245
 246         _notifyViewReadyScheduler = [[CKKSNearFutureScheduler alloc] initWithName:[NSString stringWithFormat: @"%@-ready-scheduler", self.zoneName]
 247                                                                        initialDelay:250*NSEC_PER_MSEC
 248                                                                     continuingDelay:120*NSEC_PER_SEC
 249                                                                    keepProcessAlive:true
 250                                                           dependencyDescriptionCode:CKKSResultDescriptionPendingViewChangedScheduling
 251                                                                               block:^{
 252                                                                                   STRONGIFY(self);
 253                                                                                   NSDistributedNotificationCenter *center = [self.cloudKitClassDependencies.nsdistributednotificationCenterClass defaultCenter];
 254
 255                                                                                   [center postNotificationName:@"com.apple.security.view-become-ready"
 256                                                                                                         object:nil
 257                                                                                                       userInfo:@{ @"view" : self.zoneName }
 258                                                                                                        options:0];
 259                                                                               }];
 260
 261
 262         _pendingSyncCallbacks = [[NSMutableDictionary alloc] init];
 263
 264         _lockStateTracker = lockStateTracker;
 265         _savedTLKNotifier = savedTLKNotifier;
 266
 267         _keyHierarchyConditions = [[NSMutableDictionary alloc] init];
 268         [CKKSZoneKeyStateMap() enumerateKeysAndObjectsUsingBlock:^(CKKSZoneKeyState * _Nonnull key, NSNumber * _Nonnull obj, BOOL * _Nonnull stop) {
 269             [self.keyHierarchyConditions setObject: [[CKKSCondition alloc] init] forKey:key];
 270         }];
 271
 272         // Use the keyHierarchyState setter to modify the zone key state map
 273         self.keyHierarchyState = SecCKKSZoneKeyStateLoggedOut;
 274
 275         _keyHierarchyError = nil;
 276         _keyHierarchyOperationGroup = nil;
 277         _keyStateMachineOperation = nil;
 278         _keyStateFetchRequested = false;
 279         _keyStateProcessRequested = false;
 280         _tlkCreationRequested = false;
 281
 282         _waitingQueue = [[NSOperationQueue alloc] init];
 283         _waitingQueue.maxConcurrentOperationCount = 5;
 284
 285         _keyStateReadyDependency = [self createKeyStateReadyDependency: @"Key state has become ready for the first time." ckoperationGroup:[CKOperationGroup CKKSGroupWithName:@"initial-key-state-ready-scan"]];
 286
 287         _keyStateNonTransientDependency = [self createKeyStateNontransientDependency];
 288
 289         dispatch_time_t initialOutgoingQueueDelay = SecCKKSReduceRateLimiting() ? NSEC_PER_MSEC * 200 : NSEC_PER_SEC * 1;
 290         dispatch_time_t continuingOutgoingQueueDelay = SecCKKSReduceRateLimiting() ? NSEC_PER_MSEC * 200 : NSEC_PER_SEC * 30;
 291         _outgoingQueueOperationScheduler = [[CKKSNearFutureScheduler alloc] initWithName:[NSString stringWithFormat: @"%@-outgoing-queue-scheduler", self.zoneName]
 292                                                                             initialDelay:initialOutgoingQueueDelay
 293                                                                          continuingDelay:continuingOutgoingQueueDelay
 294                                                                         keepProcessAlive:false
 295                                                                dependencyDescriptionCode:CKKSResultDescriptionPendingOutgoingQueueScheduling
 296                                                                                    block:^{}];
 297
 298
 299         dispatch_time_t initialKeyHierachyPokeDelay = SecCKKSReduceRateLimiting() ? NSEC_PER_MSEC * 100 : NSEC_PER_MSEC * 500;
 300         dispatch_time_t continuingKeyHierachyPokeDelay = SecCKKSReduceRateLimiting() ? NSEC_PER_MSEC * 200 : NSEC_PER_SEC * 5;
 301         _pokeKeyStateMachineScheduler = [[CKKSNearFutureScheduler alloc] initWithName:[NSString stringWithFormat: @"%@-reprocess-scheduler", self.zoneName]
 302                                                                          initialDelay:initialKeyHierachyPokeDelay
 303                                                                       continuingDelay:continuingKeyHierachyPokeDelay
 304                                                                      keepProcessAlive:true
 305                                                             dependencyDescriptionCode:CKKSResultDescriptionPendingKeyHierachyPokeScheduling
 306                                                                                      block:^{
 307                                                                                          STRONGIFY(self);
 308                                                                                          [self dispatchSyncWithAccountKeys: ^bool{
 309                                                                                              STRONGIFY(self);
 310
 311                                                                                              [self _onqueueAdvanceKeyStateMachineToState:nil withError:nil];
 312                                                                                              return true;
 313                                                                                          }];
 314                                                                                      }];
 315     }
 316     return self;
 317 }
 318
 319 - (NSString*)description {
 320     return [NSString stringWithFormat:@"<%@: %@ (%@)>", NSStringFromClass([self class]), self.zoneName, self.keyHierarchyState];
 321 }
 322
 323 - (NSString*)debugDescription {
 324     return [NSString stringWithFormat:@"<%@: %@ (%@) %p>", NSStringFromClass([self class]), self.zoneName, self.keyHierarchyState, self];
 325 }
 326
 327 - (CKKSZoneKeyState*)keyHierarchyState {
 328     return _keyHierarchyState;
 329 }
 330
 331 - (void)setKeyHierarchyState:(CKKSZoneKeyState *)keyHierarchyState {
 332     if((keyHierarchyState == nil && _keyHierarchyState == nil) || ([keyHierarchyState isEqualToString:_keyHierarchyState])) {
 333         // No change, do nothing.
 334     } else {
 335         // Fixup the condition variables as part of setting this state
 336         if(_keyHierarchyState) {
 337             self.keyHierarchyConditions[_keyHierarchyState] = [[CKKSCondition alloc] init];
 338         }
 339
 340         _keyHierarchyState = keyHierarchyState;
 341
 342         if(keyHierarchyState) {
 343             [self.keyHierarchyConditions[keyHierarchyState] fulfill];
 344         }
 345     }
 346 }
 347
 348 - (NSString *)lastActiveTLKUUID
 349 {
 350     return self.activeTLK;
 351 }
 352
 353 - (void)_onqueueResetSetup:(CKKSZoneKeyState*)newState resetMessage:(NSString*)resetMessage ckoperationGroup:(CKOperationGroup*)group {
 354     [super resetSetup];
 355
 356     self.keyHierarchyState = newState;
 357     self.keyHierarchyError = nil;
 358
 359     [self.keyStateMachineOperation cancel];
 360     self.keyStateMachineOperation = nil;
 361
 362     self.keyStateFetchRequested = false;
 363     self.keyStateProcessRequested = false;
 364
 365     self.keyHierarchyOperationGroup = group;
 366
 367     [self ensureKeyStateReadyDependency:resetMessage];
 368
 369     NSOperation* oldKSNTD = self.keyStateNonTransientDependency;
 370     self.keyStateNonTransientDependency = [self createKeyStateNontransientDependency];
 371     if(oldKSNTD) {
 372         [oldKSNTD addDependency:self.keyStateNonTransientDependency];
 373         [self.waitingQueue addOperation:oldKSNTD];
 374     }
 375 }
 376
 377 - (void)ensureKeyStateReadyDependency:(NSString*)resetMessage {
 378     NSOperation* oldKSRD = self.keyStateReadyDependency;
 379     self.keyStateReadyDependency = [self createKeyStateReadyDependency:resetMessage ckoperationGroup:self.keyHierarchyOperationGroup];
 380     if(oldKSRD) {
 381         [oldKSRD addDependency:self.keyStateReadyDependency];
 382         [self.waitingQueue addOperation:oldKSRD];
 383     }
 384 }
 385
 386 - (CKKSResultOperation*)createPendingInitializationOperation {
 387
 388     WEAKIFY(self);
 389     CKKSResultOperation* initializationOp = [CKKSGroupOperation named:@"view-initialization" withBlockTakingSelf:^(CKKSGroupOperation * _Nonnull strongOp) {
 390         STRONGIFY(self);
 391
 392         __block CKKSResultOperation* zoneCreationOperation = nil;
 393         [self dispatchSync:^bool {
 394             CKKSZoneStateEntry* ckse = [CKKSZoneStateEntry state: self.zoneName];
 395             zoneCreationOperation = [self handleCKLogin:ckse.ckzonecreated zoneSubscribed:ckse.ckzonesubscribed];
 396             return true;
 397         }];
 398
 399         CKKSResultOperation* viewInitializationOperation = [CKKSResultOperation named:@"view-initialization" withBlockTakingSelf:^(CKKSResultOperation * _Nonnull strongInternalOp) {
 400             STRONGIFY(self);
 401             if(!self) {
 402                 ckkserror("ckks", self, "received callback for released object");
 403                 return;
 404             }
 405
 406             [self dispatchSyncWithAccountKeys: ^bool {
 407                 ckksnotice("ckks", self, "Zone setup progress: %@ %d %@ %d %@",
 408                            [CKKSAccountStateTracker stringFromAccountStatus:self.accountStatus],
 409                            self.zoneCreated, self.zoneCreatedError, self.zoneSubscribed, self.zoneSubscribedError);
 410
 411                 NSError* error = nil;
 412                 CKKSZoneStateEntry* ckse = [CKKSZoneStateEntry state: self.zoneName];
 413                 ckse.ckzonecreated = self.zoneCreated;
 414                 ckse.ckzonesubscribed = self.zoneSubscribed;
 415
 416                 // Although, if the zone subscribed error says there's no zone, mark down that there's no zone
 417                 if(self.zoneSubscribedError &&
 418                    [self.zoneSubscribedError.domain isEqualToString:CKErrorDomain] && self.zoneSubscribedError.code == CKErrorPartialFailure) {
 419                     NSError* subscriptionError = self.zoneSubscribedError.userInfo[CKPartialErrorsByItemIDKey][self.zoneID];
 420                     if(subscriptionError && [subscriptionError.domain isEqualToString:CKErrorDomain] && subscriptionError.code == CKErrorZoneNotFound) {
 421
 422                         ckkserror("ckks", self, "zone subscription error appears to say the zone doesn't exist, fixing status: %@", self.zoneSubscribedError);
 423                         ckse.ckzonecreated = false;
 424                     }
 425                 }
 426
 427                 [ckse saveToDatabase: &error];
 428                 if(error) {
 429                     ckkserror("ckks", self, "couldn't save zone creation status for %@: %@", self.zoneName, error);
 430                 }
 431
 432                 if(!self.zoneCreated || !self.zoneSubscribed) {
 433                     // Go into 'zonecreationfailed'
 434                     strongInternalOp.error = self.zoneCreatedError ? self.zoneCreatedError : self.zoneSubscribedError;
 435                     [self _onqueueAdvanceKeyStateMachineToState:SecCKKSZoneKeyStateZoneCreationFailed withError:strongInternalOp.error];
 436
 437                     return true;
 438                 } else {
 439                     [self _onqueueAdvanceKeyStateMachineToState:SecCKKSZoneKeyStateInitialized withError:nil];
 440                 }
 441
 442                 return true;
 443             }];
 444         }];
 445
 446         [viewInitializationOperation addDependency:zoneCreationOperation];
 447         [strongOp runBeforeGroupFinished:viewInitializationOperation];
 448     }];
 449
 450     return initializationOp;
 451 }
 452
 453 - (void)_onqueuePerformKeyStateInitialized:(CKKSZoneStateEntry*)ckse {
 454     CKKSOutgoingQueueOperation* outgoingOperation = nil;
 455     NSOperation* initialProcess = nil;
 456
 457     // Check if we believe we've synced this zone before.
 458     if(ckse.changeToken == nil) {
 459         self.keyHierarchyOperationGroup = [CKOperationGroup CKKSGroupWithName:@"initial-setup"];
 460
 461         ckksnotice("ckks", self, "No existing change token; going to try to match local items with CloudKit ones.");
 462
 463         // Onboard this keychain: there's likely items in it that we haven't synced yet.
 464         // But, there might be items in The Cloud that correspond to these items, with UUIDs that we don't know yet.
 465         // First, fetch all remote items.
 466         CKKSResultOperation* fetch = [self.zoneChangeFetcher requestSuccessfulFetch:CKKSFetchBecauseInitialStart];
 467         fetch.name = @"initial-fetch";
 468
 469         // Next, try to process them (replacing local entries)
 470         initialProcess = [self processIncomingQueue:true after:fetch];
 471         initialProcess.name = @"initial-process-incoming-queue";
 472
 473         // If all that succeeds, iterate through all keychain items and find the ones which need to be uploaded
 474         self.initialScanOperation = [self scanLocalItems:@"initial-scan-operation"
 475                                         ckoperationGroup:self.keyHierarchyOperationGroup
 476                                                    after:initialProcess];
 477
 478     } else {
 479         // Likely a restart of securityd!
 480
 481         // First off, are there any in-flight queue entries? If so, put them back into New.
 482         // If they're truly in-flight, we'll "conflict" with ourselves, but that should be fine.
 483         NSError* error = nil;
 484         [self _onqueueResetAllInflightOQE:&error];
 485         if(error) {
 486             ckkserror("ckks", self, "Couldn't reset in-flight OQEs, bad behavior ahead: %@", error);
 487         }
 488
 489         // Are there any fixups to run first?
 490         self.lastFixupOperation = [CKKSFixups fixup:ckse.lastFixup for:self];
 491         if(self.lastFixupOperation) {
 492             ckksnotice("ckksfixup", self, "We have a fixup to perform: %@", self.lastFixupOperation);
 493             [self scheduleOperation:self.lastFixupOperation];
 494         }
 495
 496         self.keyHierarchyOperationGroup = [CKOperationGroup CKKSGroupWithName:@"restart-setup"];
 497
 498         if ([CKKSManifest shouldSyncManifests]) {
 499             self.egoManifest = [CKKSEgoManifest tryCurrentEgoManifestForZone:self.zoneName];
 500         }
 501
 502         // If it's been more than 24 hours since the last fetch, fetch and process everything.
 503         // Otherwise, just kick off the local queue processing.
 504
 505         NSDate* now = [NSDate date];
 506         NSDateComponents* offset = [[NSDateComponents alloc] init];
 507         [offset setHour:-24];
 508         NSDate* deadline = [[NSCalendar currentCalendar] dateByAddingComponents:offset toDate:now options:0];
 509
 510         if(ckse.lastFetchTime == nil || [ckse.lastFetchTime compare: deadline] == NSOrderedAscending) {
 511             initialProcess = [self fetchAndProcessCKChanges:CKKSFetchBecauseSecuritydRestart after:self.lastFixupOperation];
 512
 513             // Also, kick off a scan local items: it'll find any out-of-sync issues in the local keychain
 514             self.initialScanOperation = [self scanLocalItems:@"24-hr-scan-operation"
 515                                                 ckoperationGroup:self.keyHierarchyOperationGroup
 516                                                        after:initialProcess];
 517         } else {
 518             initialProcess = [self processIncomingQueue:false after:self.lastFixupOperation];
 519         }
 520
 521         if([CKKSManifest shouldSyncManifests]) {
 522             if (!self.egoManifest && !self.initialScanOperation) {
 523                 ckksnotice("ckksmanifest", self, "No ego manifest on restart; rescanning");
 524                 self.initialScanOperation = [self scanLocalItems:@"initial-scan-operation"
 525                                                 ckoperationGroup:self.keyHierarchyOperationGroup
 526                                                            after:initialProcess];
 527             }
 528         }
 529
 530         // Process outgoing queue after re-start
 531         outgoingOperation = [self processOutgoingQueueAfter:self.lastFixupOperation ckoperationGroup:self.keyHierarchyOperationGroup];
 532     }
 533
 534     /*
 535      * Launch time is determined by when the zone have:
 536      *  1. keystate have become ready
 537      *  2. scan local items (if needed)
 538      *  3. processed all outgoing item (if needed)
 539      */
 540
 541     WEAKIFY(self);
 542     NSBlockOperation *seemReady = [NSBlockOperation named:[NSString stringWithFormat:@"seemsReadyForSyncing-%@", self.zoneName] withBlock:^void{
 543         STRONGIFY(self);
 544         NSError *error = nil;
 545         ckksnotice("launch", self, "Launch complete");
 546         NSNumber *zoneSize = [CKKSMirrorEntry counts:self.zoneID error:&error];
 547         if (zoneSize) {
 548             zoneSize = @(SecBucket1Significant([zoneSize longValue]));
 549             [self.launch addAttribute:@"zonesize" value:zoneSize];
 550         }
 551         [self.launch launch];
 552
 553         /*
 554          * Since we think we are ready, signal to CK that its to check for PCS identities again, and create the
 555          * since before we completed this operation, we would probably have failed with a timeout because
 556          * we where busy downloading items from CloudKit and then processing them.
 557          */
 558         [self.notifyViewReadyScheduler trigger];
 559     }];
 560
 561     [seemReady addNullableDependency:self.keyStateReadyDependency];
 562     [seemReady addNullableDependency:outgoingOperation];
 563     [seemReady addNullableDependency:self.initialScanOperation];
 564     [seemReady addNullableDependency:initialProcess];
 565
 566     [self scheduleOperation: seemReady];
 567 }
 568
 569 - (bool)_onqueueResetLocalData: (NSError * __autoreleasing *) error {
 570     dispatch_assert_queue(self.queue);
 571
 572     NSError* localerror = nil;
 573     bool setError = false; // Ugly, but this is the only way to return the first error given
 574
 575     CKKSZoneStateEntry* ckse = [CKKSZoneStateEntry state: self.zoneName];
 576     ckse.ckzonecreated = false;
 577     ckse.ckzonesubscribed = false; // I'm actually not sure about this: can you be subscribed to a non-existent zone?
 578     ckse.changeToken = NULL;
 579     [ckse saveToDatabase: &localerror];
 580     if(localerror) {
 581         ckkserror("ckks", self, "couldn't reset zone status for %@: %@", self.zoneName, localerror);
 582         if(error && !setError) {
 583             *error = localerror; setError = true;
 584         }
 585     }
 586
 587     [CKKSMirrorEntry deleteAll:self.zoneID error: &localerror];
 588     if(localerror) {
 589         ckkserror("ckks", self, "couldn't delete all CKKSMirrorEntry: %@", localerror);
 590         if(error && !setError) {
 591             *error = localerror; setError = true;
 592         }
 593     }
 594
 595     [CKKSOutgoingQueueEntry deleteAll:self.zoneID error: &localerror];
 596     if(localerror) {
 597         ckkserror("ckks", self, "couldn't delete all CKKSOutgoingQueueEntry: %@", localerror);
 598         if(error && !setError) {
 599             *error = localerror; setError = true;
 600         }
 601     }
 602
 603     [CKKSIncomingQueueEntry deleteAll:self.zoneID error: &localerror];
 604     if(localerror) {
 605         ckkserror("ckks", self, "couldn't delete all CKKSIncomingQueueEntry: %@", localerror);
 606         if(error && !setError) {
 607             *error = localerror; setError = true;
 608         }
 609     }
 610
 611     [CKKSKey deleteAll:self.zoneID error: &localerror];
 612     if(localerror) {
 613         ckkserror("ckks", self, "couldn't delete all CKKSKey: %@", localerror);
 614         if(error && !setError) {
 615             *error = localerror; setError = true;
 616         }
 617     }
 618
 619     [CKKSTLKShareRecord deleteAll:self.zoneID error: &localerror];
 620     if(localerror) {
 621         ckkserror("ckks", self, "couldn't delete all CKKSTLKShare: %@", localerror);
 622         if(error && !setError) {
 623             *error = localerror; setError = true;
 624         }
 625     }
 626
 627     [CKKSCurrentKeyPointer deleteAll:self.zoneID error: &localerror];
 628     if(localerror) {
 629         ckkserror("ckks", self, "couldn't delete all CKKSCurrentKeyPointer: %@", localerror);
 630         if(error && !setError) {
 631             *error = localerror; setError = true;
 632         }
 633     }
 634
 635     [CKKSCurrentItemPointer deleteAll:self.zoneID error: &localerror];
 636     if(localerror) {
 637         ckkserror("ckks", self, "couldn't delete all CKKSCurrentItemPointer: %@", localerror);
 638         if(error && !setError) {
 639             *error = localerror; setError = true;
 640         }
 641     }
 642
 643     [CKKSDeviceStateEntry deleteAll:self.zoneID error:&localerror];
 644     if(localerror) {
 645         ckkserror("ckks", self, "couldn't delete all CKKSDeviceStateEntry: %@", localerror);
 646         if(error && !setError) {
 647             *error = localerror; setError = true;
 648         }
 649     }
 650
 651     return (localerror == nil && !setError);
 652 }
 653
 654 - (CKKSResultOperation*)createPendingResetLocalDataOperation {
 655     @synchronized(self.localResetOperations) {
 656         CKKSResultOperation* pendingResetLocalOperation = (CKKSResultOperation*) [self findFirstPendingOperation:self.localResetOperations];
 657         if(!pendingResetLocalOperation) {
 658             WEAKIFY(self);
 659             pendingResetLocalOperation = [CKKSResultOperation named:@"reset-local" withBlockTakingSelf:^(CKKSResultOperation * _Nonnull strongOp) {
 660                 STRONGIFY(self);
 661                 __block NSError* error = nil;
 662
 663                 [self dispatchSync: ^bool{
 664                     [self _onqueueResetLocalData: &error];
 665                     return true;
 666                 }];
 667
 668                 strongOp.error = error;
 669             }];
 670             [pendingResetLocalOperation linearDependencies:self.localResetOperations];
 671         }
 672         return pendingResetLocalOperation;
 673     }
 674 }
 675
 676 - (CKKSResultOperation*)resetLocalData {
 677     // Not overly thread-safe, but a single read is okay
 678     CKKSAccountStatus accountStatus = self.accountStatus;
 679     ckksnotice("ckksreset", self, "Requesting local data reset");
 680
 681     // If we're currently signed in, the reset operation will be handled by the CKKS key state machine, and a reset should end up in 'ready'
 682     if(accountStatus == CKKSAccountStatusAvailable) {
 683         WEAKIFY(self);
 684         CKKSGroupOperation* resetOperationGroup = [CKKSGroupOperation named:@"local-reset" withBlockTakingSelf:^(CKKSGroupOperation *strongOp) {
 685             STRONGIFY(self);
 686
 687             __block CKKSResultOperation* resetOperation = nil;
 688
 689             [self dispatchSyncWithAccountKeys:^bool {
 690                 self.keyStateLocalResetRequested = true;
 691                 resetOperation = [self createPendingResetLocalDataOperation];
 692                 [self _onqueueAdvanceKeyStateMachineToState:nil withError:nil];
 693                 return true;
 694             }];
 695
 696             [strongOp dependOnBeforeGroupFinished:resetOperation];
 697         }];
 698         [self scheduleOperationWithoutDependencies:resetOperationGroup];
 699
 700         CKKSGroupOperation* viewReset = [CKKSGroupOperation named:@"local-data-reset" withBlockTakingSelf:^(CKKSGroupOperation *strongOp) {
 701             STRONGIFY(self);
 702             // Now that the local reset finished, wait for the key hierarchy state machine to churn
 703             ckksnotice("ckksreset", self, "waiting for key hierarchy to become nontransient (after local reset)");
 704             CKKSResultOperation* waitOp = [CKKSResultOperation named:@"waiting-for-local-reset" withBlock:^{}];
 705             [waitOp timeout: 60*NSEC_PER_SEC];
 706             [waitOp addNullableDependency:self.keyStateNonTransientDependency];
 707
 708             [strongOp runBeforeGroupFinished:waitOp];
 709         }];
 710         [viewReset addSuccessDependency:resetOperationGroup];
 711
 712         [self scheduleOperationWithoutDependencies:viewReset];
 713         return viewReset;
 714     } else {
 715         // Since we're logged out, we must run the reset ourselves
 716         WEAKIFY(self);
 717         CKKSResultOperation* pendingResetLocalOperation = [CKKSResultOperation named:@"reset-local"
 718                                                                  withBlockTakingSelf:^(CKKSResultOperation * _Nonnull strongOp) {
 719             STRONGIFY(self);
 720             __block NSError* error = nil;
 721
 722             [self dispatchSync: ^bool{
 723                 [self _onqueueResetLocalData: &error];
 724                 return true;
 725             }];
 726
 727             strongOp.error = error;
 728         }];
 729         [self scheduleOperationWithoutDependencies:pendingResetLocalOperation];
 730         return pendingResetLocalOperation;
 731     }
 732 }
 733
 734 - (CKKSResultOperation*)createPendingDeleteZoneOperation:(CKOperationGroup*)operationGroup {
 735     @synchronized(self.cloudkitDeleteZoneOperations) {
 736         CKKSResultOperation* pendingDeleteOperation = (CKKSResultOperation*) [self findFirstPendingOperation:self.cloudkitDeleteZoneOperations];
 737         if(!pendingDeleteOperation) {
 738             pendingDeleteOperation = [self deleteCloudKitZoneOperation:operationGroup];
 739             [pendingDeleteOperation linearDependencies:self.cloudkitDeleteZoneOperations];
 740         }
 741         return pendingDeleteOperation;
 742     }
 743 }
 744
 745 - (CKKSResultOperation*)resetCloudKitZone:(CKOperationGroup*)operationGroup {
 746     [self.accountStateKnown wait:(SecCKKSTestsEnabled() ? 1*NSEC_PER_SEC : 10*NSEC_PER_SEC)];
 747
 748     // Not overly thread-safe, but a single read is okay
 749     if(self.accountStatus != CKKSAccountStatusAvailable) {
 750         // No CK account? goodbye!
 751         ckksnotice("ckksreset", self, "Requesting reset of CK zone, but no CK account exists");
 752         CKKSResultOperation* errorOp = [CKKSResultOperation named:@"fail" withBlockTakingSelf:^(CKKSResultOperation * _Nonnull op) {
 753             op.error = [NSError errorWithDomain:CKKSErrorDomain
 754                                           code:CKKSNotLoggedIn
 755                                    description:@"User is not signed into iCloud."];
 756         }];
 757
 758         [self scheduleOperationWithoutDependencies:errorOp];
 759         return errorOp;
 760     }
 761
 762     // Actually running the delete operation will be handled by the CKKS key state machine
 763     ckksnotice("ckksreset", self, "Requesting reset of CK zone (logged in)");
 764
 765     __block CKKSResultOperation* deleteOperation = nil;
 766     [self dispatchSyncWithAccountKeys:^bool {
 767         self.keyStateCloudKitDeleteRequested = true;
 768         deleteOperation = [self createPendingDeleteZoneOperation:operationGroup];
 769         [self _onqueueAdvanceKeyStateMachineToState:nil withError:nil];
 770         return true;
 771     }];
 772
 773     WEAKIFY(self);
 774     CKKSGroupOperation* viewReset = [CKKSGroupOperation named:[NSString stringWithFormat:@"cloudkit-view-reset-%@", self.zoneName]
 775                                           withBlockTakingSelf:^(CKKSGroupOperation *strongOp) {
 776         STRONGIFY(self);
 777         // Now that the delete finished, wait for the key hierarchy state machine
 778         ckksnotice("ckksreset", self, "waiting for key hierarchy to become nontransient (after cloudkit reset)");
 779         CKKSResultOperation* waitOp = [CKKSResultOperation named:@"waiting-for-reset" withBlock:^{}];
 780         [waitOp timeout: 60*NSEC_PER_SEC];
 781         [waitOp addNullableDependency:self.keyStateNonTransientDependency];
 782
 783         [strongOp runBeforeGroupFinished:waitOp];
 784     }];
 785
 786     [viewReset timeout:30*NSEC_PER_SEC];
 787     [viewReset addDependency:deleteOperation];
 788     [self.waitingQueue addOperation:viewReset];
 789
 790     return viewReset;
 791 }
 792
 793 - (void)_onqueueKeyStateMachineRequestFetch {
 794     dispatch_assert_queue(self.queue);
 795
 796     // We're going to set this flag, then nudge the key state machine.
 797     // If it was idle, then it should launch a fetch. If there was an active process, this flag will stay high
 798     // and the fetch will be launched later.
 799
 800     self.keyStateFetchRequested = true;
 801     [self _onqueueAdvanceKeyStateMachineToState: nil withError: nil];
 802 }
 803
 804 - (void)keyStateMachineRequestProcess {
 805     // Since bools are atomic, we don't need to get on-queue here
 806     // Just set the flag high and hope
 807     self.keyStateProcessRequested = true;
 808     [self.pokeKeyStateMachineScheduler trigger];
 809 }
 810
 811 - (void)_onqueueKeyStateMachineRequestProcess {
 812     dispatch_assert_queue(self.queue);
 813
 814     // Set the request flag, then nudge the key state machine.
 815     // If it was idle, then it should launch a process. If there was an active process, this flag will stay high
 816     // and the process will be launched later.
 817
 818     self.keyStateProcessRequested = true;
 819     [self _onqueueAdvanceKeyStateMachineToState: nil withError: nil];
 820 }
 821
 822 - (CKKSResultOperation*)createKeyStateReadyDependency:(NSString*)message ckoperationGroup:(CKOperationGroup*)group {
 823     WEAKIFY(self);
 824     CKKSResultOperation* keyStateReadyDependency = [CKKSResultOperation operationWithBlock:^{
 825         STRONGIFY(self);
 826         if(!self) {
 827             return;
 828         }
 829         ckksnotice("ckkskey", self, "%@", message);
 830
 831         [self dispatchSync:^bool {
 832             if(self.droppedItems) {
 833                 // While we weren't in 'ready', keychain modifications might have come in and were dropped on the floor. Find them!
 834                 ckksnotice("ckkskey", self, "Launching scan operation for missed items");
 835                 [self scanLocalItems:@"ready-again-scan" ckoperationGroup:group after:nil];
 836             }
 837             return true;
 838         }];
 839     }];
 840     keyStateReadyDependency.name = [NSString stringWithFormat: @"%@-key-state-ready", self.zoneName];
 841     keyStateReadyDependency.descriptionErrorCode = CKKSResultDescriptionPendingKeyReady;
 842     return keyStateReadyDependency;
 843 }
 844
 845 - (CKKSResultOperation*)createKeyStateNontransientDependency {
 846     WEAKIFY(self);
 847     return [CKKSResultOperation named:[NSString stringWithFormat: @"%@-key-state-nontransient", self.zoneName] withBlock:^{
 848         STRONGIFY(self);
 849         ckksnotice("ckkskey", self, "Key state is now non-transient");
 850     }];
 851 }
 852
 853 // The operations suggested by this state machine should call _onqueueAdvanceKeyStateMachineToState once they are complete.
 854 // At no other time should keyHierarchyState be modified.
 855
 856 // Note that this function cannot rely on doing any database work; it might get rolled back, especially in an error state
 857 - (void)_onqueueAdvanceKeyStateMachineToState: (CKKSZoneKeyState*) state withError: (NSError*) error {
 858     dispatch_assert_queue(self.queue);
 859     WEAKIFY(self);
 860
 861     // Resetting back to 'loggedout' takes all precedence.
 862     if([state isEqual:SecCKKSZoneKeyStateLoggedOut]) {
 863         ckksnotice("ckkskey", self, "Resetting the key hierarchy state machine back to '%@'", state);
 864
 865         [self _onqueueResetSetup:SecCKKSZoneKeyStateLoggedOut
 866                     resetMessage:@"Key state has become ready for the first time (after reset)."
 867                 ckoperationGroup:[CKOperationGroup CKKSGroupWithName:@"key-state-after-logout"]];
 868
 869         [self _onqueueHandleKeyStateNonTransientDependency:nil];
 870         self.launch = nil;
 871         return;
 872     }
 873
 874     [self.launch addEvent:state];
 875
 876     // Resetting back to 'initialized' also takes precedence
 877     if([state isEqual:SecCKKSZoneKeyStateInitializing]) {
 878         ckksnotice("ckkskey", self, "Resetting the key hierarchy state machine back to '%@'", state);
 879
 880         [self _onqueueResetSetup:SecCKKSZoneKeyStateInitializing
 881                     resetMessage:@"Key state has become ready for the first time (after re-initializing)."
 882                 ckoperationGroup:[CKOperationGroup CKKSGroupWithName:@"key-state-reset-to-initializing"]];
 883
 884         // Begin initialization, but rate-limit it
 885         self.keyStateMachineOperation = [self createPendingInitializationOperation];
 886         [self.keyStateMachineOperation addNullableDependency:self.zoneModifier.cloudkitRetryAfter.operationDependency];
 887         [self.zoneModifier.cloudkitRetryAfter trigger];
 888         [self scheduleOperation:self.keyStateMachineOperation];
 889
 890         [self _onqueueHandleKeyStateNonTransientDependency:nil];
 891         return;
 892     }
 893
 894     // Resetting to 'waitfortrust' also takes precedence
 895     if([state isEqualToString:SecCKKSZoneKeyStateWaitForTrust]) {
 896         if([self.keyHierarchyState isEqualToString:SecCKKSZoneKeyStateLoggedOut]) {
 897             ckksnotice("ckks", self, "Asked to waitfortrust, but we're already in loggedout. Ignoring...");
 898             return;
 899         }
 900
 901         ckksnotice("ckks", self, "Entering waitfortrust");
 902         self.keyHierarchyState = SecCKKSZoneKeyStateWaitForTrust;
 903         self.keyHierarchyError = nil;
 904         self.keyStateMachineOperation = nil;
 905
 906         [self ensureKeyStateReadyDependency:@"Key state has become ready for the first time (after lacking trust)."];
 907
 908         if(self.trustStatus == CKKSAccountStatusAvailable) {
 909             // Note: we go to initialized here, since to enter waitfortrust CKKS has already gone through initializing
 910             // initialized should refetch only if needed.
 911             ckksnotice("ckks", self, "CKKS is trusted, moving to initialized");
 912             self.keyStateMachineOperation = [self operationToEnterState:SecCKKSZoneKeyStateInitialized
 913                                                           keyStateError:nil
 914                                                                   named:@"re-enter initialized"];
 915             [self scheduleOperation:self.keyStateMachineOperation];
 916         }
 917
 918         // In wait for trust, we might have a keyset. Who knows!
 919         CKKSCurrentKeySet* keyset = [CKKSCurrentKeySet loadForZone:self.zoneID];
 920         [self _onqueueHandleKeyStateNonTransientDependency:keyset];
 921
 922         return;
 923     }
 924
 925     // Cancels and error states take precedence
 926     if([self.keyHierarchyState isEqualToString: SecCKKSZoneKeyStateError] ||
 927        [self.keyHierarchyState isEqualToString: SecCKKSZoneKeyStateCancelled] ||
 928        self.keyHierarchyError != nil) {
 929         // Error state: nowhere to go. Early-exit.
 930         ckkserror("ckkskey", self, "Asked to advance state machine from non-exit state %@ (to %@): %@", self.keyHierarchyState, state, self.keyHierarchyError);
 931         return;
 932     }
 933
 934     if([state isEqual: SecCKKSZoneKeyStateError]) {
 935         // But wait! Is this a "we're locked" error?
 936         if(error && [self.lockStateTracker isLockedError:error]) {
 937             ckkserror("ckkskey", self, "advised of 'keychain locked' error, ignoring: coming from state (%@): %@", self.keyHierarchyState, error);
 938             // After the next unlock, fake that we received the last zone transition
 939             CKKSZoneKeyState* lastState = self.keyHierarchyState;
 940             self.keyStateMachineOperation = [NSBlockOperation named:@"key-state-after-unlock" withBlock:^{
 941                 STRONGIFY(self);
 942                 if(!self) {
 943                     return;
 944                 }
 945                 [self dispatchSyncWithAccountKeys:^bool{
 946                     [self _onqueueAdvanceKeyStateMachineToState:lastState withError:nil];
 947                     return true;
 948                 }];
 949             }];
 950             state = nil;
 951
 952             self.keyHierarchyState = SecCKKSZoneKeyStateWaitForUnlock;
 953
 954             [self.keyStateMachineOperation addNullableDependency:self.lockStateTracker.unlockDependency];
 955             [self scheduleOperation:self.keyStateMachineOperation];
 956
 957             [self _onqueueHandleKeyStateNonTransientDependency:nil];
 958             return;
 959
 960         } else {
 961             // Error state: record the error and exit early
 962             ckkserror("ckkskey", self, "advised of error: coming from state (%@): %@", self.keyHierarchyState, error);
 963
 964             [[CKKSAnalytics logger] logUnrecoverableError:error
 965                                                  forEvent:CKKSEventStateError
 966                                                    inView:self
 967                                            withAttributes:@{ @"previousKeyHierarchyState" : self.keyHierarchyState }];
 968
 969
 970             self.keyHierarchyState = SecCKKSZoneKeyStateError;
 971             self.keyHierarchyError = error;
 972
 973             [self _onqueueHandleKeyStateNonTransientDependency:nil];
 974             return;
 975         }
 976     }
 977
 978     if([state isEqual: SecCKKSZoneKeyStateCancelled]) {
 979         ckkserror("ckkskey", self, "advised of cancel: coming from state (%@): %@", self.keyHierarchyState, error);
 980         self.keyHierarchyState = SecCKKSZoneKeyStateCancelled;
 981         self.keyHierarchyError = error;
 982
 983         // Cancel the key ready dependency. Strictly Speaking, this will cause errors down the line, but we're in a cancel state: those operations should be canceled anyway.
 984         self.keyHierarchyOperationGroup = nil;
 985         [self.keyStateReadyDependency cancel];
 986         self.keyStateReadyDependency = nil;
 987
 988         [self.keyStateNonTransientDependency cancel];
 989         self.keyStateNonTransientDependency = nil;
 990         return;
 991     }
 992
 993     // Now that the current or new state isn't an error or a cancel, proceed.
 994     if(self.keyStateMachineOperation && ![self.keyStateMachineOperation isFinished]) {
 995         if(state == nil) {
 996             // we started this operation to move the state machine. Since you aren't asking for a state transition, and there's an active operation, no need to do anything
 997             ckksnotice("ckkskey", self, "Not advancing state machine: waiting for %@", self.keyStateMachineOperation);
 998             return;
 999         }
1000     }
1001
1002     if(state) {
1003         ckksnotice("ckkskey", self, "Preparing to advance key hierarchy state machine from %@ to %@", self.keyHierarchyState, state);
1004         self.keyStateMachineOperation = nil;
1005     } else {
1006         ckksnotice("ckkskey", self, "Key hierarchy state machine is being poked; currently %@", self.keyHierarchyState);
1007         state = self.keyHierarchyState;
1008     }
1009
1010 #if DEBUG
1011     // During testing, keep the developer honest: this function should always have the self identities, unless the account has lost trust
1012     if(self.trustStatus == CKKSAccountStatusAvailable && ![state isEqualToString:SecCKKSZoneKeyStateLoggedOut]) {
1013         bool hasSelfIdentities = false;
1014         NSAssert(self.currentTrustStates.count > 0, @"Should have at least one trust state");
1015         for(CKKSPeerProviderState* state in self.currentTrustStates) {
1016             if(state.currentSelfPeersError == nil || state.currentSelfPeersError.code != CKKSNoPeersAvailable) {
1017                 hasSelfIdentities = true;
1018             }
1019         }
1020
1021         NSAssert(hasSelfIdentities, @"Must have viable (or errored) self peers to advance key state");
1022     }
1023 #endif
1024
1025     // Do any of these state transitions below want to change which state we're in?
1026     CKKSZoneKeyState* nextState = nil;
1027     NSError* nextError = nil;
1028
1029     // Many of our decisions below will be based on what keys exist. Help them out.
1030     CKKSCurrentKeySet* keyset = [CKKSCurrentKeySet loadForZone:self.zoneID];
1031     NSError* localerror = nil;
1032     NSArray<CKKSKey*>* localKeys = [CKKSKey localKeys:self.zoneID error:&localerror];
1033     NSArray<CKKSKey*>* remoteKeys = [CKKSKey remoteKeys:self.zoneID error: &localerror];
1034
1035     // We also are checking for OutgoingQueueEntries in the reencrypt state; this is a sign that our key hierarchy is out of date.
1036     NSInteger outdatedOQEs = [CKKSOutgoingQueueEntry countByState:SecCKKSStateReencrypt zone:self.zoneID error:&localerror];
1037
1038     SecADSetValueForScalarKey((__bridge CFStringRef) SecCKKSAggdViewKeyCount, [localKeys count]);
1039
1040     if(localerror) {
1041         ckkserror("ckkskey", self, "couldn't fetch keys and OQEs from local database, entering error state: %@", localerror);
1042         self.keyHierarchyState = SecCKKSZoneKeyStateError;
1043         self.keyHierarchyError = localerror;
1044         [self _onqueueHandleKeyStateNonTransientDependency:nil];
1045         return;
1046     }
1047
1048 #if !defined(NDEBUG)
1049     NSArray<CKKSKey*>* allKeys = [CKKSKey allKeys:self.zoneID error:&localerror];
1050     ckksdebug("ckkskey", self, "All keys: %@", allKeys);
1051 #endif
1052
1053     NSError* hierarchyError = nil;
1054
1055     if(self.keyStateCloudKitDeleteRequested || [state isEqualToString:SecCKKSZoneKeyStateResettingZone]) {
1056         // CloudKit reset requests take precedence over all other state transitions
1057         ckksnotice("ckkskey", self, "Deleting the CloudKit Zone");
1058         CKKSGroupOperation* op = [[CKKSGroupOperation alloc] init];
1059
1060         CKKSResultOperation* deleteOp = [self createPendingDeleteZoneOperation:self.keyHierarchyOperationGroup];
1061         [op runBeforeGroupFinished: deleteOp];
1062
1063         NSOperation* nextStateOp = [CKKSResultOperation named:@"inspect-zone-delete" withBlockTakingSelf:^(CKKSResultOperation * _Nonnull op) {
1064             STRONGIFY(self);
1065             [self dispatchSyncWithAccountKeys:^bool {
1066                 // Did the delete op succeed?
1067                 if(deleteOp.error == nil) {
1068                     ckksnotice("ckkskey", self, "Zone deletion operation complete! Proceeding to reset local data");
1069                     [self _onqueueAdvanceKeyStateMachineToState:SecCKKSZoneKeyStateResettingLocalData withError:nil];
1070                     return true;
1071                 }
1072
1073                 ckksnotice("ckkskey", self, "Zone deletion operation failed, will retry: %@", deleteOp.error);
1074                 [self _onqueueAdvanceKeyStateMachineToState:SecCKKSZoneKeyStateResettingZone withError:nil];
1075
1076                 return true;
1077             }];
1078         }];
1079
1080         [nextStateOp addDependency:deleteOp];
1081         [op runBeforeGroupFinished:nextStateOp];
1082
1083         self.keyStateMachineOperation = op;
1084         self.keyStateCloudKitDeleteRequested = false;
1085
1086         // Also, pending operations should be cancelled
1087         [self cancelPendingOperations];
1088
1089     } else if(self.keyStateLocalResetRequested || [state isEqualToString:SecCKKSZoneKeyStateResettingLocalData]) {
1090         // Local reset requests take precedence over all other state transitions
1091         ckksnotice("ckkskey", self, "Resetting local data");
1092         CKKSGroupOperation* op = [[CKKSGroupOperation alloc] init];
1093
1094         CKKSResultOperation* resetOp = [self createPendingResetLocalDataOperation];
1095         [op runBeforeGroupFinished: resetOp];
1096
1097         NSOperation* nextStateOp = [self operationToEnterState:SecCKKSZoneKeyStateInitializing keyStateError:nil named:@"state-resetting-initialize"];
1098         [nextStateOp addDependency:resetOp];
1099         [op runBeforeGroupFinished:nextStateOp];
1100
1101         self.keyStateMachineOperation = op;
1102         self.keyStateLocalResetRequested = false;
1103
1104     } else if([state isEqualToString:SecCKKSZoneKeyStateZoneCreationFailed]) {
1105         //Prepare to go back into initializing, as soon as the cloudkitRetryAfter is happy
1106         self.keyStateMachineOperation = [self operationToEnterState:SecCKKSZoneKeyStateInitializing keyStateError:nil named:@"recover-from-cloudkit-failure"];
1107         [self.keyStateMachineOperation addNullableDependency:self.zoneModifier.cloudkitRetryAfter.operationDependency];
1108         [self.zoneModifier.cloudkitRetryAfter trigger];
1109
1110     } else if([state isEqualToString:SecCKKSZoneKeyStateWaitForTrust]) {
1111         // Actually entering this state should have been handled above, so let's check if we can exit it here...
1112         if(self.trustStatus == CKKSAccountStatusAvailable) {
1113             ckksnotice("ckkskey", self, "Beginning trusted state machine operation");
1114             nextState = SecCKKSZoneKeyStateInitialized;
1115
1116         } else if (self.tlkCreationRequested) {
1117             ckksnotice("ckkskey", self, "No trust, but TLK creation is requested. Moving to fetchcomplete.");
1118             nextState = SecCKKSZoneKeyStateFetchComplete;
1119
1120         } else {
1121             ckksnotice("ckkskey", self, "Remaining in 'waitfortrust'");
1122         }
1123
1124     } else if([state isEqualToString: SecCKKSZoneKeyStateReady]) {
1125         if(self.keyStateProcessRequested || [remoteKeys count] > 0) {
1126             // We've either received some remote keys from the last fetch, or someone has requested a reprocess.
1127             ckksnotice("ckkskey", self, "Kicking off a key reprocess based on request:%d and remote key count %lu", self.keyStateProcessRequested, (unsigned long)[remoteKeys count]);
1128             nextState = SecCKKSZoneKeyStateProcess;
1129
1130         } else if(self.keyStateFullRefetchRequested) {
1131             // In ready, but someone has requested a full fetch. Kick it off.
1132             ckksnotice("ckkskey", self, "Kicking off a full key refetch based on request:%d", self.keyStateFullRefetchRequested);
1133             nextState = SecCKKSZoneKeyStateNeedFullRefetch;
1134
1135         } else if(self.keyStateFetchRequested) {
1136             // In ready, but someone has requested a fetch. Kick it off.
1137             ckksnotice("ckkskey", self, "Kicking off a key refetch based on request:%d", self.keyStateFetchRequested);
1138             nextState = SecCKKSZoneKeyStateFetch; // Don't go to 'ready', go to 'initialized', since we want to fetch again
1139         } else if (self.trustStatus != CKKSAccountStatusAvailable) {
1140             ckksnotice("ckkskey", self, "Asked to go into ready, but there's no trust; going into waitfortrust");
1141             nextState = SecCKKSZoneKeyStateWaitForTrust;
1142         } else if (self.trustedPeersSetChanged) {
1143             ckksnotice("ckkskey", self, "Received a nudge that the trusted peers set might have changed! Reprocessing.");
1144             nextState = SecCKKSZoneKeyStateProcess;
1145             self.trustedPeersSetChanged = false;
1146         }
1147
1148         // TODO: kick off a key roll if one has been requested
1149
1150         if(!self.keyStateMachineOperation && !nextState) {
1151             // We think we're ready. Double check.
1152             CKKSZoneKeyState* checkedstate = [self _onqueueEnsureKeyHierarchyHealth:keyset error:&hierarchyError];
1153             if(![checkedstate isEqualToString:SecCKKSZoneKeyStateReady] || hierarchyError) {
1154                 // Things is bad. Kick off a heal to fix things up.
1155                 ckksnotice("ckkskey", self, "Thought we were ready, but the key hierarchy is %@: %@", checkedstate, hierarchyError);
1156                 nextState = checkedstate;
1157                 if([nextState isEqualToString:SecCKKSZoneKeyStateError]) {
1158                     nextError = hierarchyError;
1159                 }
1160             }
1161         }
1162
1163     } else if([state isEqualToString: SecCKKSZoneKeyStateInitialized]) {
1164         // We're initialized and CloudKit is ready. If we're trusted, see what needs done. Otherwise, wait.
1165
1166         // Note: we might be still 'untrusted' at this point. The state machine is responsible for not entering 'ready' until
1167         // we are trusted.
1168         // This is acceptable only if the key state machine does not make new TLKs without being trusted!
1169
1170         // Set this state, for test use
1171         self.keyHierarchyState = SecCKKSZoneKeyStateInitialized;
1172
1173         CKKSZoneStateEntry* ckse = [CKKSZoneStateEntry state:self.zoneName];
1174         [self _onqueuePerformKeyStateInitialized:ckse];
1175
1176         // We need to either:
1177         //  Wait for the fixup operation to occur
1178         //  Go into 'ready'
1179         //  Or start a key state fetch
1180         if(self.lastFixupOperation && ![self.lastFixupOperation isFinished]) {
1181             nextState = SecCKKSZoneKeyStateWaitForFixupOperation;
1182         } else {
1183             // Check if we have an existing key hierarchy in keyset
1184             if(keyset.error && !([keyset.error.domain isEqual: @"securityd"] && keyset.error.code == errSecItemNotFound)) {
1185                 ckkserror("ckkskey", self, "Error examining existing key hierarchy: %@", error);
1186             }
1187
1188             if(keyset.tlk && keyset.classA && keyset.classC && !keyset.error) {
1189                 // This is likely a restart of securityd, and we think we're ready. Double check.
1190
1191                 CKKSZoneKeyState* checkedstate = [self _onqueueEnsureKeyHierarchyHealth:keyset error:&hierarchyError];
1192                 if([checkedstate isEqualToString:SecCKKSZoneKeyStateReady] && !hierarchyError) {
1193                     ckksnotice("ckkskey", self, "Already have existing key hierarchy for %@; using it.", self.zoneID.zoneName);
1194                 } else {
1195                     ckksnotice("ckkskey", self, "Initial scan shows key hierarchy is %@: %@", checkedstate, hierarchyError);
1196                 }
1197                 nextState = checkedstate;
1198
1199             } else {
1200                 // We have no local key hierarchy. One might exist in CloudKit, or it might not.
1201                 ckksnotice("ckkskey", self, "No existing key hierarchy for %@. Check if there's one in CloudKit...", self.zoneID.zoneName);
1202                 nextState = SecCKKSZoneKeyStateFetch;
1203             }
1204         }
1205
1206     } else if([state isEqualToString:SecCKKSZoneKeyStateFetch]) {
1207         ckksnotice("ckkskey", self, "Starting a key hierarchy fetch");
1208         [self _onqueueKeyHierarchyFetch];
1209
1210     } else if([state isEqualToString: SecCKKSZoneKeyStateNeedFullRefetch]) {
1211         ckksnotice("ckkskey", self, "Starting a key hierarchy full refetch");
1212         [self _onqueueKeyHierarchyRefetch];
1213
1214     } else if([state isEqualToString:SecCKKSZoneKeyStateWaitForFixupOperation]) {
1215         // We should enter 'initialized' when the fixup operation completes
1216         ckksnotice("ckkskey", self, "Waiting for the fixup operation: %@", self.lastFixupOperation);
1217
1218         self.keyStateMachineOperation = [NSBlockOperation named:@"key-state-after-fixup" withBlock:^{
1219             STRONGIFY(self);
1220             [self dispatchSyncWithAccountKeys:^bool{
1221                 ckksnotice("ckkskey", self, "Fixup operation complete! Restarting key hierarchy machinery");
1222                 [self _onqueueAdvanceKeyStateMachineToState:SecCKKSZoneKeyStateInitialized withError:nil];
1223                 return true;
1224             }];
1225         }];
1226         [self.keyStateMachineOperation addNullableDependency:self.lastFixupOperation];
1227
1228     } else if([state isEqualToString: SecCKKSZoneKeyStateFetchComplete]) {
1229         // We've just completed a fetch of everything. Are there any remote keys?
1230         if(remoteKeys.count > 0u) {
1231             // Process the keys we received.
1232             self.keyStateMachineOperation = [[CKKSProcessReceivedKeysStateMachineOperation alloc] initWithCKKSKeychainView: self];
1233         } else if( (keyset.currentTLKPointer || keyset.currentClassAPointer || keyset.currentClassCPointer) &&
1234                   !(keyset.tlk && keyset.classA && keyset.classC)) {
1235             // Huh. We appear to have current key pointers, but the keys themselves don't exist. That's weird.
1236             // Transfer to the "unhealthy" state to request a fix
1237             ckksnotice("ckkskey", self, "We appear to have current key pointers but no keys to match them: %@ Moving to 'unhealthy'", keyset);
1238             nextState = SecCKKSZoneKeyStateUnhealthy;
1239         } else {
1240             // No remote keys, and the pointers look sane? Do we have an existing key hierarchy?
1241             CKKSZoneKeyState* checkedstate = [self _onqueueEnsureKeyHierarchyHealth:keyset error:&hierarchyError];
1242             if([checkedstate isEqualToString:SecCKKSZoneKeyStateReady] && !hierarchyError) {
1243                 ckksnotice("ckkskey", self, "After fetch, everything looks good.");
1244                 nextState = checkedstate;
1245
1246             } else if(localKeys.count == 0 && remoteKeys.count == 0) {
1247                 ckksnotice("ckkskey", self, "After fetch, we don't have any key hierarchy. Entering a waiting state: %@", hierarchyError ?: @"no error");
1248                 nextState = SecCKKSZoneKeyStateWaitForTLKCreation;
1249             } else {
1250                 ckksnotice("ckkskey", self, "After fetch, we have a possibly unhealthy key hierarchy. Moving to %@: %@", checkedstate, hierarchyError ?: @"no error");
1251                 nextState = checkedstate;
1252             }
1253         }
1254
1255     } else if([state isEqualToString:SecCKKSZoneKeyStateWaitForTLKCreation]) {
1256
1257         if(self.tlkCreationRequested) {
1258             self.tlkCreationRequested = false;
1259             ckksnotice("ckkskey", self, "TLK creation requested; kicking off operation");
1260             self.keyStateMachineOperation = [[CKKSNewTLKOperation alloc] initWithCKKSKeychainView: self ckoperationGroup:self.keyHierarchyOperationGroup];
1261
1262         } else if(self.keyStateProcessRequested) {
1263             ckksnotice("ckkskey", self, "We believe we need to create TLKs but we also received a key nudge; moving to key state Process.");
1264             nextState = SecCKKSZoneKeyStateProcess;
1265
1266         } else {
1267             ckksnotice("ckkskey", self, "We believe we need to create TLKs; waiting for Octagon (via %@)", self.suggestTLKUpload);
1268             [self.suggestTLKUpload trigger];
1269         }
1270
1271
1272     } else if([state isEqualToString:SecCKKSZoneKeyStateWaitForTLKUpload]) {
1273         ckksnotice("ckkskey", self, "We believe we have TLKs that need uploading");
1274
1275
1276         if(self.keyStateProcessRequested) {
1277             if(keyset.currentTLKPointer.currentKeyUUID) {
1278                 ckksnotice("ckkskey", self, "Received a nudge that our TLK records might be here (and there's some current TLK pointer)");
1279                 nextState = SecCKKSZoneKeyStateProcess;
1280             } else {
1281                 ckksnotice("ckkskey", self, "Received a nudge that our TLK records might be here, but there's no TLK pointer. Staying in WaitForTLKUpload.");
1282                 self.keyStateProcessRequested = false;
1283             }
1284         }
1285
1286         if(nextState == nil) {
1287             ckksnotice("ckkskey", self, "Alerting any listener of our proposed keyset: %@", self.lastNewTLKOperation.keyset);
1288             [self _onqueueRunKeysetProviderOperations:self.lastNewTLKOperation.keyset];
1289
1290             ckksnotice("ckkskey", self, "Notifying Octagon again, just in case");
1291             [self.suggestTLKUpload trigger];
1292         }
1293
1294     } else if([state isEqualToString: SecCKKSZoneKeyStateWaitForTLK]) {
1295         // We're in a hold state: waiting for the TLK bytes to arrive.
1296
1297         if(self.keyStateProcessRequested) {
1298             // Someone has requsted a reprocess! Go to the correct state.
1299             ckksnotice("ckkskey", self, "Received a nudge that our TLK might be here! Reprocessing.");
1300             nextState = SecCKKSZoneKeyStateProcess;
1301
1302         } else if(self.trustedPeersSetChanged) {
1303             // Hmm, maybe this trust set change will cause us to recover this TLK (due to a previously-untrusted share becoming trusted). Worth a shot!
1304             ckksnotice("ckkskey", self, "Received a nudge that the trusted peers set might have changed! Reprocessing.");
1305             nextState = SecCKKSZoneKeyStateProcess;
1306             self.trustedPeersSetChanged = false;
1307
1308         } else {
1309             // Should we nuke this zone?
1310             if(self.trustStatus == CKKSAccountStatusAvailable) {
1311                 if([self _onqueueOtherDevicesReportHavingTLKs:keyset]) {
1312                     ckksnotice("ckkskey", self, "Other devices report having TLK(%@). Entering a waiting state", keyset.currentTLKPointer);
1313                 } else {
1314                     ckksnotice("ckkskey", self, "No other devices have TLK(%@). Beginning zone reset...", keyset.currentTLKPointer);
1315                     self.keyHierarchyOperationGroup = [CKOperationGroup CKKSGroupWithName:@"tlk-missing"];
1316                     nextState = SecCKKSZoneKeyStateResettingZone;
1317                 }
1318             } else {
1319                 ckksnotice("ckkskey", self, "This device isn't trusted, so don't modify the existing TLK(%@)", keyset.currentTLKPointer);
1320                 nextState = SecCKKSZoneKeyStateWaitForTrust;
1321             }
1322         }
1323
1324     } else if([state isEqualToString: SecCKKSZoneKeyStateWaitForUnlock]) {
1325         ckksnotice("ckkskey", self, "Requested to enter waitforunlock");
1326         self.keyStateMachineOperation = [self operationToEnterState:SecCKKSZoneKeyStateInitialized keyStateError:nil named:@"key-state-after-unlock"];
1327         [self.keyStateMachineOperation addNullableDependency: self.lockStateTracker.unlockDependency];
1328
1329     } else if([state isEqualToString: SecCKKSZoneKeyStateReadyPendingUnlock]) {
1330         ckksnotice("ckkskey", self, "Believe we're ready, but rechecking after unlock");
1331         self.keyStateMachineOperation = [self operationToEnterState:SecCKKSZoneKeyStateInitialized keyStateError:nil named:@"key-state-after-unlock"];
1332         [self.keyStateMachineOperation addNullableDependency: self.lockStateTracker.unlockDependency];
1333
1334     } else if([state isEqualToString: SecCKKSZoneKeyStateBadCurrentPointers]) {
1335         // The current key pointers are broken, but we're not sure why.
1336         ckksnotice("ckkskey", self, "Our current key pointers are reported broken. Attempting a fix!");
1337         self.keyStateMachineOperation = [[CKKSHealKeyHierarchyOperation alloc] initWithCKKSKeychainView: self ckoperationGroup:self.keyHierarchyOperationGroup];
1338
1339     } else if([state isEqualToString: SecCKKSZoneKeyStateNewTLKsFailed]) {
1340         ckksnotice("ckkskey", self, "Creating new TLKs didn't work. Attempting to refetch!");
1341         [self _onqueueKeyHierarchyFetch];
1342
1343     } else if([state isEqualToString: SecCKKSZoneKeyStateHealTLKSharesFailed]) {
1344         ckksnotice("ckkskey", self, "Creating new TLK shares didn't work. Attempting to refetch!");
1345         [self _onqueueKeyHierarchyFetch];
1346
1347     } else if([state isEqualToString:SecCKKSZoneKeyStateUnhealthy]) {
1348         if(self.trustStatus != CKKSAccountStatusAvailable) {
1349             ckksnotice("ckkskey", self, "Looks like the key hierarchy is unhealthy, but we're untrusted.");
1350             nextState = SecCKKSZoneKeyStateWaitForTrust;
1351
1352         } else {
1353             ckksnotice("ckkskey", self, "Looks like the key hierarchy is unhealthy. Launching fix.");
1354             self.keyStateMachineOperation = [[CKKSHealKeyHierarchyOperation alloc] initWithCKKSKeychainView:self ckoperationGroup:self.keyHierarchyOperationGroup];
1355         }
1356
1357     } else if([state isEqualToString:SecCKKSZoneKeyStateHealTLKShares]) {
1358         ckksnotice("ckksshare", self, "Key hierarchy is okay, but not shared appropriately. Launching fix.");
1359         self.keyStateMachineOperation = [[CKKSHealTLKSharesOperation alloc] initWithCKKSKeychainView:self
1360                                                                                     ckoperationGroup:self.keyHierarchyOperationGroup];
1361
1362     } else if([state isEqualToString:SecCKKSZoneKeyStateProcess]) {
1363         ckksnotice("ckksshare", self, "Launching key state process");
1364         self.keyStateMachineOperation = [[CKKSProcessReceivedKeysStateMachineOperation alloc] initWithCKKSKeychainView: self];
1365
1366         // Since we're starting a reprocess, this is answering all previous requests.
1367         self.keyStateProcessRequested = false;
1368
1369     } else {
1370         ckkserror("ckks", self, "asked to advance state machine to unknown state: %@", state);
1371         self.keyHierarchyState = state;
1372         [self _onqueueHandleKeyStateNonTransientDependency:keyset];
1373         return;
1374     }
1375
1376     // Handle the key state ready dependency
1377     // If we're in ready and not entering a non-ready state, we should activate the ready dependency. Otherwise, we should create it.
1378     if(([state isEqualToString:SecCKKSZoneKeyStateReady] || [state isEqualToString:SecCKKSZoneKeyStateReadyPendingUnlock]) &&
1379        (nextState == nil || [nextState isEqualToString:SecCKKSZoneKeyStateReady] || [nextState isEqualToString:SecCKKSZoneKeyStateReadyPendingUnlock])) {
1380
1381         // Ready enough!
1382         [[CKKSAnalytics logger] setDateProperty:[NSDate date] forKey:CKKSAnalyticsLastKeystateReady inView:self];
1383
1384         if(self.keyStateReadyDependency) {
1385             [self scheduleOperation: self.keyStateReadyDependency];
1386             self.keyStateReadyDependency = nil;
1387         }
1388
1389         // If there are any OQEs waiting to be encrypted, launch an op to fix them
1390         if(outdatedOQEs > 0) {
1391             ckksnotice("ckksreencrypt", self, "Reencrypting outgoing items as the key hierarchy is ready");
1392             CKKSReencryptOutgoingItemsOperation* op = [[CKKSReencryptOutgoingItemsOperation alloc] initWithCKKSKeychainView:self ckoperationGroup:self.keyHierarchyOperationGroup];
1393             [self scheduleOperation:op];
1394         }
1395     } else {
1396         // Not in ready: we need a key state ready dependency
1397         if(self.keyStateReadyDependency == nil || [self.keyStateReadyDependency isFinished]) {
1398             self.keyHierarchyOperationGroup = [CKOperationGroup CKKSGroupWithName:@"key-state-broken"];
1399             self.keyStateReadyDependency = [self createKeyStateReadyDependency:@"Key state has become ready again." ckoperationGroup:self.keyHierarchyOperationGroup];
1400         }
1401     }
1402
1403     NSAssert(!((self.keyStateMachineOperation != nil) &&
1404                (nextState != nil)),
1405              @"Should have a machine operation or a next state, not both");
1406
1407     // Start any operations, or log that we aren't
1408     if(self.keyStateMachineOperation) {
1409         [self scheduleOperation: self.keyStateMachineOperation];
1410         ckksnotice("ckkskey", self, "Now in key state: %@", state);
1411         self.keyHierarchyState = state;
1412
1413     } else if([state isEqualToString:SecCKKSZoneKeyStateError]) {
1414         ckksnotice("ckkskey", self, "Entering key state 'error'");
1415         self.keyHierarchyState = state;
1416
1417     } else if(nextState == nil) {
1418         ckksnotice("ckkskey", self, "Entering key state: %@", state);
1419         self.keyHierarchyState = state;
1420
1421     } else if(![state isEqualToString: nextState]) {
1422         ckksnotice("ckkskey", self, "Staying in state %@, but proceeding to %@ as soon as possible", self.keyHierarchyState, nextState);
1423         self.keyStateMachineOperation = [self operationToEnterState:nextState keyStateError:nextError named:[NSString stringWithFormat:@"next-key-state-%@", nextState]];
1424         [self scheduleOperation: self.keyStateMachineOperation];
1425
1426     } else {
1427         // Nothing to do and not in a waiting state? This is likely a bug, but, hey: pretend to be in ready!
1428         if(!([state isEqualToString:SecCKKSZoneKeyStateReady] || [state isEqualToString:SecCKKSZoneKeyStateReadyPendingUnlock])) {
1429             ckkserror("ckkskey", self, "No action to take in state %@; BUG, but: maybe we're ready?", state);
1430             nextState = SecCKKSZoneKeyStateReady;
1431             self.keyStateMachineOperation = [self operationToEnterState:nextState keyStateError:nil named:@"next-key-state"];
1432             [self scheduleOperation: self.keyStateMachineOperation];
1433         }
1434     }
1435
1436     [self _onqueueHandleKeyStateNonTransientDependency:keyset];
1437 }
1438
1439 - (void)_onqueueHandleKeyStateNonTransientDependency:(CKKSCurrentKeySet* _Nullable)keyset {
1440     dispatch_assert_queue(self.queue);
1441
1442     if(CKKSKeyStateTransient(self.keyHierarchyState)) {
1443         if(self.keyStateNonTransientDependency == nil || [self.keyStateNonTransientDependency isFinished]) {
1444             self.keyStateNonTransientDependency = [self createKeyStateNontransientDependency];
1445         }
1446     } else {
1447         // Nontransient: go for it
1448         if(self.keyStateNonTransientDependency) {
1449             [self scheduleOperation: self.keyStateNonTransientDependency];
1450             self.keyStateNonTransientDependency = nil;
1451         }
1452
1453         if(keyset && keyset.currentTLKPointer.currentKeyUUID) {
1454             [self _onqueueRunKeysetProviderOperations:keyset];
1455         } else {
1456             ckksnotice("ckkskey", self, "State machine is nontransient, but no keyset...");
1457         }
1458     }
1459 }
1460
1461 - (NSOperation*)operationToEnterState:(CKKSZoneKeyState*)state keyStateError:(NSError* _Nullable)keyStateError named:(NSString*)name {
1462     WEAKIFY(self);
1463
1464     return [NSBlockOperation named:name withBlock:^{
1465         STRONGIFY(self);
1466         if(!self) {
1467             return;
1468         }
1469         [self dispatchSyncWithAccountKeys:^bool{
1470             [self _onqueueAdvanceKeyStateMachineToState:state withError:keyStateError];
1471             return true;
1472         }];
1473     }];
1474 }
1475
1476 - (BOOL)otherDevicesReportHavingTLKs:(CKKSCurrentKeySet*)keyset
1477 {
1478     __block BOOL report = false;
1479     [self dispatchSync:^bool{
1480         report = [self _onqueueOtherDevicesReportHavingTLKs:keyset];
1481         return true;
1482     }];
1483     return report ? YES : NO;
1484 }
1485
1486 - (bool)_onqueueOtherDevicesReportHavingTLKs:(CKKSCurrentKeySet*)keyset
1487 {
1488     dispatch_assert_queue(self.queue);
1489
1490     //Has there been any activity indicating that other trusted devices have keys in the past 45 days, or untrusted devices in the past 4?
1491     // (We chose 4 as devices attempt to upload their device state every 3 days. If a device is unceremoniously kicked out of circle, we normally won't immediately reset.)
1492     NSDate* now = [NSDate date];
1493     NSDateComponents* trustedOffset = [[NSDateComponents alloc] init];
1494     [trustedOffset setDay:-45];
1495     NSDate* trustedDeadline = [[NSCalendar currentCalendar] dateByAddingComponents:trustedOffset toDate:now options:0];
1496
1497     NSDateComponents* untrustedOffset = [[NSDateComponents alloc] init];
1498     [untrustedOffset setDay:-4];
1499     NSDate* untrustedDeadline = [[NSCalendar currentCalendar] dateByAddingComponents:untrustedOffset toDate:now options:0];
1500
1501
1502     NSMutableSet<NSString*>* trustedPeerIDs = [NSMutableSet set];
1503     for(CKKSPeerProviderState* trustState in self.currentTrustStates) {
1504         for(id<CKKSPeer> peer in trustState.currentTrustedPeers) {
1505             [trustedPeerIDs addObject:peer.peerID];
1506         }
1507     }
1508
1509     NSError* localerror = nil;
1510
1511     NSArray<CKKSDeviceStateEntry*>* allDeviceStates = [CKKSDeviceStateEntry allInZone:self.zoneID error:&localerror];
1512     if(localerror) {
1513         ckkserror("ckkskey", self, "Error fetching device states: %@", localerror);
1514         localerror = nil;
1515         return true;
1516     }
1517     for(CKKSDeviceStateEntry* device in allDeviceStates) {
1518         // The peerIDs in CDSEs aren't written with the peer prefix. Make sure we match both.
1519         NSString* sosPeerID = device.circlePeerID ? [CKKSSOSPeerPrefix stringByAppendingString:device.circlePeerID] : nil;
1520
1521         if([trustedPeerIDs containsObject:device.circlePeerID] ||
1522            [trustedPeerIDs containsObject:sosPeerID] ||
1523            [trustedPeerIDs containsObject:device.octagonPeerID]) {
1524             // Is this a recent DSE? If it's older than the deadline, skip it
1525             if([device.storedCKRecord.modificationDate compare:trustedDeadline] == NSOrderedAscending) {
1526                 ckksnotice("ckkskey", self, "Trusted device state (%@) is too old; ignoring", device);
1527                 continue;
1528             }
1529         } else {
1530             // Device is untrusted. How does it fare with the untrustedDeadline?
1531             if([device.storedCKRecord.modificationDate compare:untrustedDeadline] == NSOrderedAscending) {
1532                 ckksnotice("ckkskey", self, "Device (%@) is not trusted and from too long ago; ignoring device state (%@)", device.circlePeerID, device);
1533                 continue;
1534             } else {
1535                 ckksnotice("ckkskey", self, "Device (%@) is not trusted, but very recent. Including in heuristic: %@", device.circlePeerID, device);
1536             }
1537         }
1538
1539         if([device.keyState isEqualToString:SecCKKSZoneKeyStateReady] ||
1540            [device.keyState isEqualToString:SecCKKSZoneKeyStateReadyPendingUnlock]) {
1541             ckksnotice("ckkskey", self, "Other device (%@) has keys; it should send them to us", device);
1542             return true;
1543         }
1544     }
1545
1546     NSArray<CKKSTLKShareRecord*>* tlkShares = [CKKSTLKShareRecord allForUUID:keyset.currentTLKPointer.currentKeyUUID
1547                                                           zoneID:self.zoneID
1548                                                            error:&localerror];
1549     if(localerror) {
1550         ckkserror("ckkskey", self, "Error fetching device states: %@", localerror);
1551         localerror = nil;
1552         return false;
1553     }
1554
1555     for(CKKSTLKShareRecord* tlkShare in tlkShares) {
1556         if([trustedPeerIDs containsObject:tlkShare.senderPeerID] &&
1557            [tlkShare.storedCKRecord.modificationDate compare:trustedDeadline] == NSOrderedDescending) {
1558             ckksnotice("ckkskey", self, "Trusted TLK Share (%@) created recently; other devices have keys and should send them to us", tlkShare);
1559             return true;
1560         }
1561     }
1562
1563     // Okay, how about the untrusted deadline?
1564     for(CKKSTLKShareRecord* tlkShare in tlkShares) {
1565         if([tlkShare.storedCKRecord.modificationDate compare:untrustedDeadline] == NSOrderedDescending) {
1566             ckksnotice("ckkskey", self, "Untrusted TLK Share (%@) created very recently; other devices might have keys and should rejoin the circle (and send them to us)", tlkShare);
1567             return true;
1568         }
1569     }
1570
1571     return false;
1572 }
1573
1574 // For this key, who doesn't yet have a valid CKKSTLKShare for it?
1575 // Note that we really want a record sharing the TLK to ourselves, so this function might return
1576 // a non-empty set even if all peers have the TLK: it wants us to make a record for ourself.
1577 - (NSSet<id<CKKSPeer>>*)_onqueueFindPeers:(CKKSPeerProviderState*)trustState
1578                              missingShare:(CKKSKey*)key
1579                            afterUploading:(NSSet<CKKSTLKShareRecord*>* _Nullable)newShares
1580                                     error:(NSError* __autoreleasing*)error
1581 {
1582     dispatch_assert_queue(self.queue);
1583
1584     if(!key) {
1585         ckkserror("ckksshare", self, "Attempting to find missing shares for nil key");
1586         return [NSSet set];
1587     }
1588
1589     if(trustState.currentTrustedPeersError) {
1590         ckkserror("ckksshare", self, "Couldn't find missing shares because trusted peers aren't available: %@", trustState.currentTrustedPeersError);
1591         if(error) {
1592             *error = trustState.currentTrustedPeersError;
1593         }
1594         return [NSSet set];
1595     }
1596     if(trustState.currentSelfPeersError) {
1597         ckkserror("ckksshare", self, "Couldn't find missing shares because self peers aren't available: %@", trustState.currentSelfPeersError);
1598         if(error) {
1599             *error = trustState.currentSelfPeersError;
1600         }
1601         return [NSSet set];
1602     }
1603
1604     NSMutableSet<id<CKKSPeer>>* peersMissingShares = [NSMutableSet set];
1605
1606     // Ensure that the 'self peer' is one of the current trusted peers. Otherwise, any TLKShare we create
1607     // won't be considered trusted the next time through...
1608     if(![trustState.currentTrustedPeerIDs containsObject:trustState.currentSelfPeers.currentSelf.peerID]) {
1609         ckkserror("ckksshare", self, "current self peer (%@) is not in the set of trusted peers: %@",
1610                   trustState.currentSelfPeers.currentSelf.peerID,
1611                   trustState.currentTrustedPeerIDs);
1612
1613         if(error) {
1614             *error = [NSError errorWithDomain:CKKSErrorDomain
1615                                          code:CKKSLackingTrust
1616                                   description:[NSString stringWithFormat:@"current self peer (%@) is not in the set of trusted peers",
1617                                                trustState.currentSelfPeers.currentSelf.peerID]];
1618         }
1619
1620         return nil;
1621     }
1622
1623     for(id<CKKSRemotePeerProtocol> peer in trustState.currentTrustedPeers) {
1624         if(![peer shouldHaveView:self.zoneName]) {
1625             ckkserror("ckksshare", self, "Peer (%@) is not supposed to have view, skipping", peer);
1626             continue;
1627         }
1628
1629         NSError* peerError = nil;
1630         // Find all the shares for this peer for this key
1631         NSArray<CKKSTLKShareRecord*>* currentPeerShares = [CKKSTLKShareRecord allFor:peer.peerID
1632                                                                  keyUUID:key.uuid
1633                                                                   zoneID:self.zoneID
1634                                                                    error:&peerError];
1635
1636         if(peerError) {
1637             ckkserror("ckksshare", self, "Couldn't load shares for peer %@: %@", peer, peerError);
1638             if(error) {
1639                 *error = peerError;
1640             }
1641             return nil;
1642         }
1643
1644         // Include the new shares, too....
1645         NSArray<CKKSTLKShareRecord*>* possiblePeerShares = newShares ? [currentPeerShares arrayByAddingObjectsFromArray:[newShares allObjects]] : currentPeerShares;
1646
1647         // Determine if we think this peer has enough things shared to them
1648         bool alreadyShared = false;
1649         for(CKKSTLKShareRecord* existingPeerShare in possiblePeerShares) {
1650             // Ensure this share is to this peer...
1651             if(![existingPeerShare.share.receiverPeerID isEqualToString:peer.peerID]) {
1652                 continue;
1653             }
1654
1655             // If an SOS Peer sent this share, is its signature still valid? Or did the signing key change?
1656             if([existingPeerShare.senderPeerID hasPrefix:CKKSSOSPeerPrefix]) {
1657                 NSError* signatureError = nil;
1658                 if(![existingPeerShare signatureVerifiesWithPeerSet:trustState.currentTrustedPeers error:&signatureError]) {
1659                     ckksnotice("ckksshare", self, "Existing TLKShare's signature doesn't verify with current peer set: %@ %@", signatureError, existingPeerShare);
1660                     continue;
1661                 }
1662             }
1663
1664             if([existingPeerShare.tlkUUID isEqualToString:key.uuid] && [trustState.currentTrustedPeerIDs containsObject:existingPeerShare.senderPeerID]) {
1665                 // Was this shared to us?
1666                 if([peer.peerID isEqualToString: trustState.currentSelfPeers.currentSelf.peerID]) {
1667                     // We only count this as 'found' if we did the sharing and it's to our current keys
1668                     NSData* currentKey = trustState.currentSelfPeers.currentSelf.publicEncryptionKey.keyData;
1669
1670                     if([existingPeerShare.senderPeerID isEqualToString:trustState.currentSelfPeers.currentSelf.peerID] &&
1671                        [existingPeerShare.share.receiverPublicEncryptionKeySPKI isEqual:currentKey]) {
1672                         ckksnotice("ckksshare", self, "Local peer %@ is shared %@ via self: %@", peer, key, existingPeerShare);
1673                         alreadyShared = true;
1674                         break;
1675                     } else {
1676                         ckksnotice("ckksshare", self, "Local peer %@ is shared %@ via trusted %@, but that's not good enough", peer, key, existingPeerShare);
1677                     }
1678
1679                 } else {
1680                     // Was this shared to the remote peer's current keys?
1681                     NSData* currentKeySPKI = peer.publicEncryptionKey.keyData;
1682
1683                     if([existingPeerShare.share.receiverPublicEncryptionKeySPKI isEqual:currentKeySPKI]) {
1684                         // Some other peer has a trusted share. Cool!
1685                         ckksnotice("ckksshare", self, "Peer %@ is shared %@ via trusted %@", peer, key, existingPeerShare);
1686                         alreadyShared = true;
1687                         break;
1688                     } else {
1689                         ckksnotice("ckksshare", self, "Peer %@ has a share for %@, but to old keys: %@", peer, key, existingPeerShare);
1690                     }
1691                 }
1692             }
1693         }
1694
1695         if(!alreadyShared) {
1696             // Add this peer to our set, if it has an encryption key to receive the share
1697             if(peer.publicEncryptionKey) {
1698                 [peersMissingShares addObject:peer];
1699             }
1700         }
1701     }
1702
1703     if(peersMissingShares.count > 0u) {
1704         // Log each and every one of the things
1705         ckksnotice("ckksshare", self, "Missing TLK shares for %lu peers: %@", (unsigned long)peersMissingShares.count, peersMissingShares);
1706         ckksnotice("ckksshare", self, "Self peers are (%@) %@", trustState.currentSelfPeersError ?: @"no error", trustState.currentSelfPeers);
1707         ckksnotice("ckksshare", self, "Trusted peers are (%@) %@", trustState.currentTrustedPeersError ?: @"no error", trustState.currentTrustedPeers);
1708     }
1709
1710     return peersMissingShares;
1711 }
1712
1713 - (BOOL)_onqueueAreNewSharesSufficient:(NSSet<CKKSTLKShareRecord*>*)newShares
1714                             currentTLK:(CKKSKey*)key
1715                                  error:(NSError* __autoreleasing*)error
1716 {
1717     dispatch_assert_queue(self.queue);
1718
1719     for(CKKSPeerProviderState* trustState in self.currentTrustStates) {
1720         NSError* localError = nil;
1721         NSSet<id<CKKSPeer>>* peersMissingShares = [self _onqueueFindPeers:trustState
1722                                                              missingShare:key
1723                                                            afterUploading:newShares
1724                                                                     error:&localError];
1725         if(peersMissingShares == nil || localError) {
1726             if(trustState.essential) {
1727                 if(error) {
1728                     *error = localError;
1729                 }
1730                 return NO;
1731             } else {
1732                 ckksnotice("ckksshare", self, "Failed to find peers for nonessential system: %@", trustState);
1733                 // Not a hard failure.
1734             }
1735         }
1736
1737         if(peersMissingShares.count > 0) {
1738             ckksnotice("ckksshare", self, "New share set is missing shares for peers: %@", peersMissingShares);
1739             return NO;
1740         }
1741     }
1742
1743     return YES;
1744 }
1745
1746 - (NSSet<CKKSTLKShareRecord*>*)_onqueueCreateMissingKeyShares:(CKKSKey*)key
1747                                                         error:(NSError* __autoreleasing*)error
1748 {
1749     NSError* localerror = nil;
1750     NSSet<CKKSTLKShareRecord*>* newShares = nil;
1751
1752     // If any one of our trust states succeed, this function doesn't have an error
1753     for(CKKSPeerProviderState* trustState in self.currentTrustStates) {
1754         NSError* stateError = nil;
1755
1756         NSSet<CKKSTLKShareRecord*>* newTrustShares = [self _onqueueCreateMissingKeyShares:key
1757                                                                                     peers:trustState
1758                                                                                     error:&stateError];
1759
1760
1761         if(newTrustShares && !stateError) {
1762             newShares = newShares ? [newShares setByAddingObjectsFromSet:newTrustShares] : newTrustShares;
1763         } else {
1764             ckksnotice("ckksshare", self, "Unable to create shares for trust set %@: %@", trustState, stateError);
1765             if(localerror == nil) {
1766                 localerror = stateError;
1767             }
1768         }
1769     }
1770
1771     // Only report an error if none of the trust states were able to succeed
1772     if(newShares) {
1773         return newShares;
1774     } else {
1775         if(error && localerror) {
1776             *error = localerror;
1777         }
1778         return nil;
1779     }
1780 }
1781
1782 - (NSSet<CKKSTLKShareRecord*>*)_onqueueCreateMissingKeyShares:(CKKSKey*)key
1783                                                         peers:(CKKSPeerProviderState*)trustState
1784                                                         error:(NSError* __autoreleasing*)error
1785 {
1786     dispatch_assert_queue(self.queue);
1787
1788     if(trustState.currentTrustedPeersError) {
1789         ckkserror("ckksshare", self, "Couldn't create missing shares because trusted peers aren't available: %@", trustState.currentTrustedPeersError);
1790         if(error) {
1791             *error = trustState.currentTrustedPeersError;
1792         }
1793         return nil;
1794     }
1795     if(trustState.currentSelfPeersError) {
1796         ckkserror("ckksshare", self, "Couldn't create missing shares because self peers aren't available: %@", trustState.currentSelfPeersError);
1797         if(error) {
1798             *error = trustState.currentSelfPeersError;
1799         }
1800         return nil;
1801     }
1802
1803     NSSet<id<CKKSPeer>>* remainingPeers = [self _onqueueFindPeers:trustState missingShare:key afterUploading:nil error:error];
1804     NSMutableSet<CKKSTLKShareRecord*>* newShares = [NSMutableSet set];
1805
1806     if(!remainingPeers) {
1807         return nil;
1808     }
1809
1810     NSError* localerror = nil;
1811
1812     if(![key ensureKeyLoaded:error]) {
1813         return nil;
1814     }
1815
1816     for(id<CKKSPeer> peer in remainingPeers) {
1817         if(!peer.publicEncryptionKey) {
1818             ckksnotice("ckksshare", self, "No need to make TLK for %@; they don't have any encryption keys", peer);
1819             continue;
1820         }
1821
1822         // Create a share for this peer.
1823         ckksnotice("ckksshare", self, "Creating share of %@ as %@ for %@", key, trustState.currentSelfPeers.currentSelf, peer);
1824         CKKSTLKShareRecord* newShare = [CKKSTLKShareRecord share:key
1825                                                   as:trustState.currentSelfPeers.currentSelf
1826                                                   to:peer
1827                                                epoch:-1
1828                                             poisoned:0
1829                                                error:&localerror];
1830
1831         if(localerror) {
1832             ckkserror("ckksshare", self, "Couldn't create new share for %@: %@", peer, localerror);
1833             if(error) {
1834                 *error = localerror;
1835             }
1836             return nil;
1837         }
1838
1839         [newShares addObject: newShare];
1840     }
1841
1842     return newShares;
1843 }
1844
1845 - (CKKSZoneKeyState*)_onqueueEnsureKeyHierarchyHealth:(CKKSCurrentKeySet*)set error:(NSError* __autoreleasing *)error {
1846     dispatch_assert_queue(self.queue);
1847
1848     if(!set.currentTLKPointer && !set.currentClassAPointer && !set.currentClassCPointer) {
1849         ckkserror("ckkskey", self, "Error examining existing key hierarchy (missing all CKPs, likely no hierarchy exists): %@", set);
1850         return SecCKKSZoneKeyStateWaitForTLKCreation;
1851     }
1852
1853     // Check keyset
1854     if(!set.tlk || !set.classA || !set.classC) {
1855         ckkserror("ckkskey", self, "Error examining existing key hierarchy (missing at least one key): %@", set);
1856         if(error) {
1857             *error = set.error;
1858         }
1859         return SecCKKSZoneKeyStateUnhealthy;
1860     }
1861
1862     NSError* localerror = nil;
1863     bool probablyOkIfUnlocked = false;
1864
1865     // keychain being locked is not a fatal error here
1866     [set.tlk loadKeyMaterialFromKeychain:&localerror];
1867     if(localerror && !([localerror.domain isEqual: @"securityd"] && localerror.code == errSecInteractionNotAllowed)) {
1868         ckkserror("ckkskey", self, "Error loading TLK(%@): %@", set.tlk, localerror);
1869         if(error) {
1870             *error = localerror;
1871         }
1872         return SecCKKSZoneKeyStateUnhealthy;
1873     } else if(localerror) {
1874         ckkserror("ckkskey", self, "Soft error loading TLK(%@), maybe locked: %@", set.tlk, localerror);
1875         probablyOkIfUnlocked = true;
1876     }
1877     localerror = nil;
1878
1879     // keychain being locked is not a fatal error here
1880     [set.classA loadKeyMaterialFromKeychain:&localerror];
1881     if(localerror && !([localerror.domain isEqual: @"securityd"] && localerror.code == errSecInteractionNotAllowed)) {
1882         ckkserror("ckkskey", self, "Error loading classA key(%@): %@", set.classA, localerror);
1883         if(error) {
1884             *error = localerror;
1885         }
1886         return SecCKKSZoneKeyStateUnhealthy;
1887     } else if(localerror) {
1888         ckkserror("ckkskey", self, "Soft error loading classA key(%@), maybe locked: %@", set.classA, localerror);
1889         probablyOkIfUnlocked = true;
1890     }
1891     localerror = nil;
1892
1893     // keychain being locked is a fatal error here, since this is class C
1894     [set.classC loadKeyMaterialFromKeychain:&localerror];
1895     if(localerror) {
1896         ckkserror("ckkskey", self, "Error loading classC(%@): %@", set.classC, localerror);
1897         if(error) {
1898             *error = localerror;
1899         }
1900         return SecCKKSZoneKeyStateUnhealthy;
1901     }
1902
1903     // Check that the classA and classC keys point to the current TLK
1904     if(![set.classA.parentKeyUUID isEqualToString: set.tlk.uuid]) {
1905         localerror = [NSError errorWithDomain:CKKSServerExtensionErrorDomain
1906                                          code:CKKSServerUnexpectedSyncKeyInChain
1907                                      userInfo:@{
1908                                                 NSLocalizedDescriptionKey: @"Current class A key does not wrap to current TLK",
1909                                                }];
1910         ckkserror("ckkskey", self, "Key hierarchy unhealthy: %@", localerror);
1911         if(error) {
1912             *error = localerror;
1913         }
1914         return SecCKKSZoneKeyStateUnhealthy;
1915     }
1916     if(![set.classC.parentKeyUUID isEqualToString: set.tlk.uuid]) {
1917         localerror = [NSError errorWithDomain:CKKSServerExtensionErrorDomain
1918                                          code:CKKSServerUnexpectedSyncKeyInChain
1919                                      userInfo:@{
1920                                                 NSLocalizedDescriptionKey: @"Current class C key does not wrap to current TLK",
1921                                                }];
1922         ckkserror("ckkskey", self, "Key hierarchy unhealthy: %@", localerror);
1923         if(error) {
1924             *error = localerror;
1925         }
1926         return SecCKKSZoneKeyStateUnhealthy;
1927     }
1928
1929     self.activeTLK = [set.tlk uuid];
1930
1931     // Now that we're pretty sure we have the keys, are they shared appropriately?
1932     // We need trust in order to proceed here
1933     if(self.currentTrustStates.count == 0u) {
1934         ckkserror("ckkskey", self, "Can't check TLKShares due to missing trust states");
1935         return SecCKKSZoneKeyStateWaitForTrust;
1936     }
1937
1938     // Check that every trusted peer has at least one TLK share
1939     // If any trust state check works, don't error out
1940     bool anyTrustStateSucceeded = false;
1941     for(CKKSPeerProviderState* trustState in self.currentTrustStates) {
1942         NSSet<id<CKKSPeer>>* missingShares = [self _onqueueFindPeers:trustState missingShare:set.tlk afterUploading:nil error:&localerror];
1943         if(localerror && [self.lockStateTracker isLockedError: localerror]) {
1944             ckkserror("ckkskey", self, "Couldn't find missing TLK shares due to lock state: %@", localerror);
1945             probablyOkIfUnlocked = true;
1946
1947         } else if(([localerror.domain isEqualToString:TrustedPeersHelperErrorDomain] && localerror.code == TrustedPeersHelperErrorNoPreparedIdentity) ||
1948                   ([localerror.domain isEqualToString:CKKSErrorDomain] && localerror.code == CKKSLackingTrust) ||
1949                   ([localerror.domain isEqualToString:CKKSErrorDomain] && localerror.code == CKKSNoPeersAvailable)) {
1950             ckkserror("ckkskey", self, "Couldn't find missing TLK shares due some trust issue: %@", localerror);
1951
1952             if(trustState.essential) {
1953                 ckkserror("ckkskey", self, "Trust state is considered essential; entering waitfortrust: %@", trustState);
1954
1955                 // Octagon can reinform us when it thinks we should start again
1956                 self.trustStatus = CKKSAccountStatusUnknown;
1957                 return SecCKKSZoneKeyStateWaitForTrust;
1958             } else {
1959                 ckkserror("ckkskey", self, "Peer provider is considered nonessential; ignoring error: %@", trustState);
1960                 continue;
1961             }
1962
1963         } else if(localerror) {
1964             ckkserror("ckkskey", self, "Error finding missing TLK shares: %@", localerror);
1965             continue;
1966         }
1967
1968         if(!missingShares || missingShares.count != 0u) {
1969             localerror = [NSError errorWithDomain:CKKSErrorDomain code:CKKSMissingTLKShare
1970                                       description:[NSString stringWithFormat:@"Missing shares for %lu peers", (unsigned long)missingShares.count]];
1971             if(error) {
1972                 *error = localerror;
1973             }
1974             return SecCKKSZoneKeyStateHealTLKShares;
1975         } else {
1976             ckksnotice("ckksshare", self, "TLK (%@) is shared correctly for trust state %@", set.tlk, trustState.peerProviderID);
1977         }
1978
1979         anyTrustStateSucceeded |= true;
1980     }
1981
1982     if(!anyTrustStateSucceeded) {
1983         if(error) {
1984             *error = localerror;
1985         }
1986
1987         return SecCKKSZoneKeyStateError;
1988     }
1989
1990     // Got to the bottom? Cool! All keys are present and accounted for.
1991     return probablyOkIfUnlocked ? SecCKKSZoneKeyStateReadyPendingUnlock : SecCKKSZoneKeyStateReady;
1992 }
1993
1994 - (void)_onqueueKeyHierarchyFetch {
1995     dispatch_assert_queue(self.queue);
1996
1997     WEAKIFY(self);
1998     self.keyStateMachineOperation = [NSBlockOperation blockOperationWithBlock: ^{
1999         STRONGIFY(self);
2000         if(!self) {
2001             ckkserror("ckks", self, "received callback for released object");
2002             return;
2003         }
2004         [self.launch addEvent:@"fetch-complete"];
2005
2006         [self dispatchSyncWithAccountKeys: ^bool{
2007             [self _onqueueAdvanceKeyStateMachineToState: SecCKKSZoneKeyStateFetchComplete withError: nil];
2008             return true;
2009         }];
2010     }];
2011     self.keyStateMachineOperation.name = @"waiting-for-fetch";
2012
2013     NSOperation* fetchOp = [self.zoneChangeFetcher requestSuccessfulFetch: CKKSFetchBecauseKeyHierarchy];
2014     [self.keyStateMachineOperation addDependency: fetchOp];
2015
2016     self.keyStateFetchRequested = false;
2017 }
2018
2019 - (void)_onqueueKeyHierarchyRefetch {
2020     dispatch_assert_queue(self.queue);
2021
2022     WEAKIFY(self);
2023     self.keyStateMachineOperation = [NSBlockOperation blockOperationWithBlock: ^{
2024         STRONGIFY(self);
2025         if(!self) {
2026             ckkserror("ckks", self, "received callback for released object");
2027             return;
2028         }
2029
2030         [self dispatchSyncWithAccountKeys: ^bool{
2031             [self _onqueueAdvanceKeyStateMachineToState: SecCKKSZoneKeyStateFetchComplete withError: nil];
2032             return true;
2033         }];
2034     }];
2035     self.keyStateMachineOperation.name = @"waiting-for-refetch";
2036
2037     NSOperation* fetchOp = [self.zoneChangeFetcher requestSuccessfulFetchForManyReasons:[NSSet setWithObjects:CKKSFetchBecauseKeyHierarchy, CKKSFetchBecauseResync, nil]];
2038     [self.keyStateMachineOperation addDependency: fetchOp];
2039
2040     self.keyStateMachineRefetched = true;
2041     self.keyStateFullRefetchRequested = false;
2042     self.keyStateFetchRequested = false;
2043 }
2044
2045 - (void) handleKeychainEventDbConnection: (SecDbConnectionRef) dbconn
2046                                    added: (SecDbItemRef) added
2047                                  deleted: (SecDbItemRef) deleted
2048                              rateLimiter: (CKKSRateLimiter*) rateLimiter
2049                             syncCallback: (SecBoolNSErrorCallback) syncCallback {
2050     if(!SecCKKSIsEnabled()) {
2051         ckksnotice("ckks", self, "Skipping handleKeychainEventDbConnection due to disabled CKKS");
2052         return;
2053     }
2054
2055     __block NSError* error = nil;
2056
2057     // Tombstones come in as item modifications or item adds. Handle modifications here.
2058     bool addedTombstone   = added   && SecDbItemIsTombstone(added);
2059     bool deletedTombstone = deleted && SecDbItemIsTombstone(deleted);
2060
2061     bool addedSync   = added   && SecDbItemIsSyncable(added);
2062     bool deletedSync = deleted && SecDbItemIsSyncable(deleted);
2063
2064     bool isAdd    = ( added && !deleted) || (added && deleted && !addedTombstone &&  deletedTombstone) || (added && deleted &&  addedSync && !deletedSync);
2065     bool isDelete = (!added &&  deleted) || (added && deleted &&  addedTombstone && !deletedTombstone) || (added && deleted && !addedSync &&  deletedSync);
2066     bool isModify = ( added &&  deleted) && (!isAdd) && (!isDelete);
2067
2068     // On an update that changes an item's primary key, SecDb modifies the existing item, then adds a new tombstone to replace the old primary key.
2069     // Therefore, we might receive an added tombstone here with no deleted item to accompany it. This should be considered a deletion.
2070     if(addedTombstone && !deleted) {
2071         isAdd = false;
2072         isDelete = true;
2073         isModify = false;
2074
2075         // Passed to withItem: below
2076         deleted = added;
2077     }
2078
2079     // If neither item is syncable, don't proceed further in the syncing system
2080     bool proceed = addedSync || deletedSync;
2081
2082     if(!proceed) {
2083         ckksnotice("ckks", self, "skipping sync of non-sync item (%d, %d)", addedSync, deletedSync);
2084         return;
2085     }
2086
2087     // It's possible to ask for an item to be deleted without adding a corresponding tombstone.
2088     // This is arguably a bug, as it generates an out-of-sync state, but it is in the API contract.
2089     // CKKS should ignore these, but log very upset messages.
2090     if(isDelete && !addedTombstone) {
2091         ckksnotice("ckks", self, "Client has asked for an item deletion to not sync. Keychain is now out of sync with account");
2092         return;
2093     }
2094
2095     // Only synchronize items which can transfer between devices
2096     NSString* protection = (__bridge NSString*)SecDbItemGetCachedValueWithName(added ? added : deleted, kSecAttrAccessible);
2097     if(! ([protection isEqualToString: (__bridge NSString*)kSecAttrAccessibleWhenUnlocked] ||
2098           [protection isEqualToString: (__bridge NSString*)kSecAttrAccessibleAfterFirstUnlock] ||
2099           [protection isEqualToString: (__bridge NSString*)kSecAttrAccessibleAlwaysPrivate])) {
2100         ckksnotice("ckks", self, "skipping sync of device-bound(%@) item", protection);
2101         return;
2102     }
2103
2104     // Our caller gave us a database connection. We must get on the local queue to ensure atomicity
2105     // Note that we're at the mercy of the surrounding db transaction, so don't try to rollback here
2106     [self dispatchSyncWithConnection: dbconn block: ^bool {
2107         // Schedule a "view changed" notification
2108         [self.notifyViewChangedScheduler trigger];
2109
2110         if(self.accountStatus == CKKSAccountStatusNoAccount) {
2111             // No account; CKKS shouldn't attempt anything.
2112             self.droppedItems = true;
2113             ckksnotice("ckks", self, "Dropping sync item modification due to CK account state; will scan to find changes later");
2114
2115             if(syncCallback) {
2116                 // We're positively not logged into CloudKit, and therefore don't expect this item to be synced anytime particularly soon.
2117                 [self callSyncCallbackWithErrorNoAccount: syncCallback];
2118             }
2119             return true;
2120         }
2121
2122         // Always record the callback, even if we can't encrypt the item right now. Maybe we'll get to it soon!
2123         if(syncCallback) {
2124             CFErrorRef cferror = NULL;
2125             NSString* uuid = (__bridge_transfer NSString*) CFRetain(SecDbItemGetValue(added, &v10itemuuid, &cferror));
2126             if(!cferror && uuid) {
2127                 self.pendingSyncCallbacks[uuid] = syncCallback;
2128             }
2129             CFReleaseNull(cferror);
2130         }
2131
2132         CKKSOutgoingQueueEntry* oqe = nil;
2133         if       (isAdd) {
2134             oqe = [CKKSOutgoingQueueEntry withItem: added   action: SecCKKSActionAdd    ckks:self error: &error];
2135         } else if(isDelete) {
2136             oqe = [CKKSOutgoingQueueEntry withItem: deleted action: SecCKKSActionDelete ckks:self error: &error];
2137         } else if(isModify) {
2138             oqe = [CKKSOutgoingQueueEntry withItem: added   action: SecCKKSActionModify ckks:self error: &error];
2139         } else {
2140             ckkserror("ckks", self, "processKeychainEventItemAdded given garbage: %@ %@", added, deleted);
2141             return true;
2142         }
2143
2144         CKOperationGroup* operationGroup = [CKOperationGroup CKKSGroupWithName:@"keychain-api-use"];
2145
2146         if(error) {
2147             ckkserror("ckks", self, "Couldn't create outgoing queue entry: %@", error);
2148             self.droppedItems = true;
2149
2150             // If the problem is 'no UUID', launch a scan operation to find and fix it
2151             // We don't want to fix it up here, in the closing moments of a transaction
2152             if([error.domain isEqualToString:CKKSErrorDomain] && error.code == CKKSNoUUIDOnItem) {
2153                 ckksnotice("ckks", self, "Launching scan operation to find UUID");
2154                 [self scanLocalItems:@"uuid-find-scan" ckoperationGroup:operationGroup after:nil];
2155             }
2156
2157             // If the problem is 'couldn't load key', tell the key hierarchy state machine to fix it
2158             if([error.domain isEqualToString:CKKSErrorDomain] && error.code == errSecItemNotFound) {
2159                 [self.pokeKeyStateMachineScheduler trigger];
2160             }
2161
2162             return true;
2163         }
2164
2165         if(rateLimiter) {
2166             NSDate* limit = nil;
2167             NSInteger value = [rateLimiter judge:oqe at:[NSDate date] limitTime:&limit];
2168             if(limit) {
2169                 oqe.waitUntil = limit;
2170                 SecPLLogRegisteredEvent(@"CKKSSyncing", @{ @"ratelimit" : @(value), @"accessgroup" : oqe.accessgroup});
2171             }
2172         }
2173
2174         [oqe saveToDatabaseWithConnection: dbconn error: &error];
2175         if(error) {
2176             ckkserror("ckks", self, "Couldn't save outgoing queue entry to database: %@", error);
2177             return true;
2178         } else {
2179             ckksnotice("ckks", self, "Saved %@ to outgoing queue", oqe);
2180         }
2181
2182         // This update supercedes all other local modifications to this item (_except_ those in-flight).
2183         // Delete all items in reencrypt or error.
2184         CKKSOutgoingQueueEntry* reencryptOQE = [CKKSOutgoingQueueEntry tryFromDatabase:oqe.uuid state:SecCKKSStateReencrypt zoneID:self.zoneID error:&error];
2185         if(error) {
2186             ckkserror("ckks", self, "Couldn't load reencrypt OQE sibling for %@: %@", oqe, error);
2187         }
2188         if(reencryptOQE) {
2189             [reencryptOQE deleteFromDatabase:&error];
2190             if(error) {
2191                 ckkserror("ckks", self, "Couldn't delete reencrypt OQE sibling(%@) for %@: %@", reencryptOQE, oqe, error);
2192             }
2193             error = nil;
2194         }
2195
2196         CKKSOutgoingQueueEntry* errorOQE = [CKKSOutgoingQueueEntry tryFromDatabase:oqe.uuid state:SecCKKSStateError zoneID:self.zoneID error:&error];
2197         if(error) {
2198             ckkserror("ckks", self, "Couldn't load error OQE sibling for %@: %@", oqe, error);
2199         }
2200         if(errorOQE) {
2201             [errorOQE deleteFromDatabase:&error];
2202             if(error) {
2203                 ckkserror("ckks", self, "Couldn't delete error OQE sibling(%@) for %@: %@", reencryptOQE, oqe, error);
2204             }
2205         }
2206
2207         [self processOutgoingQueue:operationGroup];
2208
2209         return true;
2210     }];
2211 }
2212
2213 -(void)setCurrentItemForAccessGroup:(NSData* _Nonnull)newItemPersistentRef
2214                                hash:(NSData*)newItemSHA1
2215                         accessGroup:(NSString*)accessGroup
2216                          identifier:(NSString*)identifier
2217                           replacing:(NSData* _Nullable)oldCurrentItemPersistentRef
2218                                hash:(NSData*)oldItemSHA1
2219                            complete:(void (^) (NSError* operror)) complete
2220 {
2221     if(accessGroup == nil || identifier == nil) {
2222         NSError* error = [NSError errorWithDomain:CKKSErrorDomain
2223                                              code:errSecParam
2224                                       description:@"No access group or identifier given"];
2225         ckkserror("ckkscurrent", self, "Cancelling request: %@", error);
2226         complete(error);
2227         return;
2228     }
2229
2230     // Not being in a CloudKit account is an automatic failure.
2231     // But, wait a good long while for the CloudKit account state to be known (in the case of daemon startup)
2232     [self.accountStateKnown wait:(SecCKKSTestsEnabled() ? 1*NSEC_PER_SEC : 30*NSEC_PER_SEC)];
2233
2234     if(self.accountStatus != CKKSAccountStatusAvailable) {
2235         NSError* error = [NSError errorWithDomain:CKKSErrorDomain
2236                                              code:CKKSNotLoggedIn
2237                                       description:@"User is not signed into iCloud."];
2238         ckksnotice("ckkscurrent", self, "Rejecting current item pointer set since we don't have an iCloud account.");
2239         complete(error);
2240         return;
2241     }
2242
2243     ckksnotice("ckkscurrent", self, "Starting change current pointer operation for %@-%@", accessGroup, identifier);
2244     CKKSUpdateCurrentItemPointerOperation* ucipo = [[CKKSUpdateCurrentItemPointerOperation alloc] initWithCKKSKeychainView:self
2245                                                                                                                    newItem:newItemPersistentRef
2246                                                                                                                       hash:newItemSHA1
2247                                                                                                                accessGroup:accessGroup
2248                                                                                                                 identifier:identifier
2249                                                                                                                  replacing:oldCurrentItemPersistentRef
2250                                                                                                                       hash:oldItemSHA1
2251                                                                                                           ckoperationGroup:[CKOperationGroup CKKSGroupWithName:@"currentitem-api"]];
2252
2253     WEAKIFY(self);
2254     CKKSResultOperation* returnCallback = [CKKSResultOperation operationWithBlock:^{
2255         STRONGIFY(self);
2256
2257         if(ucipo.error) {
2258             ckkserror("ckkscurrent", self, "Failed setting a current item pointer for %@ with %@", ucipo.currentPointerIdentifier, ucipo.error);
2259         } else {
2260             ckksnotice("ckkscurrent", self, "Finished setting a current item pointer for %@", ucipo.currentPointerIdentifier);
2261         }
2262         complete(ucipo.error);
2263     }];
2264     returnCallback.name = @"setCurrentItem-return-callback";
2265     [returnCallback addDependency: ucipo];
2266     [self scheduleOperation: returnCallback];
2267
2268     // Now, schedule ucipo. It modifies the CloudKit zone, so it should insert itself into the list of OutgoingQueueOperations.
2269     // Then, we won't have simultaneous zone-modifying operations.
2270     [ucipo linearDependencies:self.outgoingQueueOperations];
2271
2272     // If this operation hasn't started within 60 seconds, cancel it and return a "timed out" error.
2273     [ucipo timeout:60*NSEC_PER_SEC];
2274
2275     [self scheduleOperation:ucipo];
2276     return;
2277 }
2278
2279 -(void)getCurrentItemForAccessGroup:(NSString*)accessGroup
2280                          identifier:(NSString*)identifier
2281                     fetchCloudValue:(bool)fetchCloudValue
2282                            complete:(void (^) (NSString* uuid, NSError* operror)) complete
2283 {
2284     if(accessGroup == nil || identifier == nil) {
2285         ckksnotice("ckkscurrent", self, "Rejecting current item pointer get since no access group(%@) or identifier(%@) given", accessGroup, identifier);
2286         complete(NULL, [NSError errorWithDomain:CKKSErrorDomain
2287                                            code:errSecParam
2288                                     description:@"No access group or identifier given"]);
2289         return;
2290     }
2291
2292     // Not being in a CloudKit account is an automatic failure.
2293     // But, wait a good long while for the CloudKit account state to be known (in the case of daemon startup)
2294     [self.accountStateKnown wait:(SecCKKSTestsEnabled() ? 1*NSEC_PER_SEC : 30*NSEC_PER_SEC)];
2295
2296     if(self.accountStatus != CKKSAccountStatusAvailable) {
2297         ckksnotice("ckkscurrent", self, "Rejecting current item pointer get since we don't have an iCloud account.");
2298         complete(NULL, [NSError errorWithDomain:CKKSErrorDomain
2299                                            code:CKKSNotLoggedIn
2300                                     description:@"User is not signed into iCloud."]);
2301         return;
2302     }
2303
2304     CKKSResultOperation* fetchAndProcess = nil;
2305     if(fetchCloudValue) {
2306         fetchAndProcess = [self fetchAndProcessCKChanges:CKKSFetchBecauseCurrentItemFetchRequest];
2307     }
2308
2309     WEAKIFY(self);
2310     CKKSResultOperation* getCurrentItem = [CKKSResultOperation named:@"get-current-item-pointer" withBlock:^{
2311         if(fetchAndProcess.error) {
2312             ckksnotice("ckkscurrent", self, "Rejecting current item pointer get since fetch failed: %@", fetchAndProcess.error);
2313             complete(NULL, fetchAndProcess.error);
2314             return;
2315         }
2316
2317         STRONGIFY(self);
2318
2319         [self dispatchSync: ^bool {
2320             NSError* error = nil;
2321             NSString* currentIdentifier = [NSString stringWithFormat:@"%@-%@", accessGroup, identifier];
2322
2323             CKKSCurrentItemPointer* cip = [CKKSCurrentItemPointer fromDatabase:currentIdentifier
2324                                                                          state:SecCKKSProcessedStateLocal
2325                                                                         zoneID:self.zoneID
2326                                                                          error:&error];
2327             if(!cip || error) {
2328                 ckkserror("ckkscurrent", self, "No current item pointer for %@", currentIdentifier);
2329                 complete(nil, error);
2330                 return false;
2331             }
2332
2333             if(!cip.currentItemUUID) {
2334                 ckkserror("ckkscurrent", self, "Current item pointer is empty %@", cip);
2335                 complete(nil, [NSError errorWithDomain:CKKSErrorDomain
2336                                                   code:errSecInternalError
2337                                            description:@"Current item pointer is empty"]);
2338                 return false;
2339             }
2340
2341             ckksinfo("ckkscurrent", self, "Retrieved current item pointer: %@", cip);
2342             complete(cip.currentItemUUID, NULL);
2343             return true;
2344         }];
2345     }];
2346
2347     [getCurrentItem addNullableDependency:fetchAndProcess];
2348     [self scheduleOperation: getCurrentItem];
2349 }
2350
2351 - (CKKSKey*) keyForItem: (SecDbItemRef) item error: (NSError * __autoreleasing *) error {
2352     CKKSKeyClass* class = nil;
2353
2354     NSString* protection = (__bridge NSString*)SecDbItemGetCachedValueWithName(item, kSecAttrAccessible);
2355     if([protection isEqualToString: (__bridge NSString*)kSecAttrAccessibleWhenUnlocked]) {
2356         class = SecCKKSKeyClassA;
2357     } else if([protection isEqualToString: (__bridge NSString*)kSecAttrAccessibleAlwaysPrivate] ||
2358               [protection isEqualToString: (__bridge NSString*)kSecAttrAccessibleAfterFirstUnlock]) {
2359         class = SecCKKSKeyClassC;
2360     } else {
2361         NSError* localError = [NSError errorWithDomain:CKKSErrorDomain
2362                                                   code:CKKSInvalidKeyClass
2363                                            description:[NSString stringWithFormat:@"can't pick key class for protection %@", protection]];
2364         ckkserror("ckks", self, "can't pick key class: %@ %@", localError, item);
2365         if(error) {
2366             *error = localError;
2367         }
2368
2369         return nil;
2370     }
2371
2372     NSError* currentKeyError = nil;
2373     CKKSKey* key = [CKKSKey currentKeyForClass: class zoneID:self.zoneID error:&currentKeyError];
2374     if(!key || currentKeyError) {
2375         ckkserror("ckks", self, "Couldn't find current key for %@: %@", class, currentKeyError);
2376
2377         if(error) {
2378             *error = currentKeyError;
2379         }
2380         return nil;
2381     }
2382
2383     // and make sure it's unwrapped.
2384     NSError* loadedError = nil;
2385     if(![key ensureKeyLoaded:&loadedError]) {
2386         ckkserror("ckks", self, "Couldn't load key(%@): %@", key, loadedError);
2387         if(error) {
2388             *error = loadedError;
2389         }
2390         return nil;
2391     }
2392
2393     return key;
2394 }
2395
2396 - (CKKSResultOperation<CKKSKeySetProviderOperationProtocol>*)findKeySet
2397 {
2398     __block CKKSResultOperation<CKKSKeySetProviderOperationProtocol>* keysetOp = nil;
2399
2400     [self dispatchSyncWithAccountKeys:^bool {
2401         CKKSCurrentKeySet* keyset = [CKKSCurrentKeySet loadForZone:self.zoneID];
2402         if(keyset.currentTLKPointer.currentKeyUUID && keyset.tlk.uuid) {
2403             ckksnotice("ckks", self, "Already have keyset %@", keyset);
2404
2405             keysetOp = [[CKKSProvideKeySetOperation alloc] initWithZoneName:self.zoneName keySet:keyset];
2406             [self scheduleOperationWithoutDependencies:keysetOp];
2407             return true;
2408         } else if([self.keyHierarchyState isEqualToString:SecCKKSZoneKeyStateWaitForTLKUpload]) {
2409             CKKSCurrentKeySet* proposedKeySet = self.lastNewTLKOperation.keyset;
2410             ckksnotice("ckks", self, "Already have proposed keyset %@", proposedKeySet);
2411
2412             keysetOp = [[CKKSProvideKeySetOperation alloc] initWithZoneName:self.zoneName keySet:proposedKeySet];
2413             [self scheduleOperationWithoutDependencies:keysetOp];
2414             return true;
2415         } else {
2416             // No existing keyset (including keys) exists.
2417             // The state machine will know what to do!
2418             self.tlkCreationRequested = true;
2419
2420             ckksnotice("ckks", self, "Received a keyset request; forwarding to state machine");
2421
2422             keysetOp = (CKKSProvideKeySetOperation*) [self findFirstPendingOperation:self.keysetProviderOperations];
2423             if(!keysetOp) {
2424                 keysetOp = [[CKKSProvideKeySetOperation alloc] initWithZoneName:self.zoneName];
2425                 [self.keysetProviderOperations addObject:keysetOp];
2426
2427                 // This is an abuse of operations: they should generally run when added to a queue, not wait, but this allows recipients to set timeouts
2428                 [self scheduleOperationWithoutDependencies:keysetOp];
2429             }
2430
2431             [self _onqueueAdvanceKeyStateMachineToState:nil withError:nil];
2432         }
2433
2434         return true;
2435     }];
2436
2437     return keysetOp;
2438 }
2439
2440 - (void)_onqueueRunKeysetProviderOperations:(CKKSCurrentKeySet*)keyset
2441 {
2442     ckksnotice("ckkskey", self, "Providing keyset (%@) to listeners", keyset);
2443
2444     // We have some keyset; they can ask again if they want a new one
2445     self.tlkCreationRequested = false;
2446
2447     for(CKKSResultOperation<CKKSKeySetProviderOperationProtocol>* op in self.keysetProviderOperations) {
2448         if([op isPending]) {
2449             [op provideKeySet:keyset];
2450         }
2451     }
2452 }
2453
2454 - (void)receiveTLKUploadRecords:(NSArray<CKRecord*>*)records
2455 {
2456     // First, filter for records matching this zone
2457     NSMutableArray<CKRecord*>* zoneRecords = [NSMutableArray array];
2458     for(CKRecord* record in records) {
2459         if([record.recordID.zoneID isEqual:self.zoneID]) {
2460             [zoneRecords addObject:record];
2461         }
2462     }
2463
2464     ckksnotice("ckkskey", self, "Received a set of %lu TLK upload records", (unsigned long)zoneRecords.count);
2465
2466     if(!zoneRecords || zoneRecords.count == 0) {
2467         return;
2468     }
2469
2470     [self dispatchSyncWithAccountKeys:^bool {
2471
2472         for(CKRecord* record in zoneRecords) {
2473             [self _onqueueCKRecordChanged:record resync:false];
2474         }
2475
2476         return true;
2477     }];
2478 }
2479
2480 // Use the following method to find the first pending operation in a weak collection
2481 - (NSOperation*)findFirstPendingOperation: (NSHashTable*) table {
2482     return [self findFirstPendingOperation:table ofClass:nil];
2483 }
2484
2485 // Use the following method to find the first pending operation in a weak collection
2486 - (NSOperation*)findFirstPendingOperation: (NSHashTable*) table ofClass:(Class)class {
2487     @synchronized(table) {
2488         for(NSOperation* op in table) {
2489             if(op != nil && [op isPending] && (class == nil || [op isKindOfClass: class])) {
2490                 return op;
2491             }
2492         }
2493         return nil;
2494     }
2495 }
2496
2497 // Use the following method to count the pending operations in a weak collection
2498 - (int64_t)countPendingOperations: (NSHashTable*) table {
2499     @synchronized(table) {
2500         int count = 0;
2501         for(NSOperation* op in table) {
2502             if(op != nil && !([op isExecuting] || [op isFinished])) {
2503                 count++;
2504             }
2505         }
2506         return count;
2507     }
2508 }
2509
2510 - (NSSet<NSString*>*)_onqueuePriorityOutgoingQueueUUIDs
2511 {
2512     return [self.pendingSyncCallbacks.allKeys copy];
2513 }
2514
2515 - (CKKSOutgoingQueueOperation*)processOutgoingQueue:(CKOperationGroup*)ckoperationGroup {
2516     return [self processOutgoingQueueAfter:nil ckoperationGroup:ckoperationGroup];
2517 }
2518
2519 - (CKKSOutgoingQueueOperation*)processOutgoingQueueAfter:(CKKSResultOperation*)after ckoperationGroup:(CKOperationGroup*)ckoperationGroup {
2520     return [self processOutgoingQueueAfter:after requiredDelay:DISPATCH_TIME_FOREVER ckoperationGroup:ckoperationGroup];
2521 }
2522
2523 - (CKKSOutgoingQueueOperation*)processOutgoingQueueAfter:(CKKSResultOperation*)after
2524                                            requiredDelay:(uint64_t)requiredDelay
2525                                         ckoperationGroup:(CKOperationGroup*)ckoperationGroup
2526 {
2527     CKKSOutgoingQueueOperation* outgoingop =
2528             (CKKSOutgoingQueueOperation*) [self findFirstPendingOperation:self.outgoingQueueOperations
2529                                                                   ofClass:[CKKSOutgoingQueueOperation class]];
2530     if(outgoingop) {
2531         if(after) {
2532             [outgoingop addDependency: after];
2533         }
2534         if([outgoingop isPending]) {
2535             if(!outgoingop.ckoperationGroup && ckoperationGroup) {
2536                 outgoingop.ckoperationGroup = ckoperationGroup;
2537             } else if(ckoperationGroup) {
2538                 ckkserror("ckks", self, "Throwing away CKOperationGroup(%@) in favor of (%@)", ckoperationGroup.name, outgoingop.ckoperationGroup.name);
2539             }
2540
2541             // Will log any pending dependencies as well
2542             ckksnotice("ckksoutgoing", self, "Returning existing %@", outgoingop);
2543
2544             // Shouldn't be necessary, but can't hurt
2545             [self.outgoingQueueOperationScheduler triggerAt:requiredDelay];
2546             return outgoingop;
2547         }
2548     }
2549
2550     CKKSOutgoingQueueOperation* op = [[CKKSOutgoingQueueOperation alloc] initWithCKKSKeychainView:self ckoperationGroup:ckoperationGroup];
2551     op.name = @"outgoing-queue-operation";
2552     [op addNullableDependency:after];
2553     [op addNullableDependency:self.outgoingQueueOperationScheduler.operationDependency];
2554
2555     [self.outgoingQueueOperationScheduler triggerAt:requiredDelay];
2556
2557     [self scheduleOperation: op];
2558     ckksnotice("ckksoutgoing", self, "Scheduled %@", op);
2559     return op;
2560 }
2561
2562 - (void)processIncomingQueueAfterNextUnlock {
2563     // Thread races aren't so important here; we might end up with two or three copies of this operation, but that's okay.
2564     if(![self.processIncomingQueueAfterNextUnlockOperation isPending]) {
2565         WEAKIFY(self);
2566
2567         CKKSResultOperation* restartIncomingQueueOperation = [CKKSResultOperation operationWithBlock:^{
2568             STRONGIFY(self);
2569             // This IQO shouldn't error if the keybag has locked again. It will simply try again later.
2570             [self processIncomingQueue:false];
2571         }];
2572
2573         restartIncomingQueueOperation.name = @"reprocess-incoming-queue-after-unlock";
2574         self.processIncomingQueueAfterNextUnlockOperation = restartIncomingQueueOperation;
2575
2576         [restartIncomingQueueOperation addNullableDependency:self.lockStateTracker.unlockDependency];
2577         [self scheduleOperation: restartIncomingQueueOperation];
2578     }
2579 }
2580
2581 - (CKKSResultOperation*)resultsOfNextProcessIncomingQueueOperation {
2582     if(self.resultsOfNextIncomingQueueOperationOperation && [self.resultsOfNextIncomingQueueOperationOperation isPending]) {
2583         return self.resultsOfNextIncomingQueueOperationOperation;
2584     }
2585
2586     // Else, make a new one.
2587     self.resultsOfNextIncomingQueueOperationOperation = [CKKSResultOperation named:[NSString stringWithFormat:@"wait-for-next-incoming-queue-operation-%@", self.zoneName] withBlock:^{}];
2588     return self.resultsOfNextIncomingQueueOperationOperation;
2589 }
2590
2591 - (CKKSIncomingQueueOperation*)processIncomingQueue:(bool)failOnClassA {
2592     return [self processIncomingQueue:failOnClassA after: nil];
2593 }
2594
2595 - (CKKSIncomingQueueOperation*) processIncomingQueue:(bool)failOnClassA after: (CKKSResultOperation*) after {
2596     CKKSIncomingQueueOperation* incomingop = (CKKSIncomingQueueOperation*) [self findFirstPendingOperation:self.incomingQueueOperations];
2597     if(incomingop) {
2598         ckksinfo("ckks", self, "Skipping processIncomingQueue due to at least one pending instance");
2599         if(after) {
2600             [incomingop addNullableDependency: after];
2601         }
2602
2603         // check (again) for race condition; if the op has started we need to add another (for the dependency)
2604         if([incomingop isPending]) {
2605             incomingop.errorOnClassAFailure |= failOnClassA;
2606             return incomingop;
2607         }
2608     }
2609
2610     CKKSIncomingQueueOperation* op = [[CKKSIncomingQueueOperation alloc] initWithCKKSKeychainView:self errorOnClassAFailure:failOnClassA];
2611     op.name = @"incoming-queue-operation";
2612     if(after != nil) {
2613         [op addSuccessDependency: after];
2614     }
2615
2616     if(self.resultsOfNextIncomingQueueOperationOperation) {
2617         [self.resultsOfNextIncomingQueueOperationOperation addSuccessDependency:op];
2618         [self scheduleOperation:self.resultsOfNextIncomingQueueOperationOperation];
2619     }
2620
2621     [self scheduleOperation: op];
2622     return op;
2623 }
2624
2625 - (CKKSScanLocalItemsOperation*)scanLocalItems:(NSString*)operationName {
2626     return [self scanLocalItems:operationName ckoperationGroup:nil after:nil];
2627 }
2628
2629 - (CKKSScanLocalItemsOperation*)scanLocalItems:(NSString*)operationName ckoperationGroup:(CKOperationGroup*)operationGroup after:(NSOperation*)after {
2630     CKKSScanLocalItemsOperation* scanOperation = [[CKKSScanLocalItemsOperation alloc] initWithCKKSKeychainView:self ckoperationGroup:operationGroup];
2631     scanOperation.name = operationName;
2632
2633     [scanOperation addNullableDependency:self.lastFixupOperation];
2634     [scanOperation addNullableDependency:self.lockStateTracker.unlockDependency];
2635     [scanOperation addNullableDependency:self.keyStateReadyDependency];
2636     [scanOperation addNullableDependency:after];
2637
2638     [self scheduleOperation: scanOperation];
2639     return scanOperation;
2640 }
2641
2642 - (CKKSUpdateDeviceStateOperation*)updateDeviceState:(bool)rateLimit
2643                    waitForKeyHierarchyInitialization:(uint64_t)timeout
2644                                     ckoperationGroup:(CKOperationGroup*)ckoperationGroup {
2645
2646     WEAKIFY(self);
2647
2648     // If securityd just started, the key state might be in some transient early state. Wait a bit.
2649     CKKSResultOperation* waitForKeyReady = [CKKSResultOperation named:@"device-state-wait" withBlock:^{
2650         STRONGIFY(self);
2651         ckksnotice("ckksdevice", self, "Finished waiting for key hierarchy transient state, currently %@", self.keyHierarchyState);
2652     }];
2653
2654     [waitForKeyReady addNullableDependency:self.keyStateNonTransientDependency];
2655     [waitForKeyReady timeout:timeout];
2656     [self.waitingQueue addOperation:waitForKeyReady];
2657
2658     CKKSUpdateDeviceStateOperation* op = [[CKKSUpdateDeviceStateOperation alloc] initWithCKKSKeychainView:self rateLimit:rateLimit ckoperationGroup:ckoperationGroup];
2659     op.name = @"device-state-operation";
2660
2661     [op addDependency: waitForKeyReady];
2662
2663     // op modifies the CloudKit zone, so it should insert itself into the list of OutgoingQueueOperations.
2664     // Then, we won't have simultaneous zone-modifying operations and confuse ourselves.
2665     // However, since we might have pending OQOs, it should try to insert itself at the beginning of the linearized list
2666     [op linearDependenciesWithSelfFirst:self.outgoingQueueOperations];
2667
2668     // CKKSUpdateDeviceStateOperations are special: they should fire even if we don't believe we're in an iCloud account.
2669     // They also shouldn't block or be blocked by any other operation; our wait operation above will handle that
2670     [self scheduleOperationWithoutDependencies:op];
2671     return op;
2672 }
2673
2674 // There are some errors which won't be reported but will be reflected in the CDSE; any error coming out of here is fatal
2675 - (CKKSDeviceStateEntry*)_onqueueCurrentDeviceStateEntry: (NSError* __autoreleasing*)error {
2676     NSError* localerror = nil;
2677
2678     CKKSAccountStateTracker* accountTracker = self.accountTracker;
2679     CKKSAccountStatus hsa2Status = accountTracker.hsa2iCloudAccountStatus;
2680
2681     // We must have an HSA2 iCloud account and a CloudKit account to even create one of these
2682     if(hsa2Status != CKKSAccountStatusAvailable ||
2683        accountTracker.currentCKAccountInfo.accountStatus != CKAccountStatusAvailable) {
2684         ckkserror("ckksdevice", self, "No iCloud account active: %@ hsa2 account:%@",
2685                   accountTracker.currentCKAccountInfo,
2686                   CKKSAccountStatusToString(hsa2Status));
2687         localerror = [NSError errorWithDomain:@"securityd"
2688                                          code:errSecInternalError
2689                                      userInfo:@{NSLocalizedDescriptionKey: [NSString stringWithFormat: @"No active HSA2 iCloud account: %@", accountTracker.currentCKAccountInfo]}];
2690         if(error) {
2691             *error = localerror;
2692         }
2693         return nil;
2694     }
2695
2696     NSString* ckdeviceID = accountTracker.ckdeviceID;
2697     if(ckdeviceID == nil) {
2698         ckkserror("ckksdevice", self, "No CK device ID available; cannot make device state entry");
2699         localerror = [NSError errorWithDomain:CKKSErrorDomain
2700                                          code:CKKSNotLoggedIn
2701                                      userInfo:@{NSLocalizedDescriptionKey: [NSString stringWithFormat: @"No CK device ID: %@", accountTracker.currentCKAccountInfo]}];
2702         if(error) {
2703             *error = localerror;
2704         }
2705         return nil;
2706     }
2707
2708     CKKSDeviceStateEntry* oldcdse = [CKKSDeviceStateEntry tryFromDatabase:ckdeviceID zoneID:self.zoneID error:&localerror];
2709     if(localerror) {
2710         ckkserror("ckksdevice", self, "Couldn't read old CKKSDeviceStateEntry from database: %@", localerror);
2711         if(error) {
2712             *error = localerror;
2713         }
2714         return nil;
2715     }
2716
2717     // Find out what we think the current keys are
2718     CKKSCurrentKeyPointer* currentTLKPointer    = [CKKSCurrentKeyPointer tryFromDatabase: SecCKKSKeyClassTLK zoneID:self.zoneID error:&localerror];
2719     CKKSCurrentKeyPointer* currentClassAPointer = [CKKSCurrentKeyPointer tryFromDatabase: SecCKKSKeyClassA   zoneID:self.zoneID error:&localerror];
2720     CKKSCurrentKeyPointer* currentClassCPointer = [CKKSCurrentKeyPointer tryFromDatabase: SecCKKSKeyClassC   zoneID:self.zoneID error:&localerror];
2721     if(localerror) {
2722         // Things is broken, but the whole point of this record is to share the brokenness. Continue.
2723         ckkserror("ckksdevice", self, "Couldn't read current key pointers from database: %@; proceeding", localerror);
2724         localerror = nil;
2725     }
2726
2727     CKKSKey* suggestedTLK       = currentTLKPointer.currentKeyUUID    ? [CKKSKey tryFromDatabase:currentTLKPointer.currentKeyUUID    zoneID:self.zoneID error:&localerror] : nil;
2728     CKKSKey* suggestedClassAKey = currentClassAPointer.currentKeyUUID ? [CKKSKey tryFromDatabase:currentClassAPointer.currentKeyUUID zoneID:self.zoneID error:&localerror] : nil;
2729     CKKSKey* suggestedClassCKey = currentClassCPointer.currentKeyUUID ? [CKKSKey tryFromDatabase:currentClassCPointer.currentKeyUUID zoneID:self.zoneID error:&localerror] : nil;
2730
2731     if(localerror) {
2732         // Things is broken, but the whole point of this record is to share the brokenness. Continue.
2733         ckkserror("ckksdevice", self, "Couldn't read keys from database: %@; proceeding", localerror);
2734         localerror = nil;
2735     }
2736
2737     // Check if we posess the keys in the keychain
2738     [suggestedTLK ensureKeyLoaded:&localerror];
2739     if(localerror && [self.lockStateTracker isLockedError:localerror]) {
2740         ckkserror("ckksdevice", self, "Device is locked; couldn't read TLK from keychain. Assuming it is present and continuing; error was %@", localerror);
2741         localerror = nil;
2742     } else if(localerror) {
2743         ckkserror("ckksdevice", self, "Couldn't read TLK from keychain. We do not have a current TLK. Error was %@", localerror);
2744         suggestedTLK = nil;
2745     }
2746
2747     [suggestedClassAKey ensureKeyLoaded:&localerror];
2748     if(localerror && [self.lockStateTracker isLockedError:localerror]) {
2749         ckkserror("ckksdevice", self, "Device is locked; couldn't read ClassA key from keychain. Assuming it is present and continuing; error was %@", localerror);
2750         localerror = nil;
2751     } else if(localerror) {
2752         ckkserror("ckksdevice", self, "Couldn't read ClassA key from keychain. We do not have a current ClassA key. Error was %@", localerror);
2753         suggestedClassAKey = nil;
2754     }
2755
2756     [suggestedClassCKey ensureKeyLoaded:&localerror];
2757     // class C keys are stored class C, so uh, don't check lock state.
2758     if(localerror) {
2759         ckkserror("ckksdevice", self, "Couldn't read ClassC key from keychain. We do not have a current ClassC key. Error was %@", localerror);
2760         suggestedClassCKey = nil;
2761     }
2762
2763     // We'd like to have the circle peer ID. Give the account state tracker a fighting chance, but not having it is not an error
2764     // But, if the platform doesn't have SOS, don't bother
2765     if(OctagonPlatformSupportsSOS() && [accountTracker.accountCirclePeerIDInitialized wait:500*NSEC_PER_MSEC] != 0 && !accountTracker.accountCirclePeerID) {
2766         ckkserror("ckksdevice", self, "No SOS peer ID available");
2767     }
2768
2769     // We'd also like the Octagon status
2770     if([accountTracker.octagonInformationInitialized wait:500*NSEC_PER_MSEC] != 0 && !accountTracker.octagonPeerID) {
2771         ckkserror("ckksdevice", self, "No octagon peer ID available");
2772     }
2773
2774     // Reset the last unlock time to 'day' granularity in UTC
2775     NSCalendar* calendar = [NSCalendar calendarWithIdentifier:NSCalendarIdentifierISO8601];
2776     calendar.timeZone = [NSTimeZone timeZoneWithAbbreviation:@"UTC"];
2777     NSDate* lastUnlockDay = self.lockStateTracker.lastUnlockTime;
2778     lastUnlockDay = lastUnlockDay ? [calendar startOfDayForDate:lastUnlockDay] : nil;
2779
2780     // We only really want the oldcdse for its encodedCKRecord, so make a new cdse here
2781     CKKSDeviceStateEntry* newcdse = [[CKKSDeviceStateEntry alloc] initForDevice:ckdeviceID
2782                                                                       osVersion:SecCKKSHostOSVersion()
2783                                                                  lastUnlockTime:lastUnlockDay
2784                                                                   octagonPeerID:accountTracker.octagonPeerID
2785                                                                   octagonStatus:accountTracker.octagonStatus
2786                                                                    circlePeerID:accountTracker.accountCirclePeerID
2787                                                                    circleStatus:accountTracker.currentCircleStatus.status
2788                                                                        keyState:self.keyHierarchyState
2789                                                                  currentTLKUUID:suggestedTLK.uuid
2790                                                               currentClassAUUID:suggestedClassAKey.uuid
2791                                                               currentClassCUUID:suggestedClassCKey.uuid
2792                                                                          zoneID:self.zoneID
2793                                                                 encodedCKRecord:oldcdse.encodedCKRecord];
2794     return newcdse;
2795 }
2796
2797 - (CKKSSynchronizeOperation*) resyncWithCloud {
2798     CKKSSynchronizeOperation* op = [[CKKSSynchronizeOperation alloc] initWithCKKSKeychainView: self];
2799     [self scheduleOperation: op];
2800     return op;
2801 }
2802
2803 - (CKKSLocalSynchronizeOperation*)resyncLocal {
2804     CKKSLocalSynchronizeOperation* op = [[CKKSLocalSynchronizeOperation alloc] initWithCKKSKeychainView:self];
2805     [self scheduleOperation: op];
2806     return op;
2807 }
2808
2809 - (CKKSResultOperation*)fetchAndProcessCKChanges:(CKKSFetchBecause*)because {
2810     return [self fetchAndProcessCKChanges:because after:nil];
2811 }
2812
2813 - (CKKSResultOperation*)fetchAndProcessCKChanges:(CKKSFetchBecause*)because after:(CKKSResultOperation*)after {
2814     if(!SecCKKSIsEnabled()) {
2815         ckksinfo("ckks", self, "Skipping fetchAndProcessCKChanges due to disabled CKKS");
2816         return nil;
2817     }
2818
2819     if(after) {
2820         [self.zoneChangeFetcher holdFetchesUntil:after];
2821     }
2822
2823     // We fetched some changes; try to process them!
2824     return [self processIncomingQueue:false after:[self.zoneChangeFetcher requestSuccessfulFetch:because]];
2825 }
2826
2827 - (CKKSResultOperation*)fetchAndProcessCKChangesDueToAPNS:(CKRecordZoneNotification*)notification {
2828     if(!SecCKKSIsEnabled()) {
2829         ckksinfo("ckks", self, "Skipping fetchAndProcessCKChanges due to disabled CKKS");
2830         return nil;
2831     }
2832
2833     // We fetched some changes; try to process them!
2834     return [self processIncomingQueue:false after:[self.zoneChangeFetcher requestSuccessfulFetchDueToAPNS:notification]];
2835 }
2836
2837 // Lets the view know about a failed CloudKit write. If the error is "already have one of these records", it will
2838 // store the new records and kick off the new processing
2839 //
2840 // Note that you need to tell this function the records you wanted to save, so it can determine what needs deletion
2841 - (bool)_onqueueCKWriteFailed:(NSError*)ckerror attemptedRecordsChanged:(NSDictionary<CKRecordID*, CKRecord*>*)savedRecords {
2842     dispatch_assert_queue(self.queue);
2843
2844     NSDictionary<CKRecordID*,NSError*>* partialErrors = ckerror.userInfo[CKPartialErrorsByItemIDKey];
2845     if([ckerror.domain isEqual:CKErrorDomain] && ckerror.code == CKErrorPartialFailure && partialErrors) {
2846         // Check if this error was "you're out of date"
2847         bool recordChanged = true;
2848
2849         for(NSError* error in partialErrors.allValues) {
2850             if((![error.domain isEqual:CKErrorDomain]) || (error.code != CKErrorBatchRequestFailed && error.code != CKErrorServerRecordChanged && error.code != CKErrorUnknownItem)) {
2851                 // There's an error in there that isn't CKErrorServerRecordChanged, CKErrorBatchRequestFailed, or CKErrorUnknownItem. Don't handle nicely...
2852                 recordChanged = false;
2853             }
2854         }
2855
2856         if(recordChanged) {
2857             ckksnotice("ckks", self, "Received a ServerRecordChanged error, attempting to update new records and delete unknown ones");
2858
2859             bool updatedRecord = false;
2860
2861             for(CKRecordID* recordID in partialErrors.allKeys) {
2862                 NSError* error = partialErrors[recordID];
2863                 if([error.domain isEqual:CKErrorDomain] && error.code == CKErrorServerRecordChanged) {
2864                     CKRecord* newRecord = error.userInfo[CKRecordChangedErrorServerRecordKey];
2865                     ckksnotice("ckks", self, "On error: updating our idea of: %@", newRecord);
2866
2867                     updatedRecord |= [self _onqueueCKRecordChanged:newRecord resync:true];
2868                 } else if([error.domain isEqual:CKErrorDomain] && error.code == CKErrorUnknownItem) {
2869                     CKRecord* record = savedRecords[recordID];
2870                     ckksnotice("ckks", self, "On error: handling an unexpected delete of: %@ %@", recordID, record);
2871
2872                     updatedRecord |= [self _onqueueCKRecordDeleted:recordID recordType:record.recordType resync:true];
2873                 }
2874             }
2875
2876             if(updatedRecord) {
2877                 [self processIncomingQueue:false];
2878                 return true;
2879             }
2880         }
2881
2882         // Check if this error was the CKKS server extension rejecting the write
2883         for(CKRecordID* recordID in partialErrors.allKeys) {
2884             NSError* error = partialErrors[recordID];
2885
2886             NSError* underlyingError = error.userInfo[NSUnderlyingErrorKey];
2887             NSError* thirdLevelError = underlyingError.userInfo[NSUnderlyingErrorKey];
2888             ckksnotice("ckks", self, "Examining 'write failed' error: %@ %@ %@", error, underlyingError, thirdLevelError);
2889
2890             if([error.domain isEqualToString:CKErrorDomain] && error.code == CKErrorServerRejectedRequest &&
2891                underlyingError && [underlyingError.domain isEqualToString:CKInternalErrorDomain] && underlyingError.code == CKErrorInternalPluginError &&
2892                thirdLevelError && [thirdLevelError.domain isEqualToString:@"CloudkitKeychainService"]) {
2893
2894                 if(thirdLevelError.code == CKKSServerUnexpectedSyncKeyInChain) {
2895                     // The server thinks the classA/C synckeys don't wrap directly the to top TLK, but we don't (otherwise, we would have fixed it).
2896                     // Issue a key hierarchy fetch and see what's what.
2897                     ckkserror("ckks", self, "CKKS Server extension has told us about %@ for record %@; requesting refetch and reprocess of key hierarchy", thirdLevelError, recordID);
2898                     [self _onqueueKeyStateMachineRequestFetch];
2899                 } else {
2900                     ckkserror("ckks", self, "CKKS Server extension has told us about %@ for record %@, but we don't currently handle this error", thirdLevelError, recordID);
2901                 }
2902             }
2903         }
2904     }
2905
2906     return false;
2907 }
2908
2909 - (bool)_onqueueCKRecordDeleted:(CKRecordID*)recordID recordType:(NSString*)recordType resync:(bool)resync {
2910     dispatch_assert_queue(self.queue);
2911
2912     // TODO: resync doesn't really mean much here; what does it mean for a record to be 'deleted' if you're fetching from scratch?
2913
2914     if([recordType isEqual: SecCKRecordItemType]) {
2915         ckksinfo("ckks", self, "CloudKit notification: deleted record(%@): %@", recordType, recordID);
2916         NSError* error = nil;
2917         NSError* iqeerror = nil;
2918         CKKSMirrorEntry* ckme = [CKKSMirrorEntry fromDatabase: [recordID recordName] zoneID:self.zoneID error: &error];
2919
2920         // Deletes always succeed, not matter the generation count
2921         if(ckme) {
2922             [ckme deleteFromDatabase:&error];
2923
2924             CKKSIncomingQueueEntry* iqe = [[CKKSIncomingQueueEntry alloc] initWithCKKSItem:ckme.item action:SecCKKSActionDelete state:SecCKKSStateNew];
2925             [iqe saveToDatabase:&iqeerror];
2926             if(iqeerror) {
2927                 ckkserror("ckks", self, "Couldn't save incoming queue entry: %@", iqeerror);
2928             }
2929         }
2930         ckksinfo("ckks", self, "CKKSMirrorEntry was deleted: %@ %@ error: %@", recordID, ckme, error);
2931         // TODO: actually pass error back up
2932         return (error == nil);
2933
2934     } else if([recordType isEqual: SecCKRecordCurrentItemType]) {
2935         ckksinfo("ckks", self, "CloudKit notification: deleted current item pointer(%@): %@", recordType, recordID);
2936         NSError* error = nil;
2937
2938         [[CKKSCurrentItemPointer tryFromDatabase:[recordID recordName] state:SecCKKSProcessedStateRemote zoneID:self.zoneID error:&error] deleteFromDatabase:&error];
2939         [[CKKSCurrentItemPointer fromDatabase:[recordID recordName]    state:SecCKKSProcessedStateLocal  zoneID:self.zoneID error:&error] deleteFromDatabase:&error];
2940
2941         ckksinfo("ckks", self, "CKKSCurrentItemPointer was deleted: %@ error: %@", recordID, error);
2942         return (error == nil);
2943
2944     } else if([recordType isEqual: SecCKRecordIntermediateKeyType]) {
2945         // TODO: handle in some interesting way
2946         return true;
2947     } else if([recordType isEqual: SecCKRecordTLKShareType]) {
2948         NSError* error = nil;
2949         ckksinfo("ckks", self, "CloudKit notification: deleted tlk share record(%@): %@", recordType, recordID);
2950         CKKSTLKShareRecord* share = [CKKSTLKShareRecord tryFromDatabaseFromCKRecordID:recordID error:&error];
2951         [share deleteFromDatabase:&error];
2952
2953         if(error) {
2954             ckkserror("ckks", self, "CK notification: Couldn't delete deleted TLKShare: %@ %@", recordID,  error);
2955         }
2956         return (error == nil);
2957
2958     } else if([recordType isEqual: SecCKRecordDeviceStateType]) {
2959         NSError* error = nil;
2960         ckksinfo("ckks", self, "CloudKit notification: deleted device state record(%@): %@", recordType, recordID);
2961
2962         CKKSDeviceStateEntry* cdse = [CKKSDeviceStateEntry tryFromDatabaseFromCKRecordID:recordID error:&error];
2963         [cdse deleteFromDatabase: &error];
2964         ckksinfo("ckks", self, "CKKSCurrentItemPointer(%@) was deleted: %@ error: %@", cdse, recordID, error);
2965
2966         return (error == nil);
2967
2968     } else if ([recordType isEqualToString:SecCKRecordManifestType]) {
2969         ckksinfo("ckks", self, "CloudKit notification: deleted manifest record (%@): %@", recordType, recordID);
2970
2971         NSError* error = nil;
2972         CKKSManifest* manifest = [CKKSManifest manifestForRecordName:recordID.recordName error:&error];
2973         if (manifest) {
2974             [manifest deleteFromDatabase:&error];
2975         }
2976
2977         ckksinfo("ckks", self, "CKKSManifest was deleted: %@ %@ error: %@", recordID, manifest, error);
2978         // TODO: actually pass error back up
2979         return error == nil;
2980     }
2981
2982     else {
2983         ckkserror("ckksfetch", self, "unknown record type: %@ %@", recordType, recordID);
2984         return false;
2985     }
2986 }
2987
2988 - (bool)_onqueueCKRecordChanged:(CKRecord*)record resync:(bool)resync {
2989     dispatch_assert_queue(self.queue);
2990
2991     @autoreleasepool {
2992         ckksnotice("ckksfetch", self, "Processing record modification(%@): %@", record.recordType, record);
2993
2994         if([[record recordType] isEqual: SecCKRecordItemType]) {
2995             [self _onqueueCKRecordItemChanged:record resync:resync];
2996             return true;
2997         } else if([[record recordType] isEqual: SecCKRecordCurrentItemType]) {
2998             [self _onqueueCKRecordCurrentItemPointerChanged:record resync:resync];
2999             return true;
3000         } else if([[record recordType] isEqual: SecCKRecordIntermediateKeyType]) {
3001             [self _onqueueCKRecordKeyChanged:record resync:resync];
3002             return true;
3003         } else if ([[record recordType] isEqual: SecCKRecordTLKShareType]) {
3004             [self _onqueueCKRecordTLKShareChanged:record resync:resync];
3005             return true;
3006         } else if([[record recordType] isEqualToString: SecCKRecordCurrentKeyType]) {
3007             [self _onqueueCKRecordCurrentKeyPointerChanged:record resync:resync];
3008             return true;
3009         } else if ([[record recordType] isEqualToString:SecCKRecordManifestType]) {
3010             [self _onqueueCKRecordManifestChanged:record resync:resync];
3011             return true;
3012         } else if ([[record recordType] isEqualToString:SecCKRecordManifestLeafType]) {
3013             [self _onqueueCKRecordManifestLeafChanged:record resync:resync];
3014             return true;
3015         } else if ([[record recordType] isEqualToString:SecCKRecordDeviceStateType]) {
3016             [self _onqueueCKRecordDeviceStateChanged:record resync:resync];
3017             return true;
3018         } else {
3019             ckkserror("ckksfetch", self, "unknown record type: %@ %@", [record recordType], record);
3020             return false;
3021         }
3022     }
3023 }
3024
3025 - (void)_onqueueCKRecordItemChanged:(CKRecord*)record resync:(bool)resync {
3026     dispatch_assert_queue(self.queue);
3027
3028     NSError* error = nil;
3029     // Find if we knew about this record in the past
3030     bool update = false;
3031     CKKSMirrorEntry* ckme = [CKKSMirrorEntry tryFromDatabase: [[record recordID] recordName] zoneID:self.zoneID error:&error];
3032
3033     if(error) {
3034         ckkserror("ckks", self, "error loading a CKKSMirrorEntry from database: %@", error);
3035         // TODO: quit?
3036     }
3037
3038     if(resync) {
3039         if(!ckme) {
3040             ckkserror("ckksresync", self, "BUG: No local item matching resynced CloudKit record: %@", record);
3041         } else if(![ckme matchesCKRecord:record]) {
3042             ckkserror("ckksresync", self, "BUG: Local item doesn't match resynced CloudKit record: %@ %@", ckme, record);
3043         } else {
3044             ckksnotice("ckksresync", self, "Already know about this item record, updating anyway: %@", record.recordID);
3045         }
3046     }
3047
3048     if(ckme && ckme.item && ckme.item.generationCount > [record[SecCKRecordGenerationCountKey] unsignedLongLongValue]) {
3049         ckkserror("ckks", self, "received a record from CloudKit with a bad generation count: %@ (%ld > %@)", ckme.uuid,
3050                  (long) ckme.item.generationCount,
3051                  record[SecCKRecordGenerationCountKey]);
3052         // Abort processing this record.
3053         return;
3054     }
3055
3056     // If we found an old version in the database; this might be an update
3057     if(ckme) {
3058         if([ckme matchesCKRecord:record] && !resync) {
3059             // This is almost certainly a record we uploaded; CKFetchChanges sends them back as new records
3060             ckksnotice("ckks", self, "CloudKit has told us of record we already know about; skipping update");
3061             return;
3062         }
3063
3064         update = true;
3065         // Set the CKKSMirrorEntry's fields to be whatever this record holds
3066         [ckme setFromCKRecord: record];
3067     } else {
3068         // Have to make a new CKKSMirrorEntry
3069         ckme = [[CKKSMirrorEntry alloc] initWithCKRecord: record];
3070     }
3071
3072     [ckme saveToDatabase: &error];
3073
3074     if(error) {
3075         ckkserror("ckks", self, "couldn't save new CKRecord to database: %@ %@", record, error);
3076     } else {
3077         ckksdebug("ckks", self, "CKKSMirrorEntry was created: %@", ckme);
3078     }
3079
3080     NSError* iqeerror = nil;
3081     CKKSIncomingQueueEntry* iqe = [[CKKSIncomingQueueEntry alloc] initWithCKKSItem:ckme.item
3082                                                                             action:(update ? SecCKKSActionModify : SecCKKSActionAdd)
3083                                                                              state:SecCKKSStateNew];
3084     [iqe saveToDatabase:&iqeerror];
3085     if(iqeerror) {
3086         ckkserror("ckks", self, "Couldn't save modified incoming queue entry: %@", iqeerror);
3087     } else {
3088         ckksdebug("ckks", self, "CKKSIncomingQueueEntry was created: %@", iqe);
3089     }
3090
3091     // A remote change has occured for this record. Delete any pending local changes; they will be overwritten.
3092     CKKSOutgoingQueueEntry* oqe = [CKKSOutgoingQueueEntry tryFromDatabase:ckme.uuid state: SecCKKSStateNew zoneID:self.zoneID error: &error];
3093     if(error) {
3094         ckkserror("ckks", self, "Couldn't load OutgoingQueueEntry: %@", error);
3095     }
3096     if(oqe) {
3097         [self _onqueueChangeOutgoingQueueEntry:oqe toState:SecCKKSStateDeleted error:&error];
3098     }
3099
3100     // Reencryptions are pending changes too
3101     oqe = [CKKSOutgoingQueueEntry tryFromDatabase:ckme.uuid state: SecCKKSStateReencrypt zoneID:self.zoneID error: &error];
3102     if(error) {
3103         ckkserror("ckks", self, "Couldn't load reencrypted OutgoingQueueEntry: %@", error);
3104     }
3105     if(oqe) {
3106         [oqe deleteFromDatabase:&error];
3107         if(error) {
3108             ckkserror("ckks", self, "Couldn't delete reencrypted oqe(%@): %@", oqe, error);
3109         }
3110     }
3111 }
3112
3113 - (void)_onqueueCKRecordKeyChanged:(CKRecord*)record resync:(bool)resync {
3114     dispatch_assert_queue(self.queue);
3115
3116     NSError* error = nil;
3117
3118     if(resync) {
3119         NSError* resyncerror = nil;
3120
3121         CKKSKey* key = [CKKSKey tryFromDatabaseAnyState:record.recordID.recordName zoneID:self.zoneID error:&resyncerror];
3122         if(resyncerror) {
3123             ckkserror("ckksresync", self, "error loading key: %@", resyncerror);
3124         }
3125         if(!key) {
3126             ckkserror("ckksresync", self, "BUG: No sync key matching resynced CloudKit record: %@", record);
3127         } else if(![key matchesCKRecord:record]) {
3128             ckkserror("ckksresync", self, "BUG: Local sync key doesn't match resynced CloudKit record(s): %@ %@", key, record);
3129         } else {
3130             ckksnotice("ckksresync", self, "Already know about this sync key, skipping update: %@", record);
3131             return;
3132         }
3133     }
3134
3135     CKKSKey* remotekey = [[CKKSKey alloc] initWithCKRecord: record];
3136
3137     // Do we already know about this key?
3138     CKKSKey* possibleLocalKey = [CKKSKey tryFromDatabase:remotekey.uuid zoneID:self.zoneID error:&error];
3139     if(error) {
3140         ckkserror("ckkskey", self, "Error findibg exsiting local key for %@: %@", remotekey, error);
3141         // Go on, assuming there isn't a local key
3142     } else if(possibleLocalKey && [possibleLocalKey matchesCKRecord:record]) {
3143         // Okay, nothing new here. Update the CKRecord and move on.
3144         // Note: If the new record doesn't match the local copy, we have to go through the whole dance below
3145         possibleLocalKey.storedCKRecord = record;
3146         [possibleLocalKey saveToDatabase:&error];
3147
3148         if(error) {
3149             ckkserror("ckkskey", self, "Couldn't update existing key: %@: %@", possibleLocalKey, error);
3150         }
3151         return;
3152     }
3153
3154     // Drop into the synckeys table as a 'remote' key, then ask for a rekey operation.
3155     remotekey.state = SecCKKSProcessedStateRemote;
3156     remotekey.currentkey = false;
3157
3158     [remotekey saveToDatabase:&error];
3159     if(error) {
3160         ckkserror("ckkskey", self, "Couldn't save key record to database: %@: %@", remotekey, error);
3161         ckksinfo("ckkskey", self, "CKRecord was %@", record);
3162     }
3163
3164     // We've saved a new key in the database; trigger a rekey operation.
3165     [self _onqueueKeyStateMachineRequestProcess];
3166 }
3167
3168 - (void)_onqueueCKRecordTLKShareChanged:(CKRecord*)record resync:(bool)resync {
3169     dispatch_assert_queue(self.queue);
3170
3171     NSError* error = nil;
3172     if(resync) {
3173         // TODO fill in
3174     }
3175
3176     // CKKSTLKShares get saved with no modification
3177     CKKSTLKShareRecord* share = [[CKKSTLKShareRecord alloc] initWithCKRecord:record];
3178     [share saveToDatabase:&error];
3179     if(error) {
3180         ckkserror("ckksshare", self, "Couldn't save new TLK share to database: %@ %@", share, error);
3181     }
3182
3183     [self _onqueueKeyStateMachineRequestProcess];
3184 }
3185
3186 - (void)_onqueueCKRecordCurrentKeyPointerChanged:(CKRecord*)record resync:(bool)resync {
3187     dispatch_assert_queue(self.queue);
3188
3189     // Pull out the old CKP, if it exists
3190     NSError* ckperror = nil;
3191     CKKSCurrentKeyPointer* oldckp = [CKKSCurrentKeyPointer tryFromDatabase:((CKKSKeyClass*) record.recordID.recordName) zoneID:self.zoneID error:&ckperror];
3192     if(ckperror) {
3193         ckkserror("ckkskey", self, "error loading ckp: %@", ckperror);
3194     }
3195
3196     if(resync) {
3197         if(!oldckp) {
3198             ckkserror("ckksresync", self, "BUG: No current key pointer matching resynced CloudKit record: %@", record);
3199         } else if(![oldckp matchesCKRecord:record]) {
3200             ckkserror("ckksresync", self, "BUG: Local current key pointer doesn't match resynced CloudKit record: %@ %@", oldckp, record);
3201         } else {
3202             ckksnotice("ckksresync", self, "Current key pointer has 'changed', but it matches our local copy: %@", record);
3203         }
3204     }
3205
3206     NSError* error = nil;
3207     CKKSCurrentKeyPointer* currentkey = [[CKKSCurrentKeyPointer alloc] initWithCKRecord: record];
3208
3209     [currentkey saveToDatabase: &error];
3210     if(error) {
3211         ckkserror("ckkskey", self, "Couldn't save current key pointer to database: %@: %@", currentkey, error);
3212         ckksinfo("ckkskey", self, "CKRecord was %@", record);
3213     }
3214
3215     if([oldckp matchesCKRecord:record]) {
3216         ckksnotice("ckkskey", self, "Current key pointer modification doesn't change anything interesting; skipping reprocess: %@", record);
3217     } else {
3218         // We've saved a new key in the database; trigger a rekey operation.
3219         [self _onqueueKeyStateMachineRequestProcess];
3220     }
3221 }
3222
3223 - (void)_onqueueCKRecordCurrentItemPointerChanged:(CKRecord*)record resync:(bool)resync {
3224     dispatch_assert_queue(self.queue);
3225
3226     if(resync) {
3227         NSError* ciperror = nil;
3228         CKKSCurrentItemPointer* localcip  = [CKKSCurrentItemPointer tryFromDatabase:record.recordID.recordName state:SecCKKSProcessedStateLocal  zoneID:self.zoneID error:&ciperror];
3229         CKKSCurrentItemPointer* remotecip = [CKKSCurrentItemPointer tryFromDatabase:record.recordID.recordName state:SecCKKSProcessedStateRemote zoneID:self.zoneID error:&ciperror];
3230         if(ciperror) {
3231             ckkserror("ckksresync", self, "error loading cip: %@", ciperror);
3232         }
3233         if(!(localcip || remotecip)) {
3234             ckkserror("ckksresync", self, "BUG: No current item pointer matching resynced CloudKit record: %@", record);
3235         } else if(! ([localcip matchesCKRecord:record] || [remotecip matchesCKRecord:record]) ) {
3236             ckkserror("ckksresync", self, "BUG: Local current item pointer doesn't match resynced CloudKit record(s): %@ %@ %@", localcip, remotecip, record);
3237         } else {
3238             ckksnotice("ckksresync", self, "Already know about this current item pointer, skipping update: %@", record);
3239             return;
3240         }
3241     }
3242
3243     NSError* error = nil;
3244     CKKSCurrentItemPointer* cip = [[CKKSCurrentItemPointer alloc] initWithCKRecord: record];
3245     cip.state = SecCKKSProcessedStateRemote;
3246
3247     [cip saveToDatabase: &error];
3248     if(error) {
3249         ckkserror("currentitem", self, "Couldn't save current item pointer to database: %@: %@ %@", cip, error, record);
3250     }
3251 }
3252
3253 - (void)_onqueueCKRecordManifestChanged:(CKRecord*)record resync:(bool)resync
3254 {
3255     NSError* error = nil;
3256     CKKSPendingManifest* manifest = [[CKKSPendingManifest alloc] initWithCKRecord:record];
3257     [manifest saveToDatabase:&error];
3258     if (error) {
3259         ckkserror("CKKS", self, "Failed to save fetched manifest record to database: %@: %@", manifest, error);
3260         ckksinfo("CKKS", self, "manifest CKRecord was %@", record);
3261     }
3262 }
3263
3264 - (void)_onqueueCKRecordManifestLeafChanged:(CKRecord*)record resync:(bool)resync
3265 {
3266     NSError* error = nil;
3267     CKKSManifestLeafRecord* manifestLeaf = [[CKKSManifestPendingLeafRecord alloc] initWithCKRecord:record];
3268     [manifestLeaf saveToDatabase:&error];
3269     if (error) {
3270         ckkserror("CKKS", self, "Failed to save fetched manifest leaf record to database: %@: %@", manifestLeaf, error);
3271         ckksinfo("CKKS", self, "manifest leaf CKRecord was %@", record);
3272     }
3273 }
3274
3275 - (void)_onqueueCKRecordDeviceStateChanged:(CKRecord*)record resync:(bool)resync {
3276     if(resync) {
3277         NSError* dserror = nil;
3278         CKKSDeviceStateEntry* cdse  = [CKKSDeviceStateEntry tryFromDatabase:record.recordID.recordName zoneID:self.zoneID error:&dserror];
3279         if(dserror) {
3280             ckkserror("ckksresync", self, "error loading cdse: %@", dserror);
3281         }
3282         if(!cdse) {
3283             ckkserror("ckksresync", self, "BUG: No current device state entry matching resynced CloudKit record: %@", record);
3284         } else if(![cdse matchesCKRecord:record]) {
3285             ckkserror("ckksresync", self, "BUG: Local current device state entry doesn't match resynced CloudKit record(s): %@ %@", cdse, record);
3286         } else {
3287             ckksnotice("ckksresync", self, "Already know about this current item pointer, skipping update: %@", record);
3288             return;
3289         }
3290     }
3291
3292     NSError* error = nil;
3293     CKKSDeviceStateEntry* cdse = [[CKKSDeviceStateEntry alloc] initWithCKRecord:record];
3294     [cdse saveToDatabase:&error];
3295     if (error) {
3296         ckkserror("ckksdevice", self, "Failed to save device record to database: %@: %@ %@", cdse, error, record);
3297     }
3298 }
3299
3300 - (bool)_onqueueResetAllInflightOQE:(NSError**)error {
3301     NSError* localError = nil;
3302
3303     while(true) {
3304         NSArray<CKKSOutgoingQueueEntry*> * inflightQueueEntries = [CKKSOutgoingQueueEntry fetch:SecCKKSOutgoingQueueItemsAtOnce
3305                                                                                           state:SecCKKSStateInFlight
3306                                                                                          zoneID:self.zoneID
3307                                                                                           error:&localError];
3308
3309         if(localError != nil) {
3310             ckkserror("ckks", self, "Error finding inflight outgoing queue records: %@", localError);
3311             if(error) {
3312                 *error = localError;
3313             }
3314             return false;
3315         }
3316
3317         if([inflightQueueEntries count] == 0u) {
3318             break;
3319         }
3320
3321         for(CKKSOutgoingQueueEntry* oqe in inflightQueueEntries) {
3322             [self _onqueueChangeOutgoingQueueEntry:oqe toState:SecCKKSStateNew error:&localError];
3323
3324             if(localError) {
3325                 ckkserror("ckks", self, "Error fixing up inflight OQE(%@): %@", oqe, localError);
3326                 if(error) {
3327                     *error = localError;
3328                 }
3329                 return false;
3330             }
3331         }
3332     }
3333
3334     return true;
3335 }
3336
3337 - (bool)_onqueueChangeOutgoingQueueEntry: (CKKSOutgoingQueueEntry*) oqe toState: (NSString*) state error: (NSError* __autoreleasing*) error {
3338     dispatch_assert_queue(self.queue);
3339
3340     NSError* localerror = nil;
3341
3342     if([state isEqualToString: SecCKKSStateDeleted]) {
3343         // Hurray, this must be a success
3344         SecBoolNSErrorCallback callback = self.pendingSyncCallbacks[oqe.uuid];
3345         if(callback) {
3346             callback(true, nil);
3347             self.pendingSyncCallbacks[oqe.uuid] = nil;
3348         }
3349
3350         [oqe deleteFromDatabase: &localerror];
3351         if(localerror) {
3352             ckkserror("ckks", self, "Couldn't delete %@: %@", oqe, localerror);
3353         }
3354
3355     } else if([oqe.state isEqualToString:SecCKKSStateInFlight] && [state isEqualToString:SecCKKSStateNew]) {
3356         // An in-flight OQE is moving to new? See if it's been superceded
3357         CKKSOutgoingQueueEntry* newOQE = [CKKSOutgoingQueueEntry tryFromDatabase:oqe.uuid state:SecCKKSStateNew zoneID:self.zoneID error:&localerror];
3358         if(localerror) {
3359             ckkserror("ckksoutgoing", self, "Couldn't fetch an overwriting OQE, assuming one doesn't exist: %@", localerror);
3360             newOQE = nil;
3361         }
3362
3363         if(newOQE) {
3364             ckksnotice("ckksoutgoing", self, "New modification has come in behind inflight %@; dropping failed change", oqe);
3365             // recurse for that lovely code reuse
3366             [self _onqueueChangeOutgoingQueueEntry:oqe toState:SecCKKSStateDeleted error:&localerror];
3367             if(localerror) {
3368                 ckkserror("ckksoutgoing", self, "Couldn't delete in-flight OQE: %@", localerror);
3369                 if(error) {
3370                     *error = localerror;
3371                 }
3372             }
3373         } else {
3374             oqe.state = state;
3375             [oqe saveToDatabase: &localerror];
3376             if(localerror) {
3377                 ckkserror("ckks", self, "Couldn't save %@ as %@: %@", oqe, state, localerror);
3378             }
3379         }
3380
3381     } else {
3382         oqe.state = state;
3383         [oqe saveToDatabase: &localerror];
3384         if(localerror) {
3385             ckkserror("ckks", self, "Couldn't save %@ as %@: %@", oqe, state, localerror);
3386         }
3387     }
3388
3389     if(error && localerror) {
3390         *error = localerror;
3391     }
3392     return localerror == nil;
3393 }
3394
3395 - (bool)_onqueueErrorOutgoingQueueEntry: (CKKSOutgoingQueueEntry*) oqe itemError: (NSError*) itemError error: (NSError* __autoreleasing*) error {
3396     dispatch_assert_queue(self.queue);
3397
3398     SecBoolNSErrorCallback callback = self.pendingSyncCallbacks[oqe.uuid];
3399     if(callback) {
3400         callback(false, itemError);
3401         self.pendingSyncCallbacks[oqe.uuid] = nil;
3402     }
3403     NSError* localerror = nil;
3404
3405     // Now, delete the OQE: it's never coming back
3406     [oqe deleteFromDatabase:&localerror];
3407     if(localerror) {
3408         ckkserror("ckks", self, "Couldn't delete %@ (due to error %@): %@", oqe, itemError, localerror);
3409     }
3410
3411     if(error && localerror) {
3412         *error = localerror;
3413     }
3414     return localerror == nil;
3415 }
3416
3417 - (bool)_onqueueUpdateLatestManifestWithError:(NSError**)error
3418 {
3419     dispatch_assert_queue(self.queue);
3420     CKKSManifest* manifest = [CKKSManifest latestTrustedManifestForZone:self.zoneName error:error];
3421     if (manifest) {
3422         self.latestManifest = manifest;
3423         return true;
3424     }
3425     else {
3426         return false;
3427     }
3428 }
3429
3430 - (bool)_onqueueWithAccountKeysCheckTLK:(CKKSKey*)proposedTLK error:(NSError* __autoreleasing *)error {
3431     dispatch_assert_queue(self.queue);
3432     // First, if we have a local identity, check for any TLK shares
3433     NSError* localerror = nil;
3434
3435     if(![proposedTLK wrapsSelf]) {
3436         localerror = [NSError errorWithDomain:CKKSErrorDomain code:CKKSKeyNotSelfWrapped description:[NSString stringWithFormat:@"Potential TLK %@ doesn't wrap itself: %@", proposedTLK, proposedTLK.parentKeyUUID] underlying:NULL];
3437         ckkserror("ckksshare", self, "%@", localerror);
3438         if (error) {
3439             *error = localerror;
3440         }
3441     } else {
3442         bool tlkShares = [self _onqueueWithAccountKeysCheckTLKFromShares:proposedTLK error:&localerror];
3443         // We only want to error out if a positive error occurred. "No shares" is okay.
3444         if(!tlkShares || localerror) {
3445             bool noTrustedTLKShares = [localerror.domain isEqualToString:CKKSErrorDomain] && localerror.code == CKKSNoTrustedTLKShares;
3446             bool noSelfPeer = [localerror.domain isEqualToString:CKKSErrorDomain] && localerror.code == CKKSNoEncryptionKey;
3447             bool noTrust = [localerror.domain isEqualToString:CKKSErrorDomain] && localerror.code == CKKSLackingTrust;
3448
3449             // If this error was something worse than 'couldn't unwrap for reasons including there not being data', report it
3450             if(!(noTrustedTLKShares || noSelfPeer || noTrust)) {
3451                 if(error) {
3452                     *error = localerror;
3453                 }
3454                 ckkserror("ckksshare", self, "Errored unwrapping TLK with TLKShares: %@", localerror);
3455                 return false;
3456             } else {
3457                 ckkserror("ckksshare", self, "Non-fatal error unwrapping TLK with TLKShares: %@", localerror);
3458             }
3459         }
3460     }
3461
3462     if([proposedTLK loadKeyMaterialFromKeychain:error]) {
3463         // Hurray!
3464         return true;
3465     } else {
3466         return false;
3467     }
3468 }
3469
3470 // This version only examines if this TLK is recoverable from TLK shares
3471 - (bool)_onqueueWithAccountKeysCheckTLKFromShares:(CKKSKey*)proposedTLK error:(NSError* __autoreleasing *)error {
3472     // But being recoverable from any trust set is okay
3473     NSError* localerror = nil;
3474
3475     if(self.currentTrustStates.count == 0u) {
3476         if(error) {
3477             *error = [NSError errorWithDomain:CKKSErrorDomain
3478                                          code:CKKSLackingTrust
3479                                   description:@"No current trust states; can't check TLK"];
3480         }
3481         return false;
3482     }
3483
3484     for(CKKSPeerProviderState* trustState in self.currentTrustStates) {
3485         ckkserror("ckksshare", self, "Checking TLK from trust state %@", trustState);
3486         bool recovered = [self _onqueueWithAccountKeysWithPeers:trustState
3487                                                        checkTLK:proposedTLK
3488                                                           error:&localerror];
3489
3490         if(recovered) {
3491             ckkserror("ckksshare", self, "Recovered the TLK");
3492             return true;
3493         }
3494
3495         ckkserror("ckksshare", self, "Unable to recover TLK from trust set: %@", localerror);
3496     }
3497
3498     // Only report the last error
3499     if(error && localerror) {
3500         *error = localerror;
3501     }
3502     return false;
3503 }
3504
3505 - (bool)_onqueueWithAccountKeysWithPeers:(CKKSPeerProviderState*)trustState
3506                                 checkTLK:(CKKSKey*)proposedTLK
3507                                   error:(NSError* __autoreleasing *)error
3508 {
3509     NSError* localerror = NULL;
3510     if(!trustState.currentSelfPeers.currentSelf || trustState.currentSelfPeersError) {
3511         ckkserror("ckksshare", self, "Don't have self peers for %@: %@", trustState.peerProviderID, trustState.currentSelfPeersError);
3512         if(error) {
3513             if([self.lockStateTracker isLockedError:trustState.currentSelfPeersError]) {
3514                 // Locked error should propagate
3515                 *error = trustState.currentSelfPeersError;
3516             } else {
3517                 *error = [NSError errorWithDomain:CKKSErrorDomain
3518                                              code:CKKSNoEncryptionKey
3519                                       description:@"No current self peer"
3520                                        underlying:trustState.currentSelfPeersError];
3521             }
3522         }
3523         return false;
3524     }
3525
3526     if(!trustState.currentTrustedPeers || trustState.currentTrustedPeersError) {
3527         ckkserror("ckksshare", self, "Don't have trusted peers: %@", trustState.currentTrustedPeersError);
3528         if(error) {
3529             *error = [NSError errorWithDomain:CKKSErrorDomain
3530                                          code:CKKSNoPeersAvailable
3531                                   description:@"No trusted peers"
3532                                    underlying:trustState.currentTrustedPeersError];
3533         }
3534         return false;
3535     }
3536
3537     NSError* lastShareError = nil;
3538
3539     for(id<CKKSSelfPeer> selfPeer in trustState.currentSelfPeers.allSelves) {
3540         NSArray<CKKSTLKShareRecord*>* possibleShares = [CKKSTLKShareRecord allFor:selfPeer.peerID
3541                                                               keyUUID:proposedTLK.uuid
3542                                                                zoneID:self.zoneID
3543                                                                 error:&localerror];
3544         if(localerror) {
3545             ckkserror("ckksshare", self, "Error fetching CKKSTLKShares for %@: %@", selfPeer, localerror);
3546         }
3547
3548         if(possibleShares.count == 0) {
3549             ckksnotice("ckksshare", self, "No CKKSTLKShares to %@ for %@", selfPeer, proposedTLK);
3550             continue;
3551         }
3552
3553         for(CKKSTLKShareRecord* possibleShare in possibleShares) {
3554             NSError* possibleShareError = nil;
3555             ckksnotice("ckksshare", self, "Checking possible TLK share %@ as %@", possibleShare, selfPeer);
3556
3557             CKKSKey* possibleKey = [possibleShare recoverTLK:selfPeer
3558                                                 trustedPeers:trustState.currentTrustedPeers
3559                                                        error:&possibleShareError];
3560
3561             if(possibleShareError) {
3562                 ckkserror("ckksshare", self, "Unable to unwrap TLKShare(%@) as %@: %@",
3563                           possibleShare, selfPeer, possibleShareError);
3564                 ckkserror("ckksshare", self, "Current trust set: %@", trustState.currentTrustedPeers);
3565                 lastShareError = possibleShareError;
3566                 continue;
3567             }
3568
3569             bool result = [proposedTLK trySelfWrappedKeyCandidate:possibleKey.aessivkey error:&possibleShareError];
3570             if(possibleShareError) {
3571                 ckkserror("ckksshare", self, "Unwrapped TLKShare(%@) does not unwrap proposed TLK(%@) as %@: %@",
3572                           possibleShare, proposedTLK, trustState.currentSelfPeers.currentSelf, possibleShareError);
3573                 lastShareError = possibleShareError;
3574                 continue;
3575             }
3576
3577             if(result) {
3578                 ckksnotice("ckksshare", self, "TLKShare(%@) unlocked TLK(%@) as %@",
3579                            possibleShare, proposedTLK, selfPeer);
3580
3581                 // The proposed TLK is trusted key material. Persist it as a "trusted" key.
3582                 [proposedTLK saveKeyMaterialToKeychain:true error:&possibleShareError];
3583                 if(possibleShareError) {
3584                     ckkserror("ckksshare", self, "Couldn't store the new TLK(%@) to the keychain: %@", proposedTLK, possibleShareError);
3585                     if(error) {
3586                         *error = possibleShareError;
3587                     }
3588                     return false;
3589                 }
3590
3591                 return true;
3592             }
3593         }
3594     }
3595
3596     if(error) {
3597         *error = [NSError errorWithDomain:CKKSErrorDomain
3598                                      code:CKKSNoTrustedTLKShares
3599                               description:[NSString stringWithFormat:@"No trusted TLKShares for %@", proposedTLK]
3600                                underlying:lastShareError];
3601     }
3602     return false;
3603 }
3604
3605 - (bool)dispatchSyncWithConnection:(SecDbConnectionRef _Nonnull)dbconn block:(bool (^)(void))block {
3606     CFErrorRef cferror = NULL;
3607
3608     // Take the DB transaction, then get on the local queue.
3609     // In the case of exclusive DB transactions, we don't really _need_ the local queue, but, it's here for future use.
3610     bool ret = kc_transaction_type(dbconn, kSecDbExclusiveRemoteCKKSTransactionType, &cferror, ^bool{
3611         __block bool ok = false;
3612
3613         dispatch_sync(self.queue, ^{
3614             ok = block();
3615         });
3616
3617         return ok;
3618     });
3619
3620     if(cferror) {
3621         ckkserror("ckks", self, "error doing database transaction, major problems ahead: %@", cferror);
3622     }
3623     return ret;
3624 }
3625
3626 - (void)dispatchSync: (bool (^)(void)) block {
3627     // important enough to block this thread. Must get a connection first, though!
3628
3629     // Please don't jetsam us...
3630     os_transaction_t transaction = os_transaction_create([[NSString stringWithFormat:@"com.apple.securityd.ckks.%@", self.zoneName] UTF8String]);
3631
3632     CFErrorRef cferror = NULL;
3633     kc_with_dbt(true, &cferror, ^bool (SecDbConnectionRef dbt) {
3634         return [self dispatchSyncWithConnection:dbt block:block];
3635     });
3636     if(cferror) {
3637         ckkserror("ckks", self, "error getting database connection, major problems ahead: %@", cferror);
3638     }
3639
3640     (void)transaction;
3641 }
3642
3643 - (void)dispatchSyncWithAccountKeys:(bool (^)(void))block
3644 {
3645     [self dispatchSyncWithPeerProviders:self.currentPeerProviders override:false block:block];
3646 }
3647
3648 - (void)dispatchSyncWithPeerProviders:(NSArray<id<CKKSPeerProvider>>*)peerProviders
3649                              override:(bool)overridePeerProviders
3650                                 block:(bool (^)(void))block
3651 {
3652     [SOSAccount performOnQuietAccountQueue: ^{
3653         NSArray<id<CKKSPeerProvider>>* actualPeerProviders = overridePeerProviders ? peerProviders : self.currentPeerProviders;
3654         NSMutableArray<CKKSPeerProviderState*>* trustStates = [NSMutableArray array];
3655
3656         for(id<CKKSPeerProvider> provider in actualPeerProviders) {
3657             ckksnotice("ckks", self, "Fetching account keys for provider %@", provider);
3658
3659             NSError* selfPeersError = nil;
3660             CKKSSelves* currentSelfPeers = [provider fetchSelfPeers:&selfPeersError];
3661
3662             NSError* trustedPeersError = nil;
3663             NSSet<id<CKKSRemotePeerProtocol>>* currentTrustedPeers = [provider fetchTrustedPeers:&trustedPeersError];
3664
3665             [trustStates addObject:[[CKKSPeerProviderState alloc] initWithPeerProviderID:provider.providerID
3666                                                                                essential:provider.essential
3667                                                                                selfPeers:currentSelfPeers
3668                                                                           selfPeersError:selfPeersError
3669                                                                             trustedPeers:currentTrustedPeers
3670                                                                        trustedPeersError:trustedPeersError]];
3671         }
3672
3673         [self dispatchSync:^bool{
3674             if(overridePeerProviders) {
3675                 self.currentPeerProviders = peerProviders;
3676             }
3677             self.currentTrustStates = trustStates;
3678
3679             __block bool result = false;
3680             [SOSAccount performWhileHoldingAccountQueue:^{ // so any calls through SOS account will know they can perform their work without dispatching to the account queue, which we already hold
3681                 result = block();
3682             }];
3683
3684             // Forget the peers; they might have class A key material
3685             NSMutableArray<CKKSPeerProviderState*>* noTrustStates = [NSMutableArray array];
3686             for(id<CKKSPeerProvider> provider in peerProviders) {
3687                 (void)provider;
3688                 [noTrustStates addObject:[CKKSPeerProviderState noPeersState:provider]];
3689             }
3690             self.currentTrustStates = noTrustStates;
3691
3692             return result;
3693         }];
3694     }];
3695 }
3696
3697 #pragma mark - CKKSZoneUpdateReceiver
3698
3699 - (void)notifyZoneChange: (CKRecordZoneNotification*) notification {
3700     ckksnotice("ckks", self, "received a zone change notification for %@ %@", self, notification);
3701
3702     [self fetchAndProcessCKChangesDueToAPNS:notification];
3703 }
3704
3705 - (void)superHandleCKLogin {
3706     [super handleCKLogin];
3707 }
3708
3709 - (void)handleCKLogin {
3710     ckksnotice("ckks", self, "received a notification of CK login");
3711     if(!SecCKKSIsEnabled()) {
3712         ckksnotice("ckks", self, "Skipping CloudKit initialization due to disabled CKKS");
3713         return;
3714     }
3715
3716     WEAKIFY(self);
3717     CKKSResultOperation* login = [CKKSResultOperation named:@"ckks-login" withBlock:^{
3718         STRONGIFY(self);
3719
3720         [self dispatchSyncWithAccountKeys:^bool{
3721             [self superHandleCKLogin];
3722
3723             // Reset key hierarchy state machine to initializing
3724             [self _onqueueAdvanceKeyStateMachineToState:SecCKKSZoneKeyStateInitializing withError:nil];
3725             return true;
3726         }];
3727
3728         // Change our condition variables to reflect that we think we're logged in
3729         self.loggedOut = [[CKKSCondition alloc] initToChain:self.loggedOut];
3730         [self.loggedIn fulfill];
3731         [self.accountStateKnown fulfill];
3732     }];
3733
3734     [self scheduleAccountStatusOperation:login];
3735 }
3736
3737 - (void)superHandleCKLogout {
3738     [super handleCKLogout];
3739 }
3740
3741 - (void)handleCKLogout {
3742     WEAKIFY(self);
3743     CKKSResultOperation* logout = [CKKSResultOperation named:@"ckks-logout" withBlock: ^{
3744         STRONGIFY(self);
3745         if(!self) {
3746             return;
3747         }
3748         [self dispatchSync:^bool {
3749             ckksnotice("ckks", self, "received a notification of CK logout");
3750             [self superHandleCKLogout];
3751
3752             NSError* error = nil;
3753             [self _onqueueResetLocalData: &error];
3754             if(error) {
3755                 ckkserror("ckks", self, "error while resetting local data: %@", error);
3756             }
3757
3758             [self _onqueueAdvanceKeyStateMachineToState:SecCKKSZoneKeyStateLoggedOut withError:nil];
3759
3760             self.loggedIn = [[CKKSCondition alloc] initToChain: self.loggedIn];
3761             [self.loggedOut fulfill];
3762             [self.accountStateKnown fulfill];
3763
3764             // Tell all pending sync clients that we don't expect to ever sync
3765             for(NSString* callbackUUID in self.pendingSyncCallbacks.allKeys) {
3766                 [self callSyncCallbackWithErrorNoAccount:self.pendingSyncCallbacks[callbackUUID]];
3767                 self.pendingSyncCallbacks[callbackUUID] = nil;
3768             }
3769
3770             return true;
3771         }];
3772     }];
3773
3774     [self scheduleAccountStatusOperation: logout];
3775 }
3776
3777 - (void)callSyncCallbackWithErrorNoAccount:(SecBoolNSErrorCallback)syncCallback {
3778     CKKSAccountStatus accountStatus = self.accountStatus;
3779     dispatch_async(self.queue, ^{
3780         syncCallback(false, [NSError errorWithDomain:@"securityd"
3781                                                 code:errSecNotLoggedIn
3782                                             userInfo:@{NSLocalizedDescriptionKey:
3783                                                            [NSString stringWithFormat: @"No iCloud account available(%d); item is not expected to sync", (int)accountStatus]}]);
3784     });
3785 }
3786
3787 #pragma mark - Trust operations
3788
3789 - (void)beginTrustedOperation:(NSArray<id<CKKSPeerProvider>>*)peerProviders
3790              suggestTLKUpload:(CKKSNearFutureScheduler*)suggestTLKUpload
3791 {
3792     for(id<CKKSPeerProvider> peerProvider in peerProviders) {
3793         [peerProvider registerForPeerChangeUpdates:self];
3794     }
3795
3796     [self.launch addEvent:@"beginTrusted"];
3797
3798     [self dispatchSyncWithPeerProviders:peerProviders override:true block:^bool {
3799         ckksnotice("ckkstrust", self, "Beginning trusted operation");
3800         CKKSAccountStatus oldTrustStatus = self.trustStatus;
3801
3802         self.suggestTLKUpload = suggestTLKUpload;
3803
3804         self.trustStatus = CKKSAccountStatusAvailable;
3805         if(self.trustDependency) {
3806             [self scheduleOperation: self.trustDependency];
3807             self.trustDependency = nil;
3808         }
3809         [self _onqueueAdvanceKeyStateMachineToState:nil withError:nil];
3810
3811         if(oldTrustStatus == CKKSAccountStatusNoAccount) {
3812             ckksnotice("ckkstrust", self, "Moving from an untrusted status; we need to process incoming queue and scan for any new items");
3813
3814             // Next, try to process them (replacing local entries)
3815             CKKSIncomingQueueOperation* initialProcess = [self processIncomingQueue:true after:nil];
3816             initialProcess.name = @"initial-process-incoming-queue";
3817
3818             // If all that succeeds, iterate through all keychain items and find the ones which need to be uploaded
3819             self.initialScanOperation = [self scanLocalItems:@"newly-trusted-scan"
3820                                             ckoperationGroup:nil
3821                                                        after:initialProcess];
3822         }
3823
3824         return true;
3825     }];
3826 }
3827
3828 - (void)endTrustedOperation
3829 {
3830     [self.launch addEvent:@"endTrusted"];
3831
3832     [self dispatchSyncWithPeerProviders:nil override:true block:^bool {
3833         ckksnotice("ckkstrust", self, "Ending trusted operation");
3834
3835         self.suggestTLKUpload = nil;
3836
3837         self.trustStatus = CKKSAccountStatusNoAccount;
3838         if(!self.trustDependency) {
3839             self.trustDependency = [CKKSResultOperation named:@"wait-for-trust" withBlock:^{}];
3840         }
3841         [self _onqueueAdvanceKeyStateMachineToState:nil withError:nil];
3842         return true;
3843     }];
3844 }
3845
3846 #pragma mark - CKKSChangeFetcherClient
3847
3848 - (CKKSCloudKitFetchRequest*)participateInFetch
3849 {
3850     __block CKKSCloudKitFetchRequest* request = [[CKKSCloudKitFetchRequest alloc] init];
3851     [self dispatchSync: ^bool {
3852         if(self.accountStatus != CKKSAccountStatusAvailable) {
3853             ckksnotice("ckksfetch", self, "Not participating in fetch: not logged in");
3854             request.participateInFetch = false;
3855             return false;
3856         }
3857
3858         if(!self.zoneCreated) {
3859             ckksnotice("ckksfetch", self, "Not participating in fetch: zone not created yet");
3860             request.participateInFetch = false;
3861             return false;
3862         }
3863
3864         request.participateInFetch = true;
3865         [self.launch addEvent:@"fetch"];
3866
3867         if([self.keyHierarchyState isEqualToString:SecCKKSZoneKeyStateNeedFullRefetch]) {
3868             // We want to return a nil change tag (to force a resync)
3869             ckksnotice("ckksfetch", self, "Beginning refetch");
3870             request.changeToken = nil;
3871         } else {
3872             CKKSZoneStateEntry* ckse = [CKKSZoneStateEntry state:self.zoneName];
3873             if(!ckse) {
3874                 ckkserror("ckksfetch", self, "couldn't fetch zone change token for %@", self.zoneName);
3875                 return false;
3876             }
3877             request.changeToken = ckse.changeToken;
3878         }
3879         return true;
3880     }];
3881
3882     if (request.changeToken == nil) {
3883         self.launch.firstLaunch = true;
3884     }
3885
3886     return request;
3887 }
3888
3889 - (void)changesFetched:(NSArray<CKRecord*>*)changedRecords
3890       deletedRecordIDs:(NSArray<CKKSCloudKitDeletion*>*)deletedRecords
3891         oldChangeToken:(CKServerChangeToken*)oldChangeToken
3892         newChangeToken:(CKServerChangeToken*)newChangeToken
3893 {
3894     [self.launch addEvent:@"changes-fetched"];
3895
3896     [self dispatchSyncWithAccountKeys:^bool{
3897         // This is a resync if we already have a change token, but this fetch didn't have one
3898         CKKSZoneStateEntry* ckse = [CKKSZoneStateEntry state: self.zoneName];
3899         bool resync = ckse.changeToken && (oldChangeToken == nil);
3900
3901         for (CKRecord* record in changedRecords) {
3902             [self _onqueueCKRecordChanged:record resync:resync];
3903         }
3904
3905         for (CKKSCloudKitDeletion* deletion in deletedRecords) {
3906             [self _onqueueCKRecordDeleted:deletion.recordID recordType:deletion.recordType resync:resync];
3907         }
3908
3909         NSError* error = nil;
3910         if(resync) {
3911             // Scan through all CKMirrorEntries and determine if any exist that CloudKit didn't tell us about
3912             ckksnotice("ckksresync", self, "Comparing local UUIDs against the CloudKit list");
3913             NSMutableArray<NSString*>* uuids = [[CKKSMirrorEntry allUUIDs:self.zoneID error:&error] mutableCopy];
3914
3915             for(NSString* uuid in uuids) {
3916                 CKRecord* record = nil;
3917                 CKRecordID* recordID = [[CKRecordID alloc] initWithRecordName:uuid zoneID:self.zoneID];
3918                 for(CKRecord* r in changedRecords) {
3919                     if([r.recordID isEqual:recordID]) {
3920                         record = r;
3921                         break;
3922                     }
3923                 }
3924
3925                 if(record) {
3926                     ckksnotice("ckksresync", self, "UUID %@ is still in CloudKit; carry on.", uuid);
3927                 } else {
3928                     CKKSMirrorEntry* ckme = [CKKSMirrorEntry tryFromDatabase:uuid zoneID:self.zoneID error:&error];
3929                     if(error != nil) {
3930                         ckkserror("ckksresync", self, "Couldn't read an item from the database, but it used to be there: %@ %@", uuid, error);
3931                         continue;
3932                     }
3933                     if(!ckme) {
3934                         ckkserror("ckksresync", self, "Couldn't read ckme(%@) from database; continuing", uuid);
3935                         continue;
3936                     }
3937
3938                     ckkserror("ckksresync", self, "BUG: Local item %@ not found in CloudKit, deleting", uuid);
3939                     [self _onqueueCKRecordDeleted:ckme.item.storedCKRecord.recordID recordType:ckme.item.storedCKRecord.recordType resync:resync];
3940                 }
3941             }
3942         }
3943
3944         error = nil;
3945
3946         CKKSZoneStateEntry* state = [CKKSZoneStateEntry state:self.zoneName];
3947         state.lastFetchTime = [NSDate date]; // The last fetch happened right now!
3948         state.changeToken = newChangeToken;
3949         [state saveToDatabase:&error];
3950         if(error) {
3951             ckkserror("ckksfetch", self, "Couldn't save new server change token: %@", error);
3952         }
3953
3954         // Might as well kick off a IQO!
3955         [self processIncomingQueue:false];
3956
3957         ckksnotice("ckksfetch", self, "Finished processing changes for %@", self.zoneID);
3958
3959         return true;
3960     }];
3961 }
3962
3963 - (bool)ckErrorOrPartialError:(NSError *)error isError:(CKErrorCode)errorCode
3964 {
3965     if((error.code == errorCode) && [error.domain isEqualToString:CKErrorDomain]) {
3966         return true;
3967     } else if((error.code == CKErrorPartialFailure) && [error.domain isEqualToString:CKErrorDomain]) {
3968         NSDictionary* partialErrors = error.userInfo[CKPartialErrorsByItemIDKey];
3969
3970         NSError* partialError = partialErrors[self.zoneID];
3971         if ((partialError.code == errorCode) && [partialError.domain isEqualToString:CKErrorDomain]) {
3972             return true;
3973         }
3974     }
3975     return false;
3976 }
3977
3978 - (bool)shouldRetryAfterFetchError:(NSError*)error {
3979
3980     bool isChangeTokenExpiredError = [self ckErrorOrPartialError:error isError:CKErrorChangeTokenExpired];
3981     if(isChangeTokenExpiredError) {
3982         ckkserror("ckks", self, "Received notice that our change token is out of date (for %@). Resetting local data...", self.zoneID);
3983
3984         // This is a bit scary: we might confuse some poor key hierarchy state machine operation. But, if a key state machine
3985         // operation is waiting for a successful fetch, we need to do this reset
3986         [self dispatchSyncWithAccountKeys:^bool{
3987             NSError* error = nil;
3988             [self _onqueueResetLocalData:&error];
3989
3990             if(error) {
3991                 ckksnotice("ckksreset", self, "CloudKit-inspired local reset of %@ ended with error: %@", self.zoneID, error);
3992             } else {
3993                 ckksnotice("ckksreset", self, "CloudKit-inspired local reset of %@ ended successfully", self.zoneID);
3994             }
3995
3996             // If we're in the middle of a fetch for the key state, then the retried fetch (which should succeed) will be sufficient to progress
3997             // Otherwise, we need to poke the key hierarchy state machine: all of its data is gone
3998             if(![self.keyHierarchyState isEqualToString:SecCKKSZoneKeyStateFetch]) {
3999                 [self _onqueueKeyStateMachineRequestFetch];
4000             }
4001
4002             return true;
4003         }];
4004
4005         return true;
4006     }
4007
4008     bool isDeletedZoneError = [self ckErrorOrPartialError:error isError:CKErrorZoneNotFound];
4009     if(isDeletedZoneError) {
4010         ckkserror("ckks", self, "Received notice that our zone(%@) does not exist. Resetting local data.", self.zoneID);
4011
4012         /*
4013          * If someone delete our zone, lets just start over from the begining
4014          */
4015         [self dispatchSync: ^bool{
4016             NSError* resetError = nil;
4017
4018             [self _onqueueResetLocalData: &resetError];
4019             if(resetError) {
4020                 ckksnotice("ckksreset", self, "CloudKit-inspired local reset of %@ ended with error: %@", self.zoneID, resetError);
4021             } else {
4022                 ckksnotice("ckksreset", self, "CloudKit-inspired local reset of %@ ended successfully", self.zoneID);
4023             }
4024
4025             [self _onqueueAdvanceKeyStateMachineToState:SecCKKSZoneKeyStateInitializing withError:nil];
4026             return true;
4027         }];
4028
4029         return false;
4030     }
4031
4032     if([error.domain isEqualToString:CKErrorDomain] && (error.code == CKErrorBadContainer)) {
4033         ckkserror("ckks", self, "Received notice that our container does not exist. Nothing to do.");
4034         return false;
4035     }
4036
4037     return true;
4038 }
4039
4040 #pragma mark CKKSPeerUpdateListener
4041
4042 - (void)selfPeerChanged:(id<CKKSPeerProvider>)provider
4043 {
4044     // Currently, we have no idea what to do with this. Kick off a key reprocess?
4045     ckkserror("ckks", self, "Received update that our self identity has changed");
4046     [self keyStateMachineRequestProcess];
4047 }
4048
4049 - (void)trustedPeerSetChanged:(id<CKKSPeerProvider>)provider
4050 {
4051     // We might need to share the TLK to some new people, or we might now trust the TLKs we have.
4052     // The key state machine should handle that, so poke it.
4053     ckkserror("ckks", self, "Received update that the trust set has changed");
4054
4055     self.trustedPeersSetChanged = true;
4056     [self.pokeKeyStateMachineScheduler trigger];
4057 }
4058
4059 #pragma mark - Test Support
4060
4061 - (bool) outgoingQueueEmpty: (NSError * __autoreleasing *) error {
4062     __block bool ret = false;
4063     [self dispatchSync: ^bool{
4064         NSArray* queueEntries = [CKKSOutgoingQueueEntry all: error];
4065         ret = queueEntries && ([queueEntries count] == 0);
4066         return true;
4067     }];
4068
4069     return ret;
4070 }
4071
4072 - (CKKSResultOperation*)waitForFetchAndIncomingQueueProcessing {
4073     CKKSResultOperation* op = [self fetchAndProcessCKChanges:CKKSFetchBecauseTesting];
4074     [op waitUntilFinished];
4075     return op;
4076 }
4077
4078 - (void)waitForKeyHierarchyReadiness {
4079     if(self.keyStateReadyDependency) {
4080         [self.keyStateReadyDependency waitUntilFinished];
4081     }
4082 }
4083
4084 - (void)cancelPendingOperations {
4085     @synchronized(self.outgoingQueueOperations) {
4086         for(NSOperation* op in self.outgoingQueueOperations) {
4087             [op cancel];
4088         }
4089         [self.outgoingQueueOperations removeAllObjects];
4090     }
4091
4092     @synchronized(self.incomingQueueOperations) {
4093         for(NSOperation* op in self.incomingQueueOperations) {
4094             [op cancel];
4095         }
4096         [self.incomingQueueOperations removeAllObjects];
4097     }
4098
4099     [super cancelAllOperations];
4100 }
4101
4102 - (void)cancelAllOperations {
4103     [self.zoneSetupOperation cancel];
4104     [self.keyStateMachineOperation cancel];
4105     [self.keyStateReadyDependency cancel];
4106     [self.keyStateNonTransientDependency cancel];
4107     [self.zoneChangeFetcher cancel];
4108     [self.notifyViewChangedScheduler cancel];
4109
4110     [self cancelPendingOperations];
4111
4112     [self dispatchSync:^bool{
4113         [self _onqueueAdvanceKeyStateMachineToState: SecCKKSZoneKeyStateCancelled withError: nil];
4114         return true;
4115     }];
4116 }
4117
4118 - (void)halt {
4119     [super halt];
4120
4121     // Don't send any more notifications, either
4122     _notifierClass = nil;
4123 }
4124
4125 - (NSDictionary*)status {
4126 #define stringify(obj) CKKSNilToNSNull([obj description])
4127 #define boolstr(obj) (!!(obj) ? @"yes" : @"no")
4128     __block NSMutableDictionary* ret = nil;
4129     __block NSError* error = nil;
4130     CKKSManifest* manifest = nil;
4131
4132     ret = [[self fastStatus] mutableCopy];
4133
4134     manifest = [CKKSManifest latestTrustedManifestForZone:self.zoneName error:&error];
4135     [self dispatchSync: ^bool {
4136
4137         CKKSCurrentKeySet* keyset = [CKKSCurrentKeySet loadForZone:self.zoneID];
4138         if(keyset.error) {
4139             error = keyset.error;
4140         }
4141
4142         NSString* manifestGeneration = manifest ? [NSString stringWithFormat:@"%lu", (unsigned long)manifest.generationCount] : nil;
4143
4144         if(error) {
4145             ckkserror("ckks", self, "error during status: %@", error);
4146         }
4147         // We actually don't care about this error, especially if it's "no current key pointers"...
4148         error = nil;
4149
4150         // Map deviceStates to strings to avoid NSXPC issues. Obj-c, why is this so hard?
4151         NSArray* deviceStates = [CKKSDeviceStateEntry allInZone:self.zoneID error:&error];
4152         NSMutableArray<NSString*>* mutDeviceStates = [[NSMutableArray alloc] init];
4153         [deviceStates enumerateObjectsUsingBlock:^(id  _Nonnull obj, NSUInteger idx, BOOL * _Nonnull stop) {
4154             [mutDeviceStates addObject: [obj description]];
4155         }];
4156
4157         NSArray* tlkShares = [CKKSTLKShareRecord allForUUID:keyset.currentTLKPointer.currentKeyUUID zoneID:self.zoneID error:&error];
4158         NSMutableArray<NSString*>* mutTLKShares = [[NSMutableArray alloc] init];
4159         [tlkShares enumerateObjectsUsingBlock:^(id  _Nonnull obj, NSUInteger idx, BOOL * _Nonnull stop) {
4160             [mutTLKShares addObject: [obj description]];
4161         }];
4162
4163         [ret addEntriesFromDictionary:@{
4164                  @"statusError":         stringify(error),
4165                  @"oqe":                 CKKSNilToNSNull([CKKSOutgoingQueueEntry countsByStateInZone:self.zoneID error:&error]),
4166                  @"iqe":                 CKKSNilToNSNull([CKKSIncomingQueueEntry countsByStateInZone:self.zoneID error:&error]),
4167                  @"ckmirror":            CKKSNilToNSNull([CKKSMirrorEntry        countsByParentKey:self.zoneID error:&error]),
4168                  @"devicestates":        CKKSNilToNSNull(mutDeviceStates),
4169                  @"tlkshares":           CKKSNilToNSNull(mutTLKShares),
4170                  @"keys":                CKKSNilToNSNull([CKKSKey countsByClass:self.zoneID error:&error]),
4171                  @"currentTLK":          CKKSNilToNSNull(keyset.tlk.uuid),
4172                  @"currentClassA":       CKKSNilToNSNull(keyset.classA.uuid),
4173                  @"currentClassC":       CKKSNilToNSNull(keyset.classC.uuid),
4174                  @"currentTLKPtr":       CKKSNilToNSNull(keyset.currentTLKPointer.currentKeyUUID),
4175                  @"currentClassAPtr":    CKKSNilToNSNull(keyset.currentClassAPointer.currentKeyUUID),
4176                  @"currentClassCPtr":    CKKSNilToNSNull(keyset.currentClassCPointer.currentKeyUUID),
4177                  @"currentManifestGen":  CKKSNilToNSNull(manifestGeneration),
4178             }];
4179         return false;
4180     }];
4181     return ret;
4182 }
4183
4184 - (NSDictionary*)fastStatus {
4185
4186     __block NSDictionary* ret = nil;
4187
4188     [self dispatchSync: ^bool {
4189
4190         ret = @{
4191             @"view":                CKKSNilToNSNull(self.zoneName),
4192             @"ckaccountstatus":     self.accountStatus == CKAccountStatusCouldNotDetermine ? @"could not determine" :
4193                 self.accountStatus == CKAccountStatusAvailable         ? @"logged in" :
4194                 self.accountStatus == CKAccountStatusRestricted        ? @"restricted" :
4195                 self.accountStatus == CKAccountStatusNoAccount         ? @"logged out" : @"unknown",
4196             @"accounttracker":      stringify(self.accountTracker),
4197             @"fetcher":             stringify(self.zoneChangeFetcher),
4198             @"zoneCreated":         boolstr(self.zoneCreated),
4199             @"zoneCreatedError":    stringify(self.zoneCreatedError),
4200             @"zoneSubscribed":      boolstr(self.zoneSubscribed),
4201             @"zoneSubscribedError": stringify(self.zoneSubscribedError),
4202             @"keystate":            CKKSNilToNSNull(self.keyHierarchyState),
4203             @"keyStateError":       stringify(self.keyHierarchyError),
4204             @"statusError":         [NSNull null],
4205             @"launchSequence":      CKKSNilToNSNull([self.launch eventsByTime]),
4206
4207             @"zoneSetupOperation":                 stringify(self.zoneSetupOperation),
4208             @"keyStateOperation":                  stringify(self.keyStateMachineOperation),
4209             @"lastIncomingQueueOperation":         stringify(self.lastIncomingQueueOperation),
4210             @"lastNewTLKOperation":                stringify(self.lastNewTLKOperation),
4211             @"lastOutgoingQueueOperation":         stringify(self.lastOutgoingQueueOperation),
4212             @"lastProcessReceivedKeysOperation":   stringify(self.lastProcessReceivedKeysOperation),
4213             @"lastReencryptOutgoingItemsOperation":stringify(self.lastReencryptOutgoingItemsOperation),
4214             @"lastScanLocalItemsOperation":        stringify(self.lastScanLocalItemsOperation),
4215         };
4216         return false;
4217     }];
4218
4219     return ret;
4220 }
4221
4222 #endif /* OCTAGON */
4223 @end