keychain/ckks/CKKSKeychainView.m

   1 /*
   2  * Copyright (c) 2016 Apple Inc. All Rights Reserved.
   3  *
   4  * @APPLE_LICENSE_HEADER_START@
   5  *
   6  * This file contains Original Code and/or Modifications of Original Code
   7  * as defined in and that are subject to the Apple Public Source License
   8  * Version 2.0 (the 'License'). You may not use this file except in
   9  * compliance with the License. Please obtain a copy of the License at
  10  * http://www.opensource.apple.com/apsl/ and read it before using this
  11  * file.
  12  *
  13  * The Original Code and all software distributed under the License are
  14  * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  15  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  16  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
  17  * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
  18  * Please see the License for the specific language governing rights and
  19  * limitations under the License.
  20  *
  21  * @APPLE_LICENSE_HEADER_END@
  22  */
  23
  24 #import "CKKSKeychainView.h"
  25
  26
  27
  28 #if OCTAGON
  29 #import "CloudKitDependencies.h"
  30 #import <CloudKit/CloudKit.h>
  31 #import <CloudKit/CloudKit_Private.h>
  32 #endif
  33
  34 #import "CKKS.h"
  35 #import "keychain/ckks/CKKSStates.h"
  36 #import "OctagonAPSReceiver.h"
  37 #import "CKKSIncomingQueueEntry.h"
  38 #import "CKKSOutgoingQueueEntry.h"
  39 #import "CKKSCurrentKeyPointer.h"
  40 #import "CKKSKey.h"
  41 #import "CKKSMirrorEntry.h"
  42 #import "CKKSZoneStateEntry.h"
  43 #import "CKKSItemEncrypter.h"
  44 #import "CKKSIncomingQueueOperation.h"
  45 #import "CKKSNewTLKOperation.h"
  46 #import "CKKSProcessReceivedKeysOperation.h"
  47 #import "CKKSFetchAllRecordZoneChangesOperation.h"
  48 #import "keychain/ckks/CKKSHealKeyHierarchyOperation.h"
  49 #import "CKKSReencryptOutgoingItemsOperation.h"
  50 #import "CKKSScanLocalItemsOperation.h"
  51 #import "CKKSSynchronizeOperation.h"
  52 #import "CKKSRateLimiter.h"
  53 #import "CKKSManifest.h"
  54 #import "CKKSManifestLeafRecord.h"
  55 #import "CKKSZoneChangeFetcher.h"
  56 #import "CKKSAnalytics.h"
  57 #import "keychain/analytics/CKKSLaunchSequence.h"
  58 #import "keychain/ckks/CKKSCloudKitClassDependencies.h"
  59 #import "keychain/ckks/CKKSDeviceStateEntry.h"
  60 #import "keychain/ckks/CKKSNearFutureScheduler.h"
  61 #import "keychain/ckks/CKKSCurrentItemPointer.h"
  62 #import "keychain/ckks/CKKSCreateCKZoneOperation.h"
  63 #import "keychain/ckks/CKKSDeleteCKZoneOperation.h"
  64 #import "keychain/ckks/CKKSUpdateCurrentItemPointerOperation.h"
  65 #import "keychain/ckks/CKKSUpdateDeviceStateOperation.h"
  66 #import "keychain/ckks/CKKSNotifier.h"
  67 #import "keychain/ckks/CloudKitCategories.h"
  68 #import "keychain/ckks/CKKSTLKShareRecord.h"
  69 #import "keychain/ckks/CKKSHealTLKSharesOperation.h"
  70 #import "keychain/ckks/CKKSLocalSynchronizeOperation.h"
  71 #import "keychain/ckks/CKKSPeerProvider.h"
  72 #import "keychain/ckks/CKKSCheckKeyHierarchyOperation.h"
  73 #import "keychain/ckks/CKKSViewManager.h"
  74 #import "keychain/categories/NSError+UsefulConstructors.h"
  75
  76 #import "keychain/ckks/CKKSLocalResetOperation.h"
  77
  78 #import "keychain/ot/OTConstants.h"
  79 #import "keychain/ot/OTDefines.h"
  80 #import "keychain/ot/OctagonCKKSPeerAdapter.h"
  81 #import "keychain/ot/ObjCImprovements.h"
  82
  83 #include <utilities/SecCFWrappers.h>
  84 #include <utilities/SecTrace.h>
  85 #include <utilities/SecDb.h>
  86 #include "keychain/securityd/SecDbItem.h"
  87 #include "keychain/securityd/SecItemDb.h"
  88 #include "keychain/securityd/SecItemSchema.h"
  89 #include "keychain/securityd/SecItemServer.h"
  90 #include <Security/SecItemPriv.h>
  91 #include "keychain/SecureObjectSync/SOSAccountTransaction.h"
  92 #include <utilities/SecPLWrappers.h>
  93 #include <os/transaction_private.h>
  94
  95 #import "keychain/trust/TrustedPeers/TPSyncingPolicy.h"
  96 #import <Security/SecItemInternal.h>
  97
  98 #if OCTAGON
  99
 100 @interface CKKSKeychainView()
 101
 102 @property (readonly) Class<CKKSNotifier> notifierClass;
 103
 104 // Slows down all outgoing queue operations
 105 @property CKKSNearFutureScheduler* outgoingQueueOperationScheduler;
 106
 107 @property CKKSResultOperation* processIncomingQueueAfterNextUnlockOperation;
 108 @property CKKSResultOperation* resultsOfNextIncomingQueueOperationOperation;
 109
 110 // An extra queue for semaphore-waiting-based NSOperations
 111 @property NSOperationQueue* waitingQueue;
 112
 113 // Scratch space for resyncs
 114 @property (nullable) NSMutableSet<NSString*>* resyncRecordsSeen;
 115
 116
 117
 118 @property NSOperationQueue* operationQueue;
 119 @property CKKSResultOperation* accountLoggedInDependency;
 120 @property BOOL halted;
 121
 122 // Make these readwrite
 123 @property NSArray<CKKSPeerProviderState*>* currentTrustStates;
 124
 125 @property NSMutableSet<CKKSFetchBecause*>* currentFetchReasons;
 126 @end
 127 #endif
 128
 129 @implementation CKKSKeychainView
 130 #if OCTAGON
 131
 132 - (instancetype)initWithContainer:(CKContainer*)container
 133                          zoneName:(NSString*)zoneName
 134                    accountTracker:(CKKSAccountStateTracker*)accountTracker
 135                  lockStateTracker:(CKKSLockStateTracker*)lockStateTracker
 136               reachabilityTracker:(CKKSReachabilityTracker*)reachabilityTracker
 137                     changeFetcher:(CKKSZoneChangeFetcher*)fetcher
 138                      zoneModifier:(CKKSZoneModifier*)zoneModifier
 139                  savedTLKNotifier:(CKKSNearFutureScheduler*)savedTLKNotifier
 140         cloudKitClassDependencies:(CKKSCloudKitClassDependencies*)cloudKitClassDependencies
 141 {
 142
 143     if((self = [super init])) {
 144         WEAKIFY(self);
 145
 146         _container = container;
 147         _zoneName = zoneName;
 148         _accountTracker = accountTracker;
 149         _reachabilityTracker = reachabilityTracker;
 150         _cloudKitClassDependencies = cloudKitClassDependencies;
 151
 152         _halted = NO;
 153
 154         _database = [_container privateCloudDatabase];
 155         _zoneID = [[CKRecordZoneID alloc] initWithZoneName:zoneName ownerName:CKCurrentUserDefaultName];
 156
 157         _accountStatus = CKKSAccountStatusUnknown;
 158         _accountLoggedInDependency = [self createAccountLoggedInDependency:@"CloudKit account logged in."];
 159
 160         _queue = dispatch_queue_create([[NSString stringWithFormat:@"CKKSQueue.%@.zone.%@", container.containerIdentifier, zoneName] UTF8String], DISPATCH_QUEUE_SERIAL_WITH_AUTORELEASE_POOL);
 161         _operationQueue = [[NSOperationQueue alloc] init];
 162
 163
 164         _loggedIn = [[CKKSCondition alloc] init];
 165         _loggedOut = [[CKKSCondition alloc] init];
 166         _accountStateKnown = [[CKKSCondition alloc] init];
 167
 168         _initiatedLocalScan = NO;
 169
 170         _trustStatus = CKKSAccountStatusUnknown;
 171
 172         _incomingQueueOperations = [NSHashTable weakObjectsHashTable];
 173         _outgoingQueueOperations = [NSHashTable weakObjectsHashTable];
 174         _scanLocalItemsOperations = [NSHashTable weakObjectsHashTable];
 175
 176         _currentTrustStates = @[];
 177
 178         _currentFetchReasons = [NSMutableSet set];
 179
 180         _launch = [[CKKSLaunchSequence alloc] initWithRocketName:@"com.apple.security.ckks.launch"];
 181         [_launch addAttribute:@"view" value:zoneName];
 182
 183         _zoneChangeFetcher = fetcher;
 184         [fetcher registerClient:self];
 185
 186         _resyncRecordsSeen = nil;
 187
 188         _notifierClass = cloudKitClassDependencies.notifierClass;
 189         _notifyViewChangedScheduler = [[CKKSNearFutureScheduler alloc] initWithName:[NSString stringWithFormat: @"%@-notify-scheduler", self.zoneName]
 190                                                             initialDelay:250*NSEC_PER_MSEC
 191                                                          continuingDelay:1*NSEC_PER_SEC
 192                                                         keepProcessAlive:true
 193                                                           dependencyDescriptionCode:CKKSResultDescriptionPendingViewChangedScheduling
 194                                                                    block:^{
 195                                                                        STRONGIFY(self);
 196                                                                        [self.notifierClass post:[NSString stringWithFormat:@"com.apple.security.view-change.%@", self.zoneName]];
 197                                                                        [self.notifierClass post:[NSString stringWithUTF8String:kSecServerKeychainChangedNotification]];
 198
 199
 200                                                                        // Ugly, but: the Manatee and Engram views need to send a fake 'PCS' view change.
 201                                                                        // TODO: make this data-driven somehow
 202                                                                        if([self.zoneName isEqualToString:@"Manatee"] ||
 203                                                                           [self.zoneName isEqualToString:@"Engram"] ||
 204                                                                           [self.zoneName isEqualToString:@"ApplePay"] ||
 205                                                                           [self.zoneName isEqualToString:@"Home"] ||
 206                                                                           [self.zoneName isEqualToString:@"LimitedPeersAllowed"]) {
 207                                                                            [self.notifierClass post:@"com.apple.security.view-change.PCS"];
 208                                                                        }
 209                                                                    }];
 210
 211         _notifyViewReadyScheduler = [[CKKSNearFutureScheduler alloc] initWithName:[NSString stringWithFormat: @"%@-ready-scheduler", self.zoneName]
 212                                                                        initialDelay:250*NSEC_PER_MSEC
 213                                                                     continuingDelay:120*NSEC_PER_SEC
 214                                                                    keepProcessAlive:true
 215                                                           dependencyDescriptionCode:CKKSResultDescriptionPendingViewChangedScheduling
 216                                                                               block:^{
 217                                                                                   STRONGIFY(self);
 218                                                                                   NSDistributedNotificationCenter *center = [self.cloudKitClassDependencies.nsdistributednotificationCenterClass defaultCenter];
 219
 220                                                                                   [center postNotificationName:@"com.apple.security.view-become-ready"
 221                                                                                                         object:nil
 222                                                                                                       userInfo:@{ @"view" : self.zoneName ?: @"unknown" }
 223                                                                                                        options:0];
 224                                                                               }];
 225
 226
 227         _lockStateTracker = lockStateTracker;
 228
 229         _stateMachine = [[OctagonStateMachine alloc] initWithName:[NSString stringWithFormat:@"ckks-%@", self.zoneName]
 230                                                            states:[NSSet setWithArray:[CKKSZoneKeyStateMap() allKeys]]
 231                                                             flags:CKKSAllStateFlags()
 232                                                      initialState:SecCKKSZoneKeyStateWaitForCloudKitAccountStatus
 233                                                             queue:self.queue
 234                                                       stateEngine:self
 235                                                  lockStateTracker:lockStateTracker];
 236         [_stateMachine startOperation];
 237
 238         _waitingQueue = [[NSOperationQueue alloc] init];
 239         _waitingQueue.maxConcurrentOperationCount = 5;
 240
 241         _keyStateReadyDependency = [self createKeyStateReadyDependency: @"Key state has become ready for the first time."];
 242
 243         dispatch_time_t initialOutgoingQueueDelay = SecCKKSReduceRateLimiting() ? NSEC_PER_MSEC * 200 : NSEC_PER_SEC * 1;
 244         dispatch_time_t continuingOutgoingQueueDelay = SecCKKSReduceRateLimiting() ? NSEC_PER_MSEC * 200 : NSEC_PER_SEC * 30;
 245         _outgoingQueueOperationScheduler = [[CKKSNearFutureScheduler alloc] initWithName:[NSString stringWithFormat: @"%@-outgoing-queue-scheduler", self.zoneName]
 246                                                                             initialDelay:initialOutgoingQueueDelay
 247                                                                          continuingDelay:continuingOutgoingQueueDelay
 248                                                                         keepProcessAlive:false
 249                                                                dependencyDescriptionCode:CKKSResultDescriptionPendingOutgoingQueueScheduling
 250                                                                                    block:^{}];
 251
 252         _operationDependencies = [[CKKSOperationDependencies alloc] initWithZoneID:self.zoneID
 253                                                                       zoneModifier:zoneModifier
 254                                                                   ckoperationGroup:nil
 255                                                                        flagHandler:_stateMachine
 256                                                                     launchSequence:_launch
 257                                                                   lockStateTracker:_lockStateTracker
 258                                                                reachabilityTracker:reachabilityTracker
 259                                                                      peerProviders:@[]
 260                                                                   databaseProvider:self
 261                                                         notifyViewChangedScheduler:_notifyViewChangedScheduler
 262                                                                   savedTLKNotifier:savedTLKNotifier];
 263     }
 264     return self;
 265 }
 266
 267 - (NSString*)description {
 268     return [NSString stringWithFormat:@"<%@: %@ (%@)>", NSStringFromClass([self class]), self.zoneName, self.keyHierarchyState];
 269 }
 270
 271 - (NSString*)debugDescription {
 272     return [NSString stringWithFormat:@"<%@: %@ (%@) %p>", NSStringFromClass([self class]), self.zoneName, self.keyHierarchyState, self];
 273 }
 274
 275 - (CKKSZoneKeyState*)keyHierarchyState {
 276     return self.stateMachine.currentState;
 277 }
 278
 279 - (NSMutableDictionary<CKKSZoneKeyState*, CKKSCondition*>*)keyHierarchyConditions
 280 {
 281     return self.stateMachine.stateConditions;
 282 }
 283
 284 - (void)ensureKeyStateReadyDependency:(NSString*)resetMessage {
 285     NSOperation* oldKSRD = self.keyStateReadyDependency;
 286     self.keyStateReadyDependency = [self createKeyStateReadyDependency:resetMessage];
 287     if(oldKSRD) {
 288         [oldKSRD addDependency:self.keyStateReadyDependency];
 289         [self.waitingQueue addOperation:oldKSRD];
 290     }
 291 }
 292
 293 - (CKKSResultOperation<OctagonStateTransitionOperationProtocol>*)performInitializedOperation
 294 {
 295     WEAKIFY(self);
 296     return [OctagonStateTransitionOperation named:@"ckks-initialized-operation"
 297                                         intending:SecCKKSZoneKeyStateBecomeReady
 298                                        errorState:SecCKKSZoneKeyStateError
 299                               withBlockTakingSelf:^(OctagonStateTransitionOperation * _Nonnull op) {
 300         STRONGIFY(self);
 301         [self dispatchSyncWithSQLTransaction:^CKKSDatabaseTransactionResult{
 302             CKKSOutgoingQueueOperation* outgoingOperation = nil;
 303             CKKSIncomingQueueOperation* initialProcess = nil;
 304             CKKSScanLocalItemsOperation* initialScan = nil;
 305
 306             CKKSZoneStateEntry* ckse = [CKKSZoneStateEntry state:self.zoneName];
 307
 308             // Check if we believe we've synced this zone before.
 309             if(ckse.changeToken == nil) {
 310                 self.operationDependencies.ckoperationGroup = [CKOperationGroup CKKSGroupWithName:@"initial-setup"];
 311
 312                 ckksnotice("ckks", self, "No existing change token; going to try to match local items with CloudKit ones.");
 313
 314                 // Onboard this keychain: there's likely items in it that we haven't synced yet.
 315                 // But, there might be items in The Cloud that correspond to these items, with UUIDs that we don't know yet.
 316                 // First, fetch all remote items.
 317
 318                 [self.currentFetchReasons addObject:CKKSFetchBecauseInitialStart];
 319                 op.nextState = SecCKKSZoneKeyStateBeginFetch;
 320
 321                 // Next, try to process them (replacing local entries). This will wait for the key state to be ready.
 322                 initialProcess = [self processIncomingQueue:true after:nil];
 323
 324                 // If all that succeeds, iterate through all keychain items and find the ones which need to be uploaded
 325                 initialScan = [self scanLocalItems:@"initial-scan-operation"
 326                                   ckoperationGroup:self.operationDependencies.ckoperationGroup
 327                                              after:initialProcess];
 328
 329             } else {
 330                 // Likely a restart of securityd!
 331
 332                 // Are there any fixups to run first?
 333                 self.lastFixupOperation = [CKKSFixups fixup:ckse.lastFixup for:self];
 334                 if(self.lastFixupOperation) {
 335                     ckksnotice("ckksfixup", self, "We have a fixup to perform: %@", self.lastFixupOperation);
 336                     [self scheduleOperation:self.lastFixupOperation];
 337                     op.nextState = SecCKKSZoneKeyStateWaitForFixupOperation;
 338                     return CKKSDatabaseTransactionCommit;
 339                 }
 340
 341                 // First off, are there any in-flight queue entries? If so, put them back into New.
 342                 // If they're truly in-flight, we'll "conflict" with ourselves, but that should be fine.
 343                 NSError* error = nil;
 344                 [self _onqueueResetAllInflightOQE:&error];
 345                 if(error) {
 346                     ckkserror("ckks", self, "Couldn't reset in-flight OQEs, bad behavior ahead: %@", error);
 347                 }
 348
 349                 // Are there any entries waiting for reencryption? If so, set the flag.
 350                 error = nil;
 351                 NSArray<CKKSOutgoingQueueEntry*>* reencryptOQEs = [CKKSOutgoingQueueEntry allInState:SecCKKSStateReencrypt
 352                                                                                               zoneID:self.zoneID
 353                                                                                                error:&error];
 354                 if(error) {
 355                     ckkserror("ckks", self, "Couldn't load reencrypt OQEs, bad behavior ahead: %@", error);
 356                 }
 357                 if(reencryptOQEs.count > 0) {
 358                     [self.stateMachine _onqueueHandleFlag:CKKSFlagItemReencryptionNeeded];
 359                 }
 360
 361                 self.operationDependencies.ckoperationGroup = [CKOperationGroup CKKSGroupWithName:@"restart-setup"];
 362
 363                 // If it's been more than 24 hours since the last fetch, fetch and process everything.
 364                 // Or, if we think we were interrupted in the middle of fetching, fetch some more.
 365                 // Otherwise, just kick off the local queue processing.
 366
 367                 NSDate* now = [NSDate date];
 368                 NSDateComponents* offset = [[NSDateComponents alloc] init];
 369                 [offset setHour:-24];
 370                 NSDate* deadline = [[NSCalendar currentCalendar] dateByAddingComponents:offset toDate:now options:0];
 371
 372                 if(ckse.lastFetchTime == nil ||
 373                    [ckse.lastFetchTime compare: deadline] == NSOrderedAscending ||
 374                    ckse.moreRecordsInCloudKit) {
 375
 376                     op.nextState = SecCKKSZoneKeyStateBeginFetch;
 377
 378                 } else {
 379                     // Check if we have an existing key hierarchy in keyset
 380                     CKKSCurrentKeySet* keyset = [CKKSCurrentKeySet loadForZone:self.zoneID];
 381                     if(keyset.error && !([keyset.error.domain isEqual: @"securityd"] && keyset.error.code == errSecItemNotFound)) {
 382                         ckkserror("ckkskey", self, "Error examining existing key hierarchy: %@", keyset.error);
 383                     }
 384
 385                     if(keyset.tlk && keyset.classA && keyset.classC && !keyset.error) {
 386                         // This is likely a restart of securityd, and we think we're ready. Double check.
 387                         op.nextState = SecCKKSZoneKeyStateBecomeReady;
 388
 389                     } else {
 390                         ckksnotice("ckkskey", self, "No existing key hierarchy for %@. Check if there's one in CloudKit...", self.zoneID.zoneName);
 391                         op.nextState = SecCKKSZoneKeyStateBeginFetch;
 392                     }
 393                 }
 394
 395                 if(ckse.lastLocalKeychainScanTime == nil || [ckse.lastLocalKeychainScanTime compare:deadline] == NSOrderedAscending) {
 396                     // TODO handle with a state flow
 397                     ckksnotice("ckksscan", self, "CKKS scan last occurred at %@; beginning a new one", ckse.lastLocalKeychainScanTime);
 398                     initialScan = [self scanLocalItems:ckse.lastLocalKeychainScanTime == nil ? @"initial-scan-operation" : @"24-hr-scan-operation"
 399                                       ckoperationGroup:self.operationDependencies.ckoperationGroup
 400                                                  after:nil];
 401                 }
 402
 403                 // Process outgoing queue after re-start
 404                 outgoingOperation = [self processOutgoingQueueAfter:nil ckoperationGroup:self.operationDependencies.ckoperationGroup];
 405             }
 406
 407             /*
 408              * Launch time is determined by when the zone have:
 409              *  1. keystate have become ready
 410              *  2. scan local items (if needed)
 411              *  3. processed all outgoing item (if needed)
 412              * TODO: this should move, once queue processing becomes part of the state machine
 413              */
 414
 415             WEAKIFY(self);
 416             NSBlockOperation *seemReady = [NSBlockOperation named:[NSString stringWithFormat:@"seemsReadyForSyncing-%@", self.zoneName] withBlock:^void{
 417                 STRONGIFY(self);
 418                 NSError *error = nil;
 419                 ckksnotice("launch", self, "Launch complete");
 420                 NSNumber *zoneSize = [CKKSMirrorEntry counts:self.zoneID error:&error];
 421                 if (zoneSize) {
 422                     zoneSize = @(SecBucket1Significant([zoneSize longValue]));
 423                     [self.launch addAttribute:@"zonesize" value:zoneSize];
 424                 }
 425                 [self.launch launch];
 426
 427                 /*
 428                  * Since we think we are ready, signal to CK that its to check for PCS identities again, and create the
 429                  * since before we completed this operation, we would probably have failed with a timeout because
 430                  * we where busy downloading items from CloudKit and then processing them.
 431                  */
 432                 [self.notifyViewReadyScheduler trigger];
 433             }];
 434
 435             [seemReady addNullableDependency:self.keyStateReadyDependency];
 436             [seemReady addNullableDependency:outgoingOperation];
 437             [seemReady addNullableDependency:initialScan];
 438             [seemReady addNullableDependency:initialProcess];
 439             [self scheduleOperation:seemReady];
 440
 441             return CKKSDatabaseTransactionCommit;
 442         }];
 443     }];
 444 }
 445
 446 - (CKKSResultOperation*)resetLocalData {
 447     ckksnotice("ckksreset", self, "Requesting local data reset");
 448
 449     return [self.stateMachine doWatchedStateMachineRPC:@"ckks-local-reset"
 450                                    sourceStates:[NSSet setWithArray:@[
 451                                        // TODO: possibly every state?
 452                                        SecCKKSZoneKeyStateReady,
 453                                        SecCKKSZoneKeyStateWaitForTLK,
 454                                        SecCKKSZoneKeyStateWaitForTrust,
 455                                        SecCKKSZoneKeyStateWaitForTLKUpload,
 456                                        SecCKKSZoneKeyStateLoggedOut,
 457                                    ]]
 458                                            path:[OctagonStateTransitionPath pathFromDictionary:@{
 459                                                SecCKKSZoneKeyStateResettingLocalData: @{
 460                                                    SecCKKSZoneKeyStateInitializing: @{
 461                                                        SecCKKSZoneKeyStateInitialized: [OctagonStateTransitionPathStep success],
 462                                                        SecCKKSZoneKeyStateLoggedOut: [OctagonStateTransitionPathStep success],
 463                                                    }
 464                                                }
 465                                            }]
 466                                                  reply:^(NSError * _Nonnull error) {}];
 467 }
 468
 469 - (CKKSResultOperation*)resetCloudKitZone:(CKOperationGroup*)operationGroup
 470 {
 471     [self.accountStateKnown wait:(SecCKKSTestsEnabled() ? 1*NSEC_PER_SEC : 10*NSEC_PER_SEC)];
 472
 473     // Not overly thread-safe, but a single read is okay
 474     if(self.accountStatus != CKKSAccountStatusAvailable) {
 475         // No CK account? goodbye!
 476         ckksnotice("ckksreset", self, "Requesting reset of CK zone, but no CK account exists");
 477         CKKSResultOperation* errorOp = [CKKSResultOperation named:@"fail" withBlockTakingSelf:^(CKKSResultOperation * _Nonnull op) {
 478             op.error = [NSError errorWithDomain:CKKSErrorDomain
 479                                           code:CKKSNotLoggedIn
 480                                    description:@"User is not signed into iCloud."];
 481         }];
 482
 483         [self scheduleOperationWithoutDependencies:errorOp];
 484         return errorOp;
 485     }
 486
 487     ckksnotice("ckksreset", self, "Requesting reset of CK zone (logged in)");
 488
 489     NSDictionary* localResetPath = @{
 490         SecCKKSZoneKeyStateInitializing: @{
 491             SecCKKSZoneKeyStateInitialized: [OctagonStateTransitionPathStep success],
 492             SecCKKSZoneKeyStateLoggedOut: [OctagonStateTransitionPathStep success],
 493         },
 494     };
 495
 496     // If the zone delete doesn't work, try it up to two more times
 497
 498     return [self.stateMachine doWatchedStateMachineRPC:@"ckks-cloud-reset"
 499                                           sourceStates:[NSSet setWithArray:@[
 500                                               // TODO: possibly every state?
 501                                               SecCKKSZoneKeyStateReady,
 502                                               SecCKKSZoneKeyStateInitialized,
 503                                               SecCKKSZoneKeyStateFetchComplete,
 504                                               SecCKKSZoneKeyStateWaitForTLK,
 505                                               SecCKKSZoneKeyStateWaitForTrust,
 506                                               SecCKKSZoneKeyStateWaitForTLKUpload,
 507                                               SecCKKSZoneKeyStateLoggedOut,
 508                                           ]]
 509                                                   path:[OctagonStateTransitionPath pathFromDictionary:@{
 510                                                       SecCKKSZoneKeyStateResettingZone: @{
 511                                                           SecCKKSZoneKeyStateResettingLocalData: localResetPath,
 512                                                           SecCKKSZoneKeyStateResettingZone: @{
 513                                                               SecCKKSZoneKeyStateResettingLocalData: localResetPath,
 514                                                               SecCKKSZoneKeyStateResettingZone: @{
 515                                                                  SecCKKSZoneKeyStateResettingLocalData: localResetPath,
 516                                                               }
 517                                                           }
 518                                                       }
 519                                                   }]
 520                                                  reply:^(NSError * _Nonnull error) {}];
 521 }
 522
 523 - (void)keyStateMachineRequestProcess {
 524     [self.stateMachine handleFlag:CKKSFlagKeyStateProcessRequested];
 525 }
 526
 527 - (CKKSResultOperation*)createKeyStateReadyDependency:(NSString*)message {
 528     WEAKIFY(self);
 529     CKKSResultOperation* keyStateReadyDependency = [CKKSResultOperation operationWithBlock:^{
 530         STRONGIFY(self);
 531         ckksnotice("ckkskey", self, "CKKS became ready: %@", message);
 532     }];
 533     keyStateReadyDependency.name = [NSString stringWithFormat: @"%@-key-state-ready", self.zoneName];
 534     keyStateReadyDependency.descriptionErrorCode = CKKSResultDescriptionPendingKeyReady;
 535     return keyStateReadyDependency;
 536 }
 537
 538 - (void)_onqueuePokeKeyStateMachine
 539 {
 540     dispatch_assert_queue(self.queue);
 541     [self.stateMachine _onqueuePokeStateMachine];
 542 }
 543
 544 - (CKKSResultOperation<OctagonStateTransitionOperationProtocol>* _Nullable)_onqueueNextStateMachineTransition:(OctagonState*)currentState
 545                                                                                                         flags:(OctagonFlags*)flags
 546                                                                                                  pendingFlags:(id<OctagonStateOnqueuePendingFlagHandler>)pendingFlagHandler
 547 {
 548     dispatch_assert_queue(self.queue);
 549
 550     // Resetting back to 'loggedout' takes all precedence.
 551     if([flags _onqueueContains:CKKSFlagCloudKitLoggedOut]) {
 552         [flags _onqueueRemoveFlag:CKKSFlagCloudKitLoggedOut];
 553         ckksnotice("ckkskey", self, "CK account is not present");
 554
 555         [self ensureKeyStateReadyDependency:@"cloudkit-account-not-present"];
 556         return [[CKKSLocalResetOperation alloc] initWithDependencies:self.operationDependencies
 557                                                        intendedState:SecCKKSZoneKeyStateLoggedOut
 558                                                           errorState:SecCKKSZoneKeyStateError];
 559     }
 560
 561     if([flags _onqueueContains:CKKSFlagCloudKitZoneMissing]) {
 562         [flags _onqueueRemoveFlag:CKKSFlagCloudKitZoneMissing];
 563
 564         [self ensureKeyStateReadyDependency:@"cloudkit-zone-missing"];
 565         // The zone is gone! Let's reset our local state, which will feed into recreating the zone
 566         return [OctagonStateTransitionOperation named:@"ck-zone-missing"
 567                                              entering:SecCKKSZoneKeyStateResettingLocalData];
 568     }
 569
 570     if([flags _onqueueContains:CKKSFlagChangeTokenExpired]) {
 571         [flags _onqueueRemoveFlag:CKKSFlagChangeTokenExpired];
 572
 573         [self ensureKeyStateReadyDependency:@"cloudkit-change-token-expired"];
 574         // Our change token is invalid! We'll have to refetch the world, so let's delete everything locally.
 575         return [OctagonStateTransitionOperation named:@"ck-token-expired"
 576                                              entering:SecCKKSZoneKeyStateResettingLocalData];
 577     }
 578
 579     if([currentState isEqualToString:SecCKKSZoneKeyStateLoggedOut]) {
 580         if([flags _onqueueContains:CKKSFlagCloudKitLoggedIn] || self.accountStatus == CKKSAccountStatusAvailable) {
 581             [flags _onqueueRemoveFlag:CKKSFlagCloudKitLoggedIn];
 582
 583             ckksnotice("ckkskey", self, "CloudKit account now present");
 584             return [OctagonStateTransitionOperation named:@"ck-sign-in"
 585                                                  entering:SecCKKSZoneKeyStateInitializing];
 586         }
 587
 588         if([flags _onqueueContains:CKKSFlag24hrNotification]) {
 589             [flags _onqueueRemoveFlag:CKKSFlag24hrNotification];
 590         }
 591         return nil;
 592     }
 593
 594     if([currentState isEqualToString: SecCKKSZoneKeyStateWaitForCloudKitAccountStatus]) {
 595         if([flags _onqueueContains:CKKSFlagCloudKitLoggedIn] || self.accountStatus == CKKSAccountStatusAvailable) {
 596             [flags _onqueueRemoveFlag:CKKSFlagCloudKitLoggedIn];
 597
 598             ckksnotice("ckkskey", self, "CloudKit account now present");
 599             return [OctagonStateTransitionOperation named:@"ck-sign-in"
 600                                                  entering:SecCKKSZoneKeyStateInitializing];
 601         }
 602
 603         if([flags _onqueueContains:CKKSFlagCloudKitLoggedOut]) {
 604             [flags _onqueueRemoveFlag:CKKSFlagCloudKitLoggedOut];
 605             ckksnotice("ckkskey", self, "No account available");
 606
 607             return [[CKKSLocalResetOperation alloc] initWithDependencies:self.operationDependencies
 608                                                            intendedState:SecCKKSZoneKeyStateLoggedOut
 609                                                               errorState:SecCKKSZoneKeyStateError];
 610         }
 611         return nil;
 612     }
 613
 614     [self.launch addEvent:currentState];
 615
 616     if([currentState isEqual:SecCKKSZoneKeyStateInitializing]) {
 617         if(self.accountStatus == CKKSAccountStatusNoAccount) {
 618             ckksnotice("ckkskey", self, "CloudKit account is missing. Departing!");
 619             return [[CKKSLocalResetOperation alloc] initWithDependencies:self.operationDependencies
 620                                                            intendedState:SecCKKSZoneKeyStateLoggedOut
 621                                                               errorState:SecCKKSZoneKeyStateError];
 622         }
 623
 624         // Begin zone creation, but rate-limit it
 625         CKKSCreateCKZoneOperation* pendingInitializeOp = [[CKKSCreateCKZoneOperation alloc] initWithDependencies:self.operationDependencies
 626                                                                                                    intendedState:SecCKKSZoneKeyStateInitialized
 627                                                                                                       errorState:SecCKKSZoneKeyStateZoneCreationFailed];
 628         [pendingInitializeOp addNullableDependency:self.operationDependencies.zoneModifier.cloudkitRetryAfter.operationDependency];
 629         [self.operationDependencies.zoneModifier.cloudkitRetryAfter trigger];
 630
 631         return pendingInitializeOp;
 632     }
 633
 634     if([currentState isEqualToString:SecCKKSZoneKeyStateWaitForFixupOperation]) {
 635         // TODO: fixup operations should become part of the state machine
 636         ckksnotice("ckkskey", self, "Waiting for the fixup operation: %@", self.lastFixupOperation);
 637         OctagonStateTransitionOperation* op = [OctagonStateTransitionOperation named:@"wait-for-fixup" entering:SecCKKSZoneKeyStateInitialized];
 638         [op addNullableDependency:self.lastFixupOperation];
 639         return op;
 640     }
 641
 642     if([currentState isEqualToString:SecCKKSZoneKeyStateInitialized]) {
 643         // We're initialized and CloudKit is ready. If we're trusted, see what needs done. Otherwise, wait.
 644         return [self performInitializedOperation];
 645     }
 646
 647     // In error? You probably aren't getting out.
 648     if([currentState isEqualToString:SecCKKSZoneKeyStateError]) {
 649         if([flags _onqueueContains:CKKSFlagCloudKitLoggedIn]) {
 650             [flags _onqueueRemoveFlag:CKKSFlagCloudKitLoggedIn];
 651
 652             // Worth one last shot. Reset everything locally, and try again.
 653             return [[CKKSLocalResetOperation alloc] initWithDependencies:self.operationDependencies
 654                                                            intendedState:SecCKKSZoneKeyStateInitializing
 655                                                               errorState:SecCKKSZoneKeyStateError];
 656         }
 657
 658         ckkserror("ckkskey", self, "Staying in error state %@", currentState);
 659         return nil;
 660     }
 661
 662     if([currentState isEqualToString:SecCKKSZoneKeyStateResettingZone]) {
 663         ckksnotice("ckkskey", self, "Deleting the CloudKit Zone");
 664
 665         [self ensureKeyStateReadyDependency:@"ck-zone-reset"];
 666         return [[CKKSDeleteCKZoneOperation alloc] initWithDependencies:self.operationDependencies
 667                                                          intendedState:SecCKKSZoneKeyStateResettingLocalData
 668                                                             errorState:SecCKKSZoneKeyStateResettingZone];
 669     }
 670
 671     if([currentState isEqualToString:SecCKKSZoneKeyStateResettingLocalData]) {
 672         ckksnotice("ckkskey", self, "Resetting local data");
 673
 674         [self ensureKeyStateReadyDependency:@"local-data-reset"];
 675         return [[CKKSLocalResetOperation alloc] initWithDependencies:self.operationDependencies
 676                                                        intendedState:SecCKKSZoneKeyStateInitializing
 677                                                           errorState:SecCKKSZoneKeyStateError];
 678     }
 679
 680     if([currentState isEqualToString:SecCKKSZoneKeyStateZoneCreationFailed]) {
 681         //Prepare to go back into initializing, as soon as the cloudkitRetryAfter is happy
 682         OctagonStateTransitionOperation* op = [OctagonStateTransitionOperation named:@"recover-from-cloudkit-failure" entering:SecCKKSZoneKeyStateInitializing];
 683
 684         [op addNullableDependency:self.operationDependencies.zoneModifier.cloudkitRetryAfter.operationDependency];
 685         [self.operationDependencies.zoneModifier.cloudkitRetryAfter trigger];
 686
 687         return op;
 688     }
 689
 690     if([currentState isEqualToString:SecCKKSZoneKeyStateLoseTrust]) {
 691         if([flags _onqueueContains:CKKSFlagBeginTrustedOperation]) {
 692             [flags _onqueueRemoveFlag:CKKSFlagBeginTrustedOperation];
 693             // This was likely a race between some operation and the beginTrustedOperation call! Skip changing state and try again.
 694             return [OctagonStateTransitionOperation named:@"begin-trusted-operation" entering:SecCKKSZoneKeyStateInitialized];
 695         }
 696
 697         // If our current state is "trusted", fall out
 698         if(self.trustStatus == CKKSAccountStatusAvailable) {
 699             self.trustStatus = CKKSAccountStatusUnknown;
 700         }
 701         return [OctagonStateTransitionOperation named:@"trust-loss" entering:SecCKKSZoneKeyStateWaitForTrust];
 702     }
 703
 704     if([currentState isEqualToString:SecCKKSZoneKeyStateWaitForTrust]) {
 705         if(self.trustStatus == CKKSAccountStatusAvailable) {
 706             ckksnotice("ckkskey", self, "Beginning trusted state machine operation");
 707             return [OctagonStateTransitionOperation named:@"begin-trusted-operation" entering:SecCKKSZoneKeyStateInitialized];
 708         }
 709
 710         if([flags _onqueueContains:CKKSFlagKeyStateProcessRequested]) {
 711             [flags _onqueueRemoveFlag:CKKSFlagKeyStateProcessRequested];
 712             return [OctagonStateTransitionOperation named:@"begin-trusted-operation" entering:SecCKKSZoneKeyStateProcess];
 713         }
 714
 715         if([flags _onqueueContains:CKKSFlag24hrNotification]) {
 716             [flags _onqueueRemoveFlag:CKKSFlag24hrNotification];
 717         }
 718
 719         return nil;
 720     }
 721
 722     if([currentState isEqualToString:SecCKKSZoneKeyStateBecomeReady]) {
 723         return [[CKKSCheckKeyHierarchyOperation alloc] initWithDependencies:self.operationDependencies
 724                                                               intendedState:SecCKKSZoneKeyStateReady
 725                                                                  errorState:SecCKKSZoneKeyStateError];
 726     }
 727
 728     if([currentState isEqualToString:SecCKKSZoneKeyStateReady]) {
 729         // If we're ready, we can ignore the begin trusted flag
 730         [flags _onqueueRemoveFlag:CKKSFlagBeginTrustedOperation];
 731
 732         if(self.keyStateFullRefetchRequested) {
 733             // In ready, but something has requested a full refetch.
 734             ckksnotice("ckkskey", self, "Kicking off a full key refetch based on request:%d", self.keyStateFullRefetchRequested);
 735             [self ensureKeyStateReadyDependency:@"key-state-full-refetch"];
 736             return [OctagonStateTransitionOperation named:@"full-refetch" entering:SecCKKSZoneKeyStateNeedFullRefetch];
 737         }
 738
 739         if([flags _onqueueContains:CKKSFlagFetchRequested]) {
 740             [flags _onqueueRemoveFlag:CKKSFlagFetchRequested];
 741             ckksnotice("ckkskey", self, "Kicking off a key refetch based on request");
 742             [self ensureKeyStateReadyDependency:@"key-state-fetch"];
 743             return [OctagonStateTransitionOperation named:@"fetch-requested" entering:SecCKKSZoneKeyStateBeginFetch];
 744         }
 745
 746         if([flags _onqueueContains:CKKSFlagKeyStateProcessRequested]) {
 747             [flags _onqueueRemoveFlag:CKKSFlagKeyStateProcessRequested];
 748             ckksnotice("ckkskey", self, "Kicking off a key reprocess based on request");
 749             [self ensureKeyStateReadyDependency:@"key-state-process"];
 750             return [OctagonStateTransitionOperation named:@"key-process" entering:SecCKKSZoneKeyStateProcess];
 751         }
 752
 753         if(self.trustStatus != CKKSAccountStatusAvailable) {
 754             ckksnotice("ckkskey", self, "In ready, but there's no trust; going into waitfortrust");
 755             [self ensureKeyStateReadyDependency:@"trust loss"];
 756             return [OctagonStateTransitionOperation named:@"trust-gone" entering:SecCKKSZoneKeyStateLoseTrust];
 757         }
 758
 759         if([flags _onqueueContains:CKKSFlagTrustedPeersSetChanged]) {
 760             [flags _onqueueRemoveFlag:CKKSFlagTrustedPeersSetChanged];
 761             ckksnotice("ckkskey", self, "Received a nudge that the trusted peers set might have changed! Reprocessing.");
 762             [self ensureKeyStateReadyDependency:@"Peer set changed"];
 763             return [OctagonStateTransitionOperation named:@"trusted-peers-changed" entering:SecCKKSZoneKeyStateProcess];
 764         }
 765
 766         if([flags _onqueueContains:CKKSFlag24hrNotification]) {
 767             [flags _onqueueRemoveFlag:CKKSFlag24hrNotification];
 768
 769             // We'd like to trigger our 24-hr backup fetch and scan.
 770             // That's currently part of the Initialized state, so head that way
 771             return [OctagonStateTransitionOperation named:@"24-hr-check" entering:SecCKKSZoneKeyStateInitialized];
 772         }
 773
 774         if([flags _onqueueContains:CKKSFlagItemReencryptionNeeded]) {
 775             [flags _onqueueRemoveFlag:CKKSFlagItemReencryptionNeeded];
 776
 777             // TODO: this should be part of the state machine
 778             CKKSReencryptOutgoingItemsOperation* op = [[CKKSReencryptOutgoingItemsOperation alloc] initWithDependencies:self.operationDependencies
 779                                                                                                                    ckks:self
 780                                                                                                           intendedState:SecCKKSZoneKeyStateReady
 781                                                                                                              errorState:SecCKKSZoneKeyStateError];
 782             [self scheduleOperation:op];
 783             // fall through.
 784         }
 785
 786         if([flags _onqueueContains:CKKSFlagProcessIncomingQueue]) {
 787             [flags _onqueueRemoveFlag:CKKSFlagProcessIncomingQueue];
 788             // TODO: this should be part of the state machine
 789
 790             [self processIncomingQueue:true];
 791             //return [OctagonStateTransitionOperation named:@"process-outgoing" entering:SecCKKSZoneKeyStateProcessIncomingQueue];
 792         }
 793
 794         if([flags _onqueueContains:CKKSFlagScanLocalItems]) {
 795             [flags _onqueueRemoveFlag:CKKSFlagScanLocalItems];
 796             ckksnotice("ckkskey", self, "Launching a scan operation to find dropped items");
 797
 798             // TODO: this should be a state flow
 799             [self scanLocalItems:@"per-request"];
 800             // fall through
 801         }
 802
 803         if([flags _onqueueContains:CKKSFlagProcessOutgoingQueue]) {
 804             [flags _onqueueRemoveFlag:CKKSFlagProcessOutgoingQueue];
 805
 806             [self processOutgoingQueue:nil];
 807             // TODO: this should be a state flow.
 808             //return [OctagonStateTransitionOperation named:@"process-outgoing" entering:SecCKKSZoneKeyStateProcessOutgoingQueue];
 809             // fall through
 810         }
 811
 812         // TODO: kick off a key roll if one has been requested
 813
 814
 815         // If we reach this point, we're in ready, and will stay there.
 816         // Tell the launch and the viewReadyScheduler about that.
 817
 818         [self.launch launch];
 819
 820         [[CKKSAnalytics logger] setDateProperty:[NSDate date] forKey:CKKSAnalyticsLastKeystateReady zoneName:self.zoneName];
 821         if(self.keyStateReadyDependency) {
 822             [self scheduleOperation:self.keyStateReadyDependency];
 823             self.keyStateReadyDependency = nil;
 824         }
 825
 826         return nil;
 827     }
 828
 829     if([currentState isEqualToString:SecCKKSZoneKeyStateReadyPendingUnlock]) {
 830         if([flags _onqueueContains:CKKSFlagDeviceUnlocked]) {
 831             [flags _onqueueRemoveFlag:CKKSFlagDeviceUnlocked];
 832             [self ensureKeyStateReadyDependency:@"Device unlocked"];
 833             return [OctagonStateTransitionOperation named:@"key-state-ready-after-unlock" entering:SecCKKSZoneKeyStateBecomeReady];
 834         }
 835
 836         if([flags _onqueueContains:CKKSFlagProcessOutgoingQueue]) {
 837             [flags _onqueueRemoveFlag:CKKSFlagProcessOutgoingQueue];
 838             [self processOutgoingQueue:nil];
 839             // TODO: this should become part of the key state hierarchy
 840         }
 841
 842         // Ready enough!
 843
 844         [[CKKSAnalytics logger] setDateProperty:[NSDate date] forKey:CKKSAnalyticsLastKeystateReady zoneName:self.zoneName];
 845         if(self.keyStateReadyDependency) {
 846             [self scheduleOperation:self.keyStateReadyDependency];
 847             self.keyStateReadyDependency = nil;
 848         }
 849
 850         OctagonPendingFlag* unlocked = [[OctagonPendingFlag alloc] initWithFlag:CKKSFlagDeviceUnlocked
 851                                                                      conditions:OctagonPendingConditionsDeviceUnlocked];
 852         [pendingFlagHandler _onqueueHandlePendingFlag:unlocked];
 853         return nil;
 854     }
 855
 856     if([currentState isEqualToString:SecCKKSZoneKeyStateBeginFetch]) {
 857         ckksnotice("ckkskey", self, "Starting a key hierarchy fetch");
 858         [flags _onqueueRemoveFlag:CKKSFlagFetchComplete];
 859
 860         WEAKIFY(self);
 861
 862         NSSet<CKKSFetchBecause*>* fetchReasons = self.currentFetchReasons ?
 863             [self.currentFetchReasons setByAddingObject:CKKSFetchBecauseKeyHierarchy] :
 864             [NSSet setWithObject:CKKSFetchBecauseKeyHierarchy];
 865
 866         CKKSResultOperation* fetchOp = [self.zoneChangeFetcher requestSuccessfulFetchForManyReasons:fetchReasons];
 867         CKKSResultOperation* flagOp = [CKKSResultOperation named:@"post-fetch"
 868                                                        withBlock:^{
 869             STRONGIFY(self);
 870             [self.stateMachine handleFlag:CKKSFlagFetchComplete];
 871         }];
 872         [flagOp addDependency:fetchOp];
 873         [self scheduleOperation:flagOp];
 874
 875         return [OctagonStateTransitionOperation named:@"waiting-for-fetch" entering:SecCKKSZoneKeyStateFetch];
 876     }
 877
 878     if([currentState isEqualToString:SecCKKSZoneKeyStateFetch]) {
 879         if([flags _onqueueContains:CKKSFlagFetchComplete]) {
 880             [flags _onqueueRemoveFlag:CKKSFlagFetchComplete];
 881             return [OctagonStateTransitionOperation named:@"fetch-complete" entering:SecCKKSZoneKeyStateFetchComplete];
 882         }
 883
 884         // The flags CKKSFlagCloudKitZoneMissing and CKKSFlagChangeTokenOutdated are both handled at the top of this function
 885         // So, we don't need to handle them here.
 886
 887         return nil;
 888     }
 889
 890     if([currentState isEqualToString:SecCKKSZoneKeyStateNeedFullRefetch]) {
 891          ckksnotice("ckkskey", self, "Starting a key hierarchy full refetch");
 892
 893          //TODO use states here instead of flags
 894          self.keyStateMachineRefetched = true;
 895          self.keyStateFullRefetchRequested = false;
 896
 897          return [OctagonStateTransitionOperation named:@"fetch-complete" entering:SecCKKSZoneKeyStateResettingLocalData];
 898     }
 899
 900     if([currentState isEqualToString:SecCKKSZoneKeyStateFetchComplete]) {
 901         [self.launch addEvent:@"fetch-complete"];
 902         [self.currentFetchReasons removeAllObjects];
 903
 904         return [OctagonStateTransitionOperation named:@"post-fetch-process" entering:SecCKKSZoneKeyStateProcess];
 905     }
 906
 907     if([currentState isEqualToString:SecCKKSZoneKeyStateWaitForTLKCreation]) {
 908         if([flags _onqueueContains:CKKSFlagKeyStateProcessRequested]) {
 909             [flags _onqueueRemoveFlag:CKKSFlagKeyStateProcessRequested];
 910             ckksnotice("ckkskey", self, "We believe we need to create TLKs but we also received a key nudge; moving to key state Process.");
 911             return [OctagonStateTransitionOperation named:@"wait-for-tlk-creation-process" entering:SecCKKSZoneKeyStateProcess];
 912
 913         } else if([flags _onqueueContains:CKKSFlagFetchRequested]) {
 914             [flags _onqueueRemoveFlag:CKKSFlagFetchRequested];
 915             return [OctagonStateTransitionOperation named:@"fetch-requested" entering:SecCKKSZoneKeyStateBeginFetch];
 916
 917         } else if([flags _onqueueContains:CKKSFlagTLKCreationRequested]) {
 918             [flags _onqueueRemoveFlag:CKKSFlagTLKCreationRequested];
 919
 920             // It's very likely that we're already untrusted at this point. But, sometimes we will be trusted right now, and can lose trust while waiting for the upload.
 921             // This probably should be handled by a state increase.
 922             [flags _onqueueRemoveFlag:CKKSFlagEndTrustedOperation];
 923
 924             ckksnotice("ckkskey", self, "TLK creation requested; kicking off operation");
 925             return [[CKKSNewTLKOperation alloc] initWithDependencies:self.operationDependencies
 926                                                                 ckks:self];
 927         } else if(self.lastNewTLKOperation.keyset) {
 928             // This means that we _have_ created new TLKs, and should wait for them to be uploaded. This is ugly and should probably be done with more states.
 929             return [OctagonStateTransitionOperation named:@"" entering:SecCKKSZoneKeyStateWaitForTLKUpload];
 930
 931         } else {
 932             ckksnotice("ckkskey", self, "We believe we need to create TLKs; waiting for Octagon (via %@)", self.suggestTLKUpload);
 933             [self.suggestTLKUpload trigger];
 934         }
 935     }
 936
 937     if([currentState isEqualToString:SecCKKSZoneKeyStateWaitForTLKUpload]) {
 938         ckksnotice("ckkskey", self, "We believe we have TLKs that need uploading");
 939
 940         if([flags _onqueueContains:CKKSFlagFetchRequested]) {
 941             ckksnotice("ckkskey", self, "Received a nudge to refetch CKKS");
 942             return [OctagonStateTransitionOperation named:@"tlk-upload-refetch" entering:SecCKKSZoneKeyStateBeginFetch];
 943         }
 944
 945         if([flags _onqueueContains:CKKSFlagKeyStateTLKsUploaded]) {
 946             [flags _onqueueRemoveFlag:CKKSFlagKeyStateTLKsUploaded];
 947
 948             return [OctagonStateTransitionOperation named:@"wait-for-tlk-upload-process" entering:SecCKKSZoneKeyStateProcess];
 949         }
 950
 951         if([flags _onqueueContains:CKKSFlagEndTrustedOperation]) {
 952             [flags _onqueueRemoveFlag:CKKSFlagEndTrustedOperation];
 953
 954             return [OctagonStateTransitionOperation named:@"trust-loss" entering:SecCKKSZoneKeyStateLoseTrust];
 955         }
 956
 957         if([flags _onqueueContains:CKKSFlagKeyStateProcessRequested]) {
 958             return [OctagonStateTransitionOperation named:@"wait-for-tlk-fetch-process" entering:SecCKKSZoneKeyStateProcess];
 959         }
 960
 961         // This is quite the hack, but it'll do for now.
 962         [self.operationDependencies provideKeySet:self.lastNewTLKOperation.keyset];
 963
 964         ckksnotice("ckkskey", self, "Notifying Octagon again, just in case");
 965         [self.suggestTLKUpload trigger];
 966     }
 967
 968     if([currentState isEqualToString:SecCKKSZoneKeyStateTLKMissing]) {
 969         return [self tlkMissingOperation:SecCKKSZoneKeyStateWaitForTLK];
 970     }
 971
 972     if([currentState isEqualToString:SecCKKSZoneKeyStateWaitForTLK]) {
 973         // We're in a hold state: waiting for the TLK bytes to arrive.
 974
 975         if([flags _onqueueContains:CKKSFlagKeyStateProcessRequested]) {
 976             [flags _onqueueRemoveFlag:CKKSFlagKeyStateProcessRequested];
 977             // Someone has requsted a reprocess! Go to the correct state.
 978             ckksnotice("ckkskey", self, "Received a nudge that our TLK might be here! Reprocessing.");
 979             return [OctagonStateTransitionOperation named:@"wait-for-tlk-process" entering:SecCKKSZoneKeyStateProcess];
 980
 981         } else if([flags _onqueueContains:CKKSFlagTrustedPeersSetChanged]) {
 982             [flags _onqueueRemoveFlag:CKKSFlagTrustedPeersSetChanged];
 983
 984             // Hmm, maybe this trust set change will cause us to recover this TLK (due to a previously-untrusted share becoming trusted). Worth a shot!
 985             ckksnotice("ckkskey", self, "Received a nudge that the trusted peers set might have changed! Reprocessing.");
 986             return [OctagonStateTransitionOperation named:@"wait-for-tlk-peers" entering:SecCKKSZoneKeyStateProcess];
 987         }
 988
 989         return nil;
 990     }
 991
 992     if([currentState isEqualToString:SecCKKSZoneKeyStateWaitForUnlock]) {
 993         ckksnotice("ckkskey", self, "Requested to enter waitforunlock");
 994
 995         if([flags _onqueueContains:CKKSFlagDeviceUnlocked ]) {
 996             [flags _onqueueRemoveFlag:CKKSFlagDeviceUnlocked];
 997             return [OctagonStateTransitionOperation named:@"key-state-after-unlock" entering:SecCKKSZoneKeyStateInitialized];
 998         }
 999
1000         OctagonPendingFlag* unlocked = [[OctagonPendingFlag alloc] initWithFlag:CKKSFlagDeviceUnlocked
1001                                                                      conditions:OctagonPendingConditionsDeviceUnlocked];
1002         [pendingFlagHandler _onqueueHandlePendingFlag:unlocked];
1003
1004         return nil;
1005     }
1006
1007     if([currentState isEqualToString:SecCKKSZoneKeyStateBadCurrentPointers]) {
1008         // The current key pointers are broken, but we're not sure why.
1009         ckksnotice("ckkskey", self, "Our current key pointers are reported broken. Attempting a fix!");
1010         return [[CKKSHealKeyHierarchyOperation alloc] initWithDependencies:self.operationDependencies
1011                                                                       ckks:self
1012                                                                  intending:SecCKKSZoneKeyStateBecomeReady
1013                                                                 errorState:SecCKKSZoneKeyStateError];
1014     }
1015
1016     if([currentState isEqualToString:SecCKKSZoneKeyStateNewTLKsFailed]) {
1017         ckksnotice("ckkskey", self, "Creating new TLKs didn't work. Attempting to refetch!");
1018         return [OctagonStateTransitionOperation named:@"new-tlks-failed" entering:SecCKKSZoneKeyStateBeginFetch];
1019     }
1020
1021     if([currentState isEqualToString:SecCKKSZoneKeyStateHealTLKSharesFailed]) {
1022         ckksnotice("ckkskey", self, "Creating new TLK shares didn't work. Attempting to refetch!");
1023         return [OctagonStateTransitionOperation named:@"heal-tlks-failed" entering:SecCKKSZoneKeyStateBeginFetch];
1024     }
1025
1026     if([currentState isEqualToString:SecCKKSZoneKeyStateUnhealthy]) {
1027         if(self.trustStatus != CKKSAccountStatusAvailable) {
1028             ckksnotice("ckkskey", self, "Looks like the key hierarchy is unhealthy, but we're untrusted.");
1029             return [OctagonStateTransitionOperation named:@"unhealthy-lacking-trust" entering:SecCKKSZoneKeyStateLoseTrust];
1030
1031         } else {
1032             ckksnotice("ckkskey", self, "Looks like the key hierarchy is unhealthy. Launching fix.");
1033             return [[CKKSHealKeyHierarchyOperation alloc] initWithDependencies:self.operationDependencies
1034                                                                           ckks:self
1035                                                                      intending:SecCKKSZoneKeyStateBecomeReady
1036                                                                     errorState:SecCKKSZoneKeyStateError];
1037         }
1038     }
1039
1040     if([currentState isEqualToString:SecCKKSZoneKeyStateHealTLKShares]) {
1041         ckksnotice("ckksshare", self, "Key hierarchy is okay, but not shared appropriately. Launching fix.");
1042         return [[CKKSHealTLKSharesOperation alloc] initWithOperationDependencies:self.operationDependencies
1043                                                                             ckks:self];
1044     }
1045
1046     if([currentState isEqualToString:SecCKKSZoneKeyStateProcess]) {
1047         [flags _onqueueRemoveFlag:CKKSFlagKeyStateProcessRequested];
1048
1049         ckksnotice("ckksshare", self, "Launching key state process");
1050         return [[CKKSProcessReceivedKeysOperation alloc] initWithDependencies:self.operationDependencies
1051                                                                 intendedState:SecCKKSZoneKeyStateBecomeReady
1052                                                                    errorState:SecCKKSZoneKeyStateError];
1053     }
1054
1055     return nil;
1056 }
1057
1058 - (OctagonStateTransitionOperation*)tlkMissingOperation:(CKKSZoneKeyState*)newState
1059 {
1060     WEAKIFY(self);
1061     return [OctagonStateTransitionOperation named:@"tlk-missing"
1062                                         intending:newState
1063                                        errorState:SecCKKSZoneKeyStateError
1064                               withBlockTakingSelf:^(OctagonStateTransitionOperation * _Nonnull op) {
1065         STRONGIFY(self);
1066
1067         NSArray<CKKSPeerProviderState*>* trustStates = self.operationDependencies.currentTrustStates;
1068
1069         [self.operationDependencies.databaseProvider dispatchSyncWithReadOnlySQLTransaction:^{
1070             CKKSCurrentKeySet* keyset = [CKKSCurrentKeySet loadForZone:self.zoneID];
1071
1072             if(keyset.error) {
1073                 ckkserror("ckkskey", self, "Unable to load keyset: %@", keyset.error);
1074                 op.nextState = newState;
1075
1076                 [self.operationDependencies provideKeySet:keyset];
1077                 return;
1078             }
1079
1080             if(!keyset.currentTLKPointer.currentKeyUUID) {
1081                 // In this case, there's no current TLK at all. Go into "wait for tlkcreation";
1082                 op.nextState = SecCKKSZoneKeyStateWaitForTLKCreation;
1083                 [self.operationDependencies provideKeySet:keyset];
1084                 return;
1085             }
1086
1087             if(self.trustStatus != CKKSAccountStatusAvailable) {
1088                 ckksnotice("ckkskey", self, "TLK is missing, but no trust is present.");
1089                 op.nextState = SecCKKSZoneKeyStateLoseTrust;
1090
1091                 [self.operationDependencies provideKeySet:keyset];
1092                 return;
1093             }
1094
1095             bool otherDevicesPresent = [self _onqueueOtherDevicesReportHavingTLKs:keyset
1096                                                                       trustStates:trustStates];
1097             if(otherDevicesPresent) {
1098                 // We expect this keyset to continue to exist. Send it to our listeners.
1099                 [self.operationDependencies provideKeySet:keyset];
1100
1101                 op.nextState = newState;
1102             } else {
1103                 ckksnotice("ckkskey", self, "No other devices claim to have the TLK. Resetting zone...");
1104                 op.nextState = SecCKKSZoneKeyStateResettingZone;
1105             }
1106             return;
1107         }];
1108     }];
1109 }
1110
1111 - (bool)_onqueueOtherDevicesReportHavingTLKs:(CKKSCurrentKeySet*)keyset
1112                                  trustStates:(NSArray<CKKSPeerProviderState*>*)trustStates
1113 {
1114     //Has there been any activity indicating that other trusted devices have keys in the past 45 days, or untrusted devices in the past 4?
1115     // (We chose 4 as devices attempt to upload their device state every 3 days. If a device is unceremoniously kicked out of circle, we normally won't immediately reset.)
1116     NSDate* now = [NSDate date];
1117     NSDateComponents* trustedOffset = [[NSDateComponents alloc] init];
1118     [trustedOffset setDay:-45];
1119     NSDate* trustedDeadline = [[NSCalendar currentCalendar] dateByAddingComponents:trustedOffset toDate:now options:0];
1120
1121     NSDateComponents* untrustedOffset = [[NSDateComponents alloc] init];
1122     [untrustedOffset setDay:-4];
1123     NSDate* untrustedDeadline = [[NSCalendar currentCalendar] dateByAddingComponents:untrustedOffset toDate:now options:0];
1124
1125
1126     NSMutableSet<NSString*>* trustedPeerIDs = [NSMutableSet set];
1127     for(CKKSPeerProviderState* trustState in trustStates) {
1128         for(id<CKKSPeer> peer in trustState.currentTrustedPeers) {
1129             [trustedPeerIDs addObject:peer.peerID];
1130         }
1131     }
1132
1133     NSError* localerror = nil;
1134
1135     NSArray<CKKSDeviceStateEntry*>* allDeviceStates = [CKKSDeviceStateEntry allInZone:keyset.currentTLKPointer.zoneID error:&localerror];
1136     if(localerror) {
1137         ckkserror("ckkskey", self, "Error fetching device states: %@", localerror);
1138         localerror = nil;
1139         return true;
1140     }
1141     for(CKKSDeviceStateEntry* device in allDeviceStates) {
1142         // The peerIDs in CDSEs aren't written with the peer prefix. Make sure we match both.
1143         NSString* sosPeerID = device.circlePeerID ? [CKKSSOSPeerPrefix stringByAppendingString:device.circlePeerID] : nil;
1144
1145         if([trustedPeerIDs containsObject:device.circlePeerID] ||
1146            [trustedPeerIDs containsObject:sosPeerID] ||
1147            [trustedPeerIDs containsObject:device.octagonPeerID]) {
1148             // Is this a recent DSE? If it's older than the deadline, skip it
1149             if([device.storedCKRecord.modificationDate compare:trustedDeadline] == NSOrderedAscending) {
1150                 ckksnotice("ckkskey", self, "Trusted device state (%@) is too old; ignoring", device);
1151                 continue;
1152             }
1153         } else {
1154             // Device is untrusted. How does it fare with the untrustedDeadline?
1155             if([device.storedCKRecord.modificationDate compare:untrustedDeadline] == NSOrderedAscending) {
1156                 ckksnotice("ckkskey", self, "Device (%@) is not trusted and from too long ago; ignoring device state (%@)", device.circlePeerID, device);
1157                 continue;
1158             } else {
1159                 ckksnotice("ckkskey", self, "Device (%@) is not trusted, but very recent. Including in heuristic: %@", device.circlePeerID, device);
1160             }
1161         }
1162
1163         if([device.keyState isEqualToString:SecCKKSZoneKeyStateReady] ||
1164            [device.keyState isEqualToString:SecCKKSZoneKeyStateReadyPendingUnlock]) {
1165             ckksnotice("ckkskey", self, "Other device (%@) has keys; it should send them to us", device);
1166             return true;
1167         }
1168     }
1169
1170     NSArray<CKKSTLKShareRecord*>* tlkShares = [CKKSTLKShareRecord allForUUID:keyset.currentTLKPointer.currentKeyUUID
1171                                                                       zoneID:keyset.currentTLKPointer.zoneID
1172                                                                        error:&localerror];
1173     if(localerror) {
1174         ckkserror("ckkskey", self, "Error fetching device states: %@", localerror);
1175         localerror = nil;
1176         return false;
1177     }
1178
1179     for(CKKSTLKShareRecord* tlkShare in tlkShares) {
1180         if([trustedPeerIDs containsObject:tlkShare.senderPeerID] &&
1181            [tlkShare.storedCKRecord.modificationDate compare:trustedDeadline] == NSOrderedDescending) {
1182             ckksnotice("ckkskey", self, "Trusted TLK Share (%@) created recently; other devices have keys and should send them to us", tlkShare);
1183             return true;
1184         }
1185     }
1186
1187     // Okay, how about the untrusted deadline?
1188     for(CKKSTLKShareRecord* tlkShare in tlkShares) {
1189         if([tlkShare.storedCKRecord.modificationDate compare:untrustedDeadline] == NSOrderedDescending) {
1190             ckksnotice("ckkskey", self, "Untrusted TLK Share (%@) created very recently; other devices might have keys and should rejoin the circle (and send them to us)", tlkShare);
1191             return true;
1192         }
1193     }
1194
1195     return false;
1196 }
1197
1198 - (void)handleKeychainEventDbConnection:(SecDbConnectionRef) dbconn
1199                                  source:(SecDbTransactionSource)txionSource
1200                                   added:(SecDbItemRef) added
1201                                 deleted:(SecDbItemRef) deleted
1202                             rateLimiter:(CKKSRateLimiter*) rateLimiter
1203 {
1204     if(!SecCKKSIsEnabled()) {
1205         ckksnotice("ckks", self, "Skipping handleKeychainEventDbConnection due to disabled CKKS");
1206         return;
1207     }
1208
1209     __block NSError* error = nil;
1210
1211     // Tombstones come in as item modifications or item adds. Handle modifications here.
1212     bool addedTombstone   = added   && SecDbItemIsTombstone(added);
1213     bool deletedTombstone = deleted && SecDbItemIsTombstone(deleted);
1214
1215     bool addedSync   = added   && SecDbItemIsSyncable(added);
1216     bool deletedSync = deleted && SecDbItemIsSyncable(deleted);
1217
1218     bool isTombstoneModification = addedTombstone && deletedTombstone;
1219     bool isAdd    = ( added && !deleted) || (added && deleted && !addedTombstone &&  deletedTombstone) || (added && deleted &&  addedSync && !deletedSync);
1220     bool isDelete = (!added &&  deleted) || (added && deleted &&  addedTombstone && !deletedTombstone) || (added && deleted && !addedSync &&  deletedSync);
1221     bool isModify = ( added &&  deleted) && (!isAdd) && (!isDelete);
1222
1223     // On an update that changes an item's primary key, SecDb modifies the existing item, then adds a new tombstone to replace the old primary key.
1224     // Therefore, we might receive an added tombstone here with no deleted item to accompany it. This should be considered a deletion.
1225     if(addedTombstone && !deleted) {
1226         isAdd = false;
1227         isDelete = true;
1228         isModify = false;
1229
1230         // Passed to withItem: below
1231         deleted = added;
1232     }
1233
1234     // If neither item is syncable, don't proceed further in the syncing system
1235     bool proceed = addedSync || deletedSync;
1236
1237     if(!proceed) {
1238         ckksnotice("ckks", self, "skipping sync of non-sync item (%d, %d)", addedSync, deletedSync);
1239         return;
1240     }
1241
1242     if(isTombstoneModification) {
1243         ckksnotice("ckks", self, "skipping syncing update of tombstone item (%d, %d)", addedTombstone, deletedTombstone);
1244         return;
1245     }
1246
1247     // It's possible to ask for an item to be deleted without adding a corresponding tombstone.
1248     // This is arguably a bug, as it generates an out-of-sync state, but it is in the API contract.
1249     // CKKS should ignore these, but log very upset messages.
1250     if(isDelete && !addedTombstone) {
1251         ckksnotice("ckks", self, "Client has asked for an item deletion to not sync. Keychain is now out of sync with account");
1252         return;
1253     }
1254
1255     // Only synchronize items which can transfer between devices
1256     NSString* protection = (__bridge NSString*)SecDbItemGetCachedValueWithName(added ? added : deleted, kSecAttrAccessible);
1257     if(! ([protection isEqualToString: (__bridge NSString*)kSecAttrAccessibleWhenUnlocked] ||
1258           [protection isEqualToString: (__bridge NSString*)kSecAttrAccessibleAfterFirstUnlock] ||
1259           [protection isEqualToString: (__bridge NSString*)kSecAttrAccessibleAlwaysPrivate])) {
1260         ckksnotice("ckks", self, "skipping sync of device-bound(%@) item", protection);
1261         return;
1262     }
1263
1264     if(txionSource == kSecDbSOSTransaction) {
1265         NSString* addedUUID = (__bridge NSString*)SecDbItemGetValue(added, &v10itemuuid, NULL);
1266         ckksnotice("ckks", self, "Received an incoming %@ from SOS (%@)",
1267                    isAdd ? @"addition" : (isModify ? @"modification" : @"deletion"),
1268                    addedUUID);
1269     }
1270
1271     // Our caller gave us a database connection. We must get on the local queue to ensure atomicity
1272     // Note that we're at the mercy of the surrounding db transaction, so don't try to rollback here
1273     [self dispatchSyncWithConnection:dbconn
1274                       readWriteTxion:YES
1275                                block:^CKKSDatabaseTransactionResult {
1276         // Schedule a "view changed" notification
1277         [self.notifyViewChangedScheduler trigger];
1278
1279         if(self.accountStatus == CKKSAccountStatusNoAccount) {
1280             // No account; CKKS shouldn't attempt anything.
1281             [self.stateMachine _onqueueHandleFlag:CKKSFlagScanLocalItems];
1282             ckksnotice("ckks", self, "Dropping sync item modification due to CK account state; will scan to find changes later");
1283
1284             // We're positively not logged into CloudKit, and therefore don't expect this item to be synced anytime particularly soon.
1285             NSString* uuid = (__bridge NSString*)SecDbItemGetValue(added ? added : deleted, &v10itemuuid, NULL);
1286
1287             SecBoolNSErrorCallback syncCallback = [[CKKSViewManager manager] claimCallbackForUUID:uuid];
1288             if(syncCallback) {
1289                 [CKKSViewManager callSyncCallbackWithErrorNoAccount: syncCallback];
1290             }
1291
1292             return CKKSDatabaseTransactionCommit;
1293         }
1294
1295         CKKSOutgoingQueueEntry* oqe = nil;
1296         if       (isAdd) {
1297             oqe = [CKKSOutgoingQueueEntry withItem: added   action: SecCKKSActionAdd    zoneID:self.zoneID error: &error];
1298         } else if(isDelete) {
1299             oqe = [CKKSOutgoingQueueEntry withItem: deleted action: SecCKKSActionDelete zoneID:self.zoneID error: &error];
1300         } else if(isModify) {
1301             oqe = [CKKSOutgoingQueueEntry withItem: added   action: SecCKKSActionModify zoneID:self.zoneID error: &error];
1302         } else {
1303             ckkserror("ckks", self, "processKeychainEventItemAdded given garbage: %@ %@", added, deleted);
1304             return CKKSDatabaseTransactionCommit;
1305         }
1306
1307         if(!self.itemSyncingEnabled) {
1308             // Call any callback now; they're not likely to get the sync they wanted
1309             SecBoolNSErrorCallback syncCallback = [[CKKSViewManager manager] claimCallbackForUUID:oqe.uuid];
1310             if(syncCallback) {
1311                 syncCallback(false, [NSError errorWithDomain:CKKSErrorDomain
1312                                                         code:CKKSErrorViewIsPaused
1313                                                  description:@"View is paused; item is not expected to sync"]);
1314             }
1315         }
1316
1317         CKOperationGroup* operationGroup = txionSource == kSecDbSOSTransaction
1318             ? [CKOperationGroup CKKSGroupWithName:@"sos-incoming-item"]
1319             : [CKOperationGroup CKKSGroupWithName:@"keychain-api-use"];
1320
1321         if(error) {
1322             ckkserror("ckks", self, "Couldn't create outgoing queue entry: %@", error);
1323             [self.stateMachine _onqueueHandleFlag:CKKSFlagScanLocalItems];
1324
1325             // If the problem is 'couldn't load key', tell the key hierarchy state machine to fix it
1326             if([error.domain isEqualToString:CKKSErrorDomain] && error.code == errSecItemNotFound) {
1327                 [self.stateMachine _onqueueHandleFlag:CKKSFlagKeyStateProcessRequested];
1328             }
1329
1330             return CKKSDatabaseTransactionCommit;
1331         } else if(!oqe) {
1332             ckkserror("ckks", self, "Decided that no operation needs to occur for %@", error);
1333             return CKKSDatabaseTransactionCommit;
1334         }
1335
1336         if(rateLimiter) {
1337             NSDate* limit = nil;
1338             NSInteger value = [rateLimiter judge:oqe at:[NSDate date] limitTime:&limit];
1339             if(limit) {
1340                 oqe.waitUntil = limit;
1341                 SecPLLogRegisteredEvent(@"CKKSSyncing", @{ @"ratelimit" : @(value), @"accessgroup" : oqe.accessgroup});
1342             }
1343         }
1344
1345         [oqe saveToDatabaseWithConnection: dbconn error: &error];
1346         if(error) {
1347             ckkserror("ckks", self, "Couldn't save outgoing queue entry to database: %@", error);
1348             return CKKSDatabaseTransactionCommit;
1349         } else {
1350             ckksnotice("ckks", self, "Saved %@ to outgoing queue", oqe);
1351         }
1352
1353         // This update supercedes all other local modifications to this item (_except_ those in-flight).
1354         // Delete all items in reencrypt or error.
1355         NSArray<CKKSOutgoingQueueEntry*>* siblings = [CKKSOutgoingQueueEntry allWithUUID:oqe.uuid
1356                                                                                   states:@[SecCKKSStateReencrypt, SecCKKSStateError]
1357                                                                                   zoneID:self.zoneID
1358                                                                                    error:&error];
1359         if(error) {
1360             ckkserror("ckks", self, "Couldn't load OQE siblings for %@: %@", oqe, error);
1361         }
1362
1363         for(CKKSOutgoingQueueEntry* oqeSibling in siblings) {
1364             NSError* deletionError = nil;
1365             [oqeSibling deleteFromDatabase:&deletionError];
1366             if(deletionError) {
1367                 ckkserror("ckks", self, "Couldn't delete OQE sibling(%@) for %@: %@", oqeSibling, oqe.uuid, deletionError);
1368             }
1369         }
1370
1371         // This update also supercedes any remote changes that are pending.
1372         NSError* iqeError = nil;
1373         CKKSIncomingQueueEntry* iqe = [CKKSIncomingQueueEntry tryFromDatabase:oqe.uuid zoneID:self.zoneID error:&iqeError];
1374         if(iqeError) {
1375             ckkserror("ckks", self, "Couldn't find IQE matching %@: %@", oqe.uuid, error);
1376         } else if(iqe) {
1377             [iqe deleteFromDatabase:&iqeError];
1378             if(iqeError) {
1379                 ckkserror("ckks", self, "Couldn't delete IQE matching %@: %@", oqe.uuid, error);
1380             } else {
1381                 ckksnotice("ckks", self, "Deleted IQE matching changed item %@", oqe.uuid);
1382             }
1383         }
1384
1385         [self processOutgoingQueue:operationGroup];
1386
1387         return CKKSDatabaseTransactionCommit;
1388     }];
1389 }
1390
1391 -(void)setCurrentItemForAccessGroup:(NSData* _Nonnull)newItemPersistentRef
1392                                hash:(NSData*)newItemSHA1
1393                         accessGroup:(NSString*)accessGroup
1394                          identifier:(NSString*)identifier
1395                           replacing:(NSData* _Nullable)oldCurrentItemPersistentRef
1396                                hash:(NSData*)oldItemSHA1
1397                            complete:(void (^) (NSError* operror)) complete
1398 {
1399     if(accessGroup == nil || identifier == nil) {
1400         NSError* error = [NSError errorWithDomain:CKKSErrorDomain
1401                                              code:errSecParam
1402                                       description:@"No access group or identifier given"];
1403         ckkserror("ckkscurrent", self, "Cancelling request: %@", error);
1404         complete(error);
1405         return;
1406     }
1407
1408     // Not being in a CloudKit account is an automatic failure.
1409     // But, wait a good long while for the CloudKit account state to be known (in the case of daemon startup)
1410     [self.accountStateKnown wait:(SecCKKSTestsEnabled() ? 1*NSEC_PER_SEC : 30*NSEC_PER_SEC)];
1411
1412     if(self.accountStatus != CKKSAccountStatusAvailable) {
1413         NSError* error = [NSError errorWithDomain:CKKSErrorDomain
1414                                              code:CKKSNotLoggedIn
1415                                       description:@"User is not signed into iCloud."];
1416         ckksnotice("ckkscurrent", self, "Rejecting current item pointer set since we don't have an iCloud account.");
1417         complete(error);
1418         return;
1419     }
1420
1421     ckksnotice("ckkscurrent", self, "Starting change current pointer operation for %@-%@", accessGroup, identifier);
1422     CKKSUpdateCurrentItemPointerOperation* ucipo = [[CKKSUpdateCurrentItemPointerOperation alloc] initWithCKKSKeychainView:self
1423                                                                                                                    newItem:newItemPersistentRef
1424                                                                                                                       hash:newItemSHA1
1425                                                                                                                accessGroup:accessGroup
1426                                                                                                                 identifier:identifier
1427                                                                                                                  replacing:oldCurrentItemPersistentRef
1428                                                                                                                       hash:oldItemSHA1
1429                                                                                                           ckoperationGroup:[CKOperationGroup CKKSGroupWithName:@"currentitem-api"]];
1430
1431     WEAKIFY(self);
1432     CKKSResultOperation* returnCallback = [CKKSResultOperation operationWithBlock:^{
1433         STRONGIFY(self);
1434
1435         if(ucipo.error) {
1436             ckkserror("ckkscurrent", self, "Failed setting a current item pointer for %@ with %@", ucipo.currentPointerIdentifier, ucipo.error);
1437         } else {
1438             ckksnotice("ckkscurrent", self, "Finished setting a current item pointer for %@", ucipo.currentPointerIdentifier);
1439         }
1440         complete(ucipo.error);
1441     }];
1442     returnCallback.name = @"setCurrentItem-return-callback";
1443     [returnCallback addDependency: ucipo];
1444     [self scheduleOperation: returnCallback];
1445
1446     // Now, schedule ucipo. It modifies the CloudKit zone, so it should insert itself into the list of OutgoingQueueOperations.
1447     // Then, we won't have simultaneous zone-modifying operations.
1448     [ucipo linearDependencies:self.outgoingQueueOperations];
1449
1450     // If this operation hasn't started within 60 seconds, cancel it and return a "timed out" error.
1451     [ucipo timeout:60*NSEC_PER_SEC];
1452
1453     [self scheduleOperation:ucipo];
1454     return;
1455 }
1456
1457 -(void)getCurrentItemForAccessGroup:(NSString*)accessGroup
1458                          identifier:(NSString*)identifier
1459                     fetchCloudValue:(bool)fetchCloudValue
1460                            complete:(void (^) (NSString* uuid, NSError* operror)) complete
1461 {
1462     if(accessGroup == nil || identifier == nil) {
1463         ckksnotice("ckkscurrent", self, "Rejecting current item pointer get since no access group(%@) or identifier(%@) given", accessGroup, identifier);
1464         complete(NULL, [NSError errorWithDomain:CKKSErrorDomain
1465                                            code:errSecParam
1466                                     description:@"No access group or identifier given"]);
1467         return;
1468     }
1469
1470     // Not being in a CloudKit account is an automatic failure.
1471     // But, wait a good long while for the CloudKit account state to be known (in the case of daemon startup)
1472     [self.accountStateKnown wait:(SecCKKSTestsEnabled() ? 1*NSEC_PER_SEC : 30*NSEC_PER_SEC)];
1473
1474     if(self.accountStatus != CKKSAccountStatusAvailable) {
1475         ckksnotice("ckkscurrent", self, "Rejecting current item pointer get since we don't have an iCloud account.");
1476         complete(NULL, [NSError errorWithDomain:CKKSErrorDomain
1477                                            code:CKKSNotLoggedIn
1478                                     description:@"User is not signed into iCloud."]);
1479         return;
1480     }
1481
1482     CKKSResultOperation* fetchAndProcess = nil;
1483     if(fetchCloudValue) {
1484         fetchAndProcess = [self fetchAndProcessCKChanges:CKKSFetchBecauseCurrentItemFetchRequest];
1485     }
1486
1487     WEAKIFY(self);
1488     CKKSResultOperation* getCurrentItem = [CKKSResultOperation named:@"get-current-item-pointer" withBlock:^{
1489         if(fetchAndProcess.error) {
1490             ckksnotice("ckkscurrent", self, "Rejecting current item pointer get since fetch failed: %@", fetchAndProcess.error);
1491             complete(NULL, fetchAndProcess.error);
1492             return;
1493         }
1494
1495         STRONGIFY(self);
1496
1497         [self dispatchSyncWithReadOnlySQLTransaction:^{
1498             NSError* error = nil;
1499             NSString* currentIdentifier = [NSString stringWithFormat:@"%@-%@", accessGroup, identifier];
1500
1501             CKKSCurrentItemPointer* cip = [CKKSCurrentItemPointer fromDatabase:currentIdentifier
1502                                                                          state:SecCKKSProcessedStateLocal
1503                                                                         zoneID:self.zoneID
1504                                                                          error:&error];
1505             if(!cip || error) {
1506                 if([error.domain isEqualToString:@"securityd"] && error.code == errSecItemNotFound) {
1507                     // This error is common and very, very noisy. Shorten it and don't log here (the framework should log for us)
1508                     ckksinfo("ckkscurrent", self, "No current item pointer for %@", currentIdentifier);
1509                     error = [NSError errorWithDomain:@"securityd" code:errSecItemNotFound description:[NSString stringWithFormat:@"No current item pointer found for %@", currentIdentifier]];
1510                 } else {
1511                     ckkserror("ckkscurrent", self, "No current item pointer for %@", currentIdentifier);
1512                 }
1513                 complete(nil, error);
1514                 return;
1515             }
1516
1517             if(!cip.currentItemUUID) {
1518                 ckkserror("ckkscurrent", self, "Current item pointer is empty %@", cip);
1519                 complete(nil, [NSError errorWithDomain:CKKSErrorDomain
1520                                                   code:errSecInternalError
1521                                            description:@"Current item pointer is empty"]);
1522                 return;
1523             }
1524
1525             ckksinfo("ckkscurrent", self, "Retrieved current item pointer: %@", cip);
1526             complete(cip.currentItemUUID, NULL);
1527             return;
1528         }];
1529     }];
1530
1531     [getCurrentItem addNullableDependency:fetchAndProcess];
1532     [self scheduleOperation: getCurrentItem];
1533 }
1534
1535 - (CKKSResultOperation<CKKSKeySetProviderOperationProtocol>*)findKeySet:(BOOL)refetchBeforeReturningKeySet
1536 {
1537     __block CKKSResultOperation<CKKSKeySetProviderOperationProtocol>* keysetOp = nil;
1538     __block BOOL moveFromWaitForTrust = NO;
1539
1540     [self dispatchSyncWithReadOnlySQLTransaction:^{
1541         keysetOp = (CKKSProvideKeySetOperation*)[self findFirstPendingOperation:self.operationDependencies.keysetProviderOperations];
1542         if(!keysetOp) {
1543             keysetOp = [[CKKSProvideKeySetOperation alloc] initWithZoneName:self.zoneName];
1544             [self.operationDependencies.keysetProviderOperations addObject:keysetOp];
1545
1546             // This is an abuse of operations: they should generally run when added to a queue, not wait, but this allows recipients to set timeouts
1547             [self scheduleOperationWithoutDependencies:keysetOp];
1548         }
1549
1550         if(refetchBeforeReturningKeySet) {
1551             ckksnotice("ckks", self, "Refetch requested before returning key set!");
1552
1553             [self.stateMachine _onqueueHandleFlag:CKKSFlagFetchRequested];
1554             [self.stateMachine _onqueueHandleFlag:CKKSFlagTLKCreationRequested];
1555
1556             if([self.stateMachine.currentState isEqualToString:SecCKKSZoneKeyStateWaitForTrust]) {
1557                 moveFromWaitForTrust = YES;
1558             }
1559             return;
1560         }
1561
1562         CKKSCurrentKeySet* keyset = [CKKSCurrentKeySet loadForZone:self.zoneID];
1563         if(keyset.currentTLKPointer.currentKeyUUID &&
1564            (keyset.tlk.uuid ||
1565             [self.stateMachine.currentState isEqualToString:SecCKKSZoneKeyStateWaitForTrust] ||
1566             [self.stateMachine.currentState isEqualToString:SecCKKSZoneKeyStateWaitForTLK])) {
1567             ckksnotice("ckks", self, "Already have keyset %@", keyset);
1568
1569             [keysetOp provideKeySet:keyset];
1570             return;
1571
1572         } else if([self.stateMachine.currentState isEqualToString:SecCKKSZoneKeyStateWaitForTrust]) {
1573             // No keyset exists, but we're in waitfortrust? Seems like a bug. Move us out of this state...
1574
1575             ckksnotice("ckks", self, "Received a keyset request in an odd state; forwarding to state machine");
1576             [self.stateMachine _onqueueHandleFlag:CKKSFlagTLKCreationRequested];
1577             moveFromWaitForTrust = YES;
1578
1579         } else {
1580             // The key state machine will know what to do.
1581             [self.stateMachine _onqueueHandleFlag:CKKSFlagTLKCreationRequested];
1582         };
1583     }];
1584
1585     if(moveFromWaitForTrust) {
1586         [self.stateMachine handleExternalRequest:[[OctagonStateTransitionRequest alloc] init:@"fix-bug"
1587                                                                                 sourceStates:[NSSet setWithObject:SecCKKSZoneKeyStateWaitForTrust]
1588                                                                                  serialQueue:self.queue
1589                                                                                      timeout:5 * NSEC_PER_SEC
1590                                                                                 transitionOp:[OctagonStateTransitionOperation named:@"fix-bug"
1591                                                                                                                            entering:SecCKKSZoneKeyStateWaitForTLKCreation]]];
1592     }
1593
1594     return keysetOp;
1595 }
1596
1597 - (void)receiveTLKUploadRecords:(NSArray<CKRecord*>*)records
1598 {
1599     // First, filter for records matching this zone
1600     NSMutableArray<CKRecord*>* zoneRecords = [NSMutableArray array];
1601     for(CKRecord* record in records) {
1602         if([record.recordID.zoneID isEqual:self.zoneID]) {
1603             [zoneRecords addObject:record];
1604         }
1605     }
1606
1607     ckksnotice("ckkskey", self, "Received a set of %lu TLK upload records", (unsigned long)zoneRecords.count);
1608
1609     if(!zoneRecords || zoneRecords.count == 0) {
1610         return;
1611     }
1612
1613     [self dispatchSyncWithSQLTransaction:^CKKSDatabaseTransactionResult{
1614         for(CKRecord* record in zoneRecords) {
1615             [self _onqueueCKRecordChanged:record resync:false];
1616         }
1617
1618         [self.stateMachine _onqueueHandleFlag:CKKSFlagKeyStateTLKsUploaded];
1619
1620         return CKKSDatabaseTransactionCommit;
1621     }];
1622 }
1623
1624 - (BOOL)requiresTLKUpload
1625 {
1626     __block BOOL requiresUpload = NO;
1627     dispatch_sync(self.queue, ^{
1628         // We want to return true only if we're in a state that immediately requires an upload.
1629         if(([self.keyHierarchyState isEqualToString:SecCKKSZoneKeyStateWaitForTLKUpload] ||
1630             [self.keyHierarchyState isEqualToString:SecCKKSZoneKeyStateWaitForTLKCreation])) {
1631             requiresUpload = YES;
1632         }
1633     });
1634
1635     return requiresUpload;
1636 }
1637
1638 // Use the following method to find the first pending operation in a weak collection
1639 - (NSOperation*)findFirstPendingOperation: (NSHashTable*) table {
1640     return [self findFirstPendingOperation:table ofClass:nil];
1641 }
1642
1643 // Use the following method to find the first pending operation in a weak collection
1644 - (NSOperation*)findFirstPendingOperation: (NSHashTable*) table ofClass:(Class)class {
1645     @synchronized(table) {
1646         for(NSOperation* op in table) {
1647             if(op != nil && [op isPending] && (class == nil || [op isKindOfClass: class])) {
1648                 return op;
1649             }
1650         }
1651         return nil;
1652     }
1653 }
1654
1655 - (CKKSOutgoingQueueOperation*)processOutgoingQueue:(CKOperationGroup* _Nullable)ckoperationGroup {
1656     return [self processOutgoingQueueAfter:nil ckoperationGroup:ckoperationGroup];
1657 }
1658
1659 - (CKKSOutgoingQueueOperation*)processOutgoingQueueAfter:(CKKSResultOperation* _Nullable)after
1660                                         ckoperationGroup:(CKOperationGroup* _Nullable)ckoperationGroup {
1661     return [self processOutgoingQueueAfter:after requiredDelay:DISPATCH_TIME_FOREVER ckoperationGroup:ckoperationGroup];
1662 }
1663
1664 - (CKKSOutgoingQueueOperation*)processOutgoingQueueAfter:(CKKSResultOperation* _Nullable)after
1665                                            requiredDelay:(uint64_t)requiredDelay
1666                                         ckoperationGroup:(CKOperationGroup* _Nullable)ckoperationGroup
1667 {
1668     CKKSOutgoingQueueOperation* outgoingop =
1669             (CKKSOutgoingQueueOperation*) [self findFirstPendingOperation:self.outgoingQueueOperations
1670                                                                   ofClass:[CKKSOutgoingQueueOperation class]];
1671     if(outgoingop) {
1672         if(after) {
1673             [outgoingop addDependency: after];
1674         }
1675         if([outgoingop isPending]) {
1676             if(!outgoingop.ckoperationGroup && ckoperationGroup) {
1677                 outgoingop.ckoperationGroup = ckoperationGroup;
1678             } else if(ckoperationGroup) {
1679                 ckkserror("ckks", self, "Throwing away CKOperationGroup(%@) in favor of (%@)", ckoperationGroup.name, outgoingop.ckoperationGroup.name);
1680             }
1681
1682             // Will log any pending dependencies as well
1683             ckksinfo("ckksoutgoing", self, "Returning existing %@", outgoingop);
1684
1685             // Shouldn't be necessary, but can't hurt
1686             [self.outgoingQueueOperationScheduler triggerAt:requiredDelay];
1687             return outgoingop;
1688         }
1689     }
1690
1691     CKKSOutgoingQueueOperation* op = [[CKKSOutgoingQueueOperation alloc] initWithDependencies:self.operationDependencies
1692                                                                                          ckks:self
1693                                                                                     intending:SecCKKSZoneKeyStateReady
1694                                                                                    errorState:SecCKKSZoneKeyStateUnhealthy
1695                                                                              ckoperationGroup:ckoperationGroup];
1696     op.name = @"outgoing-queue-operation";
1697     [op addNullableDependency:after];
1698     [op addNullableDependency:self.outgoingQueueOperationScheduler.operationDependency];
1699
1700     [self.outgoingQueueOperationScheduler triggerAt:requiredDelay];
1701
1702     [op linearDependencies:self.outgoingQueueOperations];
1703
1704     [self scheduleOperation: op];
1705     ckksnotice("ckksoutgoing", self, "Scheduled %@", op);
1706     return op;
1707 }
1708
1709 - (void)processIncomingQueueAfterNextUnlock {
1710     // Thread races aren't so important here; we might end up with two or three copies of this operation, but that's okay.
1711     if(![self.processIncomingQueueAfterNextUnlockOperation isPending]) {
1712         WEAKIFY(self);
1713
1714         CKKSResultOperation* restartIncomingQueueOperation = [CKKSResultOperation operationWithBlock:^{
1715             STRONGIFY(self);
1716             // This IQO shouldn't error if the keybag has locked again. It will simply try again later.
1717             [self processIncomingQueue:false];
1718         }];
1719
1720         restartIncomingQueueOperation.name = @"reprocess-incoming-queue-after-unlock";
1721         self.processIncomingQueueAfterNextUnlockOperation = restartIncomingQueueOperation;
1722
1723         [restartIncomingQueueOperation addNullableDependency:self.lockStateTracker.unlockDependency];
1724         [self scheduleOperation: restartIncomingQueueOperation];
1725     }
1726 }
1727
1728 - (CKKSResultOperation*)resultsOfNextProcessIncomingQueueOperation {
1729     if(self.resultsOfNextIncomingQueueOperationOperation && [self.resultsOfNextIncomingQueueOperationOperation isPending]) {
1730         return self.resultsOfNextIncomingQueueOperationOperation;
1731     }
1732
1733     // Else, make a new one.
1734     self.resultsOfNextIncomingQueueOperationOperation = [CKKSResultOperation named:[NSString stringWithFormat:@"wait-for-next-incoming-queue-operation-%@", self.zoneName] withBlock:^{}];
1735     return self.resultsOfNextIncomingQueueOperationOperation;
1736 }
1737
1738 - (CKKSIncomingQueueOperation*)processIncomingQueue:(bool)failOnClassA {
1739     return [self processIncomingQueue:failOnClassA after: nil];
1740 }
1741
1742 - (CKKSIncomingQueueOperation*) processIncomingQueue:(bool)failOnClassA after: (CKKSResultOperation*) after {
1743     return [self processIncomingQueue:failOnClassA after:after policyConsideredAuthoritative:false];
1744 }
1745
1746 - (CKKSIncomingQueueOperation*)processIncomingQueue:(bool)failOnClassA
1747                                               after:(CKKSResultOperation*)after
1748                       policyConsideredAuthoritative:(bool)policyConsideredAuthoritative
1749 {
1750     CKKSIncomingQueueOperation* incomingop = (CKKSIncomingQueueOperation*) [self findFirstPendingOperation:self.incomingQueueOperations];
1751     if(incomingop) {
1752         ckksinfo("ckks", self, "Skipping processIncomingQueue due to at least one pending instance");
1753         if(after) {
1754             [incomingop addNullableDependency: after];
1755         }
1756
1757         // check (again) for race condition; if the op has started we need to add another (for the dependency)
1758         if([incomingop isPending]) {
1759             incomingop.errorOnClassAFailure |= failOnClassA;
1760             incomingop.handleMismatchedViewItems |= policyConsideredAuthoritative;
1761             return incomingop;
1762         }
1763     }
1764
1765     CKKSIncomingQueueOperation* op = [[CKKSIncomingQueueOperation alloc] initWithDependencies:self.operationDependencies
1766                                                                                          ckks:self
1767                                                                                     intending:SecCKKSZoneKeyStateReady
1768                                                                                    errorState:SecCKKSZoneKeyStateUnhealthy
1769                                                                          errorOnClassAFailure:failOnClassA
1770                                                                         handleMismatchedViewItems:policyConsideredAuthoritative];
1771     op.name = @"incoming-queue-operation";
1772     if(after != nil) {
1773         [op addSuccessDependency: after];
1774     }
1775
1776     if(self.resultsOfNextIncomingQueueOperationOperation) {
1777         [self.resultsOfNextIncomingQueueOperationOperation addSuccessDependency:op];
1778         [self scheduleOperation:self.resultsOfNextIncomingQueueOperationOperation];
1779         self.resultsOfNextIncomingQueueOperationOperation = nil;
1780     }
1781
1782     [self scheduleOperation: op];
1783     return op;
1784 }
1785
1786 - (CKKSScanLocalItemsOperation*)scanLocalItems:(NSString*)operationName {
1787     return [self scanLocalItems:operationName ckoperationGroup:nil after:nil];
1788 }
1789
1790 - (CKKSScanLocalItemsOperation*)scanLocalItems:(NSString*)operationName
1791                               ckoperationGroup:(CKOperationGroup*)operationGroup
1792                                          after:(NSOperation*)after
1793 {
1794     CKKSScanLocalItemsOperation* scanOperation = (CKKSScanLocalItemsOperation*)[self findFirstPendingOperation:self.scanLocalItemsOperations];
1795
1796     if(scanOperation) {
1797         [scanOperation addNullableDependency:after];
1798
1799         // check (again) for race condition; if the op has started we need to add another (for the dependency)
1800         if([scanOperation isPending]) {
1801             scanOperation.ckoperationGroup = operationGroup;
1802
1803             scanOperation.name = [NSString stringWithFormat:@"%@::%@", scanOperation.name, operationName];
1804             return scanOperation;
1805         }
1806     }
1807
1808     scanOperation = [[CKKSScanLocalItemsOperation alloc] initWithDependencies:self.operationDependencies
1809                                                                          ckks:self
1810                                                                     intending:SecCKKSZoneKeyStateReady
1811                                                                    errorState:SecCKKSZoneKeyStateError
1812                                                              ckoperationGroup:operationGroup];
1813     scanOperation.name = operationName;
1814
1815     [scanOperation addNullableDependency:self.lastFixupOperation];
1816     [scanOperation addNullableDependency:self.lockStateTracker.unlockDependency];
1817     [scanOperation addNullableDependency:self.keyStateReadyDependency];
1818     [scanOperation addNullableDependency:after];
1819
1820     [scanOperation linearDependencies:self.scanLocalItemsOperations];
1821
1822     // This might generate items for upload. Make sure that any uploads wait until the scan is complete, so we know what to upload
1823     [scanOperation linearDependencies:self.outgoingQueueOperations];
1824
1825     [self scheduleOperation:scanOperation];
1826     self.initiatedLocalScan = YES;
1827     return scanOperation;
1828 }
1829
1830 - (CKKSUpdateDeviceStateOperation*)updateDeviceState:(bool)rateLimit
1831                    waitForKeyHierarchyInitialization:(uint64_t)timeout
1832                                     ckoperationGroup:(CKOperationGroup*)ckoperationGroup {
1833     // If securityd just started, the key state might be in some transient early state. Wait a bit.
1834     OctagonStateMultiStateArrivalWatcher* waitForTransient = [[OctagonStateMultiStateArrivalWatcher alloc] initNamed:@"rpc-watcher"
1835                                                                                                          serialQueue:self.queue
1836                                                                                                               states:CKKSKeyStateNonTransientStates()];
1837     [waitForTransient timeout:timeout];
1838     [self.stateMachine registerMultiStateArrivalWatcher:waitForTransient];
1839
1840     CKKSUpdateDeviceStateOperation* op = [[CKKSUpdateDeviceStateOperation alloc] initWithCKKSKeychainView:self rateLimit:rateLimit ckoperationGroup:ckoperationGroup];
1841     op.name = @"device-state-operation";
1842
1843     [op addDependency:waitForTransient.result];
1844
1845     // op modifies the CloudKit zone, so it should insert itself into the list of OutgoingQueueOperations.
1846     // Then, we won't have simultaneous zone-modifying operations and confuse ourselves.
1847     // However, since we might have pending OQOs, it should try to insert itself at the beginning of the linearized list
1848     [op linearDependenciesWithSelfFirst:self.outgoingQueueOperations];
1849
1850     // CKKSUpdateDeviceStateOperations are special: they should fire even if we don't believe we're in an iCloud account.
1851     // They also shouldn't block or be blocked by any other operation; our wait operation above will handle that
1852     [self scheduleOperationWithoutDependencies:op];
1853     return op;
1854 }
1855
1856 - (void)xpc24HrNotification
1857 {
1858     // Called roughly once every 24hrs
1859     [self.stateMachine handleFlag:CKKSFlag24hrNotification];
1860 }
1861
1862 // There are some errors which won't be reported but will be reflected in the CDSE; any error coming out of here is fatal
1863 - (CKKSDeviceStateEntry*)_onqueueCurrentDeviceStateEntry: (NSError* __autoreleasing*)error {
1864     dispatch_assert_queue(self.queue);
1865     NSError* localerror = nil;
1866
1867     CKKSAccountStateTracker* accountTracker = self.accountTracker;
1868     CKKSAccountStatus hsa2Status = accountTracker.hsa2iCloudAccountStatus;
1869
1870     // We must have an HSA2 iCloud account and a CloudKit account to even create one of these
1871     if(hsa2Status != CKKSAccountStatusAvailable ||
1872        accountTracker.currentCKAccountInfo.accountStatus != CKAccountStatusAvailable) {
1873         ckkserror("ckksdevice", self, "No iCloud account active: %@ hsa2 account:%@",
1874                   accountTracker.currentCKAccountInfo,
1875                   CKKSAccountStatusToString(hsa2Status));
1876         localerror = [NSError errorWithDomain:@"securityd"
1877                                          code:errSecInternalError
1878                                      userInfo:@{NSLocalizedDescriptionKey: [NSString stringWithFormat: @"No active HSA2 iCloud account: %@", accountTracker.currentCKAccountInfo]}];
1879         if(error) {
1880             *error = localerror;
1881         }
1882         return nil;
1883     }
1884
1885     NSString* ckdeviceID = accountTracker.ckdeviceID;
1886     if(ckdeviceID == nil) {
1887         ckkserror("ckksdevice", self, "No CK device ID available; cannot make device state entry");
1888         localerror = [NSError errorWithDomain:CKKSErrorDomain
1889                                          code:CKKSNotLoggedIn
1890                                      userInfo:@{NSLocalizedDescriptionKey: [NSString stringWithFormat: @"No CK device ID: %@", accountTracker.currentCKAccountInfo]}];
1891         if(error) {
1892             *error = localerror;
1893         }
1894         return nil;
1895     }
1896
1897     CKKSDeviceStateEntry* oldcdse = [CKKSDeviceStateEntry tryFromDatabase:ckdeviceID zoneID:self.zoneID error:&localerror];
1898     if(localerror) {
1899         ckkserror("ckksdevice", self, "Couldn't read old CKKSDeviceStateEntry from database: %@", localerror);
1900         if(error) {
1901             *error = localerror;
1902         }
1903         return nil;
1904     }
1905
1906     // Find out what we think the current keys are
1907     CKKSCurrentKeyPointer* currentTLKPointer    = [CKKSCurrentKeyPointer tryFromDatabase: SecCKKSKeyClassTLK zoneID:self.zoneID error:&localerror];
1908     CKKSCurrentKeyPointer* currentClassAPointer = [CKKSCurrentKeyPointer tryFromDatabase: SecCKKSKeyClassA   zoneID:self.zoneID error:&localerror];
1909     CKKSCurrentKeyPointer* currentClassCPointer = [CKKSCurrentKeyPointer tryFromDatabase: SecCKKSKeyClassC   zoneID:self.zoneID error:&localerror];
1910     if(localerror) {
1911         // Things is broken, but the whole point of this record is to share the brokenness. Continue.
1912         ckkserror("ckksdevice", self, "Couldn't read current key pointers from database: %@; proceeding", localerror);
1913         localerror = nil;
1914     }
1915
1916     CKKSKey* suggestedTLK       = currentTLKPointer.currentKeyUUID    ? [CKKSKey tryFromDatabase:currentTLKPointer.currentKeyUUID    zoneID:self.zoneID error:&localerror] : nil;
1917     CKKSKey* suggestedClassAKey = currentClassAPointer.currentKeyUUID ? [CKKSKey tryFromDatabase:currentClassAPointer.currentKeyUUID zoneID:self.zoneID error:&localerror] : nil;
1918     CKKSKey* suggestedClassCKey = currentClassCPointer.currentKeyUUID ? [CKKSKey tryFromDatabase:currentClassCPointer.currentKeyUUID zoneID:self.zoneID error:&localerror] : nil;
1919
1920     if(localerror) {
1921         // Things is broken, but the whole point of this record is to share the brokenness. Continue.
1922         ckkserror("ckksdevice", self, "Couldn't read keys from database: %@; proceeding", localerror);
1923         localerror = nil;
1924     }
1925
1926     // Check if we posess the keys in the keychain
1927     [suggestedTLK ensureKeyLoaded:&localerror];
1928     if(localerror && [self.lockStateTracker isLockedError:localerror]) {
1929         ckkserror("ckksdevice", self, "Device is locked; couldn't read TLK from keychain. Assuming it is present and continuing; error was %@", localerror);
1930         localerror = nil;
1931     } else if(localerror) {
1932         ckkserror("ckksdevice", self, "Couldn't read TLK from keychain. We do not have a current TLK. Error was %@", localerror);
1933         suggestedTLK = nil;
1934     }
1935
1936     [suggestedClassAKey ensureKeyLoaded:&localerror];
1937     if(localerror && [self.lockStateTracker isLockedError:localerror]) {
1938         ckkserror("ckksdevice", self, "Device is locked; couldn't read ClassA key from keychain. Assuming it is present and continuing; error was %@", localerror);
1939         localerror = nil;
1940     } else if(localerror) {
1941         ckkserror("ckksdevice", self, "Couldn't read ClassA key from keychain. We do not have a current ClassA key. Error was %@", localerror);
1942         suggestedClassAKey = nil;
1943     }
1944
1945     [suggestedClassCKey ensureKeyLoaded:&localerror];
1946     // class C keys are stored class C, so uh, don't check lock state.
1947     if(localerror) {
1948         ckkserror("ckksdevice", self, "Couldn't read ClassC key from keychain. We do not have a current ClassC key. Error was %@", localerror);
1949         suggestedClassCKey = nil;
1950     }
1951
1952     // We'd like to have the circle peer ID. Give the account state tracker a fighting chance, but not having it is not an error
1953     // But, if the platform doesn't have SOS, don't bother
1954     if(OctagonPlatformSupportsSOS() && [accountTracker.accountCirclePeerIDInitialized wait:500*NSEC_PER_MSEC] != 0 && !accountTracker.accountCirclePeerID) {
1955         ckkserror("ckksdevice", self, "No SOS peer ID available");
1956     }
1957
1958     // We'd also like the Octagon status
1959     if([accountTracker.octagonInformationInitialized wait:500*NSEC_PER_MSEC] != 0 && !accountTracker.octagonPeerID) {
1960         ckkserror("ckksdevice", self, "No octagon peer ID available");
1961     }
1962
1963     // Reset the last unlock time to 'day' granularity in UTC
1964     NSCalendar* calendar = [NSCalendar calendarWithIdentifier:NSCalendarIdentifierISO8601];
1965     calendar.timeZone = [NSTimeZone timeZoneWithAbbreviation:@"UTC"];
1966     NSDate* lastUnlockDay = self.lockStateTracker.lastUnlockTime;
1967     lastUnlockDay = lastUnlockDay ? [calendar startOfDayForDate:lastUnlockDay] : nil;
1968
1969     // We only really want the oldcdse for its encodedCKRecord, so make a new cdse here
1970     CKKSDeviceStateEntry* newcdse = [[CKKSDeviceStateEntry alloc] initForDevice:ckdeviceID
1971                                                                       osVersion:SecCKKSHostOSVersion()
1972                                                                  lastUnlockTime:lastUnlockDay
1973                                                                   octagonPeerID:accountTracker.octagonPeerID
1974                                                                   octagonStatus:accountTracker.octagonStatus
1975                                                                    circlePeerID:accountTracker.accountCirclePeerID
1976                                                                    circleStatus:accountTracker.currentCircleStatus.status
1977                                                                        keyState:self.keyHierarchyState
1978                                                                  currentTLKUUID:suggestedTLK.uuid
1979                                                               currentClassAUUID:suggestedClassAKey.uuid
1980                                                               currentClassCUUID:suggestedClassCKey.uuid
1981                                                                          zoneID:self.zoneID
1982                                                                 encodedCKRecord:oldcdse.encodedCKRecord];
1983     return newcdse;
1984 }
1985
1986 - (CKKSSynchronizeOperation*) resyncWithCloud {
1987     CKKSSynchronizeOperation* op = [[CKKSSynchronizeOperation alloc] initWithCKKSKeychainView: self];
1988     [self scheduleOperation: op];
1989     return op;
1990 }
1991
1992 - (CKKSLocalSynchronizeOperation*)resyncLocal {
1993     CKKSLocalSynchronizeOperation* op = [[CKKSLocalSynchronizeOperation alloc] initWithCKKSKeychainView:self];
1994     [self scheduleOperation: op];
1995     return op;
1996 }
1997
1998 - (CKKSResultOperation*)fetchAndProcessCKChanges:(CKKSFetchBecause*)because
1999 {
2000      if(!SecCKKSIsEnabled()) {
2001         ckksinfo("ckks", self, "Skipping fetchAndProcessCKChanges due to disabled CKKS");
2002         return nil;
2003     }
2004
2005      // We fetched some changes; try to process them!
2006     return [self processIncomingQueue:false after:[self.zoneChangeFetcher requestSuccessfulFetch:because]];
2007 }
2008
2009 // Lets the view know about a failed CloudKit write. If the error is "already have one of these records", it will
2010 // store the new records and kick off the new processing
2011 //
2012 // Note that you need to tell this function the records you wanted to save, so it can determine what needs deletion
2013 - (bool)_onqueueCKWriteFailed:(NSError*)ckerror attemptedRecordsChanged:(NSDictionary<CKRecordID*, CKRecord*>*)savedRecords {
2014     dispatch_assert_queue(self.queue);
2015
2016     NSDictionary<CKRecordID*,NSError*>* partialErrors = ckerror.userInfo[CKPartialErrorsByItemIDKey];
2017     if([ckerror.domain isEqual:CKErrorDomain] && ckerror.code == CKErrorPartialFailure && partialErrors) {
2018         // Check if this error was "you're out of date"
2019         bool recordChanged = true;
2020
2021         for(NSError* error in partialErrors.allValues) {
2022             if((![error.domain isEqual:CKErrorDomain]) || (error.code != CKErrorBatchRequestFailed && error.code != CKErrorServerRecordChanged && error.code != CKErrorUnknownItem)) {
2023                 // There's an error in there that isn't CKErrorServerRecordChanged, CKErrorBatchRequestFailed, or CKErrorUnknownItem. Don't handle nicely...
2024                 recordChanged = false;
2025             }
2026         }
2027
2028         if(recordChanged) {
2029             ckksnotice("ckks", self, "Received a ServerRecordChanged error, attempting to update new records and delete unknown ones");
2030
2031             bool updatedRecord = false;
2032
2033             for(CKRecordID* recordID in partialErrors.allKeys) {
2034                 NSError* error = partialErrors[recordID];
2035                 if([error.domain isEqual:CKErrorDomain] && error.code == CKErrorServerRecordChanged) {
2036                     CKRecord* newRecord = error.userInfo[CKRecordChangedErrorServerRecordKey];
2037                     ckksnotice("ckks", self, "On error: updating our idea of: %@", newRecord);
2038
2039                     updatedRecord |= [self _onqueueCKRecordChanged:newRecord resync:true];
2040                 } else if([error.domain isEqual:CKErrorDomain] && error.code == CKErrorUnknownItem) {
2041                     CKRecord* record = savedRecords[recordID];
2042                     ckksnotice("ckks", self, "On error: handling an unexpected delete of: %@ %@", recordID, record);
2043
2044                     updatedRecord |= [self _onqueueCKRecordDeleted:recordID recordType:record.recordType resync:true];
2045                 }
2046             }
2047
2048             if(updatedRecord) {
2049                 [self processIncomingQueue:false];
2050                 return true;
2051             }
2052         }
2053
2054         // Check if this error was the CKKS server extension rejecting the write
2055         for(CKRecordID* recordID in partialErrors.allKeys) {
2056             NSError* error = partialErrors[recordID];
2057
2058             NSError* underlyingError = error.userInfo[NSUnderlyingErrorKey];
2059             NSError* thirdLevelError = underlyingError.userInfo[NSUnderlyingErrorKey];
2060             ckksnotice("ckks", self, "Examining 'write failed' error: %@ %@ %@", error, underlyingError, thirdLevelError);
2061
2062             if([error.domain isEqualToString:CKErrorDomain] && error.code == CKErrorServerRejectedRequest &&
2063                underlyingError && [underlyingError.domain isEqualToString:CKInternalErrorDomain] && underlyingError.code == CKErrorInternalPluginError &&
2064                thirdLevelError && [thirdLevelError.domain isEqualToString:@"CloudkitKeychainService"]) {
2065
2066                 if(thirdLevelError.code == CKKSServerUnexpectedSyncKeyInChain) {
2067                     // The server thinks the classA/C synckeys don't wrap directly the to top TLK, but we don't (otherwise, we would have fixed it).
2068                     // Issue a key hierarchy fetch and see what's what.
2069                     ckkserror("ckks", self, "CKKS Server extension has told us about %@ for record %@; requesting refetch and reprocess of key hierarchy", thirdLevelError, recordID);
2070                     [self.stateMachine _onqueueHandleFlag:CKKSFlagFetchRequested];
2071
2072                 } else if(thirdLevelError.code == CKKSServerMissingRecord) {
2073                     // The server is concerned that there's a missing record somewhere.
2074                     // Issue a key hierarchy fetch and see what's happening
2075                     ckkserror("ckks", self, "CKKS Server extension has told us about %@ for record %@; requesting refetch and reprocess of key hierarchy", thirdLevelError, recordID);
2076                     [self.stateMachine _onqueueHandleFlag:CKKSFlagFetchRequested];
2077
2078                 } else {
2079                     ckkserror("ckks", self, "CKKS Server extension has told us about %@ for record %@, but we don't currently handle this error", thirdLevelError, recordID);
2080                 }
2081             }
2082         }
2083     }
2084
2085     return false;
2086 }
2087
2088 - (bool)_onqueueCKRecordDeleted:(CKRecordID*)recordID recordType:(NSString*)recordType resync:(bool)resync {
2089     dispatch_assert_queue(self.queue);
2090
2091     // TODO: resync doesn't really mean much here; what does it mean for a record to be 'deleted' if you're fetching from scratch?
2092
2093     if([recordType isEqual: SecCKRecordItemType]) {
2094         ckksnotice("ckks", self, "CloudKit notification: deleted record(%@): %@", recordType, recordID);
2095         NSError* error = nil;
2096         NSError* iqeerror = nil;
2097         CKKSMirrorEntry* ckme = [CKKSMirrorEntry fromDatabase: [recordID recordName] zoneID:self.zoneID error: &error];
2098
2099         // Deletes always succeed, not matter the generation count
2100         if(ckme) {
2101             [ckme deleteFromDatabase:&error];
2102
2103             CKKSIncomingQueueEntry* iqe = [[CKKSIncomingQueueEntry alloc] initWithCKKSItem:ckme.item action:SecCKKSActionDelete state:SecCKKSStateNew];
2104             [iqe saveToDatabase:&iqeerror];
2105             if(iqeerror) {
2106                 ckkserror("ckks", self, "Couldn't save incoming queue entry: %@", iqeerror);
2107             }
2108
2109             // Delete any pending local changes; this delete wins
2110             NSArray<CKKSOutgoingQueueEntry*>* siblings = [CKKSOutgoingQueueEntry allWithUUID:iqe.uuid
2111                                                                                       states:@[SecCKKSStateNew,
2112                                                                                                SecCKKSStateReencrypt,
2113                                                                                                SecCKKSStateError]
2114                                                                                       zoneID:self.zoneID
2115                                                                                        error:&error];
2116             if(error) {
2117                 ckkserror("ckks", self, "Couldn't load OQE sibling for %@: %@", iqe.uuid, error);
2118             }
2119
2120             for(CKKSOutgoingQueueEntry* oqe in siblings) {
2121                 NSError* deletionError = nil;
2122                 [oqe deleteFromDatabase:&deletionError];
2123                 if(deletionError) {
2124                     ckkserror("ckks", self, "Couldn't delete OQE sibling(%@) for %@: %@", oqe, iqe.uuid, deletionError);
2125                 }
2126             }
2127         }
2128         ckksinfo("ckks", self, "CKKSMirrorEntry was deleted: %@ %@ error: %@", recordID, ckme, error);
2129         // TODO: actually pass error back up
2130         return (error == nil);
2131
2132     } else if([recordType isEqual: SecCKRecordCurrentItemType]) {
2133         ckksinfo("ckks", self, "CloudKit notification: deleted current item pointer(%@): %@", recordType, recordID);
2134         NSError* error = nil;
2135
2136         [[CKKSCurrentItemPointer tryFromDatabase:[recordID recordName] state:SecCKKSProcessedStateRemote zoneID:self.zoneID error:&error] deleteFromDatabase:&error];
2137         [[CKKSCurrentItemPointer fromDatabase:[recordID recordName]    state:SecCKKSProcessedStateLocal  zoneID:self.zoneID error:&error] deleteFromDatabase:&error];
2138
2139         ckksinfo("ckks", self, "CKKSCurrentItemPointer was deleted: %@ error: %@", recordID, error);
2140         return (error == nil);
2141
2142     } else if([recordType isEqual: SecCKRecordIntermediateKeyType]) {
2143         // TODO: handle in some interesting way
2144         return true;
2145     } else if([recordType isEqual: SecCKRecordTLKShareType]) {
2146         NSError* error = nil;
2147         ckksinfo("ckks", self, "CloudKit notification: deleted tlk share record(%@): %@", recordType, recordID);
2148         CKKSTLKShareRecord* share = [CKKSTLKShareRecord tryFromDatabaseFromCKRecordID:recordID error:&error];
2149         [share deleteFromDatabase:&error];
2150
2151         if(error) {
2152             ckkserror("ckks", self, "CK notification: Couldn't delete deleted TLKShare: %@ %@", recordID,  error);
2153         }
2154         return (error == nil);
2155
2156     } else if([recordType isEqual: SecCKRecordDeviceStateType]) {
2157         NSError* error = nil;
2158         ckksinfo("ckks", self, "CloudKit notification: deleted device state record(%@): %@", recordType, recordID);
2159
2160         CKKSDeviceStateEntry* cdse = [CKKSDeviceStateEntry tryFromDatabaseFromCKRecordID:recordID error:&error];
2161         [cdse deleteFromDatabase: &error];
2162         ckksinfo("ckks", self, "CKKSCurrentItemPointer(%@) was deleted: %@ error: %@", cdse, recordID, error);
2163
2164         return (error == nil);
2165
2166     } else if ([recordType isEqualToString:SecCKRecordManifestType]) {
2167         ckksinfo("ckks", self, "CloudKit notification: deleted manifest record (%@): %@", recordType, recordID);
2168
2169         NSError* error = nil;
2170         CKKSManifest* manifest = [CKKSManifest manifestForRecordName:recordID.recordName error:&error];
2171         if (manifest) {
2172             [manifest deleteFromDatabase:&error];
2173         }
2174
2175         ckksinfo("ckks", self, "CKKSManifest was deleted: %@ %@ error: %@", recordID, manifest, error);
2176         // TODO: actually pass error back up
2177         return error == nil;
2178     }
2179
2180     else {
2181         ckkserror("ckksfetch", self, "unknown record type: %@ %@", recordType, recordID);
2182         return false;
2183     }
2184 }
2185
2186 - (bool)_onqueueCKRecordChanged:(CKRecord*)record resync:(bool)resync {
2187     dispatch_assert_queue(self.queue);
2188
2189     @autoreleasepool {
2190         ckksnotice("ckksfetch", self, "Processing record modification(%@): %@", record.recordType, record);
2191
2192         if([[record recordType] isEqual: SecCKRecordItemType]) {
2193             [self _onqueueCKRecordItemChanged:record resync:resync];
2194             return true;
2195         } else if([[record recordType] isEqual: SecCKRecordCurrentItemType]) {
2196             [self _onqueueCKRecordCurrentItemPointerChanged:record resync:resync];
2197             return true;
2198         } else if([[record recordType] isEqual: SecCKRecordIntermediateKeyType]) {
2199             [self _onqueueCKRecordKeyChanged:record resync:resync];
2200             return true;
2201         } else if ([[record recordType] isEqual: SecCKRecordTLKShareType]) {
2202             [self _onqueueCKRecordTLKShareChanged:record resync:resync];
2203             return true;
2204         } else if([[record recordType] isEqualToString: SecCKRecordCurrentKeyType]) {
2205             [self _onqueueCKRecordCurrentKeyPointerChanged:record resync:resync];
2206             return true;
2207         } else if ([[record recordType] isEqualToString:SecCKRecordManifestType]) {
2208             [self _onqueueCKRecordManifestChanged:record resync:resync];
2209             return true;
2210         } else if ([[record recordType] isEqualToString:SecCKRecordManifestLeafType]) {
2211             [self _onqueueCKRecordManifestLeafChanged:record resync:resync];
2212             return true;
2213         } else if ([[record recordType] isEqualToString:SecCKRecordDeviceStateType]) {
2214             [self _onqueueCKRecordDeviceStateChanged:record resync:resync];
2215             return true;
2216         } else {
2217             ckkserror("ckksfetch", self, "unknown record type: %@ %@", [record recordType], record);
2218             return false;
2219         }
2220     }
2221 }
2222
2223 - (void)_onqueueCKRecordItemChanged:(CKRecord*)record resync:(bool)resync {
2224     dispatch_assert_queue(self.queue);
2225
2226     NSError* error = nil;
2227     // Find if we knew about this record in the past
2228     bool update = false;
2229     CKKSMirrorEntry* ckme = [CKKSMirrorEntry tryFromDatabase: [[record recordID] recordName] zoneID:self.zoneID error:&error];
2230
2231     if(error) {
2232         ckkserror("ckks", self, "error loading a CKKSMirrorEntry from database: %@", error);
2233         // TODO: quit?
2234     }
2235
2236     if(resync) {
2237         if(!ckme) {
2238             ckkserror("ckksresync", self, "BUG: No local item matching resynced CloudKit record: %@", record);
2239         } else if(![ckme matchesCKRecord:record]) {
2240             ckkserror("ckksresync", self, "BUG: Local item doesn't match resynced CloudKit record: %@ %@", ckme, record);
2241         } else {
2242             ckksnotice("ckksresync", self, "Already know about this item record, updating anyway: %@", record.recordID);
2243         }
2244     }
2245
2246     if(ckme && ckme.item && ckme.item.generationCount > [record[SecCKRecordGenerationCountKey] unsignedLongLongValue]) {
2247         ckkserror("ckks", self, "received a record from CloudKit with a bad generation count: %@ (%ld > %@)", ckme.uuid,
2248                  (long) ckme.item.generationCount,
2249                  record[SecCKRecordGenerationCountKey]);
2250         // Abort processing this record.
2251         return;
2252     }
2253
2254     // If we found an old version in the database; this might be an update
2255     if(ckme) {
2256         if([ckme matchesCKRecord:record] && !resync) {
2257             // This is almost certainly a record we uploaded; CKFetchChanges sends them back as new records
2258             ckksnotice("ckks", self, "CloudKit has told us of record we already know about for %@; skipping update", ckme.uuid);
2259             return;
2260         }
2261
2262         update = true;
2263         // Set the CKKSMirrorEntry's fields to be whatever this record holds
2264         [ckme setFromCKRecord: record];
2265     } else {
2266         // Have to make a new CKKSMirrorEntry
2267         ckme = [[CKKSMirrorEntry alloc] initWithCKRecord: record];
2268     }
2269
2270     [ckme saveToDatabase: &error];
2271
2272     if(error) {
2273         ckkserror("ckks", self, "couldn't save new CKRecord to database: %@ %@", record, error);
2274     } else {
2275         ckksinfo("ckks", self, "CKKSMirrorEntry was created: %@", ckme);
2276     }
2277
2278     NSError* iqeerror = nil;
2279     CKKSIncomingQueueEntry* iqe = [[CKKSIncomingQueueEntry alloc] initWithCKKSItem:ckme.item
2280                                                                             action:(update ? SecCKKSActionModify : SecCKKSActionAdd)
2281                                                                              state:SecCKKSStateNew];
2282     [iqe saveToDatabase:&iqeerror];
2283     if(iqeerror) {
2284         ckkserror("ckks", self, "Couldn't save modified incoming queue entry: %@", iqeerror);
2285     } else {
2286         ckksinfo("ckks", self, "CKKSIncomingQueueEntry was created: %@", iqe);
2287     }
2288
2289     // A remote change has occured for this record. Delete any pending local changes; they will be overwritten.
2290     NSArray<CKKSOutgoingQueueEntry*>* siblings = [CKKSOutgoingQueueEntry allWithUUID:iqe.uuid
2291                                                                               states:@[SecCKKSStateNew,
2292                                                                                        SecCKKSStateReencrypt,
2293                                                                                        SecCKKSStateError]
2294                                                                               zoneID:self.zoneID
2295                                                                                error:&error];
2296     if(error) {
2297         ckkserror("ckks", self, "Couldn't load OQE sibling for %@: %@", iqe.uuid, error);
2298     }
2299
2300     for(CKKSOutgoingQueueEntry* oqe in siblings) {
2301         NSError* deletionError = nil;
2302         [oqe deleteFromDatabase:&deletionError];
2303         if(deletionError) {
2304             ckkserror("ckks", self, "Couldn't delete OQE sibling(%@) for %@: %@", oqe, iqe.uuid, deletionError);
2305         }
2306     }
2307 }
2308
2309 - (void)_onqueueCKRecordKeyChanged:(CKRecord*)record resync:(bool)resync {
2310     dispatch_assert_queue(self.queue);
2311
2312     NSError* error = nil;
2313
2314     if(resync) {
2315         NSError* resyncerror = nil;
2316
2317         CKKSKey* key = [CKKSKey tryFromDatabaseAnyState:record.recordID.recordName zoneID:self.zoneID error:&resyncerror];
2318         if(resyncerror) {
2319             ckkserror("ckksresync", self, "error loading key: %@", resyncerror);
2320         }
2321         if(!key) {
2322             ckkserror("ckksresync", self, "BUG: No sync key matching resynced CloudKit record: %@", record);
2323         } else if(![key matchesCKRecord:record]) {
2324             ckkserror("ckksresync", self, "BUG: Local sync key doesn't match resynced CloudKit record(s): %@ %@", key, record);
2325         } else {
2326             ckksnotice("ckksresync", self, "Already know about this sync key, skipping update: %@", record);
2327             return;
2328         }
2329     }
2330
2331     CKKSKey* remotekey = [[CKKSKey alloc] initWithCKRecord: record];
2332
2333     // Do we already know about this key?
2334     CKKSKey* possibleLocalKey = [CKKSKey tryFromDatabase:remotekey.uuid zoneID:self.zoneID error:&error];
2335     if(error) {
2336         ckkserror("ckkskey", self, "Error findibg exsiting local key for %@: %@", remotekey, error);
2337         // Go on, assuming there isn't a local key
2338     } else if(possibleLocalKey && [possibleLocalKey matchesCKRecord:record]) {
2339         // Okay, nothing new here. Update the CKRecord and move on.
2340         // Note: If the new record doesn't match the local copy, we have to go through the whole dance below
2341         possibleLocalKey.storedCKRecord = record;
2342         [possibleLocalKey saveToDatabase:&error];
2343
2344         if(error) {
2345             ckkserror("ckkskey", self, "Couldn't update existing key: %@: %@", possibleLocalKey, error);
2346         }
2347         return;
2348     }
2349
2350     // Drop into the synckeys table as a 'remote' key, then ask for a rekey operation.
2351     remotekey.state = SecCKKSProcessedStateRemote;
2352     remotekey.currentkey = false;
2353
2354     [remotekey saveToDatabase:&error];
2355     if(error) {
2356         ckkserror("ckkskey", self, "Couldn't save key record to database: %@: %@", remotekey, error);
2357         ckksinfo("ckkskey", self, "CKRecord was %@", record);
2358     }
2359
2360     // We've saved a new key in the database; trigger a rekey operation.
2361     [self.stateMachine _onqueueHandleFlag:CKKSFlagKeyStateProcessRequested];
2362 }
2363
2364 - (void)_onqueueCKRecordTLKShareChanged:(CKRecord*)record resync:(bool)resync {
2365     dispatch_assert_queue(self.queue);
2366
2367     NSError* error = nil;
2368     if(resync) {
2369         // TODO fill in
2370     }
2371
2372     // CKKSTLKShares get saved with no modification
2373     CKKSTLKShareRecord* share = [[CKKSTLKShareRecord alloc] initWithCKRecord:record];
2374     [share saveToDatabase:&error];
2375     if(error) {
2376         ckkserror("ckksshare", self, "Couldn't save new TLK share to database: %@ %@", share, error);
2377     }
2378
2379     [self.stateMachine _onqueueHandleFlag:CKKSFlagKeyStateProcessRequested];
2380 }
2381
2382 - (void)_onqueueCKRecordCurrentKeyPointerChanged:(CKRecord*)record resync:(bool)resync {
2383     dispatch_assert_queue(self.queue);
2384
2385     // Pull out the old CKP, if it exists
2386     NSError* ckperror = nil;
2387     CKKSCurrentKeyPointer* oldckp = [CKKSCurrentKeyPointer tryFromDatabase:((CKKSKeyClass*) record.recordID.recordName) zoneID:self.zoneID error:&ckperror];
2388     if(ckperror) {
2389         ckkserror("ckkskey", self, "error loading ckp: %@", ckperror);
2390     }
2391
2392     if(resync) {
2393         if(!oldckp) {
2394             ckkserror("ckksresync", self, "BUG: No current key pointer matching resynced CloudKit record: %@", record);
2395         } else if(![oldckp matchesCKRecord:record]) {
2396             ckkserror("ckksresync", self, "BUG: Local current key pointer doesn't match resynced CloudKit record: %@ %@", oldckp, record);
2397         } else {
2398             ckksnotice("ckksresync", self, "Current key pointer has 'changed', but it matches our local copy: %@", record);
2399         }
2400     }
2401
2402     NSError* error = nil;
2403     CKKSCurrentKeyPointer* currentkey = [[CKKSCurrentKeyPointer alloc] initWithCKRecord: record];
2404
2405     [currentkey saveToDatabase: &error];
2406     if(error) {
2407         ckkserror("ckkskey", self, "Couldn't save current key pointer to database: %@: %@", currentkey, error);
2408         ckksinfo("ckkskey", self, "CKRecord was %@", record);
2409     }
2410
2411     if([oldckp matchesCKRecord:record]) {
2412         ckksnotice("ckkskey", self, "Current key pointer modification doesn't change anything interesting; skipping reprocess: %@", record);
2413     } else {
2414         // We've saved a new key in the database; trigger a rekey operation.
2415         [self.stateMachine _onqueueHandleFlag:CKKSFlagKeyStateProcessRequested];
2416     }
2417 }
2418
2419 - (void)_onqueueCKRecordCurrentItemPointerChanged:(CKRecord*)record resync:(bool)resync {
2420     dispatch_assert_queue(self.queue);
2421
2422     if(resync) {
2423         NSError* ciperror = nil;
2424         CKKSCurrentItemPointer* localcip  = [CKKSCurrentItemPointer tryFromDatabase:record.recordID.recordName state:SecCKKSProcessedStateLocal  zoneID:self.zoneID error:&ciperror];
2425         CKKSCurrentItemPointer* remotecip = [CKKSCurrentItemPointer tryFromDatabase:record.recordID.recordName state:SecCKKSProcessedStateRemote zoneID:self.zoneID error:&ciperror];
2426         if(ciperror) {
2427             ckkserror("ckksresync", self, "error loading cip: %@", ciperror);
2428         }
2429         if(!(localcip || remotecip)) {
2430             ckkserror("ckksresync", self, "BUG: No current item pointer matching resynced CloudKit record: %@", record);
2431         } else if(! ([localcip matchesCKRecord:record] || [remotecip matchesCKRecord:record]) ) {
2432             ckkserror("ckksresync", self, "BUG: Local current item pointer doesn't match resynced CloudKit record(s): %@ %@ %@", localcip, remotecip, record);
2433         } else {
2434             ckksnotice("ckksresync", self, "Already know about this current item pointer, skipping update: %@", record);
2435             return;
2436         }
2437     }
2438
2439     NSError* error = nil;
2440     CKKSCurrentItemPointer* cip = [[CKKSCurrentItemPointer alloc] initWithCKRecord: record];
2441     cip.state = SecCKKSProcessedStateRemote;
2442
2443     [cip saveToDatabase: &error];
2444     if(error) {
2445         ckkserror("currentitem", self, "Couldn't save current item pointer to database: %@: %@ %@", cip, error, record);
2446     }
2447 }
2448
2449 - (void)_onqueueCKRecordManifestChanged:(CKRecord*)record resync:(bool)resync
2450 {
2451     dispatch_assert_queue(self.queue);
2452     NSError* error = nil;
2453     CKKSPendingManifest* manifest = [[CKKSPendingManifest alloc] initWithCKRecord:record];
2454     [manifest saveToDatabase:&error];
2455     if (error) {
2456         ckkserror("CKKS", self, "Failed to save fetched manifest record to database: %@: %@", manifest, error);
2457         ckksinfo("CKKS", self, "manifest CKRecord was %@", record);
2458     }
2459 }
2460
2461 - (void)_onqueueCKRecordManifestLeafChanged:(CKRecord*)record resync:(bool)resync
2462 {
2463     dispatch_assert_queue(self.queue);
2464     NSError* error = nil;
2465     CKKSManifestLeafRecord* manifestLeaf = [[CKKSManifestPendingLeafRecord alloc] initWithCKRecord:record];
2466     [manifestLeaf saveToDatabase:&error];
2467     if (error) {
2468         ckkserror("CKKS", self, "Failed to save fetched manifest leaf record to database: %@: %@", manifestLeaf, error);
2469         ckksinfo("CKKS", self, "manifest leaf CKRecord was %@", record);
2470     }
2471 }
2472
2473 - (void)_onqueueCKRecordDeviceStateChanged:(CKRecord*)record resync:(bool)resync {
2474     dispatch_assert_queue(self.queue);
2475     if(resync) {
2476         NSError* dserror = nil;
2477         CKKSDeviceStateEntry* cdse  = [CKKSDeviceStateEntry tryFromDatabase:record.recordID.recordName zoneID:self.zoneID error:&dserror];
2478         if(dserror) {
2479             ckkserror("ckksresync", self, "error loading cdse: %@", dserror);
2480         }
2481         if(!cdse) {
2482             ckkserror("ckksresync", self, "BUG: No current device state entry matching resynced CloudKit record: %@", record);
2483         } else if(![cdse matchesCKRecord:record]) {
2484             ckkserror("ckksresync", self, "BUG: Local current device state entry doesn't match resynced CloudKit record(s): %@ %@", cdse, record);
2485         } else {
2486             ckksnotice("ckksresync", self, "Already know about this current item pointer, skipping update: %@", record);
2487             return;
2488         }
2489     }
2490
2491     NSError* error = nil;
2492     CKKSDeviceStateEntry* cdse = [[CKKSDeviceStateEntry alloc] initWithCKRecord:record];
2493     [cdse saveToDatabase:&error];
2494     if (error) {
2495         ckkserror("ckksdevice", self, "Failed to save device record to database: %@: %@ %@", cdse, error, record);
2496     }
2497 }
2498
2499 - (bool)_onqueueResetAllInflightOQE:(NSError**)error {
2500     dispatch_assert_queue(self.queue);
2501     NSError* localError = nil;
2502
2503     while(true) {
2504         NSArray<CKKSOutgoingQueueEntry*> * inflightQueueEntries = [CKKSOutgoingQueueEntry fetch:SecCKKSOutgoingQueueItemsAtOnce
2505                                                                                           state:SecCKKSStateInFlight
2506                                                                                          zoneID:self.zoneID
2507                                                                                           error:&localError];
2508
2509         if(localError != nil) {
2510             ckkserror("ckks", self, "Error finding inflight outgoing queue records: %@", localError);
2511             if(error) {
2512                 *error = localError;
2513             }
2514             return false;
2515         }
2516
2517         if([inflightQueueEntries count] == 0u) {
2518             break;
2519         }
2520
2521         for(CKKSOutgoingQueueEntry* oqe in inflightQueueEntries) {
2522             [self _onqueueChangeOutgoingQueueEntry:oqe toState:SecCKKSStateNew error:&localError];
2523
2524             if(localError) {
2525                 ckkserror("ckks", self, "Error fixing up inflight OQE(%@): %@", oqe, localError);
2526                 if(error) {
2527                     *error = localError;
2528                 }
2529                 return false;
2530             }
2531         }
2532     }
2533
2534     return true;
2535 }
2536
2537 - (bool)_onqueueChangeOutgoingQueueEntry: (CKKSOutgoingQueueEntry*) oqe toState: (NSString*) state error: (NSError* __autoreleasing*) error {
2538     dispatch_assert_queue(self.queue);
2539
2540     NSError* localerror = nil;
2541
2542     if([state isEqualToString: SecCKKSStateDeleted]) {
2543         // Hurray, this must be a success
2544         SecBoolNSErrorCallback syncCallback = [[CKKSViewManager manager] claimCallbackForUUID:oqe.uuid];
2545         if(syncCallback) {
2546             syncCallback(true, nil);
2547         }
2548
2549         [oqe deleteFromDatabase: &localerror];
2550         if(localerror) {
2551             ckkserror("ckks", self, "Couldn't delete %@: %@", oqe, localerror);
2552         }
2553
2554     } else if([oqe.state isEqualToString:SecCKKSStateInFlight] && [state isEqualToString:SecCKKSStateNew]) {
2555         // An in-flight OQE is moving to new? See if it's been superceded
2556         CKKSOutgoingQueueEntry* newOQE = [CKKSOutgoingQueueEntry tryFromDatabase:oqe.uuid state:SecCKKSStateNew zoneID:self.zoneID error:&localerror];
2557         if(localerror) {
2558             ckkserror("ckksoutgoing", self, "Couldn't fetch an overwriting OQE, assuming one doesn't exist: %@", localerror);
2559             newOQE = nil;
2560         }
2561
2562         if(newOQE) {
2563             ckksnotice("ckksoutgoing", self, "New modification has come in behind inflight %@; dropping failed change", oqe);
2564             // recurse for that lovely code reuse
2565             [self _onqueueChangeOutgoingQueueEntry:oqe toState:SecCKKSStateDeleted error:&localerror];
2566             if(localerror) {
2567                 ckkserror("ckksoutgoing", self, "Couldn't delete in-flight OQE: %@", localerror);
2568                 if(error) {
2569                     *error = localerror;
2570                 }
2571             }
2572         } else {
2573             oqe.state = state;
2574             [oqe saveToDatabase: &localerror];
2575             if(localerror) {
2576                 ckkserror("ckks", self, "Couldn't save %@ as %@: %@", oqe, state, localerror);
2577             }
2578         }
2579
2580     } else {
2581         oqe.state = state;
2582         [oqe saveToDatabase: &localerror];
2583         if(localerror) {
2584             ckkserror("ckks", self, "Couldn't save %@ as %@: %@", oqe, state, localerror);
2585         }
2586     }
2587
2588     if(error && localerror) {
2589         *error = localerror;
2590     }
2591     return localerror == nil;
2592 }
2593
2594 - (bool)_onqueueErrorOutgoingQueueEntry: (CKKSOutgoingQueueEntry*) oqe itemError: (NSError*) itemError error: (NSError* __autoreleasing*) error {
2595     dispatch_assert_queue(self.queue);
2596
2597     SecBoolNSErrorCallback callback = [[CKKSViewManager manager] claimCallbackForUUID:oqe.uuid];
2598     if(callback) {
2599         callback(false, itemError);
2600     }
2601     NSError* localerror = nil;
2602
2603     // Now, delete the OQE: it's never coming back
2604     [oqe deleteFromDatabase:&localerror];
2605     if(localerror) {
2606         ckkserror("ckks", self, "Couldn't delete %@ (due to error %@): %@", oqe, itemError, localerror);
2607     }
2608
2609     if(error && localerror) {
2610         *error = localerror;
2611     }
2612     return localerror == nil;
2613 }
2614
2615 - (bool)dispatchSyncWithConnection:(SecDbConnectionRef _Nonnull)dbconn
2616                     readWriteTxion:(BOOL)readWriteTxion
2617                              block:(CKKSDatabaseTransactionResult (^)(void))block
2618 {
2619     CFErrorRef cferror = NULL;
2620
2621     // Take the DB transaction, then get on the local queue.
2622     // In the case of exclusive DB transactions, we don't really _need_ the local queue, but, it's here for future use.
2623
2624     SecDbTransactionType txtionType = readWriteTxion ? kSecDbExclusiveRemoteCKKSTransactionType : kSecDbNormalTransactionType;
2625     bool ret = kc_transaction_type(dbconn, txtionType, &cferror, ^bool{
2626         __block CKKSDatabaseTransactionResult result = CKKSDatabaseTransactionRollback;
2627
2628         CKKSSQLInTransaction = true;
2629         if(readWriteTxion) {
2630             CKKSSQLInWriteTransaction = true;
2631         }
2632
2633         dispatch_sync(self.queue, ^{
2634             result = block();
2635         });
2636
2637         if(readWriteTxion) {
2638             CKKSSQLInWriteTransaction = false;
2639         }
2640         CKKSSQLInTransaction = false;
2641         return result == CKKSDatabaseTransactionCommit;
2642     });
2643
2644     if(cferror) {
2645         ckkserror("ckks", self, "error doing database transaction, major problems ahead: %@", cferror);
2646     }
2647     return ret;
2648 }
2649
2650 - (void)dispatchSyncWithSQLTransaction:(CKKSDatabaseTransactionResult (^)(void))block
2651 {
2652     // important enough to block this thread. Must get a connection first, though!
2653
2654     // Please don't jetsam us...
2655     os_transaction_t transaction = os_transaction_create([[NSString stringWithFormat:@"com.apple.securityd.ckks.%@", self.zoneName] UTF8String]);
2656
2657     CFErrorRef cferror = NULL;
2658     kc_with_dbt(true, &cferror, ^bool (SecDbConnectionRef dbt) {
2659         return [self dispatchSyncWithConnection:dbt
2660                                  readWriteTxion:YES
2661                                           block:block];
2662
2663     });
2664     if(cferror) {
2665         ckkserror("ckks", self, "error getting database connection, major problems ahead: %@", cferror);
2666     }
2667
2668     (void)transaction;
2669 }
2670
2671 - (void)dispatchSyncWithReadOnlySQLTransaction:(void (^)(void))block
2672 {
2673     // Please don't jetsam us...
2674     os_transaction_t transaction = os_transaction_create([[NSString stringWithFormat:@"com.apple.securityd.ckks.%@", self.zoneName] UTF8String]);
2675
2676     CFErrorRef cferror = NULL;
2677
2678     // Note: we are lying to kc_with_dbt here about whether we're read-and-write or read-only.
2679     // This is because the SOS engine's queue are broken: SOSEngineSetNotifyPhaseBlock attempts
2680     // to take the SOS engine's queue while a SecDb transaction is still ongoing. But, in
2681     // SOSEngineCopyPeerConfirmedDigests, SOS takes the engine queue, then calls dsCopyManifestWithViewNameSet()
2682     // which attempts to get a read-only SecDb connection.
2683     //
2684     // The issue manifests when many CKKS read-only transactions are in-flight, and starve out
2685     // the pool of read-only connections. Then, a deadlock forms.
2686     //
2687     // By claiming to be a read-write connection here, we'll contend on the pool of writer threads,
2688     // and shouldn't starve SOS of its read thread.
2689     //
2690     // But, since we pass NO to readWriteTxion, the SQLite transaction will be of type
2691     // kSecDbNormalTransactionType, which won't block other readers.
2692
2693     kc_with_dbt(true, &cferror, ^bool (SecDbConnectionRef dbt) {
2694         return [self dispatchSyncWithConnection:dbt
2695                                  readWriteTxion:NO
2696                                           block:^CKKSDatabaseTransactionResult {
2697             block();
2698             return CKKSDatabaseTransactionCommit;
2699         }];
2700
2701     });
2702     if(cferror) {
2703         ckkserror("ckks", self, "error getting database connection, major problems ahead: %@", cferror);
2704     }
2705
2706     (void)transaction;
2707 }
2708
2709 - (BOOL)insideSQLTransaction
2710 {
2711     return CKKSSQLInTransaction;
2712 }
2713
2714 #pragma mark - CKKSZone operations
2715
2716 - (void)beginCloudKitOperation
2717 {
2718     [self.accountTracker registerForNotificationsOfCloudKitAccountStatusChange:self];
2719 }
2720
2721 - (CKKSResultOperation*)createAccountLoggedInDependency:(NSString*)message
2722 {
2723     WEAKIFY(self);
2724     CKKSResultOperation* accountLoggedInDependency = [CKKSResultOperation named:@"account-logged-in-dependency" withBlock:^{
2725         STRONGIFY(self);
2726         ckksnotice("ckkszone", self, "%@", message);
2727     }];
2728     accountLoggedInDependency.descriptionErrorCode = CKKSResultDescriptionPendingAccountLoggedIn;
2729     return accountLoggedInDependency;
2730 }
2731
2732 #pragma mark - CKKSZoneUpdateReceiverProtocol
2733
2734 - (CKKSAccountStatus)accountStatusFromCKAccountInfo:(CKAccountInfo*)info
2735 {
2736     if(!info) {
2737         return CKKSAccountStatusUnknown;
2738     }
2739     if(info.accountStatus == CKAccountStatusAvailable &&
2740        info.hasValidCredentials) {
2741         return CKKSAccountStatusAvailable;
2742     } else {
2743         return CKKSAccountStatusNoAccount;
2744     }
2745 }
2746
2747 - (void)cloudkitAccountStateChange:(CKAccountInfo* _Nullable)oldAccountInfo to:(CKAccountInfo*)currentAccountInfo
2748 {
2749     ckksnotice("ckkszone", self, "%@ Received notification of CloudKit account status change, moving from %@ to %@",
2750                self.zoneID.zoneName,
2751                oldAccountInfo,
2752                currentAccountInfo);
2753
2754     // Filter for device2device encryption and cloudkit grey mode
2755     CKKSAccountStatus oldStatus = [self accountStatusFromCKAccountInfo:oldAccountInfo];
2756     CKKSAccountStatus currentStatus = [self accountStatusFromCKAccountInfo:currentAccountInfo];
2757
2758     if(oldStatus == currentStatus) {
2759         ckksnotice("ckkszone", self, "Computed status of new CK account info is same as old status: %@", [CKKSAccountStateTracker stringFromAccountStatus:currentStatus]);
2760         return;
2761     }
2762
2763     switch(currentStatus) {
2764         case CKKSAccountStatusAvailable: {
2765             ckksnotice("ckkszone", self, "Logged into iCloud.");
2766             [self handleCKLogin];
2767
2768             if(self.accountLoggedInDependency) {
2769                 [self.operationQueue addOperation:self.accountLoggedInDependency];
2770                 self.accountLoggedInDependency = nil;
2771             };
2772         }
2773             break;
2774
2775         case CKKSAccountStatusNoAccount: {
2776             ckksnotice("ckkszone", self, "Logging out of iCloud. Shutting down.");
2777
2778             if(!self.accountLoggedInDependency) {
2779                 self.accountLoggedInDependency = [self createAccountLoggedInDependency:@"CloudKit account logged in again."];
2780             }
2781
2782             [self handleCKLogout];
2783         }
2784             break;
2785
2786         case CKKSAccountStatusUnknown: {
2787             // We really don't expect to receive this as a notification, but, okay!
2788             ckksnotice("ckkszone", self, "Account status has become undetermined. Pausing for %@", self.zoneID.zoneName);
2789
2790             if(!self.accountLoggedInDependency) {
2791                 self.accountLoggedInDependency = [self createAccountLoggedInDependency:@"CloudKit account logged in again."];
2792             }
2793
2794             [self handleCKLogout];
2795         }
2796             break;
2797     }
2798 }
2799
2800 - (void)handleCKLogin
2801 {
2802     ckksnotice("ckks", self, "received a notification of CK login");
2803     if(!SecCKKSIsEnabled()) {
2804         ckksnotice("ckks", self, "Skipping CloudKit initialization due to disabled CKKS");
2805         return;
2806     }
2807
2808     dispatch_sync(self.queue, ^{
2809         ckksinfo("ckkszone", self, "received a notification of CK login");
2810
2811         // Change our condition variables to reflect that we think we're logged in
2812         self.accountStatus = CKKSAccountStatusAvailable;
2813         self.loggedOut = [[CKKSCondition alloc] initToChain:self.loggedOut];
2814         [self.loggedIn fulfill];
2815     });
2816
2817     [self.stateMachine handleFlag:CKKSFlagCloudKitLoggedIn];
2818
2819     [self.accountStateKnown fulfill];
2820 }
2821
2822 - (void)handleCKLogout
2823 {
2824     dispatch_sync(self.queue, ^{
2825         ckksinfo("ckkszone", self, "received a notification of CK logout");
2826
2827         self.accountStatus = CKKSAccountStatusNoAccount;
2828         self.loggedIn = [[CKKSCondition alloc] initToChain:self.loggedIn];
2829         [self.loggedOut fulfill];
2830     });
2831
2832     [self.stateMachine handleFlag:CKKSFlagCloudKitLoggedOut];
2833
2834     [self.accountStateKnown fulfill];
2835 }
2836
2837 #pragma mark - Trust operations
2838
2839 - (void)beginTrustedOperation:(NSArray<id<CKKSPeerProvider>>*)peerProviders
2840              suggestTLKUpload:(CKKSNearFutureScheduler*)suggestTLKUpload
2841            requestPolicyCheck:(CKKSNearFutureScheduler*)requestPolicyCheck
2842 {
2843     for(id<CKKSPeerProvider> peerProvider in peerProviders) {
2844         [peerProvider registerForPeerChangeUpdates:self];
2845     }
2846
2847     [self.launch addEvent:@"beginTrusted"];
2848
2849     dispatch_sync(self.queue, ^{
2850         ckksnotice("ckkstrust", self, "Beginning trusted operation");
2851         self.operationDependencies.peerProviders = peerProviders;
2852
2853         CKKSAccountStatus oldTrustStatus = self.trustStatus;
2854
2855         self.suggestTLKUpload = suggestTLKUpload;
2856         self.requestPolicyCheck = requestPolicyCheck;
2857
2858         self.trustStatus = CKKSAccountStatusAvailable;
2859         [self.stateMachine _onqueueHandleFlag:CKKSFlagBeginTrustedOperation];
2860
2861         if(oldTrustStatus == CKKSAccountStatusNoAccount) {
2862             ckksnotice("ckkstrust", self, "Moving from an untrusted status; we need to process incoming queue and scan for any new items");
2863
2864             [self.stateMachine _onqueueHandleFlag:CKKSFlagProcessIncomingQueue];
2865             [self.stateMachine _onqueueHandleFlag:CKKSFlagScanLocalItems];
2866         }
2867     });
2868 }
2869
2870 - (void)endTrustedOperation
2871 {
2872     [self.launch addEvent:@"endTrusted"];
2873
2874     dispatch_sync(self.queue, ^{
2875         ckksnotice("ckkstrust", self, "Ending trusted operation");
2876
2877         self.operationDependencies.peerProviders = @[];
2878
2879         self.suggestTLKUpload = nil;
2880
2881         self.trustStatus = CKKSAccountStatusNoAccount;
2882         [self.stateMachine _onqueueHandleFlag:CKKSFlagEndTrustedOperation];
2883     });
2884 }
2885
2886 - (BOOL)itemSyncingEnabled
2887 {
2888     if(!self.operationDependencies.syncingPolicy) {
2889         ckksnotice("ckks", self, "No syncing policy loaded; item syncing is disabled");
2890         return NO;
2891     } else {
2892         return [self.operationDependencies.syncingPolicy isSyncingEnabledForView:self.zoneName];
2893     }
2894 }
2895
2896 - (void)setCurrentSyncingPolicy:(TPSyncingPolicy*)syncingPolicy policyIsFresh:(BOOL)policyIsFresh
2897 {
2898     dispatch_sync(self.queue, ^{
2899         BOOL oldEnabled = [self itemSyncingEnabled];
2900
2901         self.operationDependencies.syncingPolicy = syncingPolicy;
2902
2903         BOOL enabled = [self itemSyncingEnabled];
2904         if(enabled != oldEnabled) {
2905             ckksnotice("ckks", self, "Syncing for this view is now %@ (policy: %@)", enabled ? @"enabled" : @"paused", self.operationDependencies.syncingPolicy);
2906         }
2907
2908         if(enabled) {
2909             CKKSResultOperation* incomingOp = [self processIncomingQueue:false after:nil policyConsideredAuthoritative:policyIsFresh];
2910             [self processOutgoingQueueAfter:incomingOp ckoperationGroup:nil];
2911         }
2912     });
2913 }
2914
2915 - (void)receivedItemForWrongView
2916 {
2917     [self.requestPolicyCheck trigger];
2918 }
2919
2920 #pragma mark - CKKSChangeFetcherClient
2921
2922 - (BOOL)zoneIsReadyForFetching
2923 {
2924     __block BOOL ready = NO;
2925
2926     [self dispatchSyncWithReadOnlySQLTransaction:^{
2927         ready = (bool)[self _onQueueZoneIsReadyForFetching];
2928     }];
2929
2930     return ready;
2931 }
2932
2933 - (BOOL)_onQueueZoneIsReadyForFetching
2934 {
2935     dispatch_assert_queue(self.queue);
2936     if(self.accountStatus != CKKSAccountStatusAvailable) {
2937         ckksnotice("ckksfetch", self, "Not participating in fetch: not logged in");
2938         return NO;
2939     }
2940
2941     CKKSZoneStateEntry* ckse = [CKKSZoneStateEntry state:self.operationDependencies.zoneID.zoneName];
2942
2943     if(!ckse.ckzonecreated) {
2944         ckksnotice("ckksfetch", self, "Not participating in fetch: zone not created yet");
2945         return NO;
2946     }
2947     return YES;
2948 }
2949
2950 - (CKKSCloudKitFetchRequest*)participateInFetch
2951 {
2952     __block CKKSCloudKitFetchRequest* request = [[CKKSCloudKitFetchRequest alloc] init];
2953
2954     [self dispatchSyncWithReadOnlySQLTransaction:^{
2955         if (![self _onQueueZoneIsReadyForFetching]) {
2956             ckksnotice("ckksfetch", self, "skipping fetch since zones are not ready");
2957             return;
2958         }
2959
2960         request.participateInFetch = true;
2961         [self.launch addEvent:@"fetch"];
2962
2963         if([self.keyHierarchyState isEqualToString:SecCKKSZoneKeyStateNeedFullRefetch]) {
2964             // We want to return a nil change tag (to force a resync)
2965             ckksnotice("ckksfetch", self, "Beginning refetch");
2966             request.changeToken = nil;
2967             request.resync = true;
2968         } else {
2969             CKKSZoneStateEntry* ckse = [CKKSZoneStateEntry state:self.zoneName];
2970             if(!ckse) {
2971                 ckkserror("ckksfetch", self, "couldn't fetch zone change token for %@", self.zoneName);
2972                 return;
2973             }
2974             request.changeToken = ckse.changeToken;
2975         }
2976     }];
2977
2978     if (request.changeToken == nil) {
2979         self.launch.firstLaunch = true;
2980     }
2981
2982     return request;
2983 }
2984
2985 - (void)changesFetched:(NSArray<CKRecord*>*)changedRecords
2986       deletedRecordIDs:(NSArray<CKKSCloudKitDeletion*>*)deletedRecords
2987         newChangeToken:(CKServerChangeToken*)newChangeToken
2988             moreComing:(BOOL)moreComing
2989                 resync:(BOOL)resync
2990 {
2991     [self.launch addEvent:@"changes-fetched"];
2992
2993     if(changedRecords.count == 0 && deletedRecords.count == 0 && !moreComing && !resync) {
2994         // Early-exit, so we don't pick up the account keys or kick off an IncomingQueue operation for no changes
2995         [self dispatchSyncWithSQLTransaction:^CKKSDatabaseTransactionResult{
2996             ckksinfo("ckksfetch", self, "No record changes in this fetch");
2997
2998             NSError* error = nil;
2999             CKKSZoneStateEntry* state = [CKKSZoneStateEntry state:self.zoneName];
3000             state.lastFetchTime = [NSDate date]; // The last fetch happened right now!
3001             state.changeToken = newChangeToken;
3002             state.moreRecordsInCloudKit = moreComing;
3003             [state saveToDatabase:&error];
3004             if(error) {
3005                 ckkserror("ckksfetch", self, "Couldn't save new server change token: %@", error);
3006             }
3007             return CKKSDatabaseTransactionCommit;
3008         }];
3009         return;
3010     }
3011
3012     [self dispatchSyncWithSQLTransaction:^CKKSDatabaseTransactionResult{
3013         for (CKRecord* record in changedRecords) {
3014             [self _onqueueCKRecordChanged:record resync:resync];
3015         }
3016
3017         for (CKKSCloudKitDeletion* deletion in deletedRecords) {
3018             [self _onqueueCKRecordDeleted:deletion.recordID recordType:deletion.recordType resync:resync];
3019         }
3020
3021         NSError* error = nil;
3022         if(resync) {
3023             // If we're performing a resync, we need to keep track of everything that's actively in
3024             // CloudKit during the fetch, (so that we can find anything that's on-disk and not in CloudKit).
3025             // Please note that if, during a resync, the fetch errors, we won't be notified. If a record is in
3026             // the first refetch but not the second, it'll be added to our set, and the second resync will not
3027             // delete the record (which is a consistency violation, but only with actively changing records).
3028             // A third resync should correctly delete that record.
3029
3030             if(self.resyncRecordsSeen == nil) {
3031                 self.resyncRecordsSeen = [NSMutableSet set];
3032             }
3033             for(CKRecord* r in changedRecords) {
3034                 [self.resyncRecordsSeen addObject:r.recordID.recordName];
3035             }
3036
3037             // Is there More Coming? If not, self.resyncRecordsSeen contains everything in CloudKit. Inspect for anything extra!
3038             if(moreComing) {
3039                 ckksnotice("ckksresync", self, "In a resync, but there's More Coming. Waiting to scan for extra items.");
3040
3041             } else {
3042                 // Scan through all CKMirrorEntries and determine if any exist that CloudKit didn't tell us about
3043                 ckksnotice("ckksresync", self, "Comparing local UUIDs against the CloudKit list");
3044                 NSMutableArray<NSString*>* uuids = [[CKKSMirrorEntry allUUIDs:self.zoneID error:&error] mutableCopy];
3045
3046                 for(NSString* uuid in uuids) {
3047                     if([self.resyncRecordsSeen containsObject:uuid]) {
3048                         ckksnotice("ckksresync", self, "UUID %@ is still in CloudKit; carry on.", uuid);
3049                     } else {
3050                         CKKSMirrorEntry* ckme = [CKKSMirrorEntry tryFromDatabase:uuid zoneID:self.zoneID error:&error];
3051                         if(error != nil) {
3052                             ckkserror("ckksresync", self, "Couldn't read an item from the database, but it used to be there: %@ %@", uuid, error);
3053                             continue;
3054                         }
3055                         if(!ckme) {
3056                             ckkserror("ckksresync", self, "Couldn't read ckme(%@) from database; continuing", uuid);
3057                             continue;
3058                         }
3059
3060                         ckkserror("ckksresync", self, "BUG: Local item %@ not found in CloudKit, deleting", uuid);
3061                         [self _onqueueCKRecordDeleted:ckme.item.storedCKRecord.recordID recordType:ckme.item.storedCKRecord.recordType resync:resync];
3062                     }
3063                 }
3064
3065                 // Now that we've inspected resyncRecordsSeen, reset it for the next time through
3066                 self.resyncRecordsSeen = nil;
3067             }
3068         }
3069
3070         CKKSZoneStateEntry* state = [CKKSZoneStateEntry state:self.zoneName];
3071         state.lastFetchTime = [NSDate date]; // The last fetch happened right now!
3072         state.changeToken = newChangeToken;
3073         state.moreRecordsInCloudKit = moreComing;
3074         [state saveToDatabase:&error];
3075         if(error) {
3076             ckkserror("ckksfetch", self, "Couldn't save new server change token: %@", error);
3077         }
3078
3079         if(!moreComing) {
3080             // Might as well kick off a IQO!
3081             [self processIncomingQueue:false];
3082             ckksnotice("ckksfetch", self, "Beginning incoming processing for %@", self.zoneID);
3083         }
3084
3085         ckksnotice("ckksfetch", self, "Finished processing changes for %@", self.zoneID);
3086
3087         return CKKSDatabaseTransactionCommit;
3088     }];
3089 }
3090
3091 - (bool)ckErrorOrPartialError:(NSError *)error isError:(CKErrorCode)errorCode
3092 {
3093     if((error.code == errorCode) && [error.domain isEqualToString:CKErrorDomain]) {
3094         return true;
3095     } else if((error.code == CKErrorPartialFailure) && [error.domain isEqualToString:CKErrorDomain]) {
3096         NSDictionary* partialErrors = error.userInfo[CKPartialErrorsByItemIDKey];
3097
3098         NSError* partialError = partialErrors[self.zoneID];
3099         if ((partialError.code == errorCode) && [partialError.domain isEqualToString:CKErrorDomain]) {
3100             return true;
3101         }
3102     }
3103     return false;
3104 }
3105
3106 - (bool)shouldRetryAfterFetchError:(NSError*)error {
3107
3108     bool isChangeTokenExpiredError = [self ckErrorOrPartialError:error isError:CKErrorChangeTokenExpired];
3109     if(isChangeTokenExpiredError) {
3110         ckkserror("ckks", self, "Received notice that our change token is out of date (for %@). Resetting local data...", self.zoneID);
3111
3112         [self.stateMachine handleFlag:CKKSFlagChangeTokenExpired];
3113         return true;
3114     }
3115
3116     bool isDeletedZoneError = [self ckErrorOrPartialError:error isError:CKErrorZoneNotFound];
3117     if(isDeletedZoneError) {
3118         ckkserror("ckks", self, "Received notice that our zone(%@) does not exist. Resetting local data.", self.zoneID);
3119
3120         [self.stateMachine handleFlag:CKKSFlagCloudKitZoneMissing];
3121         return false;
3122     }
3123
3124     if([error.domain isEqualToString:CKErrorDomain] && (error.code == CKErrorBadContainer)) {
3125         ckkserror("ckks", self, "Received notice that our container does not exist. Nothing to do.");
3126         return false;
3127     }
3128
3129     return true;
3130 }
3131
3132 #pragma mark CKKSPeerUpdateListener
3133
3134 - (void)selfPeerChanged:(id<CKKSPeerProvider>)provider
3135 {
3136     // Currently, we have no idea what to do with this. Kick off a key reprocess?
3137     ckkserror("ckks", self, "Received update that our self identity has changed");
3138     [self keyStateMachineRequestProcess];
3139 }
3140
3141 - (void)trustedPeerSetChanged:(id<CKKSPeerProvider>)provider
3142 {
3143     // We might need to share the TLK to some new people, or we might now trust the TLKs we have.
3144     // The key state machine should handle that, so poke it.
3145     ckkserror("ckks", self, "Received update that the trust set has changed");
3146
3147     [self.stateMachine handleFlag:CKKSFlagTrustedPeersSetChanged];
3148 }
3149
3150 #pragma mark - Test Support
3151
3152 - (bool) outgoingQueueEmpty: (NSError * __autoreleasing *) error {
3153     __block bool ret = false;
3154     [self dispatchSyncWithReadOnlySQLTransaction:^{
3155         NSArray* queueEntries = [CKKSOutgoingQueueEntry all: error];
3156         ret = queueEntries && ([queueEntries count] == 0);
3157     }];
3158
3159     return ret;
3160 }
3161
3162 - (void)waitForFetchAndIncomingQueueProcessing
3163 {
3164     [[self.zoneChangeFetcher inflightFetch] waitUntilFinished];
3165     [self waitForOperationsOfClass:[CKKSIncomingQueueOperation class]];
3166 }
3167
3168 - (void)waitForKeyHierarchyReadiness {
3169     if(self.keyStateReadyDependency) {
3170         [self.keyStateReadyDependency waitUntilFinished];
3171     }
3172 }
3173
3174 #pragma mark - NSOperation assistance
3175
3176 - (void)scheduleOperation:(NSOperation*)op
3177 {
3178     if(self.halted) {
3179         ckkserror("ckkszone", self, "attempted to schedule an operation on a halted zone, ignoring");
3180         return;
3181     }
3182
3183     [op addNullableDependency:self.accountLoggedInDependency];
3184     [self.operationQueue addOperation: op];
3185 }
3186
3187 // to be used rarely, if at all
3188 - (bool)scheduleOperationWithoutDependencies:(NSOperation*)op
3189 {
3190     if(self.halted) {
3191         ckkserror("ckkszone", self, "attempted to schedule an non-dependent operation on a halted zone, ignoring");
3192         return false;
3193     }
3194
3195     [self.operationQueue addOperation: op];
3196     return true;
3197 }
3198
3199 - (void)waitUntilAllOperationsAreFinished
3200 {
3201     [self.operationQueue waitUntilAllOperationsAreFinished];
3202 }
3203
3204 - (void)waitForOperationsOfClass:(Class)operationClass
3205 {
3206     NSArray* operations = [self.operationQueue.operations copy];
3207     for(NSOperation* op in operations) {
3208         if([op isKindOfClass:operationClass]) {
3209             [op waitUntilFinished];
3210         }
3211     }
3212 }
3213
3214 - (void)cancelPendingOperations {
3215     @synchronized(self.outgoingQueueOperations) {
3216         for(NSOperation* op in self.outgoingQueueOperations) {
3217             [op cancel];
3218         }
3219         [self.outgoingQueueOperations removeAllObjects];
3220     }
3221
3222     @synchronized(self.incomingQueueOperations) {
3223         for(NSOperation* op in self.incomingQueueOperations) {
3224             [op cancel];
3225         }
3226         [self.incomingQueueOperations removeAllObjects];
3227     }
3228
3229     @synchronized(self.scanLocalItemsOperations) {
3230         for(NSOperation* op in self.scanLocalItemsOperations) {
3231             [op cancel];
3232         }
3233         [self.scanLocalItemsOperations removeAllObjects];
3234     }
3235 }
3236
3237 - (void)cancelAllOperations {
3238     [self.keyStateReadyDependency cancel];
3239     [self.zoneChangeFetcher cancel];
3240     [self.notifyViewChangedScheduler cancel];
3241
3242     [self cancelPendingOperations];
3243     [self.operationQueue cancelAllOperations];
3244 }
3245
3246 - (void)halt {
3247     [self.stateMachine haltOperation];
3248
3249     // Synchronously set the 'halted' bit
3250     dispatch_sync(self.queue, ^{
3251         self.halted = true;
3252     });
3253
3254     // Bring all operations down, too
3255     [self cancelAllOperations];
3256
3257     // And now, wait for all operations that are running
3258     for(NSOperation* op in self.operationQueue.operations) {
3259         if(op.isExecuting) {
3260             [op waitUntilFinished];
3261         }
3262     }
3263
3264     // Don't send any more notifications, either
3265     _notifierClass = nil;
3266 }
3267
3268 - (NSDictionary*)status {
3269 #define stringify(obj) CKKSNilToNSNull([obj description])
3270 #define boolstr(obj) (!!(obj) ? @"yes" : @"no")
3271     __block NSMutableDictionary* ret = nil;
3272     __block NSError* error = nil;
3273
3274     ret = [[self fastStatus] mutableCopy];
3275
3276     [self dispatchSyncWithReadOnlySQLTransaction:^{
3277         CKKSCurrentKeySet* keyset = [CKKSCurrentKeySet loadForZone:self.zoneID];
3278         if(keyset.error) {
3279             error = keyset.error;
3280         }
3281
3282         if(error) {
3283             ckkserror("ckks", self, "error during status: %@", error);
3284         }
3285         // We actually don't care about this error, especially if it's "no current key pointers"...
3286         error = nil;
3287
3288         // Map deviceStates to strings to avoid NSXPC issues. Obj-c, why is this so hard?
3289         NSArray* deviceStates = [CKKSDeviceStateEntry allInZone:self.zoneID error:&error];
3290         NSMutableArray<NSString*>* mutDeviceStates = [[NSMutableArray alloc] init];
3291         [deviceStates enumerateObjectsUsingBlock:^(id  _Nonnull obj, NSUInteger idx, BOOL * _Nonnull stop) {
3292             [mutDeviceStates addObject: [obj description]];
3293         }];
3294
3295         NSArray* tlkShares = [CKKSTLKShareRecord allForUUID:keyset.currentTLKPointer.currentKeyUUID zoneID:self.zoneID error:&error];
3296         NSMutableArray<NSString*>* mutTLKShares = [[NSMutableArray alloc] init];
3297         [tlkShares enumerateObjectsUsingBlock:^(id  _Nonnull obj, NSUInteger idx, BOOL * _Nonnull stop) {
3298             [mutTLKShares addObject: [obj description]];
3299         }];
3300
3301         [ret addEntriesFromDictionary:@{
3302                  @"statusError":         stringify(error),
3303                  @"oqe":                 CKKSNilToNSNull([CKKSOutgoingQueueEntry countsByStateInZone:self.zoneID error:&error]),
3304                  @"iqe":                 CKKSNilToNSNull([CKKSIncomingQueueEntry countsByStateInZone:self.zoneID error:&error]),
3305                  @"ckmirror":            CKKSNilToNSNull([CKKSMirrorEntry        countsByParentKey:self.zoneID error:&error]),
3306                  @"devicestates":        CKKSNilToNSNull(mutDeviceStates),
3307                  @"tlkshares":           CKKSNilToNSNull(mutTLKShares),
3308                  @"keys":                CKKSNilToNSNull([CKKSKey countsByClass:self.zoneID error:&error]),
3309                  @"currentTLK":          CKKSNilToNSNull(keyset.tlk.uuid),
3310                  @"currentClassA":       CKKSNilToNSNull(keyset.classA.uuid),
3311                  @"currentClassC":       CKKSNilToNSNull(keyset.classC.uuid),
3312                  @"currentTLKPtr":       CKKSNilToNSNull(keyset.currentTLKPointer.currentKeyUUID),
3313                  @"currentClassAPtr":    CKKSNilToNSNull(keyset.currentClassAPointer.currentKeyUUID),
3314                  @"currentClassCPtr":    CKKSNilToNSNull(keyset.currentClassCPointer.currentKeyUUID),
3315                  @"itemsyncing":         self.itemSyncingEnabled ? @"enabled" : @"paused",
3316             }];
3317     }];
3318     return ret;
3319 }
3320
3321 - (NSDictionary*)fastStatus {
3322
3323     __block NSDictionary* ret = nil;
3324
3325     [self dispatchSyncWithReadOnlySQLTransaction:^{
3326         CKKSZoneStateEntry* ckse = [CKKSZoneStateEntry state:self.zoneName];
3327
3328         ret = @{
3329             @"view":                CKKSNilToNSNull(self.zoneName),
3330             @"ckaccountstatus":     self.accountStatus == CKAccountStatusCouldNotDetermine ? @"could not determine" :
3331                 self.accountStatus == CKAccountStatusAvailable         ? @"logged in" :
3332                 self.accountStatus == CKAccountStatusRestricted        ? @"restricted" :
3333                 self.accountStatus == CKAccountStatusNoAccount         ? @"logged out" : @"unknown",
3334             @"accounttracker":      stringify(self.accountTracker),
3335             @"fetcher":             stringify(self.zoneChangeFetcher),
3336             @"zoneCreated":         boolstr(ckse.ckzonecreated),
3337             @"zoneSubscribed":      boolstr(ckse.ckzonesubscribed),
3338             @"keystate":            CKKSNilToNSNull(self.keyHierarchyState),
3339             @"statusError":         [NSNull null],
3340             @"launchSequence":      CKKSNilToNSNull([self.launch eventsByTime]),
3341
3342             @"lastIncomingQueueOperation":         stringify(self.lastIncomingQueueOperation),
3343             @"lastNewTLKOperation":                stringify(self.lastNewTLKOperation),
3344             @"lastOutgoingQueueOperation":         stringify(self.lastOutgoingQueueOperation),
3345             @"lastProcessReceivedKeysOperation":   stringify(self.lastProcessReceivedKeysOperation),
3346             @"lastReencryptOutgoingItemsOperation":stringify(self.lastReencryptOutgoingItemsOperation),
3347         };
3348     }];
3349
3350     return ret;
3351 }
3352
3353 #endif /* OCTAGON */
3354 @end