keychain/ckks/CKKSIncomingQueueOperation.m

   1 /*
   2  * Copyright (c) 2016-2020 Apple Inc. All Rights Reserved.
   3  *
   4  * @APPLE_LICENSE_HEADER_START@
   5  *
   6  * This file contains Original Code and/or Modifications of Original Code
   7  * as defined in and that are subject to the Apple Public Source License
   8  * Version 2.0 (the 'License'). You may not use this file except in
   9  * compliance with the License. Please obtain a copy of the License at
  10  * http://www.opensource.apple.com/apsl/ and read it before using this
  11  * file.
  12  *
  13  * The Original Code and all software distributed under the License are
  14  * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  15  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  16  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
  17  * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
  18  * Please see the License for the specific language governing rights and
  19  * limitations under the License.
  20  *
  21  * @APPLE_LICENSE_HEADER_END@
  22  */
  23
  24 #import "keychain/analytics/CKKSPowerCollection.h"
  25 #import "keychain/ckks/CKKSAnalytics.h"
  26 #import "keychain/ckks/CKKSCurrentItemPointer.h"
  27 #import "keychain/ckks/CKKSIncomingQueueEntry.h"
  28 #import "keychain/ckks/CKKSIncomingQueueOperation.h"
  29 #import "keychain/ckks/CKKSItemEncrypter.h"
  30 #import "keychain/ckks/CKKSKey.h"
  31 #import "keychain/ckks/CKKSKeychainView.h"
  32 #import "keychain/ckks/CKKSOutgoingQueueEntry.h"
  33 #import "keychain/ckks/CKKSStates.h"
  34 #import "keychain/ckks/CKKSViewManager.h"
  35 #import "keychain/ckks/CloudKitCategories.h"
  36 #import "keychain/ot/ObjCImprovements.h"
  37
  38 #include "keychain/securityd/SecItemServer.h"
  39 #include "keychain/securityd/SecItemDb.h"
  40 #include <Security/SecItemPriv.h>
  41
  42 #import <utilities/SecCoreAnalytics.h>
  43
  44 #if OCTAGON
  45
  46 @interface CKKSIncomingQueueOperation ()
  47 @property bool newOutgoingEntries;
  48 @property bool pendingClassAEntries;
  49 @property bool missingKey;
  50
  51 @property NSMutableSet<NSString*>* viewsToScan;
  52 @end
  53
  54 @implementation CKKSIncomingQueueOperation
  55 @synthesize nextState = _nextState;
  56 @synthesize intendedState = _intendedState;
  57
  58 - (instancetype)init {
  59     return nil;
  60 }
  61
  62 - (instancetype)initWithDependencies:(CKKSOperationDependencies*)dependencies
  63                                 ckks:(CKKSKeychainView*)ckks
  64                            intending:(OctagonState*)intending
  65                           errorState:(OctagonState*)errorState
  66                 errorOnClassAFailure:(bool)errorOnClassAFailure
  67                handleMismatchedViewItems:(bool)handleMismatchedViewItems
  68 {
  69     if(self = [super init]) {
  70         _deps = dependencies;
  71         _ckks = ckks;
  72
  73         _intendedState = intending;
  74         _nextState = errorState;
  75
  76         // Can't process unless we have a reasonable key hierarchy.
  77         if(ckks.keyStateReadyDependency) {
  78             [self addDependency: ckks.keyStateReadyDependency];
  79         }
  80
  81         [self addNullableDependency: ckks.holdIncomingQueueOperation];
  82
  83         _errorOnClassAFailure = errorOnClassAFailure;
  84         _pendingClassAEntries = false;
  85
  86         _handleMismatchedViewItems = handleMismatchedViewItems;
  87
  88         _viewsToScan = [NSMutableSet set];
  89
  90         [self linearDependencies:ckks.incomingQueueOperations];
  91     }
  92     return self;
  93 }
  94
  95 - (bool)processNewCurrentItemPointers:(NSArray<CKKSCurrentItemPointer*>*)queueEntries
  96 {
  97     NSError* error = nil;
  98     for(CKKSCurrentItemPointer* p in queueEntries) {
  99         @autoreleasepool {
 100             p.state = SecCKKSProcessedStateLocal;
 101
 102             [p saveToDatabase:&error];
 103             ckksnotice("ckkspointer", self.deps.zoneID, "Saving new current item pointer: %@", p);
 104             if(error) {
 105                 ckkserror("ckksincoming", self.deps.zoneID, "Error saving new current item pointer: %@ %@", error, p);
 106             }
 107         }
 108     }
 109
 110     if(queueEntries.count > 0) {
 111         [self.deps.notifyViewChangedScheduler trigger];
 112     }
 113
 114     return (error == nil);
 115 }
 116
 117 - (bool)processQueueEntries:(NSArray<CKKSIncomingQueueEntry*>*)queueEntries
 118 {
 119     CKKSKeychainView* ckks = self.ckks;
 120     dispatch_assert_queue(ckks.queue);
 121
 122     NSMutableArray* newOrChangedRecords = [[NSMutableArray alloc] init];
 123     NSMutableArray* deletedRecordIDs = [[NSMutableArray alloc] init];
 124
 125     for(id entry in queueEntries) {
 126         @autoreleasepool {
 127             NSError* error = nil;
 128
 129             CKKSIncomingQueueEntry* iqe = (CKKSIncomingQueueEntry*) entry;
 130             ckksnotice("ckksincoming", self.deps.zoneID, "ready to process an incoming queue entry: %@ %@ %@", iqe, iqe.uuid, iqe.action);
 131
 132             // Note that we currently unencrypt the item before deleting it, instead of just deleting it
 133             // This finds the class, which is necessary for the deletion process. We could just try to delete
 134             // across all classes, though...
 135             NSDictionary* attributes = [CKKSIncomingQueueOperation decryptCKKSItemToAttributes:iqe.item error:&error];
 136
 137             if(!attributes || error) {
 138                 if([self.deps.lockStateTracker isLockedError:error]) {
 139                     NSError* localerror = nil;
 140                     ckkserror("ckksincoming", self.deps.zoneID, "Keychain is locked; can't decrypt IQE %@", iqe);
 141                     CKKSKey* key = [CKKSKey tryFromDatabase:iqe.item.parentKeyUUID zoneID:self.deps.zoneID error:&localerror];
 142                     if(localerror || ([key.keyclass isEqualToString:SecCKKSKeyClassA] && self.errorOnClassAFailure)) {
 143                         self.error = error;
 144                     }
 145
 146                     // If this isn't an error, make sure it gets processed later.
 147                     if([key.keyclass isEqualToString:SecCKKSKeyClassA] && !self.errorOnClassAFailure) {
 148                         self.pendingClassAEntries = true;
 149                     }
 150
 151                 } else if ([error.domain isEqualToString:@"securityd"] && error.code == errSecItemNotFound) {
 152                     ckkserror("ckksincoming", self.deps.zoneID, "Coudn't find key in keychain; will attempt to poke key hierarchy: %@", error)
 153                     self.missingKey = true;
 154                     self.error = error;
 155
 156                 } else {
 157                     ckkserror("ckksincoming", self.deps.zoneID, "Couldn't decrypt IQE %@ for some reason: %@", iqe, error);
 158                     self.error = error;
 159                 }
 160                 self.errorItemsProcessed += 1;
 161                 continue;
 162             }
 163
 164             NSString* classStr = [attributes objectForKey: (__bridge NSString*) kSecClass];
 165             if(![classStr isKindOfClass: [NSString class]]) {
 166                 self.error = [NSError errorWithDomain:@"securityd"
 167                                                  code:errSecInternalError
 168                                              userInfo:@{NSLocalizedDescriptionKey : [NSString stringWithFormat:@"Item did not have a reasonable class: %@", classStr]}];
 169                 ckkserror("ckksincoming", self.deps.zoneID, "Synced item seems wrong: %@", self.error);
 170                 self.errorItemsProcessed += 1;
 171                 continue;
 172             }
 173
 174             const SecDbClass * classP = !classStr ? NULL : kc_class_with_name((__bridge CFStringRef) classStr);
 175
 176             if(!classP) {
 177                 ckkserror("ckksincoming", self.deps.zoneID, "unknown class in object: %@ %@", classStr, iqe);
 178                 iqe.state = SecCKKSStateError;
 179                 [iqe saveToDatabase:&error];
 180                 if(error) {
 181                     ckkserror("ckksincoming", self.deps.zoneID, "Couldn't save errored IQE to database: %@", error);
 182                     self.error = error;
 183                 }
 184                 self.errorItemsProcessed += 1;
 185                 continue;
 186             }
 187
 188             NSString* intendedView = [self.deps.syncingPolicy mapDictionaryToView:attributes];
 189             if(![self.deps.zoneID.zoneName isEqualToString:intendedView]) {
 190                 if(self.handleMismatchedViewItems) {
 191                     [self _onqueueHandleMismatchedViewItem:iqe
 192                                                 secDbClass:classP
 193                                                 attributes:attributes
 194                                               intendedView:intendedView];
 195                 } else {
 196                     ckksnotice("ckksincoming", ckks, "Received an item (%@), but our current policy claims it should be in view %@", iqe.uuid, intendedView);
 197
 198                     [self _onqueueUpdateIQE:iqe withState:SecCKKSStateMismatchedView error:&error];
 199                     if(error) {
 200                         ckkserror("ckksincoming", ckks, "Couldn't save mismatched IQE to database: %@", error);
 201                         self.errorItemsProcessed += 1;
 202                         self.error = error;
 203                     }
 204
 205                     [ckks receivedItemForWrongView];
 206                 }
 207                 continue;
 208             }
 209
 210             if([iqe.action isEqualToString: SecCKKSActionAdd] || [iqe.action isEqualToString: SecCKKSActionModify]) {
 211                 [self _onqueueHandleIQEChange:iqe
 212                                    attributes:attributes
 213                                         class:classP
 214                             sortedForThisView:YES];
 215                 [newOrChangedRecords addObject:[iqe.item CKRecordWithZoneID:self.deps.zoneID]];
 216
 217             } else if ([iqe.action isEqualToString: SecCKKSActionDelete]) {
 218                 [self _onqueueHandleIQEDelete: iqe class:classP];
 219                 [deletedRecordIDs addObject:[[CKRecordID alloc] initWithRecordName:iqe.uuid zoneID:self.deps.zoneID]];
 220             }
 221         }
 222     }
 223
 224     if(newOrChangedRecords.count > 0 || deletedRecordIDs > 0) {
 225         // Schedule a view change notification
 226         [self.deps.notifyViewChangedScheduler trigger];
 227     }
 228
 229     if(self.missingKey) {
 230         // TODO: will be removed when the IncomingQueueOperation is part of the state machine
 231         [ckks.stateMachine _onqueuePokeStateMachine];
 232         self.nextState = SecCKKSZoneKeyStateUnhealthy;
 233     }
 234
 235     return true;
 236 }
 237
 238 - (void)_onqueueHandleMismatchedViewItem:(CKKSIncomingQueueEntry*)iqe
 239                               secDbClass:(const SecDbClass*)secDbClass
 240                               attributes:(NSDictionary*)attributes
 241                             intendedView:(NSString* _Nullable)intendedView
 242 {
 243     ckksnotice("ckksincoming", self.deps.zoneID, "Received an item (%@), which should be in view %@", iqe.uuid, intendedView);
 244
 245     // Here's the plan:
 246     //
 247     // If this is an add or a modify, we will execute the modification _if we do not currently have this item_.
 248     // Then, ask the view that should handle this item to scan.
 249     //
 250     // When, we will leave the CloudKit record in the existing 'wrong' view.
 251     // This will allow garbage to collect, but should prevent item loss in complicated multi-device scenarios.
 252     //
 253     // If this is a deletion, then we will inspect the other zone's current on-disk state. If it knows about the item,
 254     // we will ignore the deletion from this view. Otherwise, we will proceed with the deletion.
 255     // Note that the deletion approach already ensures that the UUID of the deleted item matches the UUID of the CKRecord.
 256     // This protects against an item being in multiple views, and deleted from only one.
 257
 258     if([iqe.action isEqualToString:SecCKKSActionAdd] || [iqe.action isEqualToString:SecCKKSActionModify]) {
 259         CFErrorRef cferror = NULL;
 260         SecDbItemRef item = SecDbItemCreateWithAttributes(NULL, secDbClass, (__bridge CFDictionaryRef) attributes, KEYBAG_DEVICE, &cferror);
 261
 262         if(!item || cferror) {
 263             ckkserror("ckksincoming", self.deps.zoneID, "Unable to create SecDbItemRef from IQE: %@", cferror);
 264             return;
 265         }
 266
 267         [self _onqueueHandleIQEChange:iqe item:item sortedForThisView:NO];
 268         [self.viewsToScan addObject:intendedView];
 269
 270         CFReleaseNull(item);
 271
 272     } else if ([iqe.action isEqualToString:SecCKKSActionDelete]) {
 273         NSError* loadError = nil;
 274
 275         CKRecordZoneID* otherZoneID = [[CKRecordZoneID alloc] initWithZoneName:intendedView ownerName:CKCurrentUserDefaultName];
 276         CKKSMirrorEntry* ckme = [CKKSMirrorEntry tryFromDatabase:iqe.uuid zoneID:otherZoneID error:&loadError];
 277
 278         if(!ckme || loadError) {
 279             ckksnotice("ckksincoming", self.deps.zoneID, "Unable to load CKKSMirrorEntry from database* %@", loadError);
 280             return;
 281         }
 282
 283         if(ckme) {
 284             ckksnotice("ckksincoming", self.deps.zoneID, "Other view (%@) already knows about this item, dropping incoming queue entry: %@", intendedView, ckme);
 285             NSError* saveError = nil;
 286             [iqe deleteFromDatabase:&saveError];
 287             if(saveError) {
 288                 ckkserror("ckksincoming", self.deps.zoneID, "Unable to delete IQE: %@", saveError);
 289             }
 290
 291         } else {
 292             ckksnotice("ckksincoming", self.deps.zoneID, "Other view (%@) does not know about this item; processing delete for %@", intendedView, iqe);
 293             [self _onqueueHandleIQEDelete:iqe class:secDbClass];
 294         }
 295
 296     } else {
 297         // We don't recognize this action. Do nothing.
 298     }
 299 }
 300
 301 + (NSDictionary* _Nullable)decryptCKKSItemToAttributes:(CKKSItem*)item error:(NSError**)error
 302 {
 303     NSMutableDictionary* attributes = [[CKKSItemEncrypter decryptItemToDictionary:item error:error] mutableCopy];
 304     if(!attributes) {
 305         return nil;
 306     }
 307
 308     // Add the UUID (which isn't stored encrypted)
 309     attributes[(__bridge NSString*)kSecAttrUUID] = item.uuid;
 310
 311     // Add the PCS plaintext fields, if they exist
 312     if(item.plaintextPCSServiceIdentifier) {
 313         attributes[(__bridge NSString*)kSecAttrPCSPlaintextServiceIdentifier] = item.plaintextPCSServiceIdentifier;
 314     }
 315     if(item.plaintextPCSPublicKey) {
 316         attributes[(__bridge NSString*)kSecAttrPCSPlaintextPublicKey] = item.plaintextPCSPublicKey;
 317     }
 318     if(item.plaintextPCSPublicIdentity) {
 319         attributes[(__bridge NSString*)kSecAttrPCSPlaintextPublicIdentity] = item.plaintextPCSPublicIdentity;
 320     }
 321
 322     // This item is also synchronizable (by definition)
 323     [attributes setValue:@(YES) forKey:(__bridge NSString*)kSecAttrSynchronizable];
 324
 325     return attributes;
 326 }
 327
 328 - (bool)_onqueueUpdateIQE:(CKKSIncomingQueueEntry*)iqe withState:(NSString*)newState error:(NSError**)error
 329 {
 330     if (![iqe.state isEqualToString:newState]) {
 331         NSMutableDictionary* oldWhereClause = iqe.whereClauseToFindSelf.mutableCopy;
 332         oldWhereClause[@"state"] = iqe.state;
 333         iqe.state = newState;
 334         if ([iqe saveToDatabase:error]) {
 335             if (![CKKSSQLDatabaseObject deleteFromTable:[iqe.class sqlTable] where:oldWhereClause connection:NULL error:error]) {
 336                 return false;
 337             }
 338         }
 339         else {
 340             return false;
 341         }
 342     }
 343
 344     return true;
 345 }
 346
 347 - (void)main
 348 {
 349     CKKSKeychainView* ckks = self.ckks;
 350
 351     if(!ckks.itemSyncingEnabled) {
 352         ckkserror("ckksincoming", self.deps.zoneID, "Item syncing for this view is disabled");
 353         self.nextState = self.intendedState;
 354         return;
 355     }
 356
 357     WEAKIFY(self);
 358     self.completionBlock = ^(void) {
 359         STRONGIFY(self);
 360         if (!self) {
 361             ckkserror("ckksincoming", self.deps.zoneID, "received callback for released object");
 362             return;
 363         }
 364
 365         CKKSAnalytics* logger = [CKKSAnalytics logger];
 366
 367         if (!self.error) {
 368             [logger logSuccessForEvent:CKKSEventProcessIncomingQueueClassC zoneName:self.deps.zoneID.zoneName];
 369
 370             if (!self.pendingClassAEntries) {
 371                 [logger logSuccessForEvent:CKKSEventProcessIncomingQueueClassA zoneName:self.deps.zoneID.zoneName];
 372             }
 373         } else {
 374             [logger logRecoverableError:self.error
 375                                forEvent:self.errorOnClassAFailure ? CKKSEventProcessIncomingQueueClassA : CKKSEventProcessIncomingQueueClassC
 376                                zoneName:self.deps.zoneID.zoneName
 377                          withAttributes:NULL];
 378         }
 379     };
 380
 381     ckksnotice("ckksincoming", self.deps.zoneID, "Processing incoming queue");
 382
 383     // First, process all item deletions.
 384     // Then, process all modifications and additions.
 385     // Therefore, if there's both a delete and a re-add of a single Primary Key item in the queue,
 386     // we should end up with the item still existing in tthe keychain afterward.
 387     // But, since we're dropping off the queue inbetween, we might accidentally tell our clients that
 388     // their item doesn't exist. Fixing that would take quite a bit of complexity and memory.
 389
 390     BOOL success = [self loadAndProcessEntriesWithActionFilter:SecCKKSActionDelete];
 391     if(!success) {
 392         ckksnotice("ckksincoming", self.deps.zoneID, "Early-exiting from IncomingQueueOperation (after processing deletes): %@", self.error);
 393         return;
 394     }
 395
 396     success = [self loadAndProcessEntriesWithActionFilter:nil];
 397     if(!success) {
 398         ckksnotice("ckksincoming", self.deps.zoneID, "Early-exiting from IncomingQueueOperation (after processing all incoming entries): %@", self.error);
 399         return;
 400     }
 401
 402     ckksnotice("ckksincoming", self.deps.zoneID, "Processed %lu items in incoming queue (%lu errors)", (unsigned long)self.successfulItemsProcessed, (unsigned long)self.errorItemsProcessed);
 403
 404     if(![self fixMismatchedViewItems]) {
 405         ckksnotice("ckksincoming", ckks, "Early-exiting from IncomingQueueOperation due to failure fixing mismatched items");
 406         return;
 407     }
 408
 409     [self.deps.databaseProvider dispatchSyncWithSQLTransaction:^CKKSDatabaseTransactionResult{
 410         NSError* error = nil;
 411
 412         NSArray<CKKSCurrentItemPointer*>* newCIPs = [CKKSCurrentItemPointer remoteItemPointers:self.deps.zoneID error:&error];
 413         if(error || !newCIPs) {
 414             ckkserror("ckksincoming", self.deps.zoneID, "Could not load remote item pointers: %@", error);
 415         } else {
 416             if (![self processNewCurrentItemPointers:newCIPs]) {
 417                 return CKKSDatabaseTransactionRollback;
 418             }
 419             ckksnotice("ckksincoming", self.deps.zoneID, "Processed %lu items in CIP queue", (unsigned long)newCIPs.count);
 420         }
 421
 422         return CKKSDatabaseTransactionCommit;
 423     }];
 424
 425     if(self.newOutgoingEntries) {
 426         //[self.deps.flagHandler handleFlag:CKKSFlagProcessOutgoingQueue];
 427         [ckks processOutgoingQueue:[CKOperationGroup CKKSGroupWithName:@"incoming-queue-response"]];
 428     }
 429
 430     if(self.pendingClassAEntries) {
 431         [ckks processIncomingQueueAfterNextUnlock];
 432         //OctagonPendingFlag* whenUnlocked = [[OctagonPendingFlag alloc] initWithFlag:CKKSFlagProcessIncomingQueue
 433         //                                                                 conditions:OctagonPendingConditionsDeviceUnlocked];
 434         //[self.deps.flagHandler handlePendingFlag:whenUnlocked];
 435     }
 436
 437     for(NSString* viewName in self.viewsToScan) {
 438         CKKSKeychainView* view = [[CKKSViewManager manager] findView:viewName];
 439         ckksnotice("ckksincoming", ckks, "Requesting scan for %@ (%@)", view, viewName);
 440         [view scanLocalItems:@"policy-mismatch"];
 441     }
 442
 443     self.nextState = self.intendedState;
 444 }
 445
 446 - (BOOL)loadAndProcessEntriesWithActionFilter:(NSString* _Nullable)actionFilter
 447 {
 448     __block bool errored = false;
 449
 450     // Now for the tricky bit: take and drop the account queue for each batch of queue entries
 451     // This is for peak memory concerns, but also to allow keychain API clients to make changes while we're processing many items
 452     // Note that IncomingQueueOperations are no longer transactional: they can partially succeed. This might make them harder to reason about.
 453     __block NSUInteger lastCount = SecCKKSIncomingQueueItemsAtOnce;
 454     __block NSString* lastMaxUUID = nil;
 455
 456     id<CKKSDatabaseProviderProtocol> databaseProvider = self.deps.databaseProvider;
 457
 458     while(lastCount == SecCKKSIncomingQueueItemsAtOnce) {
 459         [databaseProvider dispatchSyncWithSQLTransaction:^CKKSDatabaseTransactionResult{
 460             NSArray<CKKSIncomingQueueEntry*> * queueEntries = nil;
 461             if(self.cancelled) {
 462                 ckksnotice("ckksincoming", self.deps.zoneID, "CKKSIncomingQueueOperation cancelled, quitting");
 463                 errored = true;
 464                 return CKKSDatabaseTransactionRollback;
 465             }
 466
 467             NSError* error = nil;
 468
 469             queueEntries = [CKKSIncomingQueueEntry fetch:SecCKKSIncomingQueueItemsAtOnce
 470                                           startingAtUUID:lastMaxUUID
 471                                                    state:SecCKKSStateNew
 472                                                   action:actionFilter
 473                                                   zoneID:self.deps.zoneID
 474                                                    error:&error];
 475
 476             if(error != nil) {
 477                 ckkserror("ckksincoming", self.deps.zoneID, "Error fetching incoming queue records: %@", error);
 478                 self.error = error;
 479                 return CKKSDatabaseTransactionRollback;
 480             }
 481
 482             lastCount = queueEntries.count;
 483
 484             if([queueEntries count] == 0) {
 485                 // Nothing to do! exit.
 486                 ckksinfo("ckksincoming", self.deps.zoneID, "Nothing in incoming queue to process (filter: %@)", actionFilter);
 487                 return CKKSDatabaseTransactionCommit;
 488             }
 489
 490             [CKKSPowerCollection CKKSPowerEvent:kCKKSPowerEventIncommingQueue zone:self.deps.zoneID.zoneName count:[queueEntries count]];
 491
 492             if (![self processQueueEntries:queueEntries]) {
 493                 ckksnotice("ckksincoming", self.deps.zoneID, "processQueueEntries didn't complete successfully");
 494                 errored = true;
 495                 return CKKSDatabaseTransactionRollback;
 496             }
 497
 498             // Find the highest UUID for the next fetch.
 499             for(CKKSIncomingQueueEntry* iqe in queueEntries) {
 500                 lastMaxUUID = ([lastMaxUUID compare:iqe.uuid] == NSOrderedDescending) ? lastMaxUUID : iqe.uuid;
 501             }
 502
 503             return CKKSDatabaseTransactionCommit;
 504         }];
 505
 506         if(errored) {
 507             ckksnotice("ckksincoming", self.deps.zoneID, "Early-exiting from IncomingQueueOperation");
 508             return false;
 509         }
 510     }
 511
 512     return true;
 513 }
 514 - (BOOL)fixMismatchedViewItems
 515 {
 516     if(!self.handleMismatchedViewItems) {
 517         return YES;
 518     }
 519
 520     ckksnotice("ckksincoming", self.deps.zoneID, "Handling policy-mismatched items");
 521     __block NSUInteger lastCount = SecCKKSIncomingQueueItemsAtOnce;
 522     __block NSString* lastMaxUUID = nil;
 523     __block BOOL errored = NO;
 524
 525     id<CKKSDatabaseProviderProtocol> databaseProvider = self.deps.databaseProvider;
 526
 527     while(lastCount == SecCKKSIncomingQueueItemsAtOnce) {
 528         [databaseProvider dispatchSyncWithSQLTransaction:^CKKSDatabaseTransactionResult{
 529             NSError* error = nil;
 530             NSArray<CKKSIncomingQueueEntry*>* queueEntries = [CKKSIncomingQueueEntry fetch:SecCKKSIncomingQueueItemsAtOnce
 531                                                                             startingAtUUID:lastMaxUUID
 532                                                                                      state:SecCKKSStateMismatchedView
 533                                                                                     action:nil
 534                                                                                     zoneID:self.deps.zoneID
 535                                                                                      error:&error];
 536             if(error) {
 537                 ckksnotice("ckksincoming", self.deps.zoneID, "Cannot fetch mismatched view items");
 538                 self.error = error;
 539                 errored = true;
 540                 return CKKSDatabaseTransactionRollback;
 541             }
 542
 543             lastCount = queueEntries.count;
 544
 545             if(queueEntries.count == 0) {
 546                 ckksnotice("ckksincoming", self.deps.zoneID, "No mismatched view items");
 547                 return CKKSDatabaseTransactionCommit;
 548             }
 549
 550             ckksnotice("ckksincoming", self.deps.zoneID, "Inspecting %lu mismatched items", (unsigned long)queueEntries.count);
 551
 552             if (![self processQueueEntries:queueEntries]) {
 553                 ckksnotice("ckksincoming", self.deps.zoneID, "processQueueEntries didn't complete successfully");
 554                 errored = true;
 555                 return CKKSDatabaseTransactionRollback;
 556             }
 557
 558             for(CKKSIncomingQueueEntry* iqe in queueEntries) {
 559                 lastMaxUUID = ([lastMaxUUID compare:iqe.uuid] == NSOrderedDescending) ? lastMaxUUID : iqe.uuid;
 560             }
 561
 562             return CKKSDatabaseTransactionCommit;
 563         }];
 564     }
 565
 566     return !errored;
 567 }
 568
 569 - (void)_onqueueHandleIQEChange:(CKKSIncomingQueueEntry*)iqe
 570                      attributes:(NSDictionary*)attributes
 571                           class:(const SecDbClass *)classP
 572               sortedForThisView:(BOOL)sortedForThisView
 573 {
 574     __block CFErrorRef cferror = NULL;
 575     SecDbItemRef item = SecDbItemCreateWithAttributes(NULL, classP, (__bridge CFDictionaryRef) attributes, KEYBAG_DEVICE, &cferror);
 576
 577      if(!item || cferror) {
 578          ckkserror("ckksincoming", self.deps.zoneID, "Unable to make SecDbItemRef out of attributes: %@", cferror);
 579          return;
 580      }
 581      CFReleaseNull(cferror);
 582
 583      [self _onqueueHandleIQEChange:iqe
 584                               item:item
 585                  sortedForThisView:sortedForThisView];
 586
 587     CFReleaseNull(item);
 588 }
 589
 590 - (void)_onqueueHandleIQEChange:(CKKSIncomingQueueEntry*)iqe
 591                            item:(SecDbItemRef)item
 592               sortedForThisView:(BOOL)sortedForThisView
 593 {
 594     bool ok = false;
 595     __block CFErrorRef cferror = NULL;
 596     __block NSError* error = NULL;
 597
 598     if(SecDbItemIsTombstone(item)) {
 599         ckkserror("ckksincoming", self.deps.zoneID, "Rejecting a tombstone item addition from CKKS(%@): %@", iqe.uuid, item);
 600
 601         NSError* error = nil;
 602         CKKSOutgoingQueueEntry* oqe = [CKKSOutgoingQueueEntry withItem:item action:SecCKKSActionDelete zoneID:self.deps.zoneID error:&error];
 603         [oqe saveToDatabase:&error];
 604
 605         if(error) {
 606             ckkserror("ckksincoming", self.deps.zoneID, "Unable to save new deletion OQE: %@", error);
 607         } else {
 608             [iqe deleteFromDatabase: &error];
 609             if(error) {
 610                 ckkserror("ckksincoming", self.deps.zoneID, "couldn't delete CKKSIncomingQueueEntry: %@", error);
 611                 self.error = error;
 612                 self.errorItemsProcessed += 1;
 613             } else {
 614                 self.successfulItemsProcessed += 1;
 615             }
 616         }
 617         self.newOutgoingEntries = true;
 618
 619         return;
 620     }
 621
 622     __block NSDate* moddate = (__bridge NSDate*) CFDictionaryGetValue(item->attributes, kSecAttrModificationDate);
 623
 624     ok = kc_with_dbt(true, &cferror, ^(SecDbConnectionRef dbt){
 625         bool replaceok = SecDbItemInsertOrReplace(item, dbt, &cferror, ^(SecDbItemRef olditem, SecDbItemRef *replace) {
 626             // If the UUIDs do not match, then check to be sure that the local item is known to CKKS. If not, accept the cloud value.
 627             // Otherwise, when the UUIDs do not match, then select the item with the 'lower' UUID, and tell CKKS to
 628             //   delete the item with the 'higher' UUID.
 629             // Otherwise, the cloud wins.
 630
 631             [SecCoreAnalytics sendEvent:SecCKKSAggdPrimaryKeyConflict event:@{SecCoreAnalyticsValue: @1}];
 632
 633             // Note that SecDbItemInsertOrReplace CFReleases any replace pointer it's given, so, be careful
 634
 635             if(!CFDictionaryContainsKey(olditem->attributes, kSecAttrUUID)) {
 636                 // No UUID -> no good.
 637                 ckksnotice("ckksincoming", self.deps.zoneID, "Replacing item (it doesn't have a UUID) for %@", iqe.uuid);
 638                 if(replace) {
 639                     *replace = CFRetainSafe(item);
 640                 }
 641                 return;
 642             }
 643
 644             // If this item arrived in what we believe to be the wrong view, drop the modification entirely.
 645             if(!sortedForThisView) {
 646                 ckksnotice("ckksincoming", self.deps.zoneID, "Primary key conflict; dropping CK item (arriving from wrong view) %@", item);
 647                 return;
 648             }
 649
 650             CFStringRef itemUUID    = CFDictionaryGetValue(item->attributes,    kSecAttrUUID);
 651             CFStringRef olditemUUID = CFDictionaryGetValue(olditem->attributes, kSecAttrUUID);
 652
 653             // Is the old item already somewhere in CKKS?
 654             NSError* ckmeError = nil;
 655             CKKSMirrorEntry* ckme = [CKKSMirrorEntry tryFromDatabase:(__bridge NSString*)olditemUUID
 656                                                               zoneID:self.deps.zoneID
 657                                                                error:&ckmeError];
 658
 659             if(ckmeError) {
 660                 ckksnotice("ckksincoming", self.deps.zoneID, "Unable to fetch ckme: %@", ckmeError);
 661                 // We'll just have to assume that there is a CKME, and let the comparison analysis below win
 662             }
 663
 664             CFComparisonResult compare = CFStringCompare(itemUUID, olditemUUID, 0);
 665             CKKSOutgoingQueueEntry* oqe = nil;
 666             if (compare == kCFCompareGreaterThan && (ckme || ckmeError)) {
 667                 // olditem wins; don't change olditem; delete item
 668                 ckksnotice("ckksincoming", self.deps.zoneID, "Primary key conflict; dropping CK item %@", item);
 669                 oqe = [CKKSOutgoingQueueEntry withItem:item action:SecCKKSActionDelete zoneID:self.deps.zoneID error:&error];
 670                 [oqe saveToDatabase: &error];
 671                 self.newOutgoingEntries = true;
 672                 moddate = nil;
 673             } else {
 674                 // item wins, either due to the UUID winning or the item not being in CKKS yet
 675                 ckksnotice("ckksincoming", self.deps.zoneID, "Primary key conflict; replacing %@%@ with CK item %@",
 676                            ckme ? @"" : @"non-onboarded", olditem, item);
 677                 if(replace) {
 678                     *replace = CFRetainSafe(item);
 679                     moddate = (__bridge NSDate*) CFDictionaryGetValue(item->attributes, kSecAttrModificationDate);
 680                 }
 681                 // delete olditem if UUID differs (same UUID is the normal update case)
 682                 if (compare != kCFCompareEqualTo) {
 683                     oqe = [CKKSOutgoingQueueEntry withItem:olditem action:SecCKKSActionDelete zoneID:self.deps.zoneID error:&error];
 684                     [oqe saveToDatabase: &error];
 685                     self.newOutgoingEntries = true;
 686                 }
 687             }
 688         });
 689
 690         // SecDbItemInsertOrReplace returns an error even when it succeeds.
 691         if(!replaceok && SecErrorIsSqliteDuplicateItemError(cferror)) {
 692             CFReleaseNull(cferror);
 693             replaceok = true;
 694         }
 695         return replaceok;
 696     });
 697
 698     if(cferror) {
 699         ckkserror("ckksincoming", self.deps.zoneID, "couldn't process item from IncomingQueue: %@", cferror);
 700         SecTranslateError(&error, cferror);
 701         self.error = error;
 702
 703         iqe.state = SecCKKSStateError;
 704         [iqe saveToDatabase:&error];
 705         if(error) {
 706             ckkserror("ckksincoming", self.deps.zoneID, "Couldn't save errored IQE to database: %@", error);
 707             self.error = error;
 708         }
 709         return;
 710     }
 711
 712     if(error) {
 713         ckkserror("ckksincoming", self.deps.zoneID, "Couldn't handle IQE, but why?: %@", error);
 714         self.error = error;
 715         return;
 716     }
 717
 718     if(ok) {
 719         ckksinfo("ckksincoming", self.deps.zoneID, "Correctly processed an IQE; deleting");
 720         [iqe deleteFromDatabase: &error];
 721
 722         if(error) {
 723             ckkserror("ckksincoming", self.deps.zoneID, "couldn't delete CKKSIncomingQueueEntry: %@", error);
 724             self.error = error;
 725             self.errorItemsProcessed += 1;
 726         } else {
 727             self.successfulItemsProcessed += 1;
 728         }
 729
 730         if(moddate) {
 731             // Log the number of ms it took to propagate this change
 732             uint64_t delayInMS = [[NSDate date] timeIntervalSinceDate:moddate] * 1000;
 733             [SecCoreAnalytics sendEvent:@"com.apple.self.deps.item.propagation" event:@{
 734                 @"time" : @(delayInMS)
 735             }];
 736
 737         }
 738
 739     } else {
 740         ckksnotice("ckksincoming", self.deps.zoneID, "IQE not correctly processed, but why? %@ %@", error, cferror);
 741         self.error = error;
 742
 743         iqe.state = SecCKKSStateError;
 744         [iqe saveToDatabase:&error];
 745         if(error) {
 746             ckkserror("ckksincoming", self.deps.zoneID, "Couldn't save errored IQE to database: %@", error);
 747             self.error = error;
 748         }
 749
 750         self.errorItemsProcessed += 1;
 751     }
 752 }
 753
 754 - (void)_onqueueHandleIQEDelete: (CKKSIncomingQueueEntry*) iqe class:(const SecDbClass *)classP {
 755     bool ok = false;
 756     __block CFErrorRef cferror = NULL;
 757     NSError* error = NULL;
 758     NSDictionary* queryAttributes = @{(__bridge NSString*) kSecClass: (__bridge NSString*) classP->name,
 759                                       (__bridge NSString*) kSecAttrUUID: iqe.uuid,
 760                                       (__bridge NSString*) kSecAttrSynchronizable: @(YES)};
 761     ckksnotice("ckksincoming", self.deps.zoneID, "trying to delete with query: %@", queryAttributes);
 762     Query *q = query_create_with_limit( (__bridge CFDictionaryRef) queryAttributes, NULL, kSecMatchUnlimited, NULL, &cferror);
 763     q->q_tombstone_use_mdat_from_item = true;
 764
 765     if(cferror) {
 766         ckkserror("ckksincoming", self.deps.zoneID, "couldn't create query: %@", cferror);
 767         SecTranslateError(&error, cferror);
 768         self.error = error;
 769         return;
 770     }
 771
 772     ok = kc_with_dbt(true, &cferror, ^(SecDbConnectionRef dbt) {
 773         return s3dl_query_delete(dbt, q, NULL, &cferror);
 774     });
 775
 776     if(cferror) {
 777         if(CFErrorGetCode(cferror) == errSecItemNotFound) {
 778             ckkserror("ckksincoming", self.deps.zoneID, "couldn't delete item (as it's already gone); this is okay: %@", cferror);
 779             ok = true;
 780             CFReleaseNull(cferror);
 781         } else {
 782             ckkserror("ckksincoming", self.deps.zoneID, "couldn't delete item: %@", cferror);
 783             SecTranslateError(&error, cferror);
 784             self.error = error;
 785             query_destroy(q, NULL);
 786             return;
 787         }
 788     }
 789
 790
 791     ok = query_notify_and_destroy(q, ok, &cferror);
 792
 793     if(cferror) {
 794         ckkserror("ckksincoming", self.deps.zoneID, "couldn't delete query: %@", cferror);
 795         SecTranslateError(&error, cferror);
 796         self.error = error;
 797         return;
 798     }
 799
 800     if(ok) {
 801         ckksnotice("ckksincoming", self.deps.zoneID, "Correctly processed an IQE; deleting");
 802         [iqe deleteFromDatabase: &error];
 803
 804         if(error) {
 805             ckkserror("ckksincoming", self.deps.zoneID, "couldn't delete CKKSIncomingQueueEntry: %@", error);
 806             self.error = error;
 807             self.errorItemsProcessed += 1;
 808         } else {
 809             self.successfulItemsProcessed += 1;
 810         }
 811     } else {
 812         ckkserror("ckksincoming", self.deps.zoneID, "IQE not correctly processed, but why? %@ %@", error, cferror);
 813         self.error = error;
 814         self.errorItemsProcessed += 1;
 815     }
 816 }
 817
 818 @end;
 819
 820 #endif