keychain/ckks/CKKSOutgoingQueueOperation.m

   1 /*
   2  * Copyright (c) 2016 Apple Inc. All Rights Reserved.
   3  *
   4  * @APPLE_LICENSE_HEADER_START@
   5  *
   6  * This file contains Original Code and/or Modifications of Original Code
   7  * as defined in and that are subject to the Apple Public Source License
   8  * Version 2.0 (the 'License'). You may not use this file except in
   9  * compliance with the License. Please obtain a copy of the License at
  10  * http://www.opensource.apple.com/apsl/ and read it before using this
  11  * file.
  12  *
  13  * The Original Code and all software distributed under the License are
  14  * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  15  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  16  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
  17  * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
  18  * Please see the License for the specific language governing rights and
  19  * limitations under the License.
  20  *
  21  * @APPLE_LICENSE_HEADER_END@
  22  */
  23
  24 #if OCTAGON
  25
  26 #import <CloudKit/CloudKit.h>
  27 #import <CloudKit/CloudKit_Private.h>
  28
  29 #import "keychain/ckks/CKKSAnalytics.h"
  30 #import "keychain/ckks/CKKSCurrentKeyPointer.h"
  31 #import "keychain/ckks/CKKSIncomingQueueEntry.h"
  32 #import "keychain/ckks/CKKSItemEncrypter.h"
  33 #import "keychain/ckks/CKKSKeychainView.h"
  34 #import "keychain/ckks/CKKSManifest.h"
  35 #import "keychain/ckks/CKKSOutgoingQueueEntry.h"
  36 #import "keychain/ckks/CKKSOutgoingQueueOperation.h"
  37 #import "keychain/ckks/CKKSReencryptOutgoingItemsOperation.h"
  38 #import "keychain/ckks/CKKSStates.h"
  39 #import "keychain/ckks/CKKSViewManager.h"
  40 #import "keychain/ckks/CloudKitCategories.h"
  41 #import "keychain/ot/ObjCImprovements.h"
  42
  43 #include "keychain/securityd/SecItemServer.h"
  44 #include "keychain/securityd/SecItemDb.h"
  45 #include <Security/SecItemPriv.h>
  46 #import "CKKSPowerCollection.h"
  47 #import "utilities/SecCoreAnalytics.h"
  48
  49 @interface CKKSOutgoingQueueOperation()
  50 @property CKModifyRecordsOperation* modifyRecordsOperation;
  51 @end
  52
  53 @implementation CKKSOutgoingQueueOperation
  54 @synthesize nextState = _nextState;
  55 @synthesize intendedState = _intendedState;
  56
  57 - (instancetype)initWithDependencies:(CKKSOperationDependencies*)dependencies
  58                                 ckks:(CKKSKeychainView*)ckks
  59                            intending:(OctagonState*)intending
  60                           errorState:(OctagonState*)errorState
  61                     ckoperationGroup:(CKOperationGroup*)ckoperationGroup
  62 {
  63     if(self = [super init]) {
  64         _deps = dependencies;
  65         _ckks = ckks;
  66         _ckoperationGroup = ckoperationGroup;
  67
  68         _nextState = errorState;
  69         _intendedState = intending;
  70
  71         [self addNullableDependency:ckks.holdOutgoingQueueOperation];
  72
  73         // Depend on all previous CKKSOutgoingQueueOperations
  74         [self linearDependencies:ckks.outgoingQueueOperations];
  75
  76         // We also depend on the view being setup and the key hierarchy being reasonable
  77         [self addNullableDependency:ckks.keyStateReadyDependency];
  78     }
  79     return self;
  80 }
  81
  82 - (void) groupStart {
  83     WEAKIFY(self);
  84
  85     CKKSKeychainView* ckks = self.ckks;
  86     if(!ckks) {
  87         ckkserror("ckksoutgoing", self.deps.zoneID, "no ckks object");
  88         return;
  89     }
  90
  91     if(!ckks.itemSyncingEnabled) {
  92         ckkserror("ckksoutgoing", self.deps.zoneID, "Item syncing for this view is disabled");
  93         return;
  94     }
  95
  96     ckks.lastOutgoingQueueOperation = self;
  97
  98     [self.deps.databaseProvider dispatchSyncWithSQLTransaction:^CKKSDatabaseTransactionResult{
  99
 100         NSError* error = nil;
 101
 102         NSSet<NSString*>* priorityUUIDs = [[CKKSViewManager manager] pendingCallbackUUIDs];
 103
 104         NSMutableArray<CKKSOutgoingQueueEntry*>* priorityEntries = [NSMutableArray array];
 105         NSMutableSet<NSString*>* priorityEntryUUIDs = [NSMutableSet set];
 106
 107         for(NSString* uuid in priorityUUIDs) {
 108             NSError* priorityFetchError = nil;
 109             CKKSOutgoingQueueEntry* oqe = [CKKSOutgoingQueueEntry tryFromDatabase:uuid zoneID:self.deps.zoneID error:&priorityFetchError];
 110
 111             if(priorityFetchError) {
 112                 ckkserror("ckksoutgoing", self.deps.zoneID, "Unable to fetch priority uuid %@: %@", uuid, priorityFetchError);
 113                 continue;
 114             }
 115
 116             if(!oqe) {
 117                 continue;
 118             }
 119
 120             if(![oqe.state isEqualToString:SecCKKSStateNew]) {
 121                 ckksnotice("ckksoutgoing", self.deps.zoneID, "Priority uuid %@ is not in 'new': %@", uuid, oqe);
 122                 continue;
 123             }
 124
 125             if(oqe) {
 126                 ckksnotice("ckksoutgoing", self.deps.zoneID, "Found OQE  to fetch priority uuid %@: %@", uuid, priorityFetchError);
 127                 [priorityEntries addObject:oqe];
 128                 [priorityEntryUUIDs addObject:oqe.uuid];
 129             }
 130         }
 131
 132         // We only actually care about queue items in the 'new' state
 133         NSArray<CKKSOutgoingQueueEntry*> * newEntries = [CKKSOutgoingQueueEntry fetch:SecCKKSOutgoingQueueItemsAtOnce
 134                                                                                 state:SecCKKSStateNew
 135                                                                                zoneID:self.deps.zoneID
 136                                                                                 error:&error];
 137         if(error != nil) {
 138             ckkserror("ckksoutgoing", self.deps.zoneID, "Error fetching outgoing queue records: %@", error);
 139             self.error = error;
 140             return CKKSDatabaseTransactionRollback;
 141         }
 142
 143         // Build our list of oqes to transmit
 144         NSMutableArray<CKKSOutgoingQueueEntry*>* queueEntries = [priorityEntries mutableCopy];
 145         for(CKKSOutgoingQueueEntry* oqe in newEntries) {
 146             if(queueEntries.count >= SecCKKSOutgoingQueueItemsAtOnce) {
 147                 break;
 148             } else {
 149                 // We only need to add this OQE if it wasn't already there
 150                 if(![priorityEntryUUIDs containsObject:oqe.uuid]) {
 151                     [queueEntries addObject:oqe];
 152                 }
 153             }
 154         }
 155
 156         bool fullUpload = queueEntries.count >= SecCKKSOutgoingQueueItemsAtOnce;
 157
 158         [CKKSPowerCollection CKKSPowerEvent:kCKKSPowerEventOutgoingQueue zone:self.deps.zoneID.zoneName count:[queueEntries count]];
 159
 160         ckksinfo("ckksoutgoing", self.deps.zoneID, "processing outgoing queue: %@", queueEntries);
 161
 162         NSMutableDictionary<CKRecordID*, CKRecord*>* recordsToSave = [[NSMutableDictionary alloc] init];
 163         NSMutableSet<CKRecordID*>* recordIDsModified = [[NSMutableSet alloc] init];
 164         NSMutableSet<CKKSOutgoingQueueEntry*>*oqesModified = [[NSMutableSet alloc] init];
 165         NSMutableArray<CKRecordID *>* recordIDsToDelete = [[NSMutableArray alloc] init];
 166
 167         CKKSCurrentKeyPointer* currentClassAKeyPointer = [CKKSCurrentKeyPointer fromDatabase:SecCKKSKeyClassA zoneID:self.deps.zoneID error:&error];
 168         CKKSCurrentKeyPointer* currentClassCKeyPointer = [CKKSCurrentKeyPointer fromDatabase:SecCKKSKeyClassC zoneID:self.deps.zoneID error:&error];
 169         NSMutableDictionary<CKKSKeyClass*, CKKSCurrentKeyPointer*>* currentKeysToSave = [[NSMutableDictionary alloc] init];
 170         bool needsReencrypt = false;
 171
 172         if(error != nil) {
 173             ckkserror("ckksoutgoing", self.deps.zoneID, "Couldn't load current class keys: %@", error);
 174             return CKKSDatabaseTransactionRollback;
 175         }
 176
 177         for(CKKSOutgoingQueueEntry* oqe in queueEntries) {
 178             if(self.cancelled) {
 179                 secdebug("ckksoutgoing", "CKKSOutgoingQueueOperation cancelled, quitting");
 180                 return CKKSDatabaseTransactionRollback;
 181             }
 182
 183             CKKSOutgoingQueueEntry* inflight = [CKKSOutgoingQueueEntry tryFromDatabase: oqe.uuid state:SecCKKSStateInFlight zoneID:self.deps.zoneID error:&error];
 184             if(!error && inflight) {
 185                 // There is an inflight request with this UUID. Leave this request in-queue until CloudKit returns and we resolve the inflight request.
 186                 continue;
 187             }
 188
 189             // If this item is not a delete, check the encryption status of this item.
 190             if(![oqe.action isEqualToString: SecCKKSActionDelete]) {
 191                 // Check if this item is encrypted under a current key
 192                 if([oqe.item.parentKeyUUID isEqualToString: currentClassAKeyPointer.currentKeyUUID]) {
 193                     // Excellent.
 194                     currentKeysToSave[SecCKKSKeyClassA] = currentClassAKeyPointer;
 195
 196                 } else if([oqe.item.parentKeyUUID isEqualToString: currentClassCKeyPointer.currentKeyUUID]) {
 197                     // Works for us!
 198                     currentKeysToSave[SecCKKSKeyClassC] = currentClassCKeyPointer;
 199
 200                 } else {
 201                     // This item is encrypted under an old key. Set it up for reencryption and move on.
 202                     ckksnotice("ckksoutgoing", self.deps.zoneID, "Item's encryption key (%@ %@) is neither %@ or %@", oqe, oqe.item.parentKeyUUID, currentClassAKeyPointer, currentClassCKeyPointer);
 203                     [ckks _onqueueChangeOutgoingQueueEntry:oqe toState:SecCKKSStateReencrypt error:&error];
 204                     if(error) {
 205                         ckkserror("ckksoutgoing", self.deps.zoneID, "couldn't save oqe to database: %@", error);
 206                         self.error = error;
 207                         error = nil;
 208                     }
 209                     needsReencrypt = true;
 210                     continue;
 211                 }
 212             }
 213
 214             if([oqe.action isEqualToString: SecCKKSActionAdd]) {
 215                 CKRecord* record = [oqe.item CKRecordWithZoneID:self.deps.zoneID];
 216                 recordsToSave[record.recordID] = record;
 217                 [recordIDsModified addObject: record.recordID];
 218                 [oqesModified addObject:oqe];
 219
 220                 [ckks _onqueueChangeOutgoingQueueEntry:oqe toState:SecCKKSStateInFlight error:&error];
 221                 if(error) {
 222                     ckkserror("ckksoutgoing", self.deps.zoneID, "couldn't save state for CKKSOutgoingQueueEntry: %@", error);
 223                     self.error = error;
 224                 }
 225
 226             } else if ([oqe.action isEqualToString: SecCKKSActionDelete]) {
 227                 CKRecordID* recordIDToDelete = [[CKRecordID alloc] initWithRecordName:oqe.item.uuid zoneID:self.deps.zoneID];
 228                 [recordIDsToDelete addObject: recordIDToDelete];
 229                 [recordIDsModified addObject: recordIDToDelete];
 230                 [oqesModified addObject:oqe];
 231
 232                 [ckks _onqueueChangeOutgoingQueueEntry:oqe toState:SecCKKSStateInFlight error:&error];
 233                 if(error) {
 234                     ckkserror("ckksoutgoing", self.deps.zoneID, "couldn't save state for CKKSOutgoingQueueEntry: %@", error);
 235                 }
 236
 237             } else if ([oqe.action isEqualToString: SecCKKSActionModify]) {
 238                 // Load the existing item from the ckmirror.
 239                 CKKSMirrorEntry* ckme = [CKKSMirrorEntry tryFromDatabase:oqe.item.uuid zoneID:self.deps.zoneID error:&error];
 240                 if(!ckme) {
 241                     // This is a problem: we have an update to an item that doesn't exist.
 242                     // Either: an Add operation we launched failed due to a CloudKit error (conflict?) and this is a follow-on update
 243                     //     Or: ?
 244                     ckkserror("ckksoutgoing", self.deps.zoneID, "update to a record that doesn't exist? %@", oqe.item.uuid);
 245                     // treat as an add.
 246                     CKRecord* record = [oqe.item CKRecordWithZoneID:self.deps.zoneID];
 247                     recordsToSave[record.recordID] = record;
 248                     [recordIDsModified addObject: record.recordID];
 249                     [oqesModified addObject:oqe];
 250
 251                     [ckks _onqueueChangeOutgoingQueueEntry:oqe toState:SecCKKSStateInFlight error:&error];
 252                     if(error) {
 253                         ckkserror("ckksoutgoing", self.deps.zoneID, "couldn't save state for CKKSOutgoingQueueEntry: %@", error);
 254                         self.error = error;
 255                     }
 256                 } else {
 257                     if(![oqe.item.storedCKRecord.recordChangeTag isEqual: ckme.item.storedCKRecord.recordChangeTag]) {
 258                         // The mirror entry has updated since this item was created. If we proceed, we might end up with
 259                         // a badly-authenticated record.
 260                         ckksnotice("ckksoutgoing", self.deps.zoneID, "Record (%@)'s change tag doesn't match ckmirror's change tag, reencrypting", oqe);
 261                         [ckks _onqueueChangeOutgoingQueueEntry:oqe toState:SecCKKSStateReencrypt error:&error];
 262                         if(error) {
 263                             ckkserror("ckksoutgoing", self.deps.zoneID, "couldn't save oqe to database: %@", error);
 264                             self.error = error;
 265                             error = nil;
 266                         }
 267                         needsReencrypt = true;
 268                         continue;
 269                     }
 270                     // Grab the old ckrecord and update it
 271                     CKRecord* record = [oqe.item updateCKRecord: ckme.item.storedCKRecord zoneID: self.deps.zoneID];
 272                     recordsToSave[record.recordID] = record;
 273                     [recordIDsModified addObject: record.recordID];
 274                     [oqesModified addObject:oqe];
 275
 276                     [ckks _onqueueChangeOutgoingQueueEntry:oqe toState:SecCKKSStateInFlight error:&error];
 277                     if(error) {
 278                         ckkserror("ckksoutgoing", self.deps.zoneID, "couldn't save state for CKKSOutgoingQueueEntry: %@", error);
 279                     }
 280                 }
 281             }
 282         }
 283
 284         if(needsReencrypt) {
 285             ckksnotice("ckksoutgoing", self.deps.zoneID, "An item needs reencryption!");
 286             [self.deps.flagHandler _onqueueHandleFlag:CKKSFlagItemReencryptionNeeded];
 287         }
 288
 289         if([recordsToSave count] == 0 && [recordIDsToDelete count] == 0) {
 290             // Nothing to do! exit.
 291             ckksnotice("ckksoutgoing", self.deps.zoneID, "Nothing in outgoing queue to process");
 292             if(self.ckoperationGroup) {
 293                 ckksinfo("ckksoutgoing", self.deps.zoneID, "End of operation group: %@", self.ckoperationGroup);
 294             }
 295             return true;
 296         }
 297
 298         self.itemsProcessed = recordsToSave.count;
 299
 300         NSBlockOperation* modifyComplete = [[NSBlockOperation alloc] init];
 301         modifyComplete.name = @"modifyRecordsComplete";
 302         [self dependOnBeforeGroupFinished: modifyComplete];
 303
 304         void (^modifyRecordsCompletionBlock)(NSArray<CKRecord*>*, NSArray<CKRecordID*>*, NSError*) = ^(NSArray<CKRecord *> *savedRecords, NSArray<CKRecordID *> *deletedRecordIDs, NSError *ckerror) {
 305             STRONGIFY(self);
 306             CKKSKeychainView* strongCKKS = self.ckks;
 307             if(!self || !strongCKKS) {
 308                 ckkserror("ckksoutgoing", self.deps.zoneID, "received callback for released object");
 309                 return;
 310             }
 311
 312             CKKSAnalytics* logger = [CKKSAnalytics logger];
 313
 314             [self.deps.databaseProvider dispatchSyncWithSQLTransaction:^CKKSDatabaseTransactionResult{
 315                 if(ckerror) {
 316                     ckkserror("ckksoutgoing", self.deps.zoneID, "error processing outgoing queue: %@", ckerror);
 317
 318                     [logger logRecoverableError:ckerror
 319                                        forEvent:CKKSEventProcessOutgoingQueue
 320                                        zoneName:self.deps.zoneID.zoneName
 321                                  withAttributes:NULL];
 322
 323                     // Tell CKKS about any out-of-date records
 324                     [strongCKKS _onqueueCKWriteFailed:ckerror attemptedRecordsChanged:recordsToSave];
 325
 326                     // Check if these are due to key records being out of date. We'll see a CKErrorBatchRequestFailed, with a bunch of errors inside
 327                     if([ckerror.domain isEqualToString:CKErrorDomain] && (ckerror.code == CKErrorPartialFailure)) {
 328                         NSMutableDictionary<CKRecordID*, NSError*>* failedRecords = ckerror.userInfo[CKPartialErrorsByItemIDKey];
 329
 330                         bool askForReencrypt = false;
 331
 332                         if([self _onqueueIsErrorBadEtagOnKeyPointersOnly:ckerror]) {
 333                             // The current key pointers have updated without our knowledge, so CloudKit failed this operation. Mark all records as 'needs reencryption' and kick that off.
 334                             ckksnotice("ckksoutgoing", self.deps.zoneID, "Error is simply due to current key pointers changing; marking all records as 'needs reencrypt'");
 335                             [self _onqueueModifyAllRecords:failedRecords.allKeys as:SecCKKSStateReencrypt];
 336                             askForReencrypt = true;
 337
 338                         } else if([self _onqueueIsErrorMissingSyncKey:ckerror]) {
 339                             ckksnotice("ckksoutgoing", self.deps.zoneID, "Error is due to the key records missing. Marking all as 'needs reencrypt'");
 340                             [self _onqueueModifyAllRecords:failedRecords.allKeys as:SecCKKSStateReencrypt];
 341                             askForReencrypt = true;
 342
 343                         } else {
 344                             // Iterate all failures, and reset each item
 345                             for(CKRecordID* recordID in failedRecords) {
 346                                 NSError* recordError = failedRecords[recordID];
 347
 348                                 ckksnotice("ckksoutgoing", self.deps.zoneID, "failed record: %@ %@", recordID, recordError);
 349
 350                                 if([recordError.domain isEqualToString: CKErrorDomain] && recordError.code == CKErrorServerRecordChanged) {
 351                                     if([recordID.recordName isEqualToString: SecCKKSKeyClassA] ||
 352                                        [recordID.recordName isEqualToString: SecCKKSKeyClassC]) {
 353                                         // Note that _onqueueCKWriteFailed is responsible for kicking the key state machine, so we don't need to do it here.
 354                                         askForReencrypt = true;
 355                                     } else {
 356                                         // CKErrorServerRecordChanged on an item update means that we've been overwritten.
 357                                         if([recordIDsModified containsObject:recordID]) {
 358                                             [self _onqueueModifyRecordAsError:recordID recordError:recordError];
 359                                         }
 360                                     }
 361                                 } else if([recordError.domain isEqualToString: CKErrorDomain] && recordError.code == CKErrorBatchRequestFailed) {
 362                                     // Also fine. This record only didn't succeed because something else failed.
 363                                     // OQEs should be placed back into the 'new' state, unless they've been overwritten by a new OQE. Other records should be ignored.
 364
 365                                     if([recordIDsModified containsObject:recordID]) {
 366                                         NSError* localerror = nil;
 367                                         CKKSOutgoingQueueEntry* inflightOQE = [CKKSOutgoingQueueEntry tryFromDatabase:recordID.recordName state:SecCKKSStateInFlight zoneID:recordID.zoneID error:&localerror];
 368                                         [strongCKKS _onqueueChangeOutgoingQueueEntry:inflightOQE toState:SecCKKSStateNew error:&localerror];
 369                                         if(localerror) {
 370                                             ckkserror("ckksoutgoing", self.deps.zoneID, "Couldn't clean up outgoing queue entry: %@", localerror);
 371                                         }
 372                                     }
 373
 374                                 } else {
 375                                     // Some unknown error occurred on this record. If it's an OQE, move it to the error state.
 376                                     ckkserror("ckksoutgoing", self.deps.zoneID, "Unknown error on row: %@ %@", recordID, recordError);
 377                                     if([recordIDsModified containsObject:recordID]) {
 378                                         [self _onqueueModifyRecordAsError:recordID recordError:recordError];
 379                                     }
 380                                 }
 381                             }
 382                         }
 383
 384                         if(askForReencrypt) {
 385                             [self.deps.flagHandler _onqueueHandleFlag:CKKSFlagItemReencryptionNeeded];
 386                         }
 387                     } else {
 388                         // Some non-partial error occured. We should place all "inflight" OQEs back into the outgoing queue.
 389                         ckksnotice("ckks", self.deps.zoneID, "Error is scary: putting all inflight OQEs back into state 'new'");
 390                         [self _onqueueModifyAllRecords:[recordIDsModified allObjects] as:SecCKKSStateNew];
 391                     }
 392
 393                     self.error = ckerror;
 394                     return CKKSDatabaseTransactionCommit;
 395                 }
 396
 397                 for(CKRecordID* deletedRecordID in deletedRecordIDs) {
 398                     ckksnotice("ckksoutgoing", self.deps.zoneID, "Record deletion successful for %@", deletedRecordID.recordName);
 399                 }
 400
 401                 ckksnotice("ckksoutgoing", self.deps.zoneID, "Completed processing outgoing queue (%d modifications, %d deletions)", (int)savedRecords.count, (int)deletedRecordIDs.count);
 402                 NSError* error = NULL;
 403                 CKKSPowerCollection *plstats = [[CKKSPowerCollection alloc] init];
 404
 405                 for(CKRecord* record in savedRecords) {
 406                     // Save the item records
 407                     if([record.recordType isEqualToString: SecCKRecordItemType]) {
 408                         CKKSOutgoingQueueEntry* oqe = [CKKSOutgoingQueueEntry fromDatabase:record.recordID.recordName
 409                                                                                      state:SecCKKSStateInFlight
 410                                                                                     zoneID:self.deps.zoneID
 411                                                                                      error:&error];
 412                         [strongCKKS _onqueueChangeOutgoingQueueEntry:oqe toState:SecCKKSStateDeleted error:&error];
 413                         if(error) {
 414                             ckkserror("ckksoutgoing", self.deps.zoneID, "Couldn't update %@ in outgoingqueue: %@", record.recordID.recordName, error);
 415                             self.error = error;
 416                         }
 417                         error = nil;
 418                         CKKSMirrorEntry* ckme = [[CKKSMirrorEntry alloc] initWithCKRecord: record];
 419                         [ckme saveToDatabase: &error];
 420                         if(error) {
 421                             ckkserror("ckksoutgoing", self.deps.zoneID, "Couldn't save %@ to ckmirror: %@", record.recordID.recordName, error);
 422                             self.error = error;
 423                         }
 424
 425                         [plstats storedOQE:oqe];
 426
 427                     // And the CKCurrentKeyRecords (do we need to do this? Will the server update the change tag on a save which saves nothing?)
 428                     } else if([record.recordType isEqualToString: SecCKRecordCurrentKeyType]) {
 429                         CKKSCurrentKeyPointer* currentkey = [[CKKSCurrentKeyPointer alloc] initWithCKRecord: record];
 430                         [currentkey saveToDatabase: &error];
 431                         if(error) {
 432                             ckkserror("ckksoutgoing", self.deps.zoneID, "Couldn't save %@ to currentkey: %@", record.recordID.recordName, error);
 433                             self.error = error;
 434                         }
 435
 436                     } else if ([record.recordType isEqualToString:SecCKRecordDeviceStateType]) {
 437                         CKKSDeviceStateEntry* newcdse = [[CKKSDeviceStateEntry alloc] initWithCKRecord:record];
 438                         [newcdse saveToDatabase:&error];
 439                         if(error) {
 440                             ckkserror("ckksoutgoing", self.deps.zoneID, "Couldn't save %@ to ckdevicestate: %@", record.recordID.recordName, error);
 441                             self.error = error;
 442                         }
 443
 444                     } else if ([record.recordType isEqualToString:SecCKRecordManifestType]) {
 445
 446                     } else if (![record.recordType isEqualToString:SecCKRecordManifestLeafType]) {
 447                         ckkserror("ckksoutgoing", self.deps.zoneID, "unknown record type in results: %@", record);
 448                     }
 449                 }
 450
 451                 // Delete the deleted record IDs
 452                 for(CKRecordID* ckrecordID in deletedRecordIDs) {
 453
 454                     NSError* error = nil;
 455                     CKKSOutgoingQueueEntry* oqe = [CKKSOutgoingQueueEntry fromDatabase:ckrecordID.recordName
 456                                                                                  state:SecCKKSStateInFlight
 457                                                                                 zoneID:self.deps.zoneID
 458                                                                                  error:&error];
 459                     [strongCKKS _onqueueChangeOutgoingQueueEntry:oqe toState:SecCKKSStateDeleted error:&error];
 460                     if(error) {
 461                         ckkserror("ckksoutgoing", self.deps.zoneID, "Couldn't delete %@ from outgoingqueue: %@", ckrecordID.recordName, error);
 462                         self.error = error;
 463                     }
 464                     error = nil;
 465                     CKKSMirrorEntry* ckme = [CKKSMirrorEntry tryFromDatabase: ckrecordID.recordName zoneID:self.deps.zoneID error:&error];
 466                     [ckme deleteFromDatabase: &error];
 467                     if(error) {
 468                         ckkserror("ckksoutgoing", self.deps.zoneID, "Couldn't delete %@ from ckmirror: %@", ckrecordID.recordName, error);
 469                         self.error = error;
 470                     }
 471
 472                     [plstats deletedOQE:oqe];
 473                 }
 474
 475                 [plstats commit];
 476
 477                 if(self.error) {
 478                     ckkserror("ckksoutgoing", self.deps.zoneID, "Operation failed; rolling back: %@", self.error);
 479                     [logger logRecoverableError:self.error
 480                                        forEvent:CKKSEventProcessOutgoingQueue
 481                                        zoneName:strongCKKS.zoneName
 482                                  withAttributes:NULL];
 483                     return CKKSDatabaseTransactionRollback;
 484                 } else {
 485                     [logger logSuccessForEvent:CKKSEventProcessOutgoingQueue zoneName:strongCKKS.zoneName];
 486                 }
 487                 return CKKSDatabaseTransactionCommit;
 488             }];
 489
 490             self.nextState = self.intendedState;
 491             [self.operationQueue addOperation: modifyComplete];
 492
 493             // If this was a "full upload", or we errored in any way, then kick off another queue process. There might be more items to send!
 494             if(fullUpload || self.error) {
 495                 // If we think the network is iffy, though, wait for it to come back
 496                 CKKSResultOperation* possibleNetworkDependency = nil;
 497                 CKKSReachabilityTracker* reachabilityTracker = strongCKKS.reachabilityTracker;
 498                 if(ckerror && [reachabilityTracker isNetworkError:ckerror]) {
 499                     possibleNetworkDependency = reachabilityTracker.reachabilityDependency;
 500                 }
 501
 502                 [strongCKKS processOutgoingQueueAfter:possibleNetworkDependency
 503                                         requiredDelay:fullUpload && !ckerror ? NSEC_PER_MSEC * 100 : DISPATCH_TIME_FOREVER
 504                                      ckoperationGroup:self.ckoperationGroup];
 505             }
 506         };
 507
 508         ckksinfo("ckksoutgoing", self.deps.zoneID, "Current keys to update: %@", currentKeysToSave);
 509         for(CKKSCurrentKeyPointer* keypointer in currentKeysToSave.allValues) {
 510             CKRecord* record = [keypointer CKRecordWithZoneID:self.deps.zoneID];
 511             recordsToSave[record.recordID] = record;
 512         }
 513
 514         // Piggyback on this operation to update our device state
 515         NSError* cdseError = nil;
 516         CKKSDeviceStateEntry* cdse = [ckks _onqueueCurrentDeviceStateEntry:&cdseError];
 517         CKRecord* cdseRecord = [cdse CKRecordWithZoneID:self.deps.zoneID];
 518         if(cdseError) {
 519             ckkserror("ckksoutgoing", self.deps.zoneID, "Can't make current device state: %@", cdseError);
 520         } else if(!cdseRecord) {
 521             ckkserror("ckksoutgoing", self.deps.zoneID, "Can't make current device state cloudkit record, but no reason why");
 522         } else {
 523             // Add the CDSE to the outgoing records
 524             // TODO: maybe only do this every few hours?
 525             ckksnotice("ckksoutgoing", self.deps.zoneID, "Updating device state: %@", cdse);
 526             recordsToSave[cdseRecord.recordID] = cdseRecord;
 527         }
 528
 529         ckksinfo("ckksoutgoing", self.deps.zoneID, "Saving records %@ to CloudKit zone %@", recordsToSave, self.deps.zoneID);
 530
 531         self.modifyRecordsOperation = [[CKModifyRecordsOperation alloc] initWithRecordsToSave:recordsToSave.allValues recordIDsToDelete:recordIDsToDelete];
 532         self.modifyRecordsOperation.atomic = TRUE;
 533
 534         // Until <rdar://problem/38725728> Changes to discretionary-ness (explicit or derived from QoS) should be "live", all requests should be nondiscretionary
 535         self.modifyRecordsOperation.configuration.automaticallyRetryNetworkFailures = NO;
 536         self.modifyRecordsOperation.configuration.discretionaryNetworkBehavior = CKOperationDiscretionaryNetworkBehaviorNonDiscretionary;
 537         self.modifyRecordsOperation.configuration.isCloudKitSupportOperation = YES;
 538
 539         self.modifyRecordsOperation.savePolicy = CKRecordSaveIfServerRecordUnchanged;
 540         self.modifyRecordsOperation.group = self.ckoperationGroup;
 541         ckksnotice("ckksoutgoing", self.deps.zoneID, "QoS: %d; operation group is %@", (int)self.modifyRecordsOperation.qualityOfService, self.modifyRecordsOperation.group);
 542         ckksnotice("ckksoutgoing", self.deps.zoneID, "Beginning upload for %d records, deleting %d records",
 543                    (int)recordsToSave.count,
 544                    (int)recordIDsToDelete.count);
 545
 546         for(CKRecordID* recordID in [recordsToSave allKeys]) {
 547             ckksinfo("ckksoutgoing", self.deps.zoneID, "Record to save: %@", recordID);
 548         }
 549         for(CKRecordID* recordID in recordIDsToDelete) {
 550             ckksinfo("ckksoutgoing", self.deps.zoneID, "Record to delete: %@", recordID);
 551         }
 552
 553         self.modifyRecordsOperation.perRecordCompletionBlock = ^(CKRecord *record, NSError * _Nullable error) {
 554             STRONGIFY(self);
 555
 556             if(!error) {
 557                 ckksnotice("ckksoutgoing", self.deps.zoneID, "Record upload successful for %@ (%@)", record.recordID.recordName, record.recordChangeTag);
 558             } else {
 559                 ckkserror("ckksoutgoing", self.deps.zoneID, "error on row: %@ %@", error, record);
 560             }
 561         };
 562
 563         self.modifyRecordsOperation.modifyRecordsCompletionBlock = modifyRecordsCompletionBlock;
 564         [self dependOnBeforeGroupFinished: self.modifyRecordsOperation];
 565         [ckks.database addOperation: self.modifyRecordsOperation];
 566
 567         return true;
 568     }];
 569 }
 570
 571 - (void)_onqueueModifyRecordAsError:(CKRecordID*)recordID recordError:(NSError*)itemerror {
 572     CKKSKeychainView* ckks = self.ckks;
 573     if(!ckks) {
 574         ckkserror("ckksoutgoing", self.deps.zoneID, "no CKKS object");
 575         return;
 576     }
 577
 578     dispatch_assert_queue(ckks.queue);
 579
 580     NSError* error = nil;
 581     uint64_t count = 0;
 582
 583     // At this stage, cloudkit doesn't give us record types
 584     if([recordID.recordName isEqualToString: SecCKKSKeyClassA] ||
 585        [recordID.recordName isEqualToString: SecCKKSKeyClassC] ||
 586        [recordID.recordName hasPrefix:@"Manifest:-:"] ||
 587        [recordID.recordName hasPrefix:@"ManifestLeafRecord:-:"]) {
 588         // Nothing to do here. We need a whole key refetch and synchronize.
 589     } else {
 590         CKKSOutgoingQueueEntry* oqe = [CKKSOutgoingQueueEntry fromDatabase:recordID.recordName state:SecCKKSStateInFlight zoneID:self.deps.zoneID error:&error];
 591         [ckks _onqueueErrorOutgoingQueueEntry: oqe itemError: itemerror error:&error];
 592         if(error) {
 593             ckkserror("ckksoutgoing", self.deps.zoneID, "Couldn't set OQE %@ as error: %@", recordID.recordName, error);
 594             self.error = error;
 595         }
 596         count ++;
 597     }
 598 }
 599
 600
 601 - (void)_onqueueModifyAllRecords:(NSArray<CKRecordID*>*)recordIDs as:(CKKSItemState*)state {
 602     CKKSKeychainView* ckks = self.ckks;
 603     if(!ckks) {
 604         ckkserror("ckksoutgoing", self.deps.zoneID, "no CKKS object");
 605         return;
 606     }
 607
 608     dispatch_assert_queue(ckks.queue);
 609
 610     NSError* error = nil;
 611     uint64_t count = 0;
 612
 613     for(CKRecordID* recordID in recordIDs) {
 614         // At this stage, cloudkit doesn't give us record types
 615         if([recordID.recordName isEqualToString: SecCKKSKeyClassA] ||
 616            [recordID.recordName isEqualToString: SecCKKSKeyClassC]) {
 617             // Nothing to do here. We need a whole key refetch and synchronize.
 618         } else {
 619             CKKSOutgoingQueueEntry* oqe = [CKKSOutgoingQueueEntry fromDatabase:recordID.recordName state:SecCKKSStateInFlight zoneID:self.deps.zoneID error:&error];
 620             [ckks _onqueueChangeOutgoingQueueEntry:oqe toState:state error:&error];
 621             if(error) {
 622                 ckkserror("ckksoutgoing", self.deps.zoneID, "Couldn't set OQE %@ as %@: %@", recordID.recordName, state, error);
 623                 self.error = error;
 624             }
 625             count ++;
 626         }
 627     }
 628
 629     if([state isEqualToString:SecCKKSStateReencrypt]) {
 630         [SecCoreAnalytics sendEvent:SecCKKSAggdItemReencryption event:@{SecCoreAnalyticsValue: [NSNumber numberWithUnsignedInteger:count]}];
 631     }
 632 }
 633
 634 - (BOOL)_onqueueIsErrorMissingSyncKey:(NSError*)ckerror {
 635     if([ckerror.domain isEqualToString:CKErrorDomain] && (ckerror.code == CKErrorPartialFailure)) {
 636         NSMutableDictionary<CKRecordID*, NSError*>* failedRecords = ckerror.userInfo[CKPartialErrorsByItemIDKey];
 637
 638         for(CKRecordID* recordID in failedRecords) {
 639             NSError* recordError = failedRecords[recordID];
 640
 641             if([recordError isCKKSServerPluginError:CKKSServerMissingRecord]) {
 642                 ckksnotice("ckksoutgoing", recordID.zoneID, "Error is a 'missing record' error: %@", recordError);
 643                 return YES;
 644             }
 645         }
 646     }
 647
 648     return NO;
 649 }
 650
 651 - (bool)_onqueueIsErrorBadEtagOnKeyPointersOnly:(NSError*)ckerror {
 652     CKKSKeychainView* ckks = self.ckks;
 653     dispatch_assert_queue(ckks.queue);
 654     bool anyOtherErrors = false;
 655
 656     if([ckerror.domain isEqualToString:CKErrorDomain] && (ckerror.code == CKErrorPartialFailure)) {
 657         NSMutableDictionary<CKRecordID*, NSError*>* failedRecords = ckerror.userInfo[CKPartialErrorsByItemIDKey];
 658
 659         for(CKRecordID* recordID in failedRecords) {
 660             NSError* recordError = failedRecords[recordID];
 661
 662             if([recordError.domain isEqualToString: CKErrorDomain] && recordError.code == CKErrorServerRecordChanged) {
 663                 if([recordID.recordName isEqualToString: SecCKKSKeyClassA] ||
 664                    [recordID.recordName isEqualToString: SecCKKSKeyClassC]) {
 665                     // this is fine!
 666                 } else {
 667                     // Some record other than the key pointers changed.
 668                     anyOtherErrors |= true;
 669                     break;
 670                 }
 671             } else {
 672                 // Some other error than ServerRecordChanged
 673                 anyOtherErrors |= true;
 674                 break;
 675             }
 676         }
 677     }
 678
 679     return !anyOtherErrors;
 680 }
 681
 682 @end;
 683
 684 #endif