OSX/sec/SOSCircle/SecureObjectSync/SOSPeer.c

   1 /*
   2  * Copyright (c) 2012-2014 Apple Inc. All Rights Reserved.
   3  *
   4  * @APPLE_LICENSE_HEADER_START@
   5  *
   6  * This file contains Original Code and/or Modifications of Original Code
   7  * as defined in and that are subject to the Apple Public Source License
   8  * Version 2.0 (the 'License'). You may not use this file except in
   9  * compliance with the License. Please obtain a copy of the License at
  10  * http://www.opensource.apple.com/apsl/ and read it before using this
  11  * file.
  12  *
  13  * The Original Code and all software distributed under the License are
  14  * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  15  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  16  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
  17  * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
  18  * Please see the License for the specific language governing rights and
  19  * limitations under the License.
  20  *
  21  * @APPLE_LICENSE_HEADER_END@
  22  */
  23
  24
  25 /*
  26  * SOSPeer.c -  Implementation of a secure object syncing peer
  27  */
  28 #include <Security/SecureObjectSync/SOSPeer.h>
  29
  30 #include <Security/SecureObjectSync/SOSCoder.h>
  31 #include <Security/SecureObjectSync/SOSDigestVector.h>
  32 #include <Security/SecureObjectSync/SOSInternal.h>
  33 #include <Security/SecureObjectSync/SOSTransport.h>
  34 #include <Security/SecureObjectSync/SOSViews.h>
  35 #include <Security/SecureObjectSync/SOSChangeTracker.h>
  36 #include <utilities/SecCFError.h>
  37 #include <utilities/SecCFRelease.h>
  38 #include <utilities/SecCFWrappers.h>
  39 #include <utilities/SecDb.h>
  40 #include <utilities/SecFileLocations.h>
  41 #include <utilities/SecIOFormat.h>
  42 #include <utilities/array_size.h>
  43 #include <utilities/debugging.h>
  44 #include <Security/SecureObjectSync/SOSBackupEvent.h>
  45 #include <Security/SecItemBackup.h>
  46
  47 #include <securityd/SOSCloudCircleServer.h>
  48
  49 #include <CoreFoundation/CoreFoundation.h>
  50
  51 #include <stdlib.h>
  52
  53 #include <AssertMacros.h>
  54
  55 // Backup Peer Support
  56 #include <securityd/SecKeybagSupport.h>
  57 #include <notify.h>
  58
  59
  60 //
  61 // MARK: - SOSPeerPersistence code
  62 //
  63 static CFStringRef kSOSPeerSequenceNumberKey = CFSTR("sequence-number");
  64
  65 CFStringRef kSOSPeerDataLabel = CFSTR("iCloud Peer Data Meta-data");
  66
  67 //
  68 // MARK: SOSPeerState (dictionary keys)
  69 //
  70
  71 // PeerState dictionary keys
  72 static CFStringRef kSOSPeerSendObjectsKey = CFSTR("send-objects"); // bool
  73 static CFStringRef kSOSPeerMustSendMessageKey = CFSTR("must-send"); // bool
  74 static CFStringRef kSOSPeerPendingObjectsKey = CFSTR("pending-objects"); // digest
  75 static CFStringRef kSOSPeerUnwantedManifestKey = CFSTR("unwanted-manifest"); // digest
  76 static CFStringRef kSOSPeerConfirmedManifestKey = CFSTR("confirmed-manifest");  //digest
  77 static CFStringRef kSOSPeerProposedManifestKey = CFSTR("pending-manifest"); // array of digests
  78 static CFStringRef kSOSPeerLocalManifestKey = CFSTR("local-manifest"); // array of digests
  79 static CFStringRef kSOSPeerVersionKey = CFSTR("vers"); // int
  80 static CFStringRef kSOSPeerCoderKey = CFSTR("coder"); // der encoded SOSCoder
  81
  82 //
  83 // SOSPeerMeta keys that can also be used in peerstate...
  84 //
  85 static CFStringRef kSOSPeerPeerIDKey = CFSTR("peer-id"); // string
  86 static CFStringRef kSOSPeerViewsKey = CFSTR("views"); // set (or array) of string
  87 static CFStringRef kSOSPeerKeyBagKey = CFSTR("keybag"); // data
  88
  89 /*
  90  Theory of syncing for both incoming and outgoing messages
  91
  92  A peerstate consists of:
  93  (T, U, C, P, L, D->M, M->D, H->O, O->{(Op,Or)|Oa,(Op,Or)|(Om,Oa),(Op,Or)|Om,Oi|Oi|Ob|Oa,Ob|(Om,Oa),Ob|(Ol,Ou)|Oa,(Ol,Ou)|(Om,Oa)(Ol,Ou)})
  94     T: to be sent (pendingObjects) manifest
  95     U: unwanted objects manifest
  96     C: confirmed manifest
  97     P: proposed manifest
  98     D->M? digest or manifest to optional manifest function
  99     M->D manifest to digest of manifest function
 100     H->O? hash (manifest entry) to optional object function (the datasource)
 101     O->{                    Mapping from incoming O objects to one of:
 102         (Op,Or):            Op = Peers object, Or is replaced local object.
 103         Oa,(Op,Or):         Oa = appeared local object, Or = Oa, see above for (Op, Or)
 104         (Om,Oa),(Op,Or):    Om missing local object, was apparently Oa instead see above for (Oa, Op, Or)
 105         Om,Oi:              Om missing local object, inserted Oi (nothing replaced), but Om still disapeared from manifest
 106         Oi                  Oi inserted object from peer (nothing replaced)
 107         Ob:                 Ob both remote and local object are identical, nothing changed
 108         Oa,Ob:              Oa = appeared local object equal to Oa = Ob.  Equivalent to single Oi
 109         (Om,Oa),Ob:         Om missing local object, must be O->H->Ob, Oa found in place, Ob != Oa, Oa magically appeared unrelated to O->H->Ob
 110         (Ol,Ou):            Ol local object wins from peers Ou older object => append Ou to U
 111         Oa,(Ol,Ou):         Oa appeared as a local object and Oa = Ol above
 112         (Om,Oa),(Ol,Ou):    Om removed and Oa replaced Om and Oa = Ol as above
 113     }
 114  A message consists of
 115  (B,M,E,O)
 116     B: base digest
 117     M: missing manifest
 118     E: extra manifest
 119     O: objects
 120
 121  To send a message we simply compute:
 122     B: D->M->C
 123     M: C \ L
 124     E: L \ C          # subsetted if not all of O is sent
 125     O: H->O?->O       # as size permits
 126     and change P to (C \ M:) union E: union O->H->O:
 127
 128  To receive a message we compute
 129     Op,Oi,Ob,Oo and Om,Oa,Ol
 130     C: (D->M->B \ M) union E union O->H->O
 131         A = E \ (O->H->O)
 132     T: T \ (Om union Or union A union Oa,Ob.last union (Om,Oa),Ob.last ) union (M intersect L)
 133  */
 134
 135
 136 //
 137 // MARK: SOSPeerMeta
 138 //
 139
 140 static SOSPeerMetaRef SOSPeerMetaCreate(CFStringRef peerID) {
 141     return CFRetain(peerID);
 142 }
 143
 144 static SOSPeerMetaRef SOSPeerMetaCreateWithViews(CFStringRef peerID, CFSetRef views) {
 145     const void *keys[] = { kSOSPeerPeerIDKey, kSOSPeerViewsKey };
 146     const void *values[] = { peerID, views };
 147     return CFDictionaryCreate(kCFAllocatorDefault, keys, values, array_size(keys), &kCFTypeDictionaryKeyCallBacks, &kCFTypeDictionaryValueCallBacks);
 148 }
 149
 150 static SOSPeerMetaRef SOSPeerMetaCreateWithViewsAndKeyBag(CFStringRef peerID, CFSetRef views, CFDataRef keybag) {
 151     const void *keys[] = { kSOSPeerPeerIDKey, kSOSPeerViewsKey, kSOSPeerKeyBagKey };
 152     const void *values[] = { peerID, views, keybag };
 153     return CFDictionaryCreate(kCFAllocatorDefault, keys, values, array_size(keys), &kCFTypeDictionaryKeyCallBacks, &kCFTypeDictionaryValueCallBacks);
 154 }
 155
 156 SOSPeerMetaRef SOSPeerMetaCreateWithComponents(CFStringRef peerID, CFSetRef views, CFDataRef keybag) {
 157     if (!isString(peerID))
 158         return NULL;
 159     if (views) {
 160         if (keybag)
 161             return SOSPeerMetaCreateWithViewsAndKeyBag(peerID, views, keybag);
 162         else
 163             return SOSPeerMetaCreateWithViews(peerID, views);
 164     } else
 165         return SOSPeerMetaCreate(peerID);
 166 }
 167
 168 SOSPeerMetaRef SOSPeerMetaCreateWithState(CFStringRef peerID, CFDictionaryRef state) {
 169     return SOSPeerMetaCreateWithComponents(peerID, CFDictionaryGetValue(state, kSOSPeerViewsKey), CFDictionaryGetValue(state, kSOSPeerKeyBagKey));
 170 }
 171
 172 CFStringRef SOSPeerMetaGetComponents(SOSPeerMetaRef peerMeta, CFSetRef *views, CFDataRef *keybag, CFErrorRef *error) {
 173     if (isDictionary(peerMeta)) {
 174         CFDictionaryRef meta = (CFDictionaryRef)peerMeta;
 175         CFStringRef peerID = asString(CFDictionaryGetValue(meta, kSOSPeerPeerIDKey), error);
 176         CFSetRef vns = asSet(CFDictionaryGetValue(meta, kSOSPeerViewsKey), error);
 177         if (vns && asDataOptional(CFDictionaryGetValue(meta, kSOSPeerKeyBagKey), keybag, error)) {
 178             if (views)
 179                 *views = vns;
 180             return peerID;
 181         }
 182         return NULL;
 183     } else {
 184         if (views) {
 185             // Hack so tests can pass simple peerIDs
 186             *views = SOSViewsGetV0ViewSet();
 187         }
 188         return asString(peerMeta, error);
 189     }
 190 }
 191
 192 CFTypeRef SOSPeerOrStateSetViewsKeyBagAndCreateCopy(CFTypeRef peerOrState, CFSetRef views, CFDataRef keyBag) {
 193     assert(views);
 194     if (peerOrState && CFGetTypeID(peerOrState) == SOSPeerGetTypeID()) {
 195         // Inflated peer, update its views and move on
 196         SOSPeerRef peer = (SOSPeerRef)peerOrState;
 197         SOSPeerSetViewNameSet(peer, views);
 198         SOSPeerSetKeyBag(peer, keyBag);
 199         return CFRetainSafe(peer);
 200     } else if (peerOrState && CFGetTypeID(peerOrState) == CFDictionaryGetTypeID()) {
 201         CFMutableDictionaryRef state = CFDictionaryCreateMutableCopy(kCFAllocatorDefault, 0, peerOrState);
 202         // Deserialized peer, just updated the serialized state with the new views
 203         CFDictionarySetValue(state, kSOSPeerViewsKey, views);
 204         if (keyBag)
 205             CFDictionarySetValue(state, kSOSPeerKeyBagKey, keyBag);
 206         else
 207             CFDictionaryRemoveValue(state, kSOSPeerKeyBagKey);
 208         return state;
 209     } else {
 210         // New peer, just create a state object.
 211         if (keyBag)
 212             return CFDictionaryCreateForCFTypes(kCFAllocatorDefault, kSOSPeerViewsKey, views, kSOSPeerKeyBagKey, keyBag, NULL);
 213         else
 214             return CFDictionaryCreateForCFTypes(kCFAllocatorDefault, kSOSPeerViewsKey, views, NULL);
 215     }
 216 }
 217
 218 CFTypeRef SOSPeerOrStateSetViewsAndCopyState(CFTypeRef peerOrState, CFSetRef views) {
 219     assert(views);
 220     if (peerOrState && CFGetTypeID(peerOrState) == SOSPeerGetTypeID()) {
 221         // Inflated peer, update its views and deflate it
 222         SOSPeerRef peer = (SOSPeerRef)peerOrState;
 223         SOSPeerSetViewNameSet(peer, views);
 224         return SOSPeerCopyState(peer, NULL);
 225     } else if (peerOrState && CFGetTypeID(peerOrState) == CFDictionaryGetTypeID()) {
 226         // We have a deflated peer.  Update its views and keep it deflated
 227         CFMutableDictionaryRef state = CFDictionaryCreateMutableCopy(kCFAllocatorDefault, 0, peerOrState);
 228         CFDictionarySetValue(state, kSOSPeerViewsKey, views);
 229         return state;
 230     } else {
 231         return NULL;
 232     }
 233 }
 234
 235 bool SOSPeerMapEntryIsBackup(const void *mapEntry) {
 236     if (!mapEntry) return false;
 237     if (CFGetTypeID(mapEntry) == SOSPeerGetTypeID()) {
 238         return SOSPeerGetKeyBag((SOSPeerRef)mapEntry);
 239     } else {
 240         return CFDictionaryContainsKey(mapEntry, kSOSPeerKeyBagKey);
 241     }
 242 }
 243
 244 //
 245 // MARK: - SOSManifest
 246 //
 247
 248 enum {
 249     kSOSPeerMaxManifestWindowDepth = 4
 250 };
 251
 252 static CFStringRef SOSManifestCreateOptionalDescriptionWithLabel(SOSManifestRef manifest, CFStringRef label) {
 253     if (!manifest) return CFSTR(" -  ");
 254     return CFStringCreateWithFormat(kCFAllocatorDefault, NULL, CFSTR(" %@%@"), label, manifest);
 255 }
 256
 257 static CFStringRef SOSManifestArrayCreateOptionalDescriptionWithLabel(CFArrayRef manifests, CFStringRef label) {
 258     CFIndex count = manifests ? CFArrayGetCount(manifests) : 0;
 259     if (count == 0) return CFSTR(" -  ");
 260     SOSManifestRef manifest = (SOSManifestRef)CFArrayGetValueAtIndex(manifests, 0);
 261     return CFStringCreateWithFormat(kCFAllocatorDefault, NULL, CFSTR(" %@[%" PRIdCFIndex "]%@"), label, count, manifest);
 262 }
 263
 264 static void SOSManifestArraySetManifest(CFMutableArrayRef *manifests, SOSManifestRef manifest) {
 265     if (manifest) {
 266         if (*manifests)
 267             CFArrayRemoveAllValues(*manifests);
 268         else
 269             *manifests = CFArrayCreateMutableForCFTypes(kCFAllocatorDefault);
 270         CFArrayAppendValue(*manifests, manifest);
 271     } else {
 272         CFReleaseNull(*manifests);
 273     }
 274 }
 275
 276 static void SOSManifestMutableArrayAppendManifest(CFMutableArrayRef manifests, SOSManifestRef manifest) {
 277     if (manifest) {
 278         CFIndex count = CFArrayGetCount(manifests);
 279         CFIndex ixOfManifest = CFArrayGetFirstIndexOfValue(manifests, CFRangeMake(0, count), manifest);
 280         if (ixOfManifest != 0) {
 281             // If the manifest isn't at the front of the array move it there.
 282             // If it's not in the array, remove enough entires from the end to
 283             // make room to put it in the front.
 284             if (ixOfManifest != kCFNotFound) {
 285                 CFArrayRemoveValueAtIndex(manifests, ixOfManifest);
 286             } else {
 287                 while (count >= kSOSPeerMaxManifestWindowDepth)
 288                     CFArrayRemoveValueAtIndex(manifests, --count);
 289             }
 290
 291             CFArrayInsertValueAtIndex(manifests, 0, manifest);
 292         }
 293     } else {
 294         // pending == NULL => nothing clear history
 295         CFArrayRemoveAllValues(manifests);
 296     }
 297 }
 298
 299 static void SOSManifestArrayAppendManifest(CFMutableArrayRef *manifests, SOSManifestRef manifest) {
 300     if (*manifests)
 301         SOSManifestMutableArrayAppendManifest(*manifests, manifest);
 302     else
 303         SOSManifestArraySetManifest(manifests, manifest);
 304 }
 305
 306 //
 307 // MARK: - SOSPeer
 308 //
 309
 310 struct __OpaqueSOSPeer {
 311     CFRuntimeBase _base;
 312
 313     SOSCoderRef coder;
 314     CFDataRef coderData;
 315     CFStringRef peer_id;
 316     CFSetRef views;
 317     CFIndex version;
 318     uint64_t sequenceNumber;
 319     bool mustSendMessage;
 320     bool sendObjects;
 321
 322     SOSManifestRef pendingObjects;
 323     SOSManifestRef unwantedManifest;
 324     SOSManifestRef confirmedManifest;
 325     CFMutableArrayRef proposedManifests;
 326     CFMutableArrayRef localManifests;
 327
 328     // Only backup peers have these:
 329     CFDataRef _keyBag;
 330     FILE *journalFile;
 331 };
 332
 333 CFGiblisWithCompareFor(SOSPeer)
 334
 335 static CFStringRef SOSPeerCopyFormatDescription(CFTypeRef cf, CFDictionaryRef formatOptions) {
 336     SOSPeerRef peer = (SOSPeerRef)cf;
 337     if(peer){
 338         CFStringRef po = SOSManifestCreateOptionalDescriptionWithLabel(peer->pendingObjects, CFSTR("O"));
 339         CFStringRef uo = SOSManifestCreateOptionalDescriptionWithLabel(peer->unwantedManifest, CFSTR("U"));
 340         CFStringRef co = SOSManifestCreateOptionalDescriptionWithLabel(peer->confirmedManifest, CFSTR("C"));
 341         CFStringRef pe = SOSManifestArrayCreateOptionalDescriptionWithLabel(peer->proposedManifests, CFSTR("P"));
 342         CFStringRef lo = SOSManifestArrayCreateOptionalDescriptionWithLabel(peer->localManifests, CFSTR("L"));
 343         CFStringRef desc = CFStringCreateWithFormat(kCFAllocatorDefault, NULL, CFSTR("<%@ %s%s%@%@%@%@%@>"),
 344                                                     SOSPeerGetID(peer),
 345                                                     SOSPeerMustSendMessage(peer) ? "F" : "f",
 346                                                     SOSPeerSendObjects(peer) ? "S" : "s",
 347                                                     po, uo, co, pe, lo);
 348         CFReleaseSafe(lo);
 349         CFReleaseSafe(pe);
 350         CFReleaseSafe(co);
 351         CFReleaseSafe(uo);
 352         CFReleaseSafe(po);
 353
 354         return desc;
 355     }
 356     else
 357         return CFSTR("NULL");
 358 }
 359
 360 static Boolean SOSPeerCompare(CFTypeRef cfA, CFTypeRef cfB)
 361 {
 362     SOSPeerRef peerA = (SOSPeerRef)cfA, peerB = (SOSPeerRef)cfB;
 363     // Use mainly to see if peerB is actually this device (peerA)
 364     return CFStringCompare(SOSPeerGetID(peerA), SOSPeerGetID(peerB), 0) == kCFCompareEqualTo;
 365 }
 366
 367 // Coder and coderData caching.
 368 // A Peer has either a coderData or a coder.  Upon serialization the
 369 // coder will be turned into coderData but the coder will stay instantiated
 370 // unless the peer is released.
 371 static void SOSPeerSetCoderData(SOSPeerRef peer, CFDataRef coderData){
 372     if (peer->coder) {
 373         SOSCoderDispose(peer->coder);
 374         peer->coder = NULL;
 375     }
 376     CFRetainAssign(peer->coderData, coderData);
 377 }
 378
 379 static bool SOSPeerCopyCoderData(SOSPeerRef peer, CFDataRef *coderData, CFErrorRef *error) {
 380     // TODO: We can optionally call SOSPeerSetCoderData here to clear the coder whenever its encoded,
 381     // if we assume that coders are written out to disk more often than they are used.
 382     bool ok = true;
 383     if (peer->coder) {
 384 #if 1
 385         CFErrorRef localError = NULL;
 386         ok = *coderData = SOSCoderCopyDER(peer->coder, &localError);
 387         if (!ok) {
 388             secerror("failed to der encode coder for peer %@, dropping it: %@", peer->peer_id, localError);
 389             SOSCoderDispose(peer->coder);
 390             peer->coder = NULL;
 391             CFErrorPropagate(localError, error);
 392         }
 393         return ok;
 394 #else
 395         // Alternate always delete in memory coder after der encoding it.
 396         CFAssignRetained(peer->coderData, SOSCoderCopyDER(peer->coder, error));
 397         ok = peer->coderData;
 398         SOSCoderDispose(peer->coder);
 399         peer->coder = NULL;
 400 #endif
 401     }
 402     *coderData = CFRetainSafe(peer->coderData);
 403     return ok;
 404 }
 405
 406 SOSCoderRef SOSPeerGetCoder(SOSPeerRef peer, CFErrorRef *error) {
 407     if (peer->coderData) {
 408         peer->coder = SOSCoderCreateFromData(peer->coderData, error);
 409         CFReleaseNull(peer->coderData);
 410     } else if (!peer->coder) {
 411         SOSErrorCreate(kSOSErrorPeerNotFound, error, NULL, CFSTR("No coderData nor coder for peer: %@"), peer->peer_id);
 412     }
 413     return peer->coder;
 414 }
 415
 416 bool SOSPeerEnsureCoder(SOSPeerRef peer, SOSFullPeerInfoRef myPeerInfo, SOSPeerInfoRef peerInfo, CFErrorRef *error) {
 417     if (!SOSPeerGetCoder(peer, NULL)) {
 418         secinfo("peer", "New coder for id %@.", peer->peer_id);
 419         CFErrorRef localError = NULL;
 420         peer->coder = SOSCoderCreate(peerInfo, myPeerInfo, kCFBooleanFalse, &localError);
 421         if (!peer->coder) {
 422             secerror("Failed to create coder for %@: %@", peer->peer_id, localError);
 423             CFErrorPropagate(localError, error);
 424             return false;
 425         }
 426     }
 427     return true;
 428 }
 429
 430 static bool SOSPeerGetPersistedBoolean(CFDictionaryRef persisted, CFStringRef key) {
 431     CFBooleanRef boolean = CFDictionaryGetValue(persisted, key);
 432     return boolean && CFBooleanGetValue(boolean);
 433 }
 434
 435 static CFDataRef SOSPeerGetPersistedData(CFDictionaryRef persisted, CFStringRef key) {
 436     return asData(CFDictionaryGetValue(persisted, key), NULL);
 437 }
 438
 439 static int64_t SOSPeerGetPersistedInt64(CFDictionaryRef persisted, CFStringRef key) {
 440     int64_t integer = 0;
 441     CFNumberRef number = CFDictionaryGetValue(persisted, key);
 442     if (number) {
 443         CFNumberGetValue(number, kCFNumberSInt64Type, &integer);
 444     }
 445     return integer;
 446 }
 447
 448 static void SOSPeerGetOptionalPersistedCFIndex(CFDictionaryRef persisted, CFStringRef key, CFIndex *value) {
 449     CFNumberRef number = CFDictionaryGetValue(persisted, key);
 450     if (number) {
 451         CFNumberGetValue(number, kCFNumberCFIndexType, value);
 452     }
 453 }
 454
 455 static CFSetRef SOSPeerGetPersistedViewNameSet(SOSPeerRef peer, CFDictionaryRef persisted, CFStringRef key) {
 456     CFSetRef vns = CFDictionaryGetValue(persisted, key);
 457     if (!vns) {
 458         // Engine state in db contained a v0 peer, thus it must be in the V0ViewSet.
 459         vns = SOSViewsGetV0ViewSet();
 460         secnotice("peer", "%@ had no views, inferring: %@", peer->peer_id, vns);
 461     }
 462     return vns;
 463 }
 464
 465 //
 466 // MARK: Backup Peers
 467 //
 468
 469 void SOSBackupPeerPostNotification(const char *reason) {
 470     // Let sbd know when a notable event occurs
 471     // - Disk full
 472     // - Backup bag change
 473     secnotice("backup", "posting notification to CloudServices: %s", reason?reason:"");
 474     notify_post(kSecItemBackupNotification);
 475 }
 476
 477 static bool SOSPeerDoWithJournalPath(SOSPeerRef peer, CFErrorRef *error, void(^with)(const char *journalPath)) {
 478     // TODO: Probably switch to using CFURL to construct the path.
 479     bool ok = true;
 480     char strBuffer[PATH_MAX + 1];
 481     size_t userTempLen = confstr(_CS_DARWIN_USER_TEMP_DIR, strBuffer, sizeof(strBuffer));
 482     if (userTempLen == 0) {
 483         ok = SecCheckErrno(-1, error, CFSTR("confstr on _CS_DARWIN_USER_TEMP_DIR returned an error."));
 484     } else {
 485         CFStringRef journalName = CFStringCreateWithFormat(kCFAllocatorDefault, NULL, CFSTR("%s/SOSBackup-%@"), strBuffer, SOSPeerGetID(peer));
 486         CFStringPerformWithCString(journalName, with);
 487         CFReleaseSafe(journalName);
 488     }
 489     return ok;
 490 }
 491
 492 static FILE *fopen_journal(const char *journalPath, const char *mode, CFErrorRef *error) {
 493     FILE *file = fopen(journalPath, mode);
 494     SecCheckErrno(!file, error, CFSTR("fopen %s,%s"), journalPath, mode);
 495     return file;
 496 }
 497
 498 #include <sys/stat.h>
 499
 500 #if !defined(NDEBUG)
 501 static off_t getFileSize(int fd) {
 502     return lseek(fd, 0, SEEK_END);
 503 }
 504 #endif
 505
 506 int SOSPeerHandoffFD(SOSPeerRef peer, CFErrorRef *error) {
 507     __block int fd = -1;
 508     SOSPeerDoWithJournalPath(peer, error, ^(const char *journalName) {
 509         fd = open(journalName, O_RDONLY | O_CLOEXEC);
 510         if (SecCheckErrno(fd < 0, error, CFSTR("open %s"), journalName)) {
 511             if (!SecCheckErrno(unlink(journalName), error, CFSTR("unlink %s"), journalName)) {
 512                 close(fd);
 513                 fd = -1;
 514             } else {
 515                 secdebug("backup", "Handing off file %s with fd %d of size %llu", journalName, fd, getFileSize(fd));
 516             }
 517         } else {
 518             secdebug("backup", "Handing off file %s failed, %@", journalName, error?*error:NULL);
 519         }
 520     });
 521     return fd;
 522 }
 523
 524 static CFDataRef SOSPeerCopyAKSKeyBag(SOSPeerRef peer, CFErrorRef *error) {
 525     if (CFEqual(peer->peer_id, kSOSViewKeychainV0_tomb)) {
 526         return CFRetainSafe(peer->_keyBag);
 527     } else {
 528         CFDataRef aksKeybag = NULL;
 529         SOSBackupSliceKeyBagRef backupSliceKeyBag = SOSBackupSliceKeyBagCreateFromData(kCFAllocatorDefault, peer->_keyBag, error);
 530         if (backupSliceKeyBag) {
 531             aksKeybag = SOSBSKBCopyAKSBag(backupSliceKeyBag, error);
 532             CFRelease(backupSliceKeyBag);
 533         }
 534         return aksKeybag;
 535     }
 536 }
 537
 538 bool SOSPeerAppendToJournal(SOSPeerRef peer, CFErrorRef *error, void(^with)(FILE *journalFile, keybag_handle_t kbhandle)) {
 539     __block bool ok = true;
 540     // We only need a keybag if we are writing ADDs. Since we don't know at this layer
 541     // what operations we may be doing, open keybag if we have one, otherwise don't
 542     ok &= SOSPeerDoWithJournalPath(peer, error, ^(const char *fname) {
 543         FILE *file = fopen_journal(fname, "a", error);
 544         if (file) {
 545             keybag_handle_t kbhandle = bad_keybag_handle;
 546             CFDataRef keybag = SOSPeerCopyAKSKeyBag(peer, error);
 547             ok = keybag;
 548             if (ok && (ok = ks_open_keybag(keybag, NULL, &kbhandle, error))) {
 549                 with(file, kbhandle);
 550                 if (kbhandle != bad_keybag_handle)
 551                     ok &= ks_close_keybag(kbhandle, error);
 552             }
 553             CFReleaseSafe(keybag);
 554             fclose(file);
 555         }
 556     });
 557     return ok;
 558 }
 559
 560 static bool SOSPeerTruncateJournal(SOSPeerRef peer, CFErrorRef *error, void(^with)(FILE *journalFile)) {
 561     __block bool ok = true;
 562     ok &= SOSPeerDoWithJournalPath(peer, error, ^(const char *fname) {
 563         FILE *file = fopen_journal(fname, "w", error);
 564         if (file) {
 565             with(file);
 566             fclose(file);
 567         }
 568     });
 569     return ok;
 570 }
 571
 572 bool SOSPeerSetState(SOSPeerRef p, SOSEngineRef engine, CFDictionaryRef state, CFErrorRef *error) {
 573     bool ok = true;
 574     if (state) {
 575         SOSPeerGetOptionalPersistedCFIndex(state, kSOSPeerVersionKey, &p->version);
 576
 577         p->sequenceNumber = SOSPeerGetPersistedInt64(state, kSOSPeerSequenceNumberKey);
 578         p->mustSendMessage = SOSPeerGetPersistedBoolean(state, kSOSPeerMustSendMessageKey);
 579         p->sendObjects = SOSPeerGetPersistedBoolean(state, kSOSPeerSendObjectsKey);
 580         CFRetainAssign(p->views, SOSPeerGetPersistedViewNameSet(p, state, kSOSPeerViewsKey));
 581         SOSPeerSetKeyBag(p, SOSPeerGetPersistedData(state, kSOSPeerKeyBagKey));
 582         // Don't rollback coder in memory if a transaction is rolled back, since this
 583         // might lead to reuse of an IV.
 584         if (!p->coder)
 585             SOSPeerSetCoderData(p, SOSPeerGetPersistedData(state, kSOSPeerCoderKey));
 586         CFAssignRetained(p->pendingObjects, SOSEngineCopyPersistedManifest(engine, state, kSOSPeerPendingObjectsKey));
 587         CFAssignRetained(p->unwantedManifest, SOSEngineCopyPersistedManifest(engine, state, kSOSPeerUnwantedManifestKey));
 588         CFAssignRetained(p->confirmedManifest, SOSEngineCopyPersistedManifest(engine, state, kSOSPeerConfirmedManifestKey));
 589         CFAssignRetained(p->proposedManifests, SOSEngineCopyPersistedManifestArray(engine, state, kSOSPeerProposedManifestKey, error));
 590         ok &= p->proposedManifests != NULL;
 591         CFAssignRetained(p->localManifests, SOSEngineCopyPersistedManifestArray(engine, state, kSOSPeerLocalManifestKey, error));
 592         ok &= p->localManifests != NULL;
 593     }
 594     return ok;
 595 }
 596
 597 static SOSPeerRef SOSPeerCreate_Internal(SOSEngineRef engine, CFDictionaryRef state, CFStringRef theirPeerID, CFIndex version, CFErrorRef *error) {
 598     SOSPeerRef p = CFTypeAllocate(SOSPeer, struct __OpaqueSOSPeer, kCFAllocatorDefault);
 599     p->peer_id = CFRetainSafe(theirPeerID);
 600     p->version = version;
 601     CFDictionaryRef empty = NULL;
 602     if (!state) {
 603         empty = CFDictionaryCreate(kCFAllocatorDefault, NULL, NULL, 0, &kCFTypeDictionaryKeyCallBacks, &kCFTypeDictionaryValueCallBacks);
 604         state = empty;
 605     }
 606     if (!SOSPeerSetState(p, engine, state, error)) {
 607         CFReleaseNull(p);
 608     }
 609     CFReleaseNull(empty);
 610     return p;
 611 }
 612
 613 static bool SOSPeerPersistOptionalCoder(SOSPeerRef peer, CFMutableDictionaryRef persist, CFStringRef key, CFErrorRef *error) {
 614     CFDataRef coderData = NULL;
 615     bool ok = SOSPeerCopyCoderData(peer, &coderData, error);
 616     if (coderData) {
 617         CFDictionarySetValue(persist, key, coderData);
 618         CFReleaseSafe(coderData);
 619     }
 620     return ok;
 621 }
 622
 623 static void SOSPeerPersistBool(CFMutableDictionaryRef persist, CFStringRef key, bool value) {
 624     CFDictionarySetValue(persist, key, value ? kCFBooleanTrue : kCFBooleanFalse);
 625 }
 626
 627 static void SOSPeerPersistInt64(CFMutableDictionaryRef persist, CFStringRef key, int64_t value) {
 628     CFNumberRef number = CFNumberCreate(kCFAllocatorDefault, kCFNumberSInt64Type, &value);
 629     CFDictionarySetValue(persist, key, number);
 630     CFReleaseSafe(number);
 631 }
 632
 633 static void SOSPeerPersistCFIndex(CFMutableDictionaryRef persist, CFStringRef key, CFIndex value) {
 634     CFNumberRef number = CFNumberCreate(kCFAllocatorDefault, kCFNumberCFIndexType, &value);
 635     CFDictionarySetValue(persist, key, number);
 636     CFReleaseSafe(number);
 637 }
 638
 639 static bool SOSPeerPersistOptionalManifest(CFMutableDictionaryRef persist, CFStringRef key, SOSManifestRef manifest, CFErrorRef *error) {
 640     if (!manifest)
 641         return true;
 642     CFDataRef digest = SOSManifestGetDigest(manifest, error);
 643     bool ok = digest;
 644     if (ok)
 645         CFDictionarySetValue(persist, key, digest);
 646     return ok;
 647 }
 648
 649 static bool SSOSPeerPersistManifestArray(CFMutableDictionaryRef persist, CFStringRef key, CFArrayRef manifests, CFErrorRef *error) {
 650     CFMutableArrayRef digests = CFArrayCreateMutableForCFTypes(kCFAllocatorDefault);
 651     SOSManifestRef manifest;
 652     if (manifests) CFArrayForEachC(manifests, manifest) {
 653         CFDataRef digest = SOSManifestGetDigest(manifest, error);
 654         if (!digest)
 655             CFReleaseNull(digests);
 656         if (digests) {
 657             CFArrayAppendValue(digests, digest);
 658         }
 659     }
 660     if (digests) {
 661         CFDictionarySetValue(persist, key, digests);
 662         CFRelease(digests);
 663     }
 664     return digests;
 665 }
 666
 667 static void SOSPeerPersistOptionalValue(CFMutableDictionaryRef persist, CFStringRef key, CFTypeRef value) {
 668     if (value)
 669         CFDictionarySetValue(persist, key, value);
 670 }
 671
 672 CFDictionaryRef SOSPeerCopyState(SOSPeerRef peer, CFErrorRef *error) {
 673     CFMutableDictionaryRef state = CFDictionaryCreateMutableForCFTypes(kCFAllocatorDefault);
 674     SOSPeerPersistInt64(state, kSOSPeerSequenceNumberKey, peer->sequenceNumber);
 675     if (peer->version) {
 676         SOSPeerPersistCFIndex(state, kSOSPeerVersionKey, peer->version);
 677     }
 678
 679     SOSPeerPersistBool(state, kSOSPeerMustSendMessageKey, peer->mustSendMessage);
 680     SOSPeerPersistBool(state, kSOSPeerSendObjectsKey, peer->sendObjects);
 681     SOSPeerPersistOptionalValue(state, kSOSPeerViewsKey, peer->views);
 682
 683     CFDataRef keybag = SOSPeerGetKeyBag(peer);
 684     if (keybag && !CFEqual(peer->peer_id, kSOSViewKeychainV0_tomb))
 685         SOSPeerPersistOptionalValue(state, kSOSPeerKeyBagKey, keybag);
 686
 687     if (!SOSPeerPersistOptionalCoder(peer, state, kSOSPeerCoderKey, error)
 688         || !SOSPeerPersistOptionalManifest(state, kSOSPeerPendingObjectsKey, peer->pendingObjects, error)
 689         || !SOSPeerPersistOptionalManifest(state, kSOSPeerUnwantedManifestKey, peer->unwantedManifest, error)
 690         || !SOSPeerPersistOptionalManifest(state, kSOSPeerConfirmedManifestKey, peer->confirmedManifest, error)
 691         || !SSOSPeerPersistManifestArray(state, kSOSPeerProposedManifestKey, peer->proposedManifests, error)
 692         || !SSOSPeerPersistManifestArray(state, kSOSPeerLocalManifestKey, peer->localManifests, error)) {
 693         CFReleaseNull(state);
 694     }
 695     return state;
 696 }
 697
 698 SOSPeerRef SOSPeerCreateWithState(SOSEngineRef engine, CFStringRef peer_id, CFDictionaryRef state, CFErrorRef *error) {
 699     return SOSPeerCreate_Internal(engine, state, peer_id, 0, error);
 700 }
 701
 702 static void SOSPeerDestroy(CFTypeRef cf) {
 703     SOSPeerRef peer = (SOSPeerRef)cf;
 704     CFReleaseNull(peer->peer_id);
 705     CFReleaseNull(peer->views);
 706     SOSPeerSetCoderData(peer, NULL);
 707     CFReleaseNull(peer->pendingObjects);
 708     CFReleaseNull(peer->unwantedManifest);
 709     CFReleaseNull(peer->confirmedManifest);
 710     CFReleaseNull(peer->proposedManifests);
 711     CFReleaseNull(peer->localManifests);
 712 }
 713
 714 bool SOSPeerDidConnect(SOSPeerRef peer) {
 715     SOSPeerSetMustSendMessage(peer, true);
 716     SOSPeerSetProposedManifest(peer, SOSPeerGetConfirmedManifest(peer));
 717     // TODO: Return false if nothing changed.
 718     return true;
 719 }
 720
 721 // MARK: accessors
 722
 723 CFIndex SOSPeerGetVersion(SOSPeerRef peer) {
 724     return peer->version;
 725 }
 726
 727 CFStringRef SOSPeerGetID(SOSPeerRef peer) {
 728     return peer->peer_id;
 729 }
 730
 731 CFSetRef SOSPeerGetViewNameSet(SOSPeerRef peer) {
 732     return peer->views;
 733 }
 734
 735 void SOSPeerSetViewNameSet(SOSPeerRef peer, CFSetRef views) {
 736     CFRetainAssign(peer->views, views);
 737 }
 738
 739 CFDataRef SOSPeerGetKeyBag(SOSPeerRef peer) {
 740     return peer->_keyBag;
 741 }
 742
 743 static bool SOSPeerUnlinkBackupJournal(SOSPeerRef peer, CFErrorRef *error) {
 744     __block bool ok = true;
 745     ok &= SOSPeerDoWithJournalPath(peer, error, ^(const char *journalName) {
 746         secnotice("backup", "%@ unlinking journal file %s", peer, journalName);
 747         ok &= SecCheckErrno(unlink(journalName), error, CFSTR("unlink %s"), journalName);
 748     });
 749     return ok;
 750 }
 751
 752 static bool SOSPeerWriteReset(SOSPeerRef peer, CFErrorRef *error) {
 753     __block bool ok = true;
 754     __block CFErrorRef localError = NULL;
 755     ok &= SOSPeerTruncateJournal(peer, &localError, ^(FILE *journalFile) {
 756         ok = SOSBackupEventWriteReset(journalFile, peer->_keyBag, &localError);
 757         if (ok && !peer->_keyBag)
 758             ok = SOSBackupEventWriteCompleteMarker(journalFile, 999, &localError);
 759     });
 760     if (!ok) {
 761         secwarning("%@ failed to write reset to backup journal: %@", peer->peer_id, localError);
 762         CFErrorPropagate(localError, error);
 763     }
 764
 765     // Forget we ever wrote anything to the journal.
 766     SOSPeerSetConfirmedManifest(peer, NULL);
 767     SOSPeerSetProposedManifest(peer, NULL);
 768
 769     SOSPeerSetMustSendMessage(peer, !ok);
 770     return ok;
 771 }
 772
 773 void SOSPeerKeyBagDidChange(SOSPeerRef peer) {
 774     // If !keyBag unlink the file, instead of writing a reset.
 775     // CloudServices does not want to hear about empty keybags
 776     SOSPeerSetSendObjects(peer, false);
 777     if (!peer->_keyBag) {
 778         SOSPeerUnlinkBackupJournal(peer, NULL);
 779     } else {
 780         // Attempt to write a reset (ignoring failures since it will
 781         // be pended stickily if it fails).
 782         SOSPeerWriteReset(peer, NULL);
 783         SOSCCSyncWithAllPeers();
 784     }
 785 }
 786
 787 void SOSPeerSetKeyBag(SOSPeerRef peer, CFDataRef keyBag) {
 788     if (CFEqualSafe(keyBag, peer->_keyBag)) return;
 789     bool hadKeybag = peer->_keyBag;
 790     if (!keyBag) {
 791         secwarning("%@ keybag for backup unset", SOSPeerGetID(peer));
 792     } else {
 793         secnotice("backup", "%@ backup bag: %@", SOSPeerGetID(peer), keyBag);
 794     }
 795     CFRetainAssign(peer->_keyBag, keyBag);
 796     // Don't call SOSPeerKeybagDidChange for the inital edge from NULL -> having a keybag.
 797     if (hadKeybag) {
 798         SOSPeerKeyBagDidChange(peer);
 799     }
 800 }
 801
 802 bool SOSPeerWritePendingReset(SOSPeerRef peer, CFErrorRef *error) {
 803     return !SOSPeerMustSendMessage(peer) || SOSPeerWriteReset(peer, error);
 804 }
 805
 806 uint64_t SOSPeerNextSequenceNumber(SOSPeerRef peer) {
 807     return ++peer->sequenceNumber;
 808 }
 809
 810 uint64_t SOSPeerGetMessageVersion(SOSPeerRef peer) {
 811     return SOSPeerGetVersion(peer);
 812 }
 813
 814 bool SOSPeerMustSendMessage(SOSPeerRef peer) {
 815     return peer->mustSendMessage;
 816 }
 817
 818 void SOSPeerSetMustSendMessage(SOSPeerRef peer, bool sendMessage) {
 819     peer->mustSendMessage = sendMessage;
 820 }
 821
 822 bool SOSPeerSendObjects(SOSPeerRef peer) {
 823     return peer->sendObjects;
 824 }
 825
 826 void SOSPeerSetSendObjects(SOSPeerRef peer, bool sendObjects) {
 827     peer->sendObjects = sendObjects;
 828 }
 829
 830 // MARK: Manifests
 831
 832 SOSManifestRef SOSPeerGetProposedManifest(SOSPeerRef peer) {
 833     if (peer->proposedManifests && CFArrayGetCount(peer->proposedManifests))
 834         return (SOSManifestRef)CFArrayGetValueAtIndex(peer->proposedManifests, 0);
 835     return NULL;
 836 }
 837
 838 SOSManifestRef SOSPeerGetConfirmedManifest(SOSPeerRef peer) {
 839     return peer->confirmedManifest;
 840 }
 841
 842 void SOSPeerSetConfirmedManifest(SOSPeerRef peer, SOSManifestRef confirmed) {
 843     CFRetainAssign(peer->confirmedManifest, confirmed);
 844
 845     // TODO: Clear only expired pending and local manifests from the array - this clears them all
 846     // To do so we'd have to track the messageIds we sent to our peer and when we proposed a particular manifest.
 847     // Then we simply remove the entries from messages older than the one we are confirming now
 848     //CFArrayRemoveAllValues(SOSPeerGetDigestsWithKey(peer, kSOSPeerProposedManifestKey));
 849     //CFArrayRemoveAllValues(SOSPeerGetDigestsWithKey(peer, kSOSPeerLocalManifestKey));
 850 }
 851
 852 void SOSPeerAddProposedManifest(SOSPeerRef peer, SOSManifestRef proposed) {
 853     SOSManifestArrayAppendManifest(&peer->proposedManifests, proposed);
 854 }
 855
 856 void SOSPeerSetProposedManifest(SOSPeerRef peer, SOSManifestRef proposed) {
 857     SOSManifestArraySetManifest(&peer->proposedManifests, proposed);
 858 }
 859
 860 void SOSPeerAddLocalManifest(SOSPeerRef peer, SOSManifestRef local) {
 861     SOSManifestArrayAppendManifest(&peer->localManifests, local);
 862 }
 863
 864 SOSManifestRef SOSPeerGetPendingObjects(SOSPeerRef peer) {
 865     return peer->pendingObjects;
 866 }
 867
 868 void SOSPeerSetPendingObjects(SOSPeerRef peer, SOSManifestRef pendingObjects) {
 869     CFRetainAssign(peer->pendingObjects, pendingObjects);
 870 }
 871
 872 SOSManifestRef SOSPeerGetUnwantedManifest(SOSPeerRef peer) {
 873     return peer->unwantedManifest;
 874 }
 875
 876 void SOSPeerSetUnwantedManifest(SOSPeerRef peer, SOSManifestRef unwantedManifest) {
 877     CFRetainAssign(peer->unwantedManifest, unwantedManifest);
 878 }
 879
 880 SOSManifestRef SOSPeerCopyManifestForDigest(SOSPeerRef peer, CFDataRef digest) {
 881     if (!digest) return NULL;
 882     SOSManifestRef manifest;
 883     if (peer->proposedManifests) CFArrayForEachC(peer->proposedManifests, manifest) {
 884         if (CFEqual(digest, SOSManifestGetDigest(manifest, NULL)))
 885             return CFRetainSafe(manifest);
 886     }
 887     if (peer->localManifests) CFArrayForEachC(peer->localManifests, manifest) {
 888         if (CFEqual(digest, SOSManifestGetDigest(manifest, NULL)))
 889             return CFRetainSafe(manifest);
 890     }
 891     if (peer->confirmedManifest && CFEqual(digest, SOSManifestGetDigest(peer->confirmedManifest, NULL)))
 892         return CFRetainSafe(peer->confirmedManifest);
 893
 894     return NULL;
 895 }
 896
 897 static void SOSMarkManifestInUse(struct SOSDigestVector *mdInUse, SOSManifestRef manifest) {
 898     CFDataRef digest = SOSManifestGetDigest(manifest, NULL);
 899     if (digest)
 900         SOSDigestVectorAppend(mdInUse, CFDataGetBytePtr(digest));
 901 }
 902
 903 static void SOSMarkManifestsInUse(struct SOSDigestVector *mdInUse, CFArrayRef manifests) {
 904     if (!isArray(manifests)) return;
 905     SOSManifestRef manifest = NULL;
 906     CFArrayForEachC(manifests, manifest) {
 907         SOSMarkManifestInUse(mdInUse, manifest);
 908     }
 909 }
 910
 911 // Add all digests we are using to mdInUse
 912 void SOSPeerMarkDigestsInUse(SOSPeerRef peer, struct SOSDigestVector *mdInUse) {
 913     SOSMarkManifestInUse(mdInUse, peer->pendingObjects);
 914     SOSMarkManifestInUse(mdInUse, peer->unwantedManifest);
 915     SOSMarkManifestInUse(mdInUse, peer->confirmedManifest);
 916     SOSMarkManifestsInUse(mdInUse, peer->localManifests);
 917     SOSMarkManifestsInUse(mdInUse, peer->proposedManifests);
 918 }
 919
 920 static void SOSAddManifestInUse(CFMutableDictionaryRef mfc, SOSManifestRef manifest) {
 921     CFDataRef digest = SOSManifestGetDigest(manifest, NULL);
 922     CFDataRef data = SOSManifestGetData(manifest);
 923     if (digest && data)
 924         CFDictionarySetValue(mfc, digest, data);
 925 }
 926
 927 static void SOSAddManifestsInUse(CFMutableDictionaryRef mfc, CFArrayRef manifests) {
 928     if (!isArray(manifests)) return;
 929     SOSManifestRef manifest = NULL;
 930     CFArrayForEachC(manifests, manifest) {
 931         SOSAddManifestInUse(mfc, manifest);
 932     }
 933 }
 934
 935 void SOSPeerAddManifestsInUse(SOSPeerRef peer, CFMutableDictionaryRef mfc) {
 936     SOSAddManifestInUse(mfc, peer->pendingObjects);
 937     SOSAddManifestInUse(mfc, peer->unwantedManifest);
 938     SOSAddManifestInUse(mfc, peer->confirmedManifest);
 939     SOSAddManifestsInUse(mfc, peer->localManifests);
 940     SOSAddManifestsInUse(mfc, peer->proposedManifests);
 941
 942 }
 943
 944 // absentFromRemote
 945 // AbsentLocally
 946 // additionsFromRemote
 947 // original intent was that digests only got added to pendingObjects. We only know for sure if it is something added locally via api call
 948
 949
 950 bool SOSPeerDidReceiveRemovalsAndAdditions(SOSPeerRef peer, SOSManifestRef absentFromRemote, SOSManifestRef additionsFromRemote, SOSManifestRef unwantedFromRemote,
 951                                            SOSManifestRef local, CFErrorRef *error) {
 952     // We assume that incoming manifests are all sorted, and absentFromRemote is disjoint from additionsFromRemote
 953     bool ok = true;
 954     SOSManifestRef remoteMissing = NULL, sharedRemovals = NULL, sharedAdditions = NULL;
 955
 956     // TODO: Simplify -- a lot.
 957     ok = ok && (remoteMissing = SOSManifestCreateIntersection(absentFromRemote, local, error));           // remoteMissing = absentFromRemote <Intersected> local
 958     ok = ok && (sharedRemovals = SOSManifestCreateComplement(remoteMissing, absentFromRemote, error));    // sharedRemovals = absentFromRemote - remoteMissing
 959     ok = ok && (sharedAdditions = SOSManifestCreateIntersection(additionsFromRemote, local, error));         // sharedAdditions = additionsFromRemote <Intersected> local
 960     //ok = ok && (remoteAdditions = SOSManifestCreateComplement(sharedAdditions, additionsFromRemote, error)); // remoteAdditions = additionsFromRemote - sharedAdditions
 961
 962     // remoteMissing are things we have that remote has asked for => add to pendingObjects
 963     // sharedRemovals are things we don't have that remote has asked for => remove from pendingDeletes
 964     // sharedAdditions are things we have that remote has too => remove from pendingObjects
 965     // remoteAdditions are things that remote said they have that we don't and we should probably ask for => add to pendingDeletes?
 966     // unwantedFromRemote are things we received from remote for which we already have a newer object according to the conflict resolver.
 967     secnotice("peer", "%@ RM:%@ SR:%@ SA:%@ UR:%@", peer, remoteMissing, sharedRemovals, sharedAdditions, unwantedFromRemote);
 968
 969     SOSManifestRef pendingObjectsManifest = SOSManifestCreateWithPatch(peer->pendingObjects, sharedAdditions, remoteMissing, error);
 970     SOSManifestRef unwantedManifest = SOSManifestCreateWithPatch(peer->unwantedManifest, sharedRemovals,  unwantedFromRemote, error);
 971     CFAssignRetained(peer->pendingObjects, pendingObjectsManifest); // PO = PO - sharedAdditions + remoteMissing
 972     CFAssignRetained(peer->unwantedManifest, unwantedManifest); // U = U - sharedRemovals + unwantedFromRemote
 973
 974     CFReleaseSafe(remoteMissing);
 975     CFReleaseSafe(sharedRemovals);
 976     CFReleaseSafe(sharedAdditions);
 977
 978     secnotice("peer", "%@ C:%@ U:%@ O:%@", peer, SOSPeerGetConfirmedManifest(peer), SOSPeerGetUnwantedManifest(peer), SOSPeerGetPendingObjects(peer));
 979
 980     return ok;
 981 }
 982
 983 // Called for a normal syncing peer.  Only updates pendingObjects currently.
 984 bool SOSPeerDataSourceWillCommit(SOSPeerRef peer, SOSDataSourceTransactionSource source, SOSManifestRef removals, SOSManifestRef additions, CFErrorRef *error) {
 985     bool isAPITransaction = source == kSOSDataSourceAPITransaction;
 986     SOSManifestRef unconfirmedAdditions = NULL;
 987     if (isAPITransaction && SOSManifestGetCount(additions)) {
 988         // Remove confirmed from additions, leaving us with additions to the local db that the remote peer doesn't have yet
 989         unconfirmedAdditions = SOSManifestCreateComplement(SOSPeerGetConfirmedManifest(peer), additions, error);
 990     }
 991
 992     if (SOSManifestGetCount(removals) || SOSManifestGetCount(unconfirmedAdditions)) {
 993         SOSManifestRef pendingObjectsManifest = SOSManifestCreateWithPatch(peer->pendingObjects, removals, unconfirmedAdditions, error);
 994
 995 //#if DEBUG
 996         // TODO: Only do this if debugScope "peer", notice is enabled.
 997         // if (!SecIsScopeActive(kSecLevelNotice, "peer"))
 998         {
 999             // pended == UA unless the db is renotifying of an addition for something we already have
1000             SOSManifestRef unpended = NULL, pended = NULL;
1001             SOSManifestDiff(peer->pendingObjects, pendingObjectsManifest, &unpended, &pended, error);
1002             secinfo("peer", "%@: willCommit R:%@ A:%@ UA:%@ %s O%s%@%s%@",
1003                        SOSPeerGetID(peer), removals, additions, unconfirmedAdditions,
1004                        (isAPITransaction ? "api": "sos"),
1005                        (SOSManifestGetCount(unpended) ? "-" : ""),
1006                        (SOSManifestGetCount(unpended) ? (CFStringRef)unpended : CFSTR("")),
1007                        (SOSManifestGetCount(pended) ? "+" : SOSManifestGetCount(unpended) ? "" : "="),
1008                        (SOSManifestGetCount(pended) ? (CFStringRef)pended : CFSTR("")));
1009             CFReleaseSafe(unpended);
1010             CFReleaseSafe(pended);
1011         }
1012 //#endif /* DEBUG */
1013         CFAssignRetained(peer->pendingObjects, pendingObjectsManifest);
1014     }
1015     CFReleaseSafe(unconfirmedAdditions);
1016
1017     return true;
1018 }
1019
1020 bool SOSPeerWriteAddEvent(FILE *journalFile, keybag_handle_t kbhandle, SOSDataSourceRef dataSource, SOSObjectRef object, CFErrorRef *error) {
1021     CFDictionaryRef backup_item = NULL;
1022     bool ok = ((backup_item = SOSObjectCopyBackup(dataSource, object, kbhandle, error))
1023                && SOSBackupEventWriteAdd(journalFile, backup_item, error));
1024     CFReleaseSafe(backup_item);
1025     return ok;
1026 }
1027
1028 // Called for a backup peer, should stream objects in changes right away
1029 bool SOSPeerDataSourceWillChange(SOSPeerRef peer, SOSDataSourceRef dataSource, SOSDataSourceTransactionSource source, CFArrayRef changes, CFErrorRef *error) {
1030     __block bool ok = true;
1031     ok &= SOSPeerWritePendingReset(peer, error) && SOSPeerAppendToJournal(peer, error, ^(FILE *journalFile, keybag_handle_t kbhandle) {
1032         struct SOSDigestVector dvdel = SOSDigestVectorInit;
1033         struct SOSDigestVector dvadd = SOSDigestVectorInit;
1034         SOSChangeRef change;
1035         CFArrayForEachC(changes, change) {
1036             bool isDelete = false;
1037             CFErrorRef localError = NULL;
1038             CFDataRef digest = NULL;
1039             SOSObjectRef object = NULL;
1040             bool ok = digest = SOSChangeCopyDigest(dataSource, change, &isDelete, &object, &localError);
1041             if (isDelete) {
1042                 ok &= SOSBackupEventWriteDelete(journalFile, digest, &localError);
1043                 SOSDigestVectorAppend(&dvdel, CFDataGetBytePtr(digest));
1044             } else {
1045                 ok &= SOSPeerWriteAddEvent(journalFile, kbhandle, dataSource, object, &localError);
1046                 SOSDigestVectorAppend(&dvadd, CFDataGetBytePtr(digest));
1047             }
1048             if (!ok) {
1049                 secerror("bad change %@: %@", change, localError);
1050             }
1051             CFReleaseSafe(digest);
1052             CFReleaseSafe(localError);
1053         };
1054
1055         if (ok) {
1056             // Update our proposed manifest since we just wrote stuff
1057             struct SOSDigestVector dvresult = SOSDigestVectorInit;
1058             SOSDigestVectorSort(&dvdel);
1059             SOSDigestVectorSort(&dvadd);
1060             if ((ok = SOSDigestVectorPatchSorted(SOSManifestGetDigestVector(SOSPeerGetProposedManifest(peer)), &dvdel,
1061                                                  &dvadd, &dvresult, error))) {
1062                 SOSManifestRef proposed;
1063                 ok = proposed = SOSManifestCreateWithDigestVector(&dvresult, error);
1064                 SOSPeerSetProposedManifest(peer, proposed);
1065                 CFReleaseSafe(proposed);
1066             }
1067             SOSDigestVectorFree(&dvresult);
1068         }
1069         SOSDigestVectorFree(&dvdel);
1070         SOSDigestVectorFree(&dvadd);
1071
1072         // Only Write marker if we are actually in sync now (local == propopsed).
1073         if (SOSPeerSendObjects(peer))
1074             SOSBackupEventWriteCompleteMarker(journalFile, 799, error);
1075     });
1076
1077     if (!ok) {
1078         // We were unable to stream everything out neatly
1079         SOSCCSyncWithAllPeers();
1080     }
1081     return ok;
1082 }