OSX/sec/ProjectHeaders/Security/SecureObjectSync/SOSPeer.c

   1 /*
   2  * Copyright (c) 2012-2014 Apple Inc. All Rights Reserved.
   3  *
   4  * @APPLE_LICENSE_HEADER_START@
   5  *
   6  * This file contains Original Code and/or Modifications of Original Code
   7  * as defined in and that are subject to the Apple Public Source License
   8  * Version 2.0 (the 'License'). You may not use this file except in
   9  * compliance with the License. Please obtain a copy of the License at
  10  * http://www.opensource.apple.com/apsl/ and read it before using this
  11  * file.
  12  *
  13  * The Original Code and all software distributed under the License are
  14  * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  15  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  16  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
  17  * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
  18  * Please see the License for the specific language governing rights and
  19  * limitations under the License.
  20  *
  21  * @APPLE_LICENSE_HEADER_END@
  22  */
  23
  24
  25 /*
  26  * SOSPeer.c -  Implementation of a secure object syncing peer
  27  */
  28 #include <Security/SecureObjectSync/SOSPeer.h>
  29
  30 #include <Security/SecureObjectSync/SOSCoder.h>
  31 #include <Security/SecureObjectSync/SOSDigestVector.h>
  32 #include <Security/SecureObjectSync/SOSInternal.h>
  33 #include <Security/SecureObjectSync/SOSTransport.h>
  34 #include <Security/SecureObjectSync/SOSViews.h>
  35 #include <Security/SecureObjectSync/SOSChangeTracker.h>
  36 #include <utilities/SecCFError.h>
  37 #include <utilities/SecCFRelease.h>
  38 #include <utilities/SecCFWrappers.h>
  39 #include <utilities/SecDb.h>
  40 #include <utilities/SecFileLocations.h>
  41 #include <utilities/SecIOFormat.h>
  42 #include <utilities/array_size.h>
  43 #include <utilities/debugging.h>
  44 #include <Security/SecureObjectSync/SOSBackupEvent.h>
  45 #include <Security/SecItemBackup.h>
  46
  47 #include <securityd/SOSCloudCircleServer.h>
  48
  49 #include <CoreFoundation/CoreFoundation.h>
  50
  51 #include <stdlib.h>
  52
  53 #include <AssertMacros.h>
  54
  55 // Backup Peer Support
  56 #include <securityd/SecKeybagSupport.h>
  57 #include <notify.h>
  58
  59
  60 //
  61 // MARK: - SOSPeerPersistence code
  62 //
  63 static CFStringRef kSOSPeerSequenceNumberKey = CFSTR("sequence-number");
  64 static CFStringRef kSOSPeerGetObjectsKey = CFSTR("get-objects");
  65 static CFStringRef kSOSPeerReceivedUnknownConfirmedDigestKey = CFSTR("received-unknown");
  66 static CFStringRef kSOSPeerJoinRequestedKey = CFSTR("join-requested");
  67 static CFStringRef kSOSPeerSkipHelloKey = CFSTR("skip-hello");
  68
  69 CFStringRef kSOSPeerDataLabel = CFSTR("iCloud Peer Data Meta-data");
  70
  71 //
  72 // MARK: SOSPeerState (dictionary keys)
  73 //
  74
  75 // PeerState dictionary keys
  76 static CFStringRef kSOSPeerSendObjectsKey = CFSTR("send-objects"); // bool
  77 static CFStringRef kSOSPeerMustSendMessageKey = CFSTR("must-send"); // bool
  78 static CFStringRef kSOSPeerPendingObjectsKey = CFSTR("pending-objects"); // digest
  79 static CFStringRef kSOSPeerUnwantedManifestKey = CFSTR("unwanted-manifest"); // digest
  80 static CFStringRef kSOSPeerConfirmedManifestKey = CFSTR("confirmed-manifest");  //digest
  81 static CFStringRef kSOSPeerProposedManifestKey = CFSTR("pending-manifest"); // array of digests
  82 static CFStringRef kSOSPeerLocalManifestKey = CFSTR("local-manifest"); // array of digests
  83 static CFStringRef kSOSPeerVersionKey = CFSTR("vers"); // int
  84 static CFStringRef kSOSPeerCoderKey = CFSTR("coder"); // der encoded SOSCoder
  85
  86 //
  87 // SOSPeerMeta keys that can also be used in peerstate...
  88 //
  89 static CFStringRef kSOSPeerPeerIDKey = CFSTR("peer-id"); // string
  90 static CFStringRef kSOSPeerViewsKey = CFSTR("views"); // set (or array) of string
  91 static CFStringRef kSOSPeerKeyBagKey = CFSTR("keybag"); // data
  92
  93 /*
  94  Theory of syncing for both incoming and outgoing messages
  95
  96  A peerstate consists of:
  97  (T, U, C, P, L, D->M, M->D, H->O, O->{(Op,Or)|Oa,(Op,Or)|(Om,Oa),(Op,Or)|Om,Oi|Oi|Ob|Oa,Ob|(Om,Oa),Ob|(Ol,Ou)|Oa,(Ol,Ou)|(Om,Oa)(Ol,Ou)})
  98     T: to be sent (pendingObjects) manifest
  99     U: unwanted objects manifest
 100     C: confirmed manifest
 101     P: proposed manifest
 102     D->M? digest or manifest to optional manifest function
 103     M->D manifest to digest of manifest function
 104     H->O? hash (manifest entry) to optional object function (the datasource)
 105     O->{                    Mapping from incoming O objects to one of:
 106         (Op,Or):            Op = Peers object, Or is replaced local object.
 107         Oa,(Op,Or):         Oa = appeared local object, Or = Oa, see above for (Op, Or)
 108         (Om,Oa),(Op,Or):    Om missing local object, was apparently Oa instead see above for (Oa, Op, Or)
 109         Om,Oi:              Om missing local object, inserted Oi (nothing replaced), but Om still disapeared from manifest
 110         Oi                  Oi inserted object from peer (nothing replaced)
 111         Ob:                 Ob both remote and local object are identical, nothing changed
 112         Oa,Ob:              Oa = appeared local object equal to Oa = Ob.  Equivalent to single Oi
 113         (Om,Oa),Ob:         Om missing local object, must be O->H->Ob, Oa found in place, Ob != Oa, Oa magically appeared unrelated to O->H->Ob
 114         (Ol,Ou):            Ol local object wins from peers Ou older object => append Ou to U
 115         Oa,(Ol,Ou):         Oa appeared as a local object and Oa = Ol above
 116         (Om,Oa),(Ol,Ou):    Om removed and Oa replaced Om and Oa = Ol as above
 117     }
 118  A message consists of
 119  (B,M,E,O)
 120     B: base digest
 121     M: missing manifest
 122     E: extra manifest
 123     O: objects
 124
 125  To send a message we simply compute:
 126     B: D->M->C
 127     M: C \ L
 128     E: L \ C          # subsetted if not all of O is sent
 129     O: H->O?->O       # as size permits
 130     and change P to (C \ M:) union E: union O->H->O:
 131
 132  To receive a message we compute
 133     Op,Oi,Ob,Oo and Om,Oa,Ol
 134     C: (D->M->B \ M) union E union O->H->O
 135         A = E \ (O->H->O)
 136     T: T \ (Om union Or union A union Oa,Ob.last union (Om,Oa),Ob.last ) union (M intersect L)
 137  */
 138
 139
 140 //
 141 // MARK: SOSPeerMeta
 142 //
 143
 144 static SOSPeerMetaRef SOSPeerMetaCreate(CFStringRef peerID) {
 145     return CFRetain(peerID);
 146 }
 147
 148 static SOSPeerMetaRef SOSPeerMetaCreateWithViews(CFStringRef peerID, CFSetRef views) {
 149     const void *keys[] = { kSOSPeerPeerIDKey, kSOSPeerViewsKey };
 150     const void *values[] = { peerID, views };
 151     return CFDictionaryCreate(kCFAllocatorDefault, keys, values, array_size(keys), &kCFTypeDictionaryKeyCallBacks, &kCFTypeDictionaryValueCallBacks);
 152 }
 153
 154 static SOSPeerMetaRef SOSPeerMetaCreateWithViewsAndKeyBag(CFStringRef peerID, CFSetRef views, CFDataRef keybag) {
 155     const void *keys[] = { kSOSPeerPeerIDKey, kSOSPeerViewsKey, kSOSPeerKeyBagKey };
 156     const void *values[] = { peerID, views, keybag };
 157     return CFDictionaryCreate(kCFAllocatorDefault, keys, values, array_size(keys), &kCFTypeDictionaryKeyCallBacks, &kCFTypeDictionaryValueCallBacks);
 158 }
 159
 160 SOSPeerMetaRef SOSPeerMetaCreateWithComponents(CFStringRef peerID, CFSetRef views, CFDataRef keybag) {
 161     if (!isString(peerID))
 162         return NULL;
 163     if (views) {
 164         if (keybag)
 165             return SOSPeerMetaCreateWithViewsAndKeyBag(peerID, views, keybag);
 166         else
 167             return SOSPeerMetaCreateWithViews(peerID, views);
 168     } else
 169         return SOSPeerMetaCreate(peerID);
 170 }
 171
 172 SOSPeerMetaRef SOSPeerMetaCreateWithState(CFStringRef peerID, CFDictionaryRef state) {
 173     return SOSPeerMetaCreateWithComponents(peerID, CFDictionaryGetValue(state, kSOSPeerViewsKey), CFDictionaryGetValue(state, kSOSPeerKeyBagKey));
 174 }
 175
 176 CFStringRef SOSPeerMetaGetComponents(SOSPeerMetaRef peerMeta, CFSetRef *views, CFDataRef *keybag, CFErrorRef *error) {
 177     if (isDictionary(peerMeta)) {
 178         CFDictionaryRef meta = (CFDictionaryRef)peerMeta;
 179         CFStringRef peerID = asString(CFDictionaryGetValue(meta, kSOSPeerPeerIDKey), error);
 180         CFSetRef vns = asSet(CFDictionaryGetValue(meta, kSOSPeerViewsKey), error);
 181         if (vns && asDataOptional(CFDictionaryGetValue(meta, kSOSPeerKeyBagKey), keybag, error)) {
 182             if (views)
 183                 *views = vns;
 184             return peerID;
 185         }
 186         return NULL;
 187     } else {
 188         if (views) {
 189             // Hack so tests can pass simple peerIDs
 190             *views = SOSViewsGetV0ViewSet();
 191         }
 192         return asString(peerMeta, error);
 193     }
 194 }
 195
 196 CFTypeRef SOSPeerOrStateSetViewsKeyBagAndCreateCopy(CFTypeRef peerOrState, CFSetRef views, CFDataRef keyBag) {
 197     assert(views);
 198     if (peerOrState && CFGetTypeID(peerOrState) == SOSPeerGetTypeID()) {
 199         // Inflated peer, update its views and move on
 200         SOSPeerRef peer = (SOSPeerRef)peerOrState;
 201         SOSPeerSetViewNameSet(peer, views);
 202         SOSPeerSetKeyBag(peer, keyBag);
 203         return CFRetainSafe(peer);
 204     } else if (peerOrState) {
 205         CFMutableDictionaryRef state = (CFMutableDictionaryRef)peerOrState;
 206         // Deserialized peer, just updated the serialized state with the new views
 207         CFDictionarySetValue(state, kSOSPeerViewsKey, views);
 208         if (keyBag)
 209             CFDictionarySetValue(state, kSOSPeerKeyBagKey, keyBag);
 210         else
 211             CFDictionaryRemoveValue(state, kSOSPeerKeyBagKey);
 212         return CFRetainSafe(state);
 213     } else {
 214         // New peer, just create a state object.
 215         if (keyBag)
 216             return CFDictionaryCreateForCFTypes(kCFAllocatorDefault, kSOSPeerViewsKey, views, kSOSPeerKeyBagKey, keyBag, NULL);
 217         else
 218             return CFDictionaryCreateForCFTypes(kCFAllocatorDefault, kSOSPeerViewsKey, views, NULL);
 219     }
 220 }
 221
 222 CFTypeRef SOSPeerOrStateSetViewsAndCopyState(CFTypeRef peerOrState, CFSetRef views) {
 223     assert(views);
 224     if (peerOrState && CFGetTypeID(peerOrState) == SOSPeerGetTypeID()) {
 225         // Inflated peer, update its views and deflate it
 226         SOSPeerRef peer = (SOSPeerRef)peerOrState;
 227         SOSPeerSetViewNameSet(peer, views);
 228         return SOSPeerCopyState(peer, NULL);
 229     } else if (peerOrState) {
 230         // We have a deflated peer.  Update its views and keep it deflated
 231         CFMutableDictionaryRef state = (CFMutableDictionaryRef)peerOrState;
 232         CFDictionarySetValue(state, kSOSPeerViewsKey, views);
 233         return CFRetainSafe(peerOrState);
 234     } else {
 235         return NULL;
 236     }
 237 }
 238
 239 bool SOSPeerMapEntryIsBackup(const void *mapEntry) {
 240     if (!mapEntry) return false;
 241     if (CFGetTypeID(mapEntry) == SOSPeerGetTypeID()) {
 242         return SOSPeerGetKeyBag((SOSPeerRef)mapEntry);
 243     } else {
 244         return CFDictionaryContainsKey(mapEntry, kSOSPeerKeyBagKey);
 245     }
 246 }
 247
 248 //
 249 // MARK: - SOSManifest
 250 //
 251
 252 enum {
 253     kSOSPeerMaxManifestWindowDepth = 4
 254 };
 255
 256 static CFStringRef SOSManifestCreateOptionalDescriptionWithLabel(SOSManifestRef manifest, CFStringRef label) {
 257     if (!manifest) return CFSTR(" -  ");
 258     return CFStringCreateWithFormat(kCFAllocatorDefault, NULL, CFSTR(" %@%@"), label, manifest);
 259 }
 260
 261 static CFStringRef SOSManifestArrayCreateOptionalDescriptionWithLabel(CFArrayRef manifests, CFStringRef label) {
 262     CFIndex count = manifests ? CFArrayGetCount(manifests) : 0;
 263     if (count == 0) return CFSTR(" -  ");
 264     SOSManifestRef manifest = (SOSManifestRef)CFArrayGetValueAtIndex(manifests, 0);
 265     return CFStringCreateWithFormat(kCFAllocatorDefault, NULL, CFSTR(" %@[%" PRIdCFIndex "]%@"), label, count, manifest);
 266 }
 267
 268 static void SOSManifestArraySetManifest(CFMutableArrayRef *manifests, SOSManifestRef manifest) {
 269     if (manifest) {
 270         if (*manifests)
 271             CFArrayRemoveAllValues(*manifests);
 272         else
 273             *manifests = CFArrayCreateMutableForCFTypes(kCFAllocatorDefault);
 274         CFArrayAppendValue(*manifests, manifest);
 275     } else {
 276         CFReleaseNull(*manifests);
 277     }
 278 }
 279
 280 static void SOSManifestMutableArrayAppendManifest(CFMutableArrayRef manifests, SOSManifestRef manifest) {
 281     if (manifest) {
 282         CFIndex count = CFArrayGetCount(manifests);
 283         CFIndex ixOfManifest = CFArrayGetFirstIndexOfValue(manifests, CFRangeMake(0, count), manifest);
 284         if (ixOfManifest != 0) {
 285             // If the manifest isn't at the front of the array move it there.
 286             // If it's not in the array, remove enough entires from the end to
 287             // make room to put it in the front.
 288             if (ixOfManifest != kCFNotFound) {
 289                 CFArrayRemoveValueAtIndex(manifests, ixOfManifest);
 290             } else {
 291                 while (count >= kSOSPeerMaxManifestWindowDepth)
 292                     CFArrayRemoveValueAtIndex(manifests, --count);
 293             }
 294
 295             CFArrayInsertValueAtIndex(manifests, 0, manifest);
 296         }
 297     } else {
 298         // pending == NULL => nothing clear history
 299         CFArrayRemoveAllValues(manifests);
 300     }
 301 }
 302
 303 static void SOSManifestArrayAppendManifest(CFMutableArrayRef *manifests, SOSManifestRef manifest) {
 304     if (*manifests)
 305         SOSManifestMutableArrayAppendManifest(*manifests, manifest);
 306     else
 307         SOSManifestArraySetManifest(manifests, manifest);
 308 }
 309
 310 //
 311 // MARK: - SOSPeer
 312 //
 313
 314 struct __OpaqueSOSPeer {
 315     CFRuntimeBase _base;
 316
 317     SOSCoderRef coder;
 318     CFDataRef coderData;
 319     CFStringRef peer_id;
 320     CFSetRef views;
 321     CFIndex version;
 322     uint64_t sequenceNumber;
 323     bool mustSendMessage;
 324     bool sendObjects;
 325
 326     SOSManifestRef pendingObjects;
 327     SOSManifestRef unwantedManifest;
 328     SOSManifestRef confirmedManifest;
 329     CFMutableArrayRef proposedManifests;
 330     CFMutableArrayRef localManifests;
 331
 332     // Only backup peers have these:
 333     CFDataRef _keyBag;
 334     FILE *journalFile;
 335 };
 336
 337 CFGiblisWithCompareFor(SOSPeer)
 338
 339 static CFStringRef SOSPeerCopyFormatDescription(CFTypeRef cf, CFDictionaryRef formatOptions) {
 340     SOSPeerRef peer = (SOSPeerRef)cf;
 341     if(peer){
 342         CFStringRef po = SOSManifestCreateOptionalDescriptionWithLabel(peer->pendingObjects, CFSTR("O"));
 343         CFStringRef uo = SOSManifestCreateOptionalDescriptionWithLabel(peer->unwantedManifest, CFSTR("U"));
 344         CFStringRef co = SOSManifestCreateOptionalDescriptionWithLabel(peer->confirmedManifest, CFSTR("C"));
 345         CFStringRef pe = SOSManifestArrayCreateOptionalDescriptionWithLabel(peer->proposedManifests, CFSTR("P"));
 346         CFStringRef lo = SOSManifestArrayCreateOptionalDescriptionWithLabel(peer->localManifests, CFSTR("L"));
 347         CFStringRef desc = CFStringCreateWithFormat(kCFAllocatorDefault, NULL, CFSTR("<%@ %s%s%@%@%@%@%@>"),
 348                                                     SOSPeerGetID(peer),
 349                                                     SOSPeerMustSendMessage(peer) ? "F" : "f",
 350                                                     SOSPeerSendObjects(peer) ? "S" : "s",
 351                                                     po, uo, co, pe, lo);
 352         CFReleaseSafe(lo);
 353         CFReleaseSafe(pe);
 354         CFReleaseSafe(co);
 355         CFReleaseSafe(uo);
 356         CFReleaseSafe(po);
 357
 358         return desc;
 359     }
 360     else
 361         return CFSTR("NULL");
 362 }
 363
 364 static Boolean SOSPeerCompare(CFTypeRef cfA, CFTypeRef cfB)
 365 {
 366     SOSPeerRef peerA = (SOSPeerRef)cfA, peerB = (SOSPeerRef)cfB;
 367     // Use mainly to see if peerB is actually this device (peerA)
 368     return CFStringCompare(SOSPeerGetID(peerA), SOSPeerGetID(peerB), 0) == kCFCompareEqualTo;
 369 }
 370
 371 // Coder and coderData caching.
 372 // A Peer has either a coderData or a coder.  Upon serialization the
 373 // coder will be turned into coderData but the coder will stay instantiated
 374 // unless the peer is released.
 375 static void SOSPeerSetCoderData(SOSPeerRef peer, CFDataRef coderData){
 376     if (peer->coder) {
 377         SOSCoderDispose(peer->coder);
 378         peer->coder = NULL;
 379     }
 380     CFRetainAssign(peer->coderData, coderData);
 381 }
 382
 383 static bool SOSPeerCopyCoderData(SOSPeerRef peer, CFDataRef *coderData, CFErrorRef *error) {
 384     // TODO: We can optionally call SOSPeerSetCoderData here to clear the coder whenever its encoded,
 385     // if we assume that coders are written out to disk more often than they are used.
 386     bool ok = true;
 387     if (peer->coder) {
 388 #if 1
 389         CFErrorRef localError = NULL;
 390         ok = *coderData = SOSCoderCopyDER(peer->coder, &localError);
 391         if (!ok) {
 392             secerror("failed to der encode coder for peer %@, dropping it: %@", peer->peer_id, localError);
 393             SOSCoderDispose(peer->coder);
 394             peer->coder = NULL;
 395             CFErrorPropagate(localError, error);
 396         }
 397         return ok;
 398 #else
 399         // Alternate always delete in memory coder after der encoding it.
 400         CFAssignRetained(peer->coderData, SOSCoderCopyDER(peer->coder, error));
 401         ok = peer->coderData;
 402         SOSCoderDispose(peer->coder);
 403         peer->coder = NULL;
 404 #endif
 405     }
 406     *coderData = CFRetainSafe(peer->coderData);
 407     return ok;
 408 }
 409
 410 SOSCoderRef SOSPeerGetCoder(SOSPeerRef peer, CFErrorRef *error) {
 411     if (peer->coderData) {
 412         peer->coder = SOSCoderCreateFromData(peer->coderData, error);
 413         CFReleaseNull(peer->coderData);
 414     } else if (!peer->coder) {
 415         SOSErrorCreate(kSOSErrorPeerNotFound, error, NULL, CFSTR("No coderData nor coder for peer: %@"), peer->peer_id);
 416     }
 417     return peer->coder;
 418 }
 419
 420 bool SOSPeerEnsureCoder(SOSPeerRef peer, SOSFullPeerInfoRef myPeerInfo, SOSPeerInfoRef peerInfo, CFErrorRef *error) {
 421     if (!SOSPeerGetCoder(peer, NULL)) {
 422         secinfo("peer", "New coder for id %@.", peer->peer_id);
 423         CFErrorRef localError = NULL;
 424         if(SOSPeerInfoShouldUseIDSTransport(SOSFullPeerInfoGetPeerInfo(myPeerInfo), peerInfo))
 425             peer->coder = SOSCoderCreate(peerInfo, myPeerInfo, kCFBooleanTrue, &localError);
 426         else
 427             peer->coder = SOSCoderCreate(peerInfo, myPeerInfo, kCFBooleanFalse, &localError);
 428
 429         if (!peer->coder) {
 430             secerror("Failed to create coder for %@: %@", peer->peer_id, localError);
 431             CFErrorPropagate(localError, error);
 432             return false;
 433         }
 434     }
 435     return true;
 436 }
 437
 438 static bool SOSPeerGetPersistedBoolean(CFDictionaryRef persisted, CFStringRef key) {
 439     CFBooleanRef boolean = CFDictionaryGetValue(persisted, key);
 440     return boolean && CFBooleanGetValue(boolean);
 441 }
 442
 443 static CFDataRef SOSPeerGetPersistedData(CFDictionaryRef persisted, CFStringRef key) {
 444     return asData(CFDictionaryGetValue(persisted, key), NULL);
 445 }
 446
 447 static int64_t SOSPeerGetPersistedInt64(CFDictionaryRef persisted, CFStringRef key) {
 448     int64_t integer = 0;
 449     CFNumberRef number = CFDictionaryGetValue(persisted, key);
 450     if (number) {
 451         CFNumberGetValue(number, kCFNumberSInt64Type, &integer);
 452     }
 453     return integer;
 454 }
 455
 456 static void SOSPeerGetOptionalPersistedCFIndex(CFDictionaryRef persisted, CFStringRef key, CFIndex *value) {
 457     CFNumberRef number = CFDictionaryGetValue(persisted, key);
 458     if (number) {
 459         CFNumberGetValue(number, kCFNumberCFIndexType, value);
 460     }
 461 }
 462
 463 static CFSetRef SOSPeerGetPersistedViewNameSet(SOSPeerRef peer, CFDictionaryRef persisted, CFStringRef key) {
 464     CFSetRef vns = CFDictionaryGetValue(persisted, key);
 465     if (!vns) {
 466         // Engine state in db contained a v0 peer, thus it must be in the V0ViewSet.
 467         vns = SOSViewsGetV0ViewSet();
 468         secnotice("peer", "%@ had no views, inferring: %@", peer->peer_id, vns);
 469     }
 470     return vns;
 471 }
 472
 473 //
 474 // MARK: Backup Peers
 475 //
 476
 477 void SOSBackupPeerPostNotification(const char *reason) {
 478     // Let sbd know when a notable event occurs
 479     // - Disk full
 480     // - Backup bag change
 481     secnotice("backup", "posting notification to CloudServices: %s", reason?reason:"");
 482     notify_post(kSecItemBackupNotification);
 483 }
 484
 485 static bool SOSPeerDoWithJournalPath(SOSPeerRef peer, CFErrorRef *error, void(^with)(const char *journalPath)) {
 486     // TODO: Probably switch to using CFURL to construct the path.
 487     bool ok = true;
 488     char strBuffer[PATH_MAX + 1];
 489     size_t userTempLen = confstr(_CS_DARWIN_USER_TEMP_DIR, strBuffer, sizeof(strBuffer));
 490     if (userTempLen == 0) {
 491         ok = SecCheckErrno(-1, error, CFSTR("confstr on _CS_DARWIN_USER_TEMP_DIR returned an error."));
 492     } else {
 493         CFStringRef journalName = CFStringCreateWithFormat(kCFAllocatorDefault, NULL, CFSTR("%s/SOSBackup-%@"), strBuffer, SOSPeerGetID(peer));
 494         CFStringPerformWithCString(journalName, with);
 495         CFReleaseSafe(journalName);
 496     }
 497     return ok;
 498 }
 499
 500 static FILE *fopen_journal(const char *journalPath, const char *mode, CFErrorRef *error) {
 501     FILE *file = fopen(journalPath, mode);
 502     SecCheckErrno(!file, error, CFSTR("fopen %s,%s"), journalPath, mode);
 503     return file;
 504 }
 505
 506 #include <sys/stat.h>
 507
 508 #if !defined(NDEBUG)
 509 static off_t getFileSize(int fd) {
 510     return lseek(fd, 0, SEEK_END);
 511 }
 512 #endif
 513
 514 int SOSPeerHandoffFD(SOSPeerRef peer, CFErrorRef *error) {
 515     __block int fd = -1;
 516     SOSPeerDoWithJournalPath(peer, error, ^(const char *journalName) {
 517         fd = open(journalName, O_RDONLY | O_CLOEXEC);
 518         if (SecCheckErrno(fd < 0, error, CFSTR("open %s"), journalName)) {
 519             if (!SecCheckErrno(unlink(journalName), error, CFSTR("unlink %s"), journalName)) {
 520                 close(fd);
 521                 fd = -1;
 522             } else {
 523                 secdebug("backup", "Handing off file %s with fd %d of size %llu", journalName, fd, getFileSize(fd));
 524             }
 525         } else {
 526             secdebug("backup", "Handing off file %s failed, %@", journalName, error?*error:NULL);
 527         }
 528     });
 529     return fd;
 530 }
 531
 532 static CFDataRef SOSPeerCopyAKSKeyBag(SOSPeerRef peer, CFErrorRef *error) {
 533     if (CFEqual(peer->peer_id, kSOSViewKeychainV0_tomb)) {
 534         return CFRetainSafe(peer->_keyBag);
 535     } else {
 536         CFDataRef aksKeybag = NULL;
 537         SOSBackupSliceKeyBagRef backupSliceKeyBag = SOSBackupSliceKeyBagCreateFromData(kCFAllocatorDefault, peer->_keyBag, error);
 538         if (backupSliceKeyBag) {
 539             aksKeybag = SOSBSKBCopyAKSBag(backupSliceKeyBag, error);
 540             CFRelease(backupSliceKeyBag);
 541         }
 542         return aksKeybag;
 543     }
 544 }
 545
 546 bool SOSPeerAppendToJournal(SOSPeerRef peer, CFErrorRef *error, void(^with)(FILE *journalFile, keybag_handle_t kbhandle)) {
 547     __block bool ok = true;
 548     // We only need a keybag if we are writing ADDs. Since we don't know at this layer
 549     // what operations we may be doing, open keybag if we have one, otherwise don't
 550     ok &= SOSPeerDoWithJournalPath(peer, error, ^(const char *fname) {
 551         FILE *file = fopen_journal(fname, "a", error);
 552         if (file) {
 553             keybag_handle_t kbhandle = bad_keybag_handle;
 554             CFDataRef keybag = SOSPeerCopyAKSKeyBag(peer, error);
 555             ok = keybag;
 556             if (ok && (ok = ks_open_keybag(keybag, NULL, &kbhandle, error))) {
 557                 with(file, kbhandle);
 558                 if (kbhandle != bad_keybag_handle)
 559                     ok &= ks_close_keybag(kbhandle, error);
 560             }
 561             CFReleaseSafe(keybag);
 562             fclose(file);
 563         }
 564     });
 565     return ok;
 566 }
 567
 568 static bool SOSPeerTruncateJournal(SOSPeerRef peer, CFErrorRef *error, void(^with)(FILE *journalFile)) {
 569     __block bool ok = true;
 570     ok &= SOSPeerDoWithJournalPath(peer, error, ^(const char *fname) {
 571         FILE *file = fopen_journal(fname, "w", error);
 572         if (file) {
 573             with(file);
 574             fclose(file);
 575         }
 576     });
 577     return ok;
 578 }
 579
 580 bool SOSPeerSetState(SOSPeerRef p, SOSEngineRef engine, CFDictionaryRef state, CFErrorRef *error) {
 581     bool ok = true;
 582     if (state) {
 583         SOSPeerGetOptionalPersistedCFIndex(state, kSOSPeerVersionKey, &p->version);
 584
 585         p->sequenceNumber = SOSPeerGetPersistedInt64(state, kSOSPeerSequenceNumberKey);
 586         p->mustSendMessage = SOSPeerGetPersistedBoolean(state, kSOSPeerMustSendMessageKey);
 587         p->sendObjects = SOSPeerGetPersistedBoolean(state, kSOSPeerSendObjectsKey);
 588         CFRetainAssign(p->views, SOSPeerGetPersistedViewNameSet(p, state, kSOSPeerViewsKey));
 589         SOSPeerSetKeyBag(p, SOSPeerGetPersistedData(state, kSOSPeerKeyBagKey));
 590         // Don't rollback coder in memory if a transaction is rolled back, since this
 591         // might lead to reuse of an IV.
 592         if (!p->coder)
 593             SOSPeerSetCoderData(p, SOSPeerGetPersistedData(state, kSOSPeerCoderKey));
 594         CFAssignRetained(p->pendingObjects, SOSEngineCopyPersistedManifest(engine, state, kSOSPeerPendingObjectsKey));
 595         CFAssignRetained(p->unwantedManifest, SOSEngineCopyPersistedManifest(engine, state, kSOSPeerUnwantedManifestKey));
 596         CFAssignRetained(p->confirmedManifest, SOSEngineCopyPersistedManifest(engine, state, kSOSPeerConfirmedManifestKey));
 597         CFAssignRetained(p->proposedManifests, SOSEngineCopyPersistedManifestArray(engine, state, kSOSPeerProposedManifestKey, error));
 598         ok &= p->proposedManifests != NULL;
 599         CFAssignRetained(p->localManifests, SOSEngineCopyPersistedManifestArray(engine, state, kSOSPeerLocalManifestKey, error));
 600         ok &= p->localManifests != NULL;
 601     }
 602     return ok;
 603 }
 604
 605 static SOSPeerRef SOSPeerCreate_Internal(SOSEngineRef engine, CFDictionaryRef state, CFStringRef theirPeerID, CFIndex version, CFErrorRef *error) {
 606     SOSPeerRef p = CFTypeAllocate(SOSPeer, struct __OpaqueSOSPeer, kCFAllocatorDefault);
 607     p->peer_id = CFRetainSafe(theirPeerID);
 608     p->version = version;
 609     CFDictionaryRef empty = NULL;
 610     if (!state) {
 611         empty = CFDictionaryCreate(kCFAllocatorDefault, NULL, NULL, 0, &kCFTypeDictionaryKeyCallBacks, &kCFTypeDictionaryValueCallBacks);
 612         state = empty;
 613     }
 614     if (!SOSPeerSetState(p, engine, state, error)) {
 615         CFReleaseNull(p);
 616     }
 617     CFReleaseNull(empty);
 618     return p;
 619 }
 620
 621 static bool SOSPeerPersistOptionalCoder(SOSPeerRef peer, CFMutableDictionaryRef persist, CFStringRef key, CFErrorRef *error) {
 622     CFDataRef coderData = NULL;
 623     bool ok = SOSPeerCopyCoderData(peer, &coderData, error);
 624     if (coderData) {
 625         CFDictionarySetValue(persist, key, coderData);
 626         CFReleaseSafe(coderData);
 627     }
 628     return ok;
 629 }
 630
 631 static void SOSPeerPersistBool(CFMutableDictionaryRef persist, CFStringRef key, bool value) {
 632     CFDictionarySetValue(persist, key, value ? kCFBooleanTrue : kCFBooleanFalse);
 633 }
 634
 635 static void SOSPeerPersistInt64(CFMutableDictionaryRef persist, CFStringRef key, int64_t value) {
 636     CFNumberRef number = CFNumberCreate(kCFAllocatorDefault, kCFNumberSInt64Type, &value);
 637     CFDictionarySetValue(persist, key, number);
 638     CFReleaseSafe(number);
 639 }
 640
 641 static void SOSPeerPersistCFIndex(CFMutableDictionaryRef persist, CFStringRef key, CFIndex value) {
 642     CFNumberRef number = CFNumberCreate(kCFAllocatorDefault, kCFNumberCFIndexType, &value);
 643     CFDictionarySetValue(persist, key, number);
 644     CFReleaseSafe(number);
 645 }
 646
 647 static bool SOSPeerPersistOptionalManifest(CFMutableDictionaryRef persist, CFStringRef key, SOSManifestRef manifest, CFErrorRef *error) {
 648     if (!manifest)
 649         return true;
 650     CFDataRef digest = SOSManifestGetDigest(manifest, error);
 651     bool ok = digest;
 652     if (ok)
 653         CFDictionarySetValue(persist, key, digest);
 654     return ok;
 655 }
 656
 657 static bool SSOSPeerPersistManifestArray(CFMutableDictionaryRef persist, CFStringRef key, CFArrayRef manifests, CFErrorRef *error) {
 658     CFMutableArrayRef digests = CFArrayCreateMutableForCFTypes(kCFAllocatorDefault);
 659     SOSManifestRef manifest;
 660     if (manifests) CFArrayForEachC(manifests, manifest) {
 661         CFDataRef digest = SOSManifestGetDigest(manifest, error);
 662         if (!digest)
 663             CFReleaseNull(digests);
 664         if (digests) {
 665             CFArrayAppendValue(digests, digest);
 666         }
 667     }
 668     if (digests) {
 669         CFDictionarySetValue(persist, key, digests);
 670         CFRelease(digests);
 671     }
 672     return digests;
 673 }
 674
 675 static void SOSPeerPersistOptionalValue(CFMutableDictionaryRef persist, CFStringRef key, CFTypeRef value) {
 676     if (value)
 677         CFDictionarySetValue(persist, key, value);
 678 }
 679
 680 CFDictionaryRef SOSPeerCopyState(SOSPeerRef peer, CFErrorRef *error) {
 681     CFMutableDictionaryRef state = CFDictionaryCreateMutableForCFTypes(kCFAllocatorDefault);
 682     SOSPeerPersistInt64(state, kSOSPeerSequenceNumberKey, peer->sequenceNumber);
 683     if (peer->version) {
 684         SOSPeerPersistCFIndex(state, kSOSPeerVersionKey, peer->version);
 685     }
 686
 687     SOSPeerPersistBool(state, kSOSPeerMustSendMessageKey, peer->mustSendMessage);
 688     SOSPeerPersistBool(state, kSOSPeerSendObjectsKey, peer->sendObjects);
 689     SOSPeerPersistOptionalValue(state, kSOSPeerViewsKey, peer->views);
 690
 691     CFDataRef keybag = SOSPeerGetKeyBag(peer);
 692     if (keybag && !CFEqual(peer->peer_id, kSOSViewKeychainV0_tomb))
 693         SOSPeerPersistOptionalValue(state, kSOSPeerKeyBagKey, keybag);
 694
 695     if (!SOSPeerPersistOptionalCoder(peer, state, kSOSPeerCoderKey, error)
 696         || !SOSPeerPersistOptionalManifest(state, kSOSPeerPendingObjectsKey, peer->pendingObjects, error)
 697         || !SOSPeerPersistOptionalManifest(state, kSOSPeerUnwantedManifestKey, peer->unwantedManifest, error)
 698         || !SOSPeerPersistOptionalManifest(state, kSOSPeerConfirmedManifestKey, peer->confirmedManifest, error)
 699         || !SSOSPeerPersistManifestArray(state, kSOSPeerProposedManifestKey, peer->proposedManifests, error)
 700         || !SSOSPeerPersistManifestArray(state, kSOSPeerLocalManifestKey, peer->localManifests, error)) {
 701         CFReleaseNull(state);
 702     }
 703     return state;
 704 }
 705
 706 SOSPeerRef SOSPeerCreateWithState(SOSEngineRef engine, CFStringRef peer_id, CFDictionaryRef state, CFErrorRef *error) {
 707     return SOSPeerCreate_Internal(engine, state, peer_id, 0, error);
 708 }
 709
 710 static void SOSPeerDestroy(CFTypeRef cf) {
 711     SOSPeerRef peer = (SOSPeerRef)cf;
 712     CFReleaseNull(peer->peer_id);
 713     CFReleaseNull(peer->views);
 714     SOSPeerSetCoderData(peer, NULL);
 715     CFReleaseNull(peer->pendingObjects);
 716     CFReleaseNull(peer->unwantedManifest);
 717     CFReleaseNull(peer->confirmedManifest);
 718     CFReleaseNull(peer->proposedManifests);
 719     CFReleaseNull(peer->localManifests);
 720 }
 721
 722 bool SOSPeerDidConnect(SOSPeerRef peer) {
 723     SOSPeerSetMustSendMessage(peer, true);
 724     SOSPeerSetProposedManifest(peer, SOSPeerGetConfirmedManifest(peer));
 725     // TODO: Return false if nothing changed.
 726     return true;
 727 }
 728
 729 // MARK: accessors
 730
 731 CFIndex SOSPeerGetVersion(SOSPeerRef peer) {
 732     return peer->version;
 733 }
 734
 735 CFStringRef SOSPeerGetID(SOSPeerRef peer) {
 736     return peer->peer_id;
 737 }
 738
 739 CFSetRef SOSPeerGetViewNameSet(SOSPeerRef peer) {
 740     return peer->views;
 741 }
 742
 743 void SOSPeerSetViewNameSet(SOSPeerRef peer, CFSetRef views) {
 744     CFRetainAssign(peer->views, views);
 745 }
 746
 747 CFDataRef SOSPeerGetKeyBag(SOSPeerRef peer) {
 748     return peer->_keyBag;
 749 }
 750
 751 static bool SOSPeerUnlinkBackupJournal(SOSPeerRef peer, CFErrorRef *error) {
 752     __block bool ok = true;
 753     ok &= SOSPeerDoWithJournalPath(peer, error, ^(const char *journalName) {
 754         secnotice("backup", "%@ unlinking journal file %s", peer, journalName);
 755         ok &= SecCheckErrno(unlink(journalName), error, CFSTR("unlink %s"), journalName);
 756     });
 757     return ok;
 758 }
 759
 760 static bool SOSPeerWriteReset(SOSPeerRef peer, CFErrorRef *error) {
 761     __block bool ok = true;
 762     __block CFErrorRef localError = NULL;
 763     ok &= SOSPeerTruncateJournal(peer, &localError, ^(FILE *journalFile) {
 764         ok = SOSBackupEventWriteReset(journalFile, peer->_keyBag, &localError);
 765         if (ok && !peer->_keyBag)
 766             ok = SOSBackupEventWriteCompleteMarker(journalFile, 999, &localError);
 767     });
 768     if (!ok) {
 769         secwarning("%@ failed to write reset to backup journal: %@", peer->peer_id, localError);
 770         CFErrorPropagate(localError, error);
 771     }
 772
 773     // Forget we ever wrote anything to the journal.
 774     SOSPeerSetConfirmedManifest(peer, NULL);
 775     SOSPeerSetProposedManifest(peer, NULL);
 776
 777     SOSPeerSetMustSendMessage(peer, !ok);
 778     return ok;
 779 }
 780
 781 void SOSPeerKeyBagDidChange(SOSPeerRef peer) {
 782     // If !keyBag unlink the file, instead of writing a reset.
 783     // CloudServices does not want to hear about empty keybags
 784     SOSPeerSetSendObjects(peer, false);
 785     if (!peer->_keyBag) {
 786         SOSPeerUnlinkBackupJournal(peer, NULL);
 787     } else {
 788         // Attempt to write a reset (ignoring failures since it will
 789         // be pended stickily if it fails).
 790         SOSPeerWriteReset(peer, NULL);
 791         SOSCCSyncWithAllPeers();
 792     }
 793 }
 794
 795 void SOSPeerSetKeyBag(SOSPeerRef peer, CFDataRef keyBag) {
 796     if (CFEqualSafe(keyBag, peer->_keyBag)) return;
 797     bool hadKeybag = peer->_keyBag;
 798     if (!keyBag) {
 799         secwarning("%@ keybag for backup unset", SOSPeerGetID(peer));
 800     } else {
 801         secnotice("backup", "%@ backup bag: %@", SOSPeerGetID(peer), keyBag);
 802     }
 803     CFRetainAssign(peer->_keyBag, keyBag);
 804     // Don't call SOSPeerKeybagDidChange for the inital edge from NULL -> having a keybag.
 805     if (hadKeybag) {
 806         SOSPeerKeyBagDidChange(peer);
 807     }
 808 }
 809
 810 bool SOSPeerWritePendingReset(SOSPeerRef peer, CFErrorRef *error) {
 811     return !SOSPeerMustSendMessage(peer) || SOSPeerWriteReset(peer, error);
 812 }
 813
 814 uint64_t SOSPeerNextSequenceNumber(SOSPeerRef peer) {
 815     return ++peer->sequenceNumber;
 816 }
 817
 818 uint64_t SOSPeerGetMessageVersion(SOSPeerRef peer) {
 819     return SOSPeerGetVersion(peer);
 820 }
 821
 822 bool SOSPeerMustSendMessage(SOSPeerRef peer) {
 823     return peer->mustSendMessage;
 824 }
 825
 826 void SOSPeerSetMustSendMessage(SOSPeerRef peer, bool sendMessage) {
 827     peer->mustSendMessage = sendMessage;
 828 }
 829
 830 bool SOSPeerSendObjects(SOSPeerRef peer) {
 831     return peer->sendObjects;
 832 }
 833
 834 void SOSPeerSetSendObjects(SOSPeerRef peer, bool sendObjects) {
 835     peer->sendObjects = sendObjects;
 836 }
 837
 838 // MARK: Manifests
 839
 840 SOSManifestRef SOSPeerGetProposedManifest(SOSPeerRef peer) {
 841     if (peer->proposedManifests && CFArrayGetCount(peer->proposedManifests))
 842         return (SOSManifestRef)CFArrayGetValueAtIndex(peer->proposedManifests, 0);
 843     return NULL;
 844 }
 845
 846 SOSManifestRef SOSPeerGetConfirmedManifest(SOSPeerRef peer) {
 847     return peer->confirmedManifest;
 848 }
 849
 850 void SOSPeerSetConfirmedManifest(SOSPeerRef peer, SOSManifestRef confirmed) {
 851     CFRetainAssign(peer->confirmedManifest, confirmed);
 852
 853     // TODO: Clear only expired pending and local manifests from the array - this clears them all
 854     // To do so we'd have to track the messageIds we sent to our peer and when we proposed a particular manifest.
 855     // Then we simply remove the entries from messages older than the one we are confirming now
 856     //CFArrayRemoveAllValues(SOSPeerGetDigestsWithKey(peer, kSOSPeerProposedManifestKey));
 857     //CFArrayRemoveAllValues(SOSPeerGetDigestsWithKey(peer, kSOSPeerLocalManifestKey));
 858 }
 859
 860 void SOSPeerAddProposedManifest(SOSPeerRef peer, SOSManifestRef proposed) {
 861     SOSManifestArrayAppendManifest(&peer->proposedManifests, proposed);
 862 }
 863
 864 void SOSPeerSetProposedManifest(SOSPeerRef peer, SOSManifestRef proposed) {
 865     SOSManifestArraySetManifest(&peer->proposedManifests, proposed);
 866 }
 867
 868 void SOSPeerAddLocalManifest(SOSPeerRef peer, SOSManifestRef local) {
 869     SOSManifestArrayAppendManifest(&peer->localManifests, local);
 870 }
 871
 872 SOSManifestRef SOSPeerGetPendingObjects(SOSPeerRef peer) {
 873     return peer->pendingObjects;
 874 }
 875
 876 void SOSPeerSetPendingObjects(SOSPeerRef peer, SOSManifestRef pendingObjects) {
 877     CFRetainAssign(peer->pendingObjects, pendingObjects);
 878 }
 879
 880 SOSManifestRef SOSPeerGetUnwantedManifest(SOSPeerRef peer) {
 881     return peer->unwantedManifest;
 882 }
 883
 884 void SOSPeerSetUnwantedManifest(SOSPeerRef peer, SOSManifestRef unwantedManifest) {
 885     CFRetainAssign(peer->unwantedManifest, unwantedManifest);
 886 }
 887
 888 SOSManifestRef SOSPeerCopyManifestForDigest(SOSPeerRef peer, CFDataRef digest) {
 889     if (!digest) return NULL;
 890     SOSManifestRef manifest;
 891     if (peer->proposedManifests) CFArrayForEachC(peer->proposedManifests, manifest) {
 892         if (CFEqual(digest, SOSManifestGetDigest(manifest, NULL)))
 893             return CFRetainSafe(manifest);
 894     }
 895     if (peer->localManifests) CFArrayForEachC(peer->localManifests, manifest) {
 896         if (CFEqual(digest, SOSManifestGetDigest(manifest, NULL)))
 897             return CFRetainSafe(manifest);
 898     }
 899     if (peer->confirmedManifest && CFEqual(digest, SOSManifestGetDigest(peer->confirmedManifest, NULL)))
 900         return CFRetainSafe(peer->confirmedManifest);
 901
 902     return NULL;
 903 }
 904
 905 static void SOSMarkManifestInUse(struct SOSDigestVector *mdInUse, SOSManifestRef manifest) {
 906     CFDataRef digest = SOSManifestGetDigest(manifest, NULL);
 907     if (digest)
 908         SOSDigestVectorAppend(mdInUse, CFDataGetBytePtr(digest));
 909 }
 910
 911 static void SOSMarkManifestsInUse(struct SOSDigestVector *mdInUse, CFArrayRef manifests) {
 912     if (!isArray(manifests)) return;
 913     SOSManifestRef manifest = NULL;
 914     CFArrayForEachC(manifests, manifest) {
 915         SOSMarkManifestInUse(mdInUse, manifest);
 916     }
 917 }
 918
 919 // Add all digests we are using to mdInUse
 920 void SOSPeerMarkDigestsInUse(SOSPeerRef peer, struct SOSDigestVector *mdInUse) {
 921     SOSMarkManifestInUse(mdInUse, peer->pendingObjects);
 922     SOSMarkManifestInUse(mdInUse, peer->unwantedManifest);
 923     SOSMarkManifestInUse(mdInUse, peer->confirmedManifest);
 924     SOSMarkManifestsInUse(mdInUse, peer->localManifests);
 925     SOSMarkManifestsInUse(mdInUse, peer->proposedManifests);
 926 }
 927
 928 static void SOSAddManifestInUse(CFMutableDictionaryRef mfc, SOSManifestRef manifest) {
 929     CFDataRef digest = SOSManifestGetDigest(manifest, NULL);
 930     CFDataRef data = SOSManifestGetData(manifest);
 931     if (digest && data)
 932         CFDictionarySetValue(mfc, digest, data);
 933 }
 934
 935 static void SOSAddManifestsInUse(CFMutableDictionaryRef mfc, CFArrayRef manifests) {
 936     if (!isArray(manifests)) return;
 937     SOSManifestRef manifest = NULL;
 938     CFArrayForEachC(manifests, manifest) {
 939         SOSAddManifestInUse(mfc, manifest);
 940     }
 941 }
 942
 943 void SOSPeerAddManifestsInUse(SOSPeerRef peer, CFMutableDictionaryRef mfc) {
 944     SOSAddManifestInUse(mfc, peer->pendingObjects);
 945     SOSAddManifestInUse(mfc, peer->unwantedManifest);
 946     SOSAddManifestInUse(mfc, peer->confirmedManifest);
 947     SOSAddManifestsInUse(mfc, peer->localManifests);
 948     SOSAddManifestsInUse(mfc, peer->proposedManifests);
 949
 950 }
 951
 952 // absentFromRemote
 953 // AbsentLocally
 954 // additionsFromRemote
 955 // original intent was that digests only got added to pendingObjects. We only know for sure if it is something added locally via api call
 956
 957
 958 bool SOSPeerDidReceiveRemovalsAndAdditions(SOSPeerRef peer, SOSManifestRef absentFromRemote, SOSManifestRef additionsFromRemote, SOSManifestRef unwantedFromRemote,
 959                                            SOSManifestRef local, CFErrorRef *error) {
 960     // We assume that incoming manifests are all sorted, and absentFromRemote is disjoint from additionsFromRemote
 961     bool ok = true;
 962     SOSManifestRef remoteMissing = NULL, sharedRemovals = NULL, sharedAdditions = NULL;
 963
 964     // TODO: Simplify -- a lot.
 965     ok = ok && (remoteMissing = SOSManifestCreateIntersection(absentFromRemote, local, error));           // remoteMissing = absentFromRemote <Intersected> local
 966     ok = ok && (sharedRemovals = SOSManifestCreateComplement(remoteMissing, absentFromRemote, error));    // sharedRemovals = absentFromRemote - remoteMissing
 967     ok = ok && (sharedAdditions = SOSManifestCreateIntersection(additionsFromRemote, local, error));         // sharedAdditions = additionsFromRemote <Intersected> local
 968     //ok = ok && (remoteAdditions = SOSManifestCreateComplement(sharedAdditions, additionsFromRemote, error)); // remoteAdditions = additionsFromRemote - sharedAdditions
 969
 970     // remoteMissing are things we have that remote has asked for => add to pendingObjects
 971     // sharedRemovals are things we don't have that remote has asked for => remove from pendingDeletes
 972     // sharedAdditions are things we have that remote has too => remove from pendingObjects
 973     // remoteAdditions are things that remote said they have that we don't and we should probably ask for => add to pendingDeletes?
 974     // unwantedFromRemote are things we received from remote for which we already have a newer object according to the conflict resolver.
 975     secnotice("peer", "%@ RM:%@ SR:%@ SA:%@ UR:%@", peer, remoteMissing, sharedRemovals, sharedAdditions, unwantedFromRemote);
 976
 977     SOSManifestRef pendingObjectsManifest = SOSManifestCreateWithPatch(peer->pendingObjects, sharedAdditions, remoteMissing, error);
 978     SOSManifestRef unwantedManifest = SOSManifestCreateWithPatch(peer->unwantedManifest, sharedRemovals,  unwantedFromRemote, error);
 979     CFAssignRetained(peer->pendingObjects, pendingObjectsManifest); // PO = PO - sharedAdditions + remoteMissing
 980     CFAssignRetained(peer->unwantedManifest, unwantedManifest); // U = U - sharedRemovals + unwantedFromRemote
 981
 982     CFReleaseSafe(remoteMissing);
 983     CFReleaseSafe(sharedRemovals);
 984     CFReleaseSafe(sharedAdditions);
 985
 986     secnotice("peer", "%@ C:%@ U:%@ O:%@", peer, SOSPeerGetConfirmedManifest(peer), SOSPeerGetUnwantedManifest(peer), SOSPeerGetPendingObjects(peer));
 987
 988     return ok;
 989 }
 990
 991 // Called for a normal syncing peer.  Only updates pendingObjects currently.
 992 bool SOSPeerDataSourceWillCommit(SOSPeerRef peer, SOSDataSourceTransactionSource source, SOSManifestRef removals, SOSManifestRef additions, CFErrorRef *error) {
 993     bool isAPITransaction = source == kSOSDataSourceAPITransaction;
 994     SOSManifestRef unconfirmedAdditions = NULL;
 995     if (isAPITransaction && SOSManifestGetCount(additions)) {
 996         // Remove confirmed from additions, leaving us with additions to the local db that the remote peer doesn't have yet
 997         unconfirmedAdditions = SOSManifestCreateComplement(SOSPeerGetConfirmedManifest(peer), additions, error);
 998     }
 999
1000     if (SOSManifestGetCount(removals) || SOSManifestGetCount(unconfirmedAdditions)) {
1001         SOSManifestRef pendingObjectsManifest = SOSManifestCreateWithPatch(peer->pendingObjects, removals, unconfirmedAdditions, error);
1002
1003 //#if DEBUG
1004         // TODO: Only do this if debugScope "peer", notice is enabled.
1005         // if (!SecIsScopeActive(kSecLevelNotice, "peer"))
1006         {
1007             // pended == UA unless the db is renotifying of an addition for something we already have
1008             SOSManifestRef unpended = NULL, pended = NULL;
1009             SOSManifestDiff(peer->pendingObjects, pendingObjectsManifest, &unpended, &pended, error);
1010             secinfo("peer", "%@: willCommit R:%@ A:%@ UA:%@ %s O%s%@%s%@",
1011                        SOSPeerGetID(peer), removals, additions, unconfirmedAdditions,
1012                        (isAPITransaction ? "api": "sos"),
1013                        (SOSManifestGetCount(unpended) ? "-" : ""),
1014                        (SOSManifestGetCount(unpended) ? (CFStringRef)unpended : CFSTR("")),
1015                        (SOSManifestGetCount(pended) ? "+" : SOSManifestGetCount(unpended) ? "" : "="),
1016                        (SOSManifestGetCount(pended) ? (CFStringRef)pended : CFSTR("")));
1017             CFReleaseSafe(unpended);
1018             CFReleaseSafe(pended);
1019         }
1020 //#endif /* DEBUG */
1021         CFAssignRetained(peer->pendingObjects, pendingObjectsManifest);
1022     }
1023     CFReleaseSafe(unconfirmedAdditions);
1024
1025     return true;
1026 }
1027
1028 bool SOSPeerWriteAddEvent(FILE *journalFile, keybag_handle_t kbhandle, SOSDataSourceRef dataSource, SOSObjectRef object, CFErrorRef *error) {
1029     CFDictionaryRef backup_item = NULL;
1030     bool ok = ((backup_item = SOSObjectCopyBackup(dataSource, object, kbhandle, error))
1031                && SOSBackupEventWriteAdd(journalFile, backup_item, error));
1032     CFReleaseSafe(backup_item);
1033     return ok;
1034 }
1035
1036 // Called for a backup peer, should stream objects in changes right away
1037 bool SOSPeerDataSourceWillChange(SOSPeerRef peer, SOSDataSourceRef dataSource, SOSDataSourceTransactionSource source, CFArrayRef changes, CFErrorRef *error) {
1038     __block bool ok = true;
1039     ok &= SOSPeerWritePendingReset(peer, error) && SOSPeerAppendToJournal(peer, error, ^(FILE *journalFile, keybag_handle_t kbhandle) {
1040         struct SOSDigestVector dvdel = SOSDigestVectorInit;
1041         struct SOSDigestVector dvadd = SOSDigestVectorInit;
1042         SOSChangeRef change;
1043         CFArrayForEachC(changes, change) {
1044             bool isDelete = false;
1045             CFErrorRef localError = NULL;
1046             CFDataRef digest = NULL;
1047             SOSObjectRef object = NULL;
1048             bool ok = digest = SOSChangeCopyDigest(dataSource, change, &isDelete, &object, &localError);
1049             if (isDelete) {
1050                 ok &= SOSBackupEventWriteDelete(journalFile, digest, &localError);
1051                 SOSDigestVectorAppend(&dvdel, CFDataGetBytePtr(digest));
1052             } else {
1053                 ok &= SOSPeerWriteAddEvent(journalFile, kbhandle, dataSource, object, &localError);
1054                 SOSDigestVectorAppend(&dvadd, CFDataGetBytePtr(digest));
1055             }
1056             if (!ok) {
1057                 secerror("bad change %@: %@", change, localError);
1058             }
1059             CFReleaseSafe(digest);
1060             CFReleaseSafe(localError);
1061         };
1062
1063         if (ok) {
1064             // Update our proposed manifest since we just wrote stuff
1065             struct SOSDigestVector dvresult = SOSDigestVectorInit;
1066             SOSDigestVectorSort(&dvdel);
1067             SOSDigestVectorSort(&dvadd);
1068             if ((ok = SOSDigestVectorPatchSorted(SOSManifestGetDigestVector(SOSPeerGetProposedManifest(peer)), &dvdel,
1069                                                  &dvadd, &dvresult, error))) {
1070                 SOSManifestRef proposed;
1071                 ok = proposed = SOSManifestCreateWithDigestVector(&dvresult, error);
1072                 SOSPeerSetProposedManifest(peer, proposed);
1073                 CFReleaseSafe(proposed);
1074             }
1075             SOSDigestVectorFree(&dvresult);
1076         }
1077         SOSDigestVectorFree(&dvdel);
1078         SOSDigestVectorFree(&dvadd);
1079
1080         // Only Write marker if we are actually in sync now (local == propopsed).
1081         if (SOSPeerSendObjects(peer))
1082             SOSBackupEventWriteCompleteMarker(journalFile, 799, error);
1083     });
1084
1085     if (!ok) {
1086         // We were unable to stream everything out neatly
1087         SOSCCSyncWithAllPeers();
1088     }
1089     return ok;
1090 }