keychain/securityd/SecItemServer.c

   1 /*
   2  * Copyright (c) 2006-2017 Apple Inc. All Rights Reserved.
   3  *
   4  * @APPLE_LICENSE_HEADER_START@
   5  *
   6  * This file contains Original Code and/or Modifications of Original Code
   7  * as defined in and that are subject to the Apple Public Source License
   8  * Version 2.0 (the 'License'). You may not use this file except in
   9  * compliance with the License. Please obtain a copy of the License at
  10  * http://www.opensource.apple.com/apsl/ and read it before using this
  11  * file.
  12  *
  13  * The Original Code and all software distributed under the License are
  14  * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  15  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  16  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
  17  * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
  18  * Please see the License for the specific language governing rights and
  19  * limitations under the License.
  20  *
  21  * @APPLE_LICENSE_HEADER_END@
  22  */
  23
  24 /*
  25  * SecItemServer.c - CoreFoundation-based constants and functions for
  26     access to Security items (certificates, keys, identities, and
  27     passwords.)
  28  */
  29
  30 #if TARGET_DARWINOS
  31 #undef OCTAGON
  32 #undef SECUREOBJECTSYNC
  33 #undef SHAREDWEBCREDENTIALS
  34 #endif
  35
  36 #include "keychain/securityd/SecItemServer.h"
  37
  38 #include <notify.h>
  39 #include "keychain/securityd/SecItemDataSource.h"
  40 #include "keychain/securityd/SecItemDb.h"
  41 #include "keychain/securityd/SecItemSchema.h"
  42 #include <utilities/SecDb.h>
  43 #include "keychain/securityd/SecDbKeychainItem.h"
  44 #include "keychain/securityd/SOSCloudCircleServer.h"
  45 #include <Security/SecBasePriv.h>
  46 #include <Security/SecItemPriv.h>
  47 #include <Security/SecItemInternal.h>
  48 #include "keychain/securityd/SecDbBackupManager.h"
  49 #import "keychain/SecureObjectSync/SOSChangeTracker.h"
  50 #include "keychain/SecureObjectSync/SOSDigestVector.h"
  51 #include "keychain/SecureObjectSync/SOSEngine.h"
  52 #include <Security/SecureObjectSync/SOSViews.h>
  53 #include <Security/SecTrustPriv.h>
  54 #include <Security/SecTrustInternal.h>
  55 #include <Security/SecCertificatePriv.h>
  56 #include <Security/SecEntitlements.h>
  57 #include <Security/SecSignpost.h>
  58
  59 #include <keychain/ckks/CKKS.h>
  60 #import "keychain/ot/OT.h"
  61 #import "keychain/ot/OTConstants.h"
  62 #import "keychain/escrowrequest/EscrowRequestServerHelpers.h"
  63
  64 #if USE_KEYSTORE
  65
  66 #if __has_include(<MobileKeyBag/MobileKeyBag.h>)
  67 #include <MobileKeyBag/MobileKeyBag.h>
  68 #else
  69 #include "OSX/utilities/SecAKSWrappers.h"
  70 #endif
  71
  72 #if __has_include(<Kernel/IOKit/crypto/AppleKeyStoreDefs.h>)
  73 #include <Kernel/IOKit/crypto/AppleKeyStoreDefs.h>
  74 #endif
  75
  76 #include <IOKit/IOReturn.h>
  77
  78 #endif
  79
  80 #if TARGET_OS_IPHONE && !TARGET_OS_SIMULATOR
  81 #include <utilities/SecADWrapper.h>
  82 #include <sys/time.h>
  83 #include <unistd.h>
  84 #endif
  85
  86 #include <utilities/array_size.h>
  87 #include <utilities/SecFileLocations.h>
  88 #include <utilities/SecTrace.h>
  89 #include <utilities/SecXPCError.h>
  90 #include <utilities/sec_action.h>
  91 #include <Security/SecuritydXPC.h>
  92 #include "swcagent_client.h"
  93 #include "SecPLWrappers.h"
  94
  95 #if TARGET_OS_IOS && !TARGET_OS_BRIDGE
  96 #include <SharedWebCredentials/SharedWebCredentials.h>
  97 #endif
  98
  99 #include <os/variant_private.h>
 100
 101
 102 #include "Analytics/Clients/LocalKeychainAnalytics.h"
 103
 104 /* Changed the name of the keychain changed notification, for testing */
 105 static const char *g_keychain_changed_notification = kSecServerKeychainChangedNotification;
 106
 107 void SecItemServerSetKeychainChangedNotification(const char *notification_name)
 108 {
 109     g_keychain_changed_notification = notification_name;
 110 }
 111
 112 void SecKeychainChanged() {
 113     static dispatch_once_t once;
 114     static sec_action_t action;
 115
 116     dispatch_once(&once, ^{
 117         action = sec_action_create("SecKeychainChanged", 1);
 118         sec_action_set_handler(action, ^{
 119             uint32_t result = notify_post(g_keychain_changed_notification);
 120             if (result == NOTIFY_STATUS_OK)
 121                 secnotice("item", "Sent %s", g_keychain_changed_notification);
 122             else
 123                 secerror("notify_post %s returned: %" PRIu32, g_keychain_changed_notification, result);
 124         });
 125     });
 126
 127     sec_action_perform(action);
 128 }
 129
 130 /* Return the current database version in *version. */
 131 bool SecKeychainDbGetVersion(SecDbConnectionRef dbt, int *version, CFErrorRef *error)
 132 {
 133     __block bool ok = true;
 134     __block CFErrorRef localError = NULL;
 135     __block bool found = false;
 136
 137     /*
 138      * First check for the version table itself
 139      */
 140
 141     ok &= SecDbPrepare(dbt, CFSTR("SELECT name FROM sqlite_master WHERE type='table' AND name='tversion'"), &localError, ^(sqlite3_stmt *stmt) {
 142         ok = SecDbStep(dbt, stmt, NULL, ^(bool *stop) {
 143             found = true;
 144             *stop = true;
 145         });
 146     });
 147     require_action(ok, out, SecDbError(SQLITE_CORRUPT, error, CFSTR("Failed to read sqlite_master table: %@"), localError));
 148     if (!found) {
 149         secnotice("upgr", "no tversion table, will setup a new database: %@", localError);
 150         *version = 0;
 151         goto out;
 152     }
 153
 154     /*
 155      * Now build up major.minor
 156      */
 157
 158     ok &= SecDbPrepare(dbt, CFSTR("SELECT version FROM tversion"), &localError, ^(sqlite3_stmt *stmt) {
 159         ok = SecDbStep(dbt, stmt, NULL, ^(bool *stop) {
 160             *version = sqlite3_column_int(stmt, 0);
 161             if (*version)
 162                 *stop = true;
 163         });
 164     });
 165     if (ok && (*version & 0xffff) >= 9) {
 166         ok &= SecDbPrepare(dbt, CFSTR("SELECT minor FROM tversion WHERE version = ?"), &localError, ^(sqlite3_stmt *stmt) {
 167             ok = SecDbBindInt(stmt, 1, *version, &localError) &&
 168             SecDbStep(dbt, stmt, NULL, ^(bool *stop) {
 169                 int64_t minor = sqlite3_column_int64(stmt, 0);
 170                 *version |= ((minor & 0xff) << 8) | ((minor & 0xff0000) << 8);
 171                 *stop = true;
 172             });
 173         });
 174         ok = true;
 175     }
 176 out:
 177     secnotice("upgr", "database version is: 0x%08x : %d : %@", *version, ok, localError);
 178     CFReleaseSafe(localError);
 179
 180
 181     return ok;
 182 }
 183
 184 static bool
 185 isClassD(SecDbItemRef item)
 186 {
 187     CFTypeRef accessible = SecDbItemGetCachedValueWithName(item, kSecAttrAccessible);
 188
 189     if (CFEqualSafe(accessible, kSecAttrAccessibleAlwaysPrivate) || CFEqualSafe(accessible, kSecAttrAccessibleAlwaysThisDeviceOnlyPrivate))
 190         return true;
 191     return false;
 192 }
 193
 194 #if TARGET_OS_IPHONE && !TARGET_OS_SIMULATOR
 195
 196 static int64_t
 197 measureDuration(struct timeval *start)
 198 {
 199     struct timeval stop;
 200     int64_t duration;
 201
 202     gettimeofday(&stop, NULL);
 203
 204     duration = (stop.tv_sec-start->tv_sec) * 1000;
 205     duration += (stop.tv_usec / 1000) - (start->tv_usec / 1000);
 206
 207     return SecBucket2Significant(duration);
 208 }
 209
 210 static void
 211 measureUpgradePhase1(struct timeval *start, bool success, int64_t itemsMigrated)
 212 {
 213     int64_t duration = measureDuration(start);
 214
 215     if (success) {
 216         SecADSetValueForScalarKey(CFSTR("com.apple.keychain.phase1.migrated-items-success"), itemsMigrated);
 217         SecADSetValueForScalarKey(CFSTR("com.apple.keychain.phase1.migrated-time-success"), duration);
 218     } else {
 219         SecADSetValueForScalarKey(CFSTR("com.apple.keychain.phase1.migrated-items-fail"), itemsMigrated);
 220         SecADSetValueForScalarKey(CFSTR("com.apple.keychain.phase1.migrated-time-fail"), duration);
 221     }
 222 }
 223
 224 static void
 225 measureUpgradePhase2(struct timeval *start, int64_t itemsMigrated)
 226 {
 227     int64_t duration = measureDuration(start);
 228
 229     SecADSetValueForScalarKey(CFSTR("com.apple.keychain.phase2.migrated-items"), itemsMigrated);
 230     SecADSetValueForScalarKey(CFSTR("com.apple.keychain.phase2.migrated-time"), duration);
 231 }
 232 #endif /* TARGET_OS_IPHONE && !TARGET_OS_SIMULATOR */
 233
 234 static bool DBClassesAreEqual(const SecDbClass* class1, const SecDbClass* class2)
 235 {
 236     if (CFEqual(class1->name, class2->name) && class1->itemclass == class2->itemclass) {
 237         int attrIndex = 0;
 238         const SecDbAttr* class1Attr = class1->attrs[attrIndex];
 239         const SecDbAttr* class2Attr = class2->attrs[attrIndex];
 240
 241         while (class1Attr && class2Attr) {
 242             if (CFEqual(class1Attr->name, class2Attr->name) && class1Attr->kind == class2Attr->kind && class1Attr->flags == class2Attr->flags && class1Attr->copyValue == class2Attr->copyValue && class1Attr->setValue == class2Attr->setValue) {
 243                 attrIndex++;
 244                 class1Attr = class1->attrs[attrIndex];
 245                 class2Attr = class2->attrs[attrIndex];
 246             }
 247             else {
 248                 return false;
 249             }
 250         }
 251
 252         // if everything has checked out to this point, and we've hit the end of both class's attr list, then they're equal
 253         if (class1Attr == NULL && class2Attr == NULL) {
 254             return true;
 255         }
 256     }
 257
 258     return false;
 259 }
 260
 261 static bool ShouldRenameTable(const SecDbClass* class1, const SecDbClass* class2, int oldTableVersion)
 262 {
 263     return oldTableVersion < 10 || !DBClassesAreEqual(class1, class2);
 264 }
 265
 266 #define SCHEMA_VERSION(schema) ((((schema)->minorVersion) << 8) | ((schema)->majorVersion))
 267 #define VERSION_MAJOR(version)  ((version)        & 0xff)
 268 #define VERSION_MINOR(version) (((version) >>  8) & 0xff)
 269 #define VERSION_NEW(version)    ((version)        & 0xffff)
 270 #define VERSION_OLD(version)   (((version) >> 16) & 0xffff)
 271
 272 // Goes through all tables represented by old_schema and tries to migrate all items from them into new (current version) tables.
 273 static bool UpgradeSchemaPhase1(SecDbConnectionRef dbt, const SecDbSchema *oldSchema, CFErrorRef *error)
 274 {
 275     int oldVersion = SCHEMA_VERSION(oldSchema);
 276     const SecDbSchema *newSchema = current_schema();
 277     int newVersion = SCHEMA_VERSION(newSchema);
 278     __block bool ok = true;
 279     SecDbClass const *const *oldClass;
 280     SecDbClass const *const *newClass;
 281     SecDbQueryRef query = NULL;
 282     CFMutableStringRef sql = NULL;
 283     SecDbClass* renamedOldClass = NULL;
 284 #if TARGET_OS_IPHONE && !TARGET_OS_SIMULATOR
 285     __block int64_t itemsMigrated = 0;
 286     struct timeval start;
 287
 288     gettimeofday(&start, NULL);
 289 #endif
 290
 291     // Rename existing tables to names derived from old schema names
 292     sql = CFStringCreateMutable(NULL, 0);
 293     bool oldClassDone = false;
 294     CFMutableArrayRef classIndexesForNewTables = CFArrayCreateMutable(NULL, 0, NULL);
 295     int classIndex = 0;
 296     for (oldClass = oldSchema->classes, newClass = newSchema->classes;
 297          *newClass != NULL; classIndex++, oldClass++, newClass++) {
 298         if (!oldClassDone) {
 299             oldClassDone |= (*oldClass) == NULL; // Check if the new schema has more tables than the old
 300         }
 301
 302         if (!oldClassDone && !CFEqual((*oldClass)->name, (*newClass)->name) && ShouldRenameTable(*oldClass, *newClass, oldSchema->majorVersion)) {
 303             CFStringAppendFormat(sql, NULL, CFSTR("ALTER TABLE %@ RENAME TO %@_old;"), (*newClass)->name, (*oldClass)->name);
 304             CFArrayAppendValue(classIndexesForNewTables, (void*)(long)classIndex);
 305
 306         } else if (!oldClassDone && !DBClassesAreEqual(*oldClass, *newClass)) {
 307             CFStringAppendFormat(sql, NULL, CFSTR("ALTER TABLE %@ RENAME TO %@_old;"), (*newClass)->name, (*oldClass)->name);
 308             CFArrayAppendValue(classIndexesForNewTables, (void*)(long)classIndex);
 309         }
 310
 311         if(oldClassDone && *newClass) {
 312             // These should be no-ops, unless you're upgrading a previously-upgraded database with an invalid version number
 313             CFStringAppendFormat(sql, NULL, CFSTR("DROP TABLE IF EXISTS %@;"), (*newClass)->name);
 314
 315             if (classIndexesForNewTables) {
 316                 CFArrayAppendValue(classIndexesForNewTables, (void*)(long)classIndex);
 317             }
 318         }
 319     }
 320
 321     if(CFStringGetLength(sql) > 0) {
 322         require_action_quiet(ok &= SecDbExec(dbt, sql, error), out,
 323                              LKAReportKeychainUpgradeOutcomeWithError(oldVersion, newVersion, LKAKeychainUpgradeOutcomePhase1AlterTables, error ? *error : NULL));
 324     }
 325     CFReleaseNull(sql);
 326
 327     // Drop indices that that new schemas will use
 328     sql = CFStringCreateMutable(NULL, 0);
 329     for (newClass = newSchema->classes; *newClass != NULL; newClass++) {
 330         SecDbForEachAttrWithMask((*newClass), desc, kSecDbIndexFlag | kSecDbInFlag) {
 331             CFStringAppendFormat(sql, 0, CFSTR("DROP INDEX IF EXISTS %@%@;"), (*newClass)->name, desc->name);
 332             if (desc->kind == kSecDbSyncAttr) {
 333                 CFStringAppendFormat(sql, 0, CFSTR("DROP INDEX IF EXISTS %@%@0;"), (*newClass)->name, desc->name);
 334             }
 335         }
 336     }
 337     require_action_quiet(ok &= SecDbExec(dbt, sql, error), out,
 338                          LKAReportKeychainUpgradeOutcomeWithError(oldVersion, newVersion, LKAKeychainUpgradeOutcomePhase1DropIndices, error ? *error : NULL));
 339     CFReleaseNull(sql);
 340
 341     // Create tables for new schema.
 342     require_action_quiet(ok &= SecItemDbCreateSchema(dbt, newSchema, classIndexesForNewTables, false, error), out,
 343                          LKAReportKeychainUpgradeOutcomeWithError(oldVersion, newVersion, LKAKeychainUpgradeOutcomePhase1CreateSchema, error ? *error : NULL));
 344     // Go through all classes of current schema to transfer all items to new tables.
 345     for (oldClass = oldSchema->classes, newClass = newSchema->classes;
 346          *oldClass != NULL && *newClass != NULL; oldClass++, newClass++) {
 347
 348         if (CFEqual((*oldClass)->name, (*newClass)->name) && DBClassesAreEqual(*oldClass, *newClass)) {
 349             continue;
 350         }
 351
 352         secnotice("upgr", "Upgrading table %@", (*oldClass)->name);
 353
 354         // Create a new 'old' class with a new 'old' name.
 355         int count = 0;
 356         SecDbForEachAttr(*oldClass, attr) {
 357             count++;
 358         }
 359         if(renamedOldClass) {
 360             CFReleaseNull(renamedOldClass->name);
 361             free(renamedOldClass);
 362         }
 363         renamedOldClass = (SecDbClass*) malloc(sizeof(SecDbClass) + sizeof(SecDbAttr*)*(count+1));
 364         renamedOldClass->name = CFStringCreateWithFormat(NULL, NULL, CFSTR("%@_old"), (*oldClass)->name);
 365         renamedOldClass->itemclass = (*oldClass)->itemclass;
 366         for(; count >= 0; count--) {
 367             renamedOldClass->attrs[count] = (*oldClass)->attrs[count];
 368         }
 369
 370         // SecDbItemSelect only works for item classes.
 371         if((*oldClass)->itemclass) {
 372             // Prepare query to iterate through all items in cur_class.
 373             if (query != NULL)
 374                 query_destroy(query, NULL);
 375             require_quiet(query = query_create(renamedOldClass, SecMUSRGetAllViews(), NULL, error), out);
 376
 377             ok &= SecDbItemSelect(query, dbt, error, ^bool(const SecDbAttr *attr) {
 378                 // We are interested in all attributes which are physically present in the DB.
 379                 return (attr->flags & kSecDbInFlag) != 0;
 380             }, ^bool(const SecDbAttr *attr) {
 381                 // No filtering please.
 382                 return false;
 383             }, NULL, NULL, ^(SecDbItemRef item, bool *stop) {
 384                 CFErrorRef localError = NULL;
 385
 386     #if TARGET_OS_IPHONE && !TARGET_OS_SIMULATOR
 387                 itemsMigrated++;
 388     #endif
 389                 // Switch item to the new class.
 390                 item->class = *newClass;
 391
 392                 if (isClassD(item)) {
 393                     // Decrypt the item.
 394                     ok &= SecDbItemEnsureDecrypted(item, true, &localError);
 395                     require_quiet(ok, out);
 396
 397                     // Delete SHA1 field from the item, so that it is newly recalculated before storing
 398                     // the item into the new table.
 399                     require_quiet(ok &= SecDbItemSetValue(item, SecDbClassAttrWithKind(item->class, kSecDbSHA1Attr, error),
 400                                                               kCFNull, error), out);
 401                 } else {
 402                     // Leave item encrypted, do not ever try to decrypt it since it will fail.
 403                     item->_edataState = kSecDbItemAlwaysEncrypted;
 404                 }
 405                 // Drop items with kSecAttrAccessGroupToken, as these items should not be there at all. Since agrp attribute
 406                 // is always stored as cleartext in the DB column, we can always rely on this attribute being present in item->attributes.
 407                 // <rdar://problem/33401870>
 408                 if (CFEqualSafe(SecDbItemGetCachedValueWithName(item, kSecAttrAccessGroup), kSecAttrAccessGroupToken) &&
 409                     SecDbItemGetCachedValueWithName(item, kSecAttrTokenID) == NULL) {
 410                     secnotice("upgr", "dropping item during schema upgrade due to agrp=com.apple.token: %@", item);
 411                 } else {
 412                     // Insert new item into the new table.
 413                     if (!SecDbItemInsert(item, dbt, &localError)) {
 414                         secerror("item: %@ insert during upgrade: %@", item, localError);
 415                         ok = false;
 416                     }
 417                 }
 418
 419             out:
 420                 if (localError) {
 421                     OSStatus status = SecErrorGetOSStatus(localError);
 422
 423                     switch (status) {
 424                         // continue to upgrade and don't propagate errors for insert failures
 425                         // that are typical of a single item failure
 426                         case errSecDecode:
 427                         case errSecDuplicateItem:
 428                             ok = true;
 429                             break;
 430                         case errSecInteractionNotAllowed:
 431                         case errSecAuthNeeded:
 432                             ok = true;
 433                             break;
 434                         // This does not mean the keychain is hosed, we just can't use it right now
 435 #if USE_KEYSTORE
 436                         case kAKSReturnNotReady:
 437                         case kAKSReturnTimeout:
 438 #endif
 439                         case errSecNotAvailable:
 440                             secnotice("upgr", "Bailing in phase 1 because AKS is unavailable: %@", localError);
 441                             // FALLTHROUGH
 442                         default:
 443                             ok &= CFErrorPropagate(CFRetainSafe(localError), error);
 444                             break;
 445                     }
 446                     CFReleaseSafe(localError);
 447                 }
 448
 449                 *stop = !ok;
 450
 451             });
 452
 453             require_action_quiet(ok, out,
 454                                  LKAReportKeychainUpgradeOutcomeWithError(oldVersion, newVersion, LKAKeychainUpgradeOutcomePhase1Items, error ? *error : NULL));
 455         } else {
 456             // This table does not contain secdb items, and must be transferred without using SecDbItemSelect.
 457             // For now, this code does not support removing or renaming any columns, or adding any new non-null columns.
 458             CFReleaseNull(sql);
 459             sql = CFStringCreateMutable(NULL, 0);
 460             bool comma = false;
 461
 462             CFMutableStringRef columns = CFStringCreateMutable(NULL, 0);
 463
 464             SecDbForEachAttr(renamedOldClass, attr) {
 465                 if(comma) {
 466                     CFStringAppendFormat(columns, NULL, CFSTR(","));
 467                 }
 468                 CFStringAppendFormat(columns, NULL, CFSTR("%@"), attr->name);
 469                 comma = true;
 470             }
 471
 472             CFStringAppendFormat(sql, NULL, CFSTR("INSERT OR REPLACE INTO %@ (%@) SELECT %@ FROM %@;"), (*newClass)->name, columns, columns, renamedOldClass->name);
 473
 474             CFReleaseNull(columns);
 475             require_action_quiet(ok &= SecDbExec(dbt, sql, error), out,
 476                                  LKAReportKeychainUpgradeOutcomeWithError(oldVersion, newVersion, LKAKeychainUpgradeOutcomePhase1NonItems, error ? *error : NULL));
 477         }
 478     }
 479
 480     // Remove old tables from the DB.
 481     CFReleaseNull(sql);
 482     sql = CFStringCreateMutable(NULL, 0);
 483     for (oldClass = oldSchema->classes, newClass = newSchema->classes;
 484          *oldClass != NULL && *newClass != NULL; oldClass++, newClass++) {
 485         if (CFEqual((*oldClass)->name, (*newClass)->name) && DBClassesAreEqual(*oldClass, *newClass)) {
 486             continue;
 487         }
 488
 489         CFStringAppendFormat(sql, NULL, CFSTR("DROP TABLE %@_old;"), (*oldClass)->name);
 490     }
 491
 492     if(CFStringGetLength(sql) > 0) {
 493         require_action_quiet(ok &= SecDbExec(dbt, sql, error), out,
 494                              LKAReportKeychainUpgradeOutcomeWithError(oldVersion, newVersion, LKAKeychainUpgradeOutcomePhase1DropOld, error ? *error : NULL));
 495     }
 496 out:
 497 #if TARGET_OS_IPHONE && !TARGET_OS_SIMULATOR
 498     measureUpgradePhase1(&start, ok, SecBucket2Significant(itemsMigrated));
 499 #endif
 500
 501     if (query != NULL) {
 502         query_destroy(query, NULL);
 503     }
 504     CFReleaseSafe(sql);
 505     CFReleaseNull(classIndexesForNewTables);
 506     if(renamedOldClass) {
 507         CFReleaseNull(renamedOldClass->name);
 508         free(renamedOldClass);
 509     }
 510     return ok;
 511 }
 512
 513 __thread SecDbConnectionRef threadDbt = NULL;
 514
 515 // Goes through all tables represented by old_schema and tries to migrate all items from them into new (current version) tables.
 516 static bool UpgradeItemPhase2(SecDbConnectionRef inDbt, bool *inProgress, int oldVersion, CFErrorRef *error) {
 517     SecDbConnectionRef oldDbt = threadDbt;
 518     threadDbt = inDbt;
 519     __block bool ok = true;
 520     SecDbQueryRef query = NULL;
 521 #if TARGET_OS_IPHONE && !TARGET_OS_SIMULATOR
 522     __block int64_t itemsMigrated = 0;
 523     struct timeval start;
 524
 525     gettimeofday(&start, NULL);
 526 #endif
 527
 528 #if SECDB_BACKUPS_ENABLED
 529     CFErrorRef backuperror = NULL;
 530     if (!SecDbBackupCreateOrLoadBackupInfrastructure(&backuperror)) {
 531         secerror("upgr: failed to create backup infrastructure: %@", backuperror);
 532     } else {
 533         secnotice("upgr", "setup backup infrastructure");
 534     }
 535 #else
 536     secnotice("upgr", "skipping backup setup for this platform");
 537 #endif
 538
 539     // Go through all classes in new schema
 540     const SecDbSchema *newSchema = current_schema();
 541     int newVersion = SCHEMA_VERSION(newSchema);
 542     for (const SecDbClass *const *class = newSchema->classes; *class != NULL && !*inProgress; class++) {
 543         if(!((*class)->itemclass)) {
 544             //Don't try to decrypt non-item 'classes'
 545             continue;
 546         }
 547
 548         const SecDbAttr *pdmn = SecDbClassAttrWithKind(*class, kSecDbAccessAttr, error);
 549         if (pdmn == nil) {
 550             continue;
 551         }
 552
 553         // Prepare query to go through all non-DK|DKU items
 554         if (query != NULL) {
 555             query_destroy(query, NULL);
 556         }
 557         require_action_quiet(query = query_create(*class, SecMUSRGetAllViews(), NULL, error), out, ok = false);
 558         ok &= SecDbItemSelect(query, threadDbt, error, NULL, ^bool(const SecDbAttr *attr) {
 559             // No simple per-attribute filtering.
 560             return false;
 561         }, ^bool(CFMutableStringRef sql, bool *needWhere) {
 562             // Select only non-D-class items
 563             SecDbAppendWhereOrAnd(sql, needWhere);
 564             CFStringAppendFormat(sql, NULL, CFSTR("NOT %@ IN (?,?)"), pdmn->name);
 565             return true;
 566         }, ^bool(sqlite3_stmt *stmt, int col) {
 567             return SecDbBindObject(stmt, col++, kSecAttrAccessibleAlwaysPrivate, error) &&
 568             SecDbBindObject(stmt, col++, kSecAttrAccessibleAlwaysThisDeviceOnlyPrivate, error);
 569         }, ^(SecDbItemRef item, bool *stop) {
 570             CFErrorRef localError = NULL;
 571
 572 #if TARGET_OS_IPHONE && !TARGET_OS_SIMULATOR
 573             itemsMigrated++;
 574 #endif
 575
 576             // Decrypt the item.
 577             if (SecDbItemEnsureDecrypted(item, true, &localError)) {
 578
 579                 // Delete SHA1 field from the item, so that it is newly recalculated before storing
 580                 // the item into the new table.
 581                 require_quiet(ok = SecDbItemSetValue(item, SecDbClassAttrWithKind(item->class, kSecDbSHA1Attr, error),
 582                                                      kCFNull, &localError), out);
 583                 // Drop items with kSecAttrAccessGroupToken, as these items should not be there at all. Since agrp attribute
 584                 // is always stored as cleartext in the DB column, we can always rely on this attribute being present in item->attributes.
 585                 // <rdar://problem/33401870>
 586                 if (CFEqualSafe(SecDbItemGetCachedValueWithName(item, kSecAttrAccessGroup), kSecAttrAccessGroupToken) &&
 587                     SecDbItemGetCachedValueWithName(item, kSecAttrTokenID) == NULL) {
 588                     secnotice("upgr", "dropping item during item upgrade due to agrp=com.apple.token: %@", item);
 589                     ok = SecDbItemDelete(item, threadDbt, kCFBooleanFalse, &localError);
 590                 } else {
 591                     // Replace item with the new value in the table; this will cause the item to be decoded and recoded back,
 592                     // incl. recalculation of item's hash.
 593                     ok = SecDbItemUpdate(item, item, threadDbt, false, query->q_uuid_from_primary_key, &localError);
 594                 }
 595             }
 596
 597             if (localError) {
 598                 CFIndex status = CFErrorGetCode(localError);
 599
 600                 switch (status) {
 601                     case errSecDecode: {
 602                         // Items producing errSecDecode are silently dropped - they are not decodable and lost forever.
 603                         // make sure we use a local error so that this error is not proppaged upward and cause a
 604                         // migration failure.
 605                         CFErrorRef deleteError = NULL;
 606                         (void)SecDbItemDelete(item, threadDbt, false, &deleteError);
 607                         CFReleaseNull(deleteError);
 608                         ok = true;
 609                         break;
 610                     }
 611                     case errSecInteractionNotAllowed:
 612                         // If we are still not able to decrypt the item because the class key is not released yet,
 613                         // remember that DB still needs phase2 migration to be run next time a connection is made.  Also
 614                         // stop iterating next items, it would be just waste of time because the whole iteration will be run
 615                         // next time when this phase2 will be rerun.
 616                         LKAReportKeychainUpgradeOutcome(oldVersion, newVersion, LKAKeychainUpgradeOutcomeLocked);
 617                         *inProgress = true;
 618                         *stop = true;
 619                         ok = true;
 620                         break;
 621                     case errSecAuthNeeded:
 622                         // errSecAuthNeeded means that it is an ACL-based item which requires authentication (or at least
 623                         // ACM context, which we do not have).
 624                         ok = true;
 625                         break;
 626                     case SQLITE_CONSTRAINT:         // yeah...
 627                         if (!CFEqual(kSecDbErrorDomain, CFErrorGetDomain(localError))) {
 628                             secerror("Received SQLITE_CONSTRAINT with wrong error domain. Huh? Item: %@, error: %@", item, localError);
 629                             break;
 630                         }
 631                     case errSecDuplicateItem:
 632                         // continue to upgrade and don't propagate errors for insert failures
 633                         // that are typical of a single item failure
 634                         secnotice("upgr", "Ignoring duplicate item: %@", item);
 635                         secdebug("upgr", "Duplicate item error: %@", localError);
 636                         ok = true;
 637                         break;
 638 #if USE_KEYSTORE
 639                     case kAKSReturnNotReady:
 640                     case kAKSReturnTimeout:
 641 #endif
 642                     case errSecNotAvailable:
 643                         *inProgress = true;     // We're not done, call me again later!
 644                         secnotice("upgr", "Bailing in phase 2 because AKS is unavailable: %@", localError);
 645                         // FALLTHROUGH
 646                     default:
 647                         //  Other errors should abort the migration completely.
 648                         ok = CFErrorPropagate(CFRetainSafe(localError), error);
 649                         break;
 650                 }
 651             }
 652
 653         out:
 654             CFReleaseSafe(localError);
 655             *stop = *stop || !ok;
 656
 657         });
 658         require_action(ok, out, LKAReportKeychainUpgradeOutcomeWithError(oldVersion, newVersion, LKAKeychainUpgradeOutcomePhase2, error ? *error : NULL));
 659     }
 660
 661 #if TARGET_OS_IPHONE && !TARGET_OS_SIMULATOR
 662     measureUpgradePhase2(&start, SecBucket2Significant(itemsMigrated));
 663 #endif
 664
 665 out:
 666     if (query != NULL)
 667         query_destroy(query, NULL);
 668
 669     threadDbt = oldDbt;
 670     return ok;
 671 }
 672
 673 static bool SecKeychainDbUpgradeFromVersion(SecDbConnectionRef dbt, int version, bool *inProgress, CFErrorRef *error) {
 674     __block bool didPhase2 = false;
 675     __block bool ok = true;
 676     __block CFErrorRef localError = NULL;
 677
 678     if (error)
 679         *error = NULL;
 680
 681     const SecDbSchema *newSchema = current_schema();
 682     int newVersion = SCHEMA_VERSION(newSchema);
 683     bool skipped_upgrade = false;
 684
 685     // If DB schema is the one we want, we are done.
 686     require_action_quiet(SCHEMA_VERSION(newSchema) != version, out, skipped_upgrade = true);
 687
 688     // Check if the schema of the database on disk is the same major, but newer version then what we have
 689     // in code, lets just skip this since a newer version of the OS have upgrade it. Since its the same
 690     // major, its a promise that it will be compatible.
 691     if (newSchema->majorVersion == VERSION_MAJOR(version) && newSchema->minorVersion < VERSION_MINOR(version)) {
 692         secnotice("upgr", "skipping upgrade since minor is newer");
 693         goto out;
 694     }
 695
 696     ok &= SecDbTransaction(dbt, kSecDbExclusiveTransactionType, &localError, ^(bool *commit) {
 697         CFStringRef sql = NULL;
 698         bool didPhase1 = false;
 699
 700         // Get version again once we start a transaction, someone else might change the migration state.
 701         int version2 = 0;
 702         require_quiet(ok = SecKeychainDbGetVersion(dbt, &version2, &localError), out);
 703         // Check if someone has raced us to the migration of the database
 704         require_action(version == version2, out, CFReleaseNull(localError); ok = true);
 705
 706         require_quiet(SCHEMA_VERSION(newSchema) != version2, out);
 707
 708         // If this is empty database, just create table according to schema and be done with it.
 709         require_action_quiet(version2 != 0, out, ok = SecItemDbCreateSchema(dbt, newSchema, NULL, true, &localError);
 710                              LKAReportKeychainUpgradeOutcomeWithError(version2, newVersion, LKAKeychainUpgradeOutcomeNewDb, localError));
 711
 712         int oldVersion = VERSION_OLD(version2);
 713         version2 = VERSION_NEW(version2);
 714
 715         require_action_quiet(version2 == SCHEMA_VERSION(newSchema) || oldVersion == 0, out,
 716                              ok = SecDbError(SQLITE_CORRUPT, &localError,
 717                                              CFSTR("Half migrated but obsolete DB found: found 0x%x(0x%x) but 0x%x is needed"),
 718                                              version2, oldVersion, SCHEMA_VERSION(newSchema));
 719                              LKAReportKeychainUpgradeOutcome(version2, newVersion, LKAKeychainUpgradeOutcomeObsoleteDb));
 720
 721         // Check whether we have both old and new tables in the DB.
 722         if (oldVersion == 0) {
 723             // Pure old-schema migration attempt, with full blown table renames etc (a.k.a. phase1)
 724             oldVersion = version2;
 725             version2 = SCHEMA_VERSION(newSchema);
 726
 727             // Find schema for old database.
 728             const SecDbSchema *oldSchema = NULL;
 729             for (const SecDbSchema * const *pschema = all_schemas(); *pschema; ++pschema) {
 730                 if (SCHEMA_VERSION((*pschema)) == oldVersion) {
 731                     oldSchema = *pschema;
 732                     break;
 733                 }
 734             }
 735
 736             // If we are attempting to upgrade from a version for which we have no schema, fail.
 737             require_action_quiet(oldSchema != NULL, out,
 738                                  ok = SecDbError(SQLITE_CORRUPT, &localError, CFSTR("no schema for version: 0x%x"), oldVersion);
 739                                  secerror("no schema for version 0x%x", oldVersion);
 740                                  LKAReportKeychainUpgradeOutcome(version2, newVersion, LKAKeychainUpgradeOutcomeNoSchema));
 741
 742             secnotice("upgr", "Upgrading from version 0x%x to 0x%x", oldVersion, SCHEMA_VERSION(newSchema));
 743             SecSignpostStart(SecSignpostUpgradePhase1);
 744             require_action(ok = UpgradeSchemaPhase1(dbt, oldSchema, &localError), out, secerror("upgrade: Upgrade phase1 failed: %@", localError));
 745             SecSignpostStop(SecSignpostUpgradePhase1);
 746
 747             didPhase1 = true;
 748         }
 749
 750         {
 751             CFErrorRef phase2Error = NULL;
 752
 753             SecSignpostStart(SecSignpostUpgradePhase2);
 754
 755             // Lets try to go through non-D-class items in new tables and apply decode/encode on them
 756             // If this fails the error will be ignored after doing a phase1 since but not in the second
 757             // time when we are doing phase2.
 758             ok = UpgradeItemPhase2(dbt, inProgress, version2, &phase2Error);
 759             if (!ok) {
 760                 if (didPhase1) {
 761                     *inProgress = true;
 762                     ok = true;
 763                     CFReleaseNull(phase2Error);
 764                 } else {
 765                     SecErrorPropagate(phase2Error, &localError);
 766                 }
 767             }
 768             require_action(ok, out, secerror("upgrade: Upgrade phase2 (%d) failed: %@", didPhase1, localError));
 769
 770             if (!*inProgress) {
 771                 // If either migration path we did reported that the migration was complete, signalize that
 772                 // in the version database by cleaning oldVersion (which is stored in upper halfword of the version)
 773                 secnotice("upgr", "Done upgrading from version 0x%x to 0x%x", oldVersion, SCHEMA_VERSION(newSchema));
 774                 oldVersion = 0;
 775
 776                 didPhase2 = true;
 777                 SecSignpostStop(SecSignpostUpgradePhase2);
 778             }
 779         }
 780
 781         // Update database version table.
 782         uint32_t major = (VERSION_MAJOR(version2)) | (VERSION_MAJOR(oldVersion) << 16);
 783         uint32_t minor = (VERSION_MINOR(version2)) | (VERSION_MINOR(oldVersion) << 16);
 784         secnotice("upgr", "Upgrading saving version major 0x%x minor 0x%x", major, minor);
 785         sql = CFStringCreateWithFormat(NULL, NULL, CFSTR("UPDATE tversion SET version='%d', minor='%d'"),
 786                                        major, minor);
 787         require_action_quiet(ok = SecDbExec(dbt, sql, &localError), out, secerror("upgrade: Setting version failed: %@", localError));
 788
 789     out:
 790         if (!ok) {
 791             secerror("upgrade: SecDB upgrade failed: %@", localError);
 792         }
 793         CFReleaseSafe(sql);
 794         *commit = ok;
 795     });
 796
 797     if (ok && didPhase2) {
 798 #if TARGET_OS_IPHONE && !TARGET_OS_SIMULATOR
 799         SecADSetValueForScalarKey(CFSTR("com.apple.keychain.migration-success"), 1);
 800 #endif
 801     }
 802
 803 out:
 804     if (!ok || localError) {
 805         // TODO: This logic should be inverted to a do-not-corrupt-unless default, <rdar://problem/29771874>
 806         /*
 807          * We assume that database is corrupt at this point, but we need to
 808          * check if the error we got isn't severe enough to mark the database as corrupt.
 809          * In those cases we opt out of corrupting the database.
 810          */
 811         bool markedCorrupt = true;
 812
 813         if (ok) {
 814             secwarning("upgrade: error has been set but status is true");
 815             ok = false;
 816         }
 817         secerror("upgrade: error occurred, considering marking database as corrupt: %@", localError);
 818         if (localError) {
 819             CFStringRef domain = CFErrorGetDomain(localError);
 820             CFIndex code = CFErrorGetCode(localError);
 821
 822             if ((CFEqualSafe(domain, kSecDbErrorDomain) &&
 823                 ((code & 0xff) == SQLITE_LOCKED || (code & 0xff) == SQLITE_BUSY || (code & 0xff) == SQLITE_FULL)) ||
 824 #if USE_KEYSTORE
 825                 code == kAKSReturnNotReady || code == kAKSReturnTimeout ||
 826 #endif
 827                 code == errSecNotAvailable)
 828             {
 829                 secerror("upgrade: not marking keychain database corrupt for error: %@", localError);
 830                 markedCorrupt = false;
 831                 CFReleaseNull(localError);
 832             } else {
 833                 secerror("upgrade: unable to complete upgrade, marking DB as corrupt: %@", localError);
 834             }
 835         } else {
 836             secerror("upgrade: unable to complete upgrade and no error object returned, marking DB as corrupt");
 837         }
 838         if (markedCorrupt) {
 839             secerror("upgrade: marking database as corrupt");
 840             SecDbCorrupt(dbt, localError);
 841 #if TARGET_OS_IPHONE && !TARGET_OS_SIMULATOR
 842             SecADSetValueForScalarKey(CFSTR("com.apple.keychain.migration-failure"), 1);
 843 #endif
 844         }
 845     } else {
 846         // Things seemed to go okay!
 847         if (didPhase2) {
 848             LKAReportKeychainUpgradeOutcome(version, newVersion, LKAKeychainUpgradeOutcomeSuccess);
 849         }
 850
 851         //If we're done here, we should opportunistically re-add all indices (just in case)
 852         if(skipped_upgrade || didPhase2) {
 853             // Create indices, ignoring all errors
 854             for (SecDbClass const* const* newClass = newSchema->classes; *newClass; ++newClass) {
 855                 SecDbForEachAttrWithMask((*newClass), desc, kSecDbIndexFlag | kSecDbInFlag) {
 856                     CFStringRef sql = NULL;
 857                     CFErrorRef classLocalError = NULL;
 858                     bool localOk = true;
 859
 860                     if (desc->kind == kSecDbSyncAttr) {
 861                         // Replace the complete sync index with a partial index for sync=0. Most items are sync=1, so the complete index isn't helpful for sync=1 queries.
 862                         sql = CFStringCreateWithFormat(NULL, NULL, CFSTR("DROP INDEX IF EXISTS %@%@; CREATE INDEX IF NOT EXISTS %@%@0 on %@(%@) WHERE %@=0;"),
 863                             (*newClass)->name, desc->name, (*newClass)->name, desc->name, (*newClass)->name, desc->name, desc->name);
 864                     } else {
 865                         sql = CFStringCreateWithFormat(NULL, NULL, CFSTR("CREATE INDEX IF NOT EXISTS %@%@ ON %@(%@);"), (*newClass)->name, desc->name, (*newClass)->name, desc->name);
 866                     }
 867                     localOk &= SecDbExec(dbt, sql, &classLocalError);
 868                     CFReleaseNull(sql);
 869
 870                     if(!localOk) {
 871                         secerror("upgrade: unable to opportunistically create index (%@,%@): %@", (*newClass)->name, desc->name, classLocalError);
 872                     }
 873                     CFReleaseNull(classLocalError);
 874                 }
 875             }
 876         }
 877     }
 878     if (localError) {
 879         if (error) {
 880             *error = (CFErrorRef)CFRetain(localError);
 881         }
 882         CFReleaseNull(localError);
 883     }
 884
 885     return ok;
 886 }
 887
 888 static bool accessGroupIsNetworkExtensionAndClientIsEntitled(CFStringRef accessGroup, SecurityClient* client)
 889 {
 890     return client && client->canAccessNetworkExtensionAccessGroups && accessGroup && CFStringHasSuffix(accessGroup, kSecNetworkExtensionAccessGroupSuffix);
 891 }
 892
 893 /* AUDIT[securityd](done):
 894    accessGroup (ok) is a caller provided, non NULL CFTypeRef.
 895
 896    Return true iff accessGroup is allowable according to accessGroups.
 897  */
 898 bool accessGroupsAllows(CFArrayRef accessGroups, CFStringRef accessGroup, SecurityClient* client) {
 899     /* NULL accessGroups is wildcard. */
 900     if (!accessGroups)
 901         return true;
 902     /* Make sure we have a string. */
 903     if (!isString(accessGroup))
 904         return false;
 905
 906     /* Having the special accessGroup "*" allows access to all accessGroups. */
 907     CFRange range = { 0, CFArrayGetCount(accessGroups) };
 908     if (range.length &&
 909         (CFArrayContainsValue(accessGroups, range, accessGroup) ||
 910          CFArrayContainsValue(accessGroups, range, CFSTR("*")) ||
 911          accessGroupIsNetworkExtensionAndClientIsEntitled(accessGroup, client)))
 912         return true;
 913
 914     return false;
 915 }
 916
 917 bool itemInAccessGroup(CFDictionaryRef item, CFArrayRef accessGroups) {
 918     return accessGroupsAllows(accessGroups,
 919                               CFDictionaryGetValue(item, kSecAttrAccessGroup), NULL);
 920 }
 921
 922
 923 static CF_RETURNS_RETAINED CFDataRef SecServerExportBackupableKeychain(SecDbConnectionRef dbt,
 924     SecurityClient *client,
 925     keybag_handle_t src_keybag, keybag_handle_t dest_keybag, CFErrorRef *error) {
 926     CFDataRef data_out = NULL;
 927
 928     SecSignpostStart(SecSignpostBackupKeychainBackupable);
 929
 930     /* Export everything except the items for which SecItemIsSystemBound()
 931        returns true. */
 932     CFDictionaryRef keychain = SecServerCopyKeychainPlist(dbt, client,
 933         src_keybag, dest_keybag, kSecBackupableItemFilter,
 934         error);
 935     if (keychain) {
 936         data_out = CFPropertyListCreateData(kCFAllocatorDefault, keychain,
 937                                              kCFPropertyListBinaryFormat_v1_0,
 938                                              0, error);
 939         CFRelease(keychain);
 940     }
 941     SecSignpostStop(SecSignpostBackupKeychainBackupable);
 942
 943     return data_out;
 944 }
 945
 946 static bool SecServerImportBackupableKeychain(SecDbConnectionRef dbt,
 947                                               SecurityClient *client,
 948                                               keybag_handle_t src_keybag,
 949                                               keybag_handle_t dest_keybag,
 950                                               CFDataRef data,
 951                                               CFErrorRef *error)
 952 {
 953     return kc_transaction(dbt, error, ^{
 954         bool ok = false;
 955         CFDictionaryRef keychain;
 956
 957         SecSignpostStart(SecSignpostRestoreKeychainBackupable);
 958
 959         keychain = CFPropertyListCreateWithData(kCFAllocatorDefault, data,
 960                                                 kCFPropertyListImmutable, NULL,
 961                                                 error);
 962         if (keychain) {
 963             if (isDictionary(keychain)) {
 964                 ok = SecServerImportKeychainInPlist(dbt,
 965                                                     client,
 966                                                     src_keybag,
 967                                                     dest_keybag,
 968                                                     keychain,
 969                                                     kSecBackupableItemFilter,
 970                                                     false,  // Restoring backup should not remove stuff that got into the keychain before us
 971                                                     error);
 972             } else {
 973                 ok = SecError(errSecParam, error, CFSTR("import: keychain is not a dictionary"));
 974             }
 975             CFRelease(keychain);
 976         }
 977
 978         SecSignpostStop(SecSignpostRestoreKeychainBackupable);
 979
 980         return ok;
 981     });
 982 }
 983
 984 #if USE_KEYSTORE
 985 /*
 986  * Similar to ks_open_keybag, but goes through MKB interface
 987  */
 988 static bool mkb_open_keybag(CFDataRef keybag, CFDataRef password, MKBKeyBagHandleRef *handle, bool emcs, CFErrorRef *error) {
 989     kern_return_t rc;
 990     MKBKeyBagHandleRef mkbhandle = NULL;
 991
 992     rc = MKBKeyBagCreateWithData(keybag, &mkbhandle);
 993     if (rc != kMobileKeyBagSuccess) {
 994         return SecKernError(rc, error, CFSTR("MKBKeyBagCreateWithData failed: %d"), rc);
 995     }
 996
 997     if (!emcs) {
 998         rc = MKBKeyBagUnlock(mkbhandle, password);
 999         if (rc != kMobileKeyBagSuccess) {
1000             CFRelease(mkbhandle);
1001             return SecKernError(rc, error, CFSTR("failed to unlock bag: %d"), rc);
1002         }
1003     } else {
1004         secnotice("keychainbackup", "skipping keybag unlock for EMCS");
1005     }
1006
1007     *handle = mkbhandle;
1008
1009     return true;
1010 }
1011 #endif
1012
1013
1014 static CFDataRef SecServerKeychainCreateBackup(SecDbConnectionRef dbt, SecurityClient *client, CFDataRef keybag,
1015     CFDataRef password, bool emcs, CFErrorRef *error) {
1016     CFDataRef backup = NULL;
1017     keybag_handle_t backup_keybag;
1018
1019     SecSignpostStart(SecSignpostBackupOpenKeybag);
1020
1021 #if USE_KEYSTORE
1022     MKBKeyBagHandleRef mkbhandle = NULL;
1023     require(mkb_open_keybag(keybag, password, &mkbhandle, emcs, error), out);
1024
1025     require_noerr(MKBKeyBagGetAKSHandle(mkbhandle, &backup_keybag), out);
1026
1027 #else
1028     backup_keybag = KEYBAG_NONE;
1029 #endif
1030     SecSignpostStop(SecSignpostBackupOpenKeybag);
1031     SecSignpostStart(SecSignpostBackupKeychain);
1032
1033     /* Export from system keybag to backup keybag. */
1034     backup = SecServerExportBackupableKeychain(dbt, client, KEYBAG_DEVICE, backup_keybag, error);
1035
1036 #if USE_KEYSTORE
1037 out:
1038     SecSignpostStop(SecSignpostBackupOpenKeybag);
1039
1040     if (mkbhandle)
1041         CFRelease(mkbhandle);
1042 #endif
1043     return backup;
1044 }
1045
1046 static bool SecServerKeychainRestore(SecDbConnectionRef dbt,
1047                                      SecurityClient *client,
1048                                      CFDataRef backup,
1049                                      CFDataRef keybag,
1050                                      CFDataRef password,
1051                                      CFErrorRef *error)
1052 {
1053     bool ok = false;
1054     keybag_handle_t backup_keybag;
1055
1056     secnotice("SecServerKeychainRestore", "Restoring keychain backup");
1057
1058     SecSignpostStart(SecSignpostRestoreOpenKeybag);
1059 #if USE_KEYSTORE
1060     MKBKeyBagHandleRef mkbhandle = NULL;
1061     require(mkb_open_keybag(keybag, password, &mkbhandle, false, error), out);
1062
1063     require_noerr(MKBKeyBagGetAKSHandle(mkbhandle, &backup_keybag), out);
1064 #else
1065     backup_keybag = KEYBAG_NONE;
1066 #endif
1067     SecSignpostStop(SecSignpostRestoreOpenKeybag);
1068     SecSignpostStart(SecSignpostRestoreKeychain);
1069
1070     /* Import from backup keybag to system keybag. */
1071     require(SecServerImportBackupableKeychain(dbt, client, backup_keybag, KEYBAG_DEVICE, backup, error), out);
1072
1073     ok = true;
1074 out:
1075     SecSignpostStop(SecSignpostRestoreKeychain);
1076 #if USE_KEYSTORE
1077     if (mkbhandle)
1078         CFRelease(mkbhandle);
1079 #endif
1080
1081     if (ok) {
1082         secnotice("SecServerKeychainRestore", "Restore completed successfully");
1083     } else {
1084         secwarning("SecServerKeychainRestore: Restore failed with: %@", error ? *error : NULL);
1085     }
1086
1087     return ok;
1088 }
1089
1090 // MARK - External SPI support code.
1091
1092 CFStringRef __SecKeychainCopyPath(void) {
1093     CFStringRef kcRelPath = NULL;
1094
1095     if (os_variant_is_recovery("securityd")) {
1096         kcRelPath = CFSTR("keychain-recovery-2.db");
1097     } else if (use_hwaes()) {
1098         kcRelPath = CFSTR("keychain-2.db");
1099     } else {
1100         kcRelPath = CFSTR("keychain-2-debug.db");
1101     }
1102
1103     CFStringRef kcPath = NULL;
1104     CFURLRef kcURL = SecCopyURLForFileInKeychainDirectory(kcRelPath);
1105     if (kcURL) {
1106         kcPath = CFURLCopyFileSystemPath(kcURL, kCFURLPOSIXPathStyle);
1107         CFRelease(kcURL);
1108     }
1109     return kcPath;
1110 }
1111
1112 // MARK; -
1113 // MARK: kc_dbhandle init and reset
1114
1115 SecDbRef SecKeychainDbCreate(CFStringRef path, CFErrorRef* error) {
1116     __block CFErrorRef localerror = NULL;
1117
1118     SecDbRef kc = SecDbCreate(path, 0600, true, true, true, true, kSecDbMaxIdleHandles,
1119         ^bool (SecDbRef db, SecDbConnectionRef dbconn, bool didCreate, bool *callMeAgainForNextConnection, CFErrorRef *createError)
1120     {
1121         // Upgrade from version 0 means create the schema in empty db.
1122         int version = 0;
1123         bool ok = true;
1124         if (!didCreate)
1125             ok = SecKeychainDbGetVersion(dbconn, &version, createError);
1126
1127         ok = ok && SecKeychainDbUpgradeFromVersion(dbconn, version, callMeAgainForNextConnection, createError);
1128         if (!ok)
1129             secerror("Upgrade %sfailed: %@", didCreate ? "from v0 " : "", createError ? *createError : NULL);
1130
1131         localerror = createError ? *createError : NULL;
1132
1133         if(ok) {
1134             // This block might get called many, many times due to callMeAgainForNextConnection.
1135             // When we no longer want to be called, we believe we're done. Begin the rest of initialization.
1136             if( !callMeAgainForNextConnection || !(*callMeAgainForNextConnection)) {
1137                 SecKeychainDbInitialize(db);
1138             }
1139         }
1140
1141         return ok;
1142     });
1143
1144     if (kc) {
1145         SecDbSetCorruptionReset(kc, ^{
1146             SecDbResetMetadataKeys();
1147         });
1148     }
1149
1150     if(error) {
1151         *error = localerror;
1152     }
1153
1154     return kc;
1155 }
1156
1157 SecDbRef SecKeychainDbInitialize(SecDbRef db) {
1158
1159 #if OCTAGON
1160     if(SecCKKSIsEnabled()) {
1161         // This needs to be async, otherwise we get hangs between securityd, cloudd, and apsd
1162         dispatch_async(dispatch_get_global_queue(DISPATCH_QUEUE_PRIORITY_DEFAULT, 0), ^{
1163             SecCKKSInitialize(db);
1164
1165         });
1166     }
1167
1168     if(OctagonIsEnabled() && OctagonShouldPerformInitialization()) {
1169         dispatch_async(dispatch_get_global_queue(DISPATCH_QUEUE_PRIORITY_DEFAULT, 0), ^{
1170             OctagonInitialize();
1171         });
1172     }
1173
1174     if(EscrowRequestServerIsEnabled()) {
1175         dispatch_async(dispatch_get_global_queue(DISPATCH_QUEUE_PRIORITY_DEFAULT, 0), ^{
1176             EscrowRequestServerInitialize();
1177         });
1178     }
1179 #endif
1180
1181     return db;
1182 }
1183
1184 static SecDbRef _kc_dbhandle = NULL;
1185 static dispatch_queue_t _kc_dbhandle_dispatch = NULL;
1186 static dispatch_once_t _kc_dbhandle_dispatch_onceToken = 0;
1187 static dispatch_queue_t get_kc_dbhandle_dispatch() {
1188     dispatch_once(&_kc_dbhandle_dispatch_onceToken, ^{
1189         _kc_dbhandle_dispatch = dispatch_queue_create("sec_kc_dbhandle", DISPATCH_QUEUE_SERIAL);
1190     });
1191
1192     return _kc_dbhandle_dispatch;
1193 }
1194
1195 static bool kc_dbhandle_init(CFErrorRef* error) {
1196     SecDbRef oldHandle = _kc_dbhandle;
1197     _kc_dbhandle = NULL;
1198     CFStringRef dbPath = __SecKeychainCopyPath();
1199     if (dbPath) {
1200         _kc_dbhandle = SecKeychainDbCreate(dbPath, error);
1201         CFRelease(dbPath);
1202     } else {
1203         secerror("no keychain path available");
1204     }
1205     if (oldHandle) {
1206         secerror("replaced %@ with %@", oldHandle, _kc_dbhandle);
1207         CFRelease(oldHandle);
1208     }
1209     // Having a dbhandle means we succeeded.
1210     return !!_kc_dbhandle;
1211 }
1212
1213 static SecDbRef kc_dbhandle(CFErrorRef* error)
1214 {
1215     dispatch_sync(get_kc_dbhandle_dispatch(), ^{
1216         if(_kc_dbhandle == NULL) {
1217             _SecDbServerSetup();
1218             kc_dbhandle_init(error);
1219         }
1220     });
1221     return _kc_dbhandle;
1222 }
1223
1224 /* For whitebox testing only */
1225 void SecKeychainDbReset(dispatch_block_t inbetween)
1226 {
1227     dispatch_sync(get_kc_dbhandle_dispatch(), ^{
1228         CFReleaseNull(_kc_dbhandle);
1229         SecDbResetMetadataKeys();
1230
1231         if (inbetween)
1232             inbetween();
1233     });
1234 }
1235
1236 static bool kc_acquire_dbt(bool writeAndRead, SecDbConnectionRef* dbconn, CFErrorRef *error) {
1237     SecDbRef db = kc_dbhandle(error);
1238     if (db == NULL) {
1239         if(error && !(*error)) {
1240             SecError(errSecDataNotAvailable, error, CFSTR("failed to get a db handle"));
1241         }
1242         return NULL;
1243     }
1244
1245     return SecDbConnectionAcquireRefMigrationSafe(db, !writeAndRead, dbconn, error);
1246 }
1247
1248 /* Return a per thread dbt handle for the keychain.  If create is true create
1249  the database if it does not yet exist.  If it is false, just return an
1250  error if it fails to auto-create. */
1251 bool kc_with_dbt(bool writeAndRead, CFErrorRef *error, bool (^perform)(SecDbConnectionRef dbt))
1252 {
1253     return kc_with_custom_db(writeAndRead, true, NULL, error, perform);
1254 }
1255
1256 bool kc_with_dbt_non_item_tables(bool writeAndRead, CFErrorRef* error, bool (^perform)(SecDbConnectionRef dbt))
1257 {
1258     return kc_with_custom_db(writeAndRead, false, NULL, error, perform);
1259 }
1260
1261 bool kc_with_custom_db(bool writeAndRead, bool usesItemTables, SecDbRef db, CFErrorRef *error, bool (^perform)(SecDbConnectionRef dbt))
1262 {
1263     if (db && db != kc_dbhandle(error)) {
1264         __block bool result = false;
1265         if (writeAndRead) {
1266             SecDbPerformWrite(db, error, ^(SecDbConnectionRef dbconn) {
1267                 result = perform(dbconn);
1268             });
1269         }
1270         else {
1271             SecDbPerformRead(db, error, ^(SecDbConnectionRef dbconn) {
1272                 result = perform(dbconn);
1273             });
1274         }
1275         return result;
1276     }
1277
1278     if(threadDbt) {
1279         // The kc_with_dbt upthread will clean this up when it's done.
1280         return perform(threadDbt);
1281     }
1282
1283     if (writeAndRead && usesItemTables) {
1284 #if SECUREOBJECTSYNC
1285         SecItemDataSourceFactoryGetDefault();
1286 #endif
1287     }
1288
1289     bool ok = false;
1290     if (kc_acquire_dbt(writeAndRead, &threadDbt, error)) {
1291         ok = perform(threadDbt);
1292         SecDbConnectionRelease(threadDbt);
1293         threadDbt = NULL;
1294     }
1295     return ok;
1296 }
1297
1298 static bool
1299 items_matching_issuer_parent(SecDbConnectionRef dbt, CFArrayRef accessGroups, CFDataRef musrView,
1300                              CFDataRef issuer, CFArrayRef issuers, int recurse)
1301 {
1302     Query *q;
1303     CFArrayRef results = NULL;
1304     CFIndex i, count;
1305     bool found = false;
1306
1307     if (CFArrayContainsValue(issuers, CFRangeMake(0, CFArrayGetCount(issuers)), issuer))
1308         return true;
1309
1310     /* XXX make musr supported */
1311     const void *keys[] = { kSecClass, kSecReturnRef, kSecAttrSubject };
1312     const void *vals[] = { kSecClassCertificate, kCFBooleanTrue, issuer };
1313     CFDictionaryRef query = CFDictionaryCreate(kCFAllocatorDefault, keys, vals, array_size(keys), NULL, NULL);
1314
1315     if (!query)
1316         return false;
1317
1318     CFErrorRef localError = NULL;
1319     q = query_create_with_limit(query, musrView, kSecMatchUnlimited, &localError);
1320     CFRelease(query);
1321     if (q) {
1322         s3dl_copy_matching(dbt, q, (CFTypeRef*)&results, accessGroups, &localError);
1323         query_destroy(q, &localError);
1324     }
1325     if (localError) {
1326         secerror("items matching issuer parent: %@", localError);
1327         CFReleaseNull(localError);
1328         return false;
1329     }
1330
1331     count = CFArrayGetCount(results);
1332     for (i = 0; (i < count) && !found; i++) {
1333         CFDictionaryRef cert_dict = (CFDictionaryRef)CFArrayGetValueAtIndex(results, i);
1334         CFDataRef cert_issuer = CFDictionaryGetValue(cert_dict, kSecAttrIssuer);
1335         if (CFEqual(cert_issuer, issuer))
1336             continue;
1337         if (recurse-- > 0)
1338             found = items_matching_issuer_parent(dbt, accessGroups, musrView, cert_issuer, issuers, recurse);
1339     }
1340     CFReleaseSafe(results);
1341
1342     return found;
1343 }
1344
1345 static bool
1346 _FilterWithPolicy(SecPolicyRef policy, CFDateRef date, SecCertificateRef cert)
1347 {
1348     CFDictionaryRef props = NULL;
1349     CFArrayRef keychains = NULL;
1350     CFArrayRef anchors = NULL;
1351     CFArrayRef certs = NULL;
1352     CFArrayRef chain = NULL;
1353     SecTrustRef trust = NULL;
1354
1355     SecTrustResultType  trustResult;
1356     Boolean needChain = false;
1357     __block bool ok = false;
1358
1359     if (!policy || !cert) return false;
1360
1361     certs = CFArrayCreate(NULL, (const void **)&cert, (CFIndex)1, &kCFTypeArrayCallBacks);
1362     require_noerr_quiet(SecTrustCreateWithCertificates(certs, policy, &trust), cleanup);
1363
1364     /* Set evaluation date, if specified (otherwise current date is implied) */
1365     if (date && (CFGetTypeID(date) == CFDateGetTypeID())) {
1366         require_noerr_quiet(SecTrustSetVerifyDate(trust, date), cleanup);
1367     }
1368
1369     /* Check whether this is the X509 Basic policy, which means chain building */
1370     props = SecPolicyCopyProperties(policy);
1371     if (props) {
1372         CFTypeRef oid = (CFTypeRef) CFDictionaryGetValue(props, kSecPolicyOid);
1373         if (oid && (CFEqual(oid, kSecPolicyAppleX509Basic) ||
1374                     CFEqual(oid, kSecPolicyAppleRevocation))) {
1375             needChain = true;
1376         }
1377     }
1378
1379     if (!needChain) {
1380         require_noerr_quiet(SecTrustEvaluateLeafOnly(trust, &trustResult), cleanup);
1381     } else {
1382         require_noerr_quiet(SecTrustEvaluate(trust, &trustResult), cleanup);
1383     }
1384
1385     require_quiet((trustResult == kSecTrustResultProceed ||
1386                          trustResult == kSecTrustResultUnspecified ||
1387                          trustResult == kSecTrustResultRecoverableTrustFailure), cleanup);
1388
1389     ok = true;
1390 #if TARGET_OS_IPHONE
1391     CFArrayRef properties = SecTrustCopyProperties(trust);
1392 #else
1393     CFArrayRef properties = SecTrustCopyProperties_ios(trust);
1394 #endif
1395     if (properties) {
1396         CFArrayForEach(properties, ^(const void *property) {
1397             CFDictionaryForEach((CFDictionaryRef)property, ^(const void *key, const void *value) {
1398                 if (CFEqual((CFTypeRef)key, kSecPropertyKeyType) && CFEqual((CFTypeRef)value, kSecPropertyTypeError))
1399                     ok = false;
1400             });
1401         });
1402         CFRelease(properties);
1403     }
1404
1405 cleanup:
1406     if(props) CFRelease(props);
1407     if(chain) CFRelease(chain);
1408     if(anchors) CFRelease(anchors);
1409     if(keychains) CFRelease(keychains);
1410     if(certs) CFRelease(certs);
1411     if(trust) CFRelease(trust);
1412
1413     return ok;
1414 }
1415
1416 static bool
1417 _FilterWithDate(CFDateRef validOnDate, SecCertificateRef cert)
1418 {
1419     if (!validOnDate || !cert) return false;
1420
1421     CFAbsoluteTime at, nb, na;
1422     at = CFDateGetAbsoluteTime((CFDateRef)validOnDate);
1423
1424     bool ok = true;
1425     nb = SecCertificateNotValidBefore(cert);
1426     na = SecCertificateNotValidAfter(cert);
1427
1428     if (nb == 0 || na == 0 || nb == na) {
1429         ok = false;
1430         secnotice("FilterWithDate", "certificate cannot operate");
1431     }
1432     else if (at < nb) {
1433         ok = false;
1434         secnotice("FilterWithDate", "certificate is not valid yet");
1435     }
1436     else if (at > na) {
1437         ok = false;
1438         secnotice("FilterWithDate", "certificate expired");
1439     }
1440
1441     return ok;
1442 }
1443
1444 static bool
1445 _FilterWithTrust(Boolean trustedOnly, SecCertificateRef cert)
1446 {
1447     if (!cert) return false;
1448     if (!trustedOnly) return true;
1449
1450     bool ok = false;
1451     CFArrayRef certArray = CFArrayCreate(NULL, (const void**)&cert, 1, &kCFTypeArrayCallBacks);
1452     SecTrustRef trust = NULL;
1453     SecPolicyRef policy = SecPolicyCreateBasicX509();
1454     require_quiet(policy, out);
1455
1456     require_noerr_quiet(SecTrustCreateWithCertificates(certArray, policy, &trust), out);
1457     SecTrustResultType trustResult;
1458     require_noerr_quiet(SecTrustEvaluate(trust, &trustResult), out);
1459
1460     require_quiet((trustResult == kSecTrustResultProceed ||
1461                    trustResult == kSecTrustResultUnspecified), out);
1462     ok = true;
1463 out:
1464     CFReleaseSafe(trust);
1465     CFReleaseSafe(policy);
1466     CFReleaseSafe(certArray);
1467     return ok;
1468 }
1469
1470 static SecCertificateRef
1471 CopyCertificateFromItem(Query *q, CFDictionaryRef item) {
1472     SecCertificateRef certRef = NULL;
1473     CFDictionaryRef itemValue = NULL;
1474
1475     CFTypeRef tokenID = NULL;
1476     CFDataRef certData = NULL;
1477     if (q->q_class == identity_class()) {
1478         certData = CFDictionaryGetValue(item, kSecAttrIdentityCertificateData);
1479         tokenID = CFDictionaryGetValue(item, kSecAttrIdentityCertificateTokenID);
1480     } else if (q->q_class == cert_class()) {
1481         certData = CFDictionaryGetValue(item, kSecValueData);
1482         tokenID = CFDictionaryGetValue(item, kSecAttrTokenID);
1483     }
1484
1485     require_quiet(certData, out);
1486     if (tokenID != NULL) {
1487         CFErrorRef error = NULL;
1488         itemValue = SecTokenItemValueCopy(certData, &error);
1489         require_action_quiet(itemValue, out, { secerror("function SecTokenItemValueCopy failed with: %@", error); CFReleaseSafe(error); });
1490         CFDataRef tokenCertData = CFDictionaryGetValue(itemValue, kSecTokenValueDataKey);
1491         require_action_quiet(tokenCertData, out, { secerror("token item doesn't contain token value data");});
1492         certRef = SecCertificateCreateWithData(kCFAllocatorDefault, tokenCertData);
1493     }
1494     else
1495         certRef = SecCertificateCreateWithData(kCFAllocatorDefault, certData);
1496
1497 out:
1498     CFReleaseNull(itemValue);
1499     return certRef;
1500 }
1501
1502 bool match_item(SecDbConnectionRef dbt, Query *q, CFArrayRef accessGroups, CFDictionaryRef item)
1503 {
1504     bool ok = false;
1505     SecCertificateRef certRef = NULL;
1506     if (q->q_match_issuer) {
1507         CFDataRef issuer = CFDictionaryGetValue(item, kSecAttrIssuer);
1508         if (!items_matching_issuer_parent(dbt, accessGroups, q->q_musrView, issuer, q->q_match_issuer, 10 /*max depth*/))
1509             return ok;
1510     }
1511
1512     if (q->q_match_policy && (q->q_class == identity_class() || q->q_class == cert_class())) {
1513         if (!certRef)
1514             certRef = CopyCertificateFromItem(q, item);
1515         require_quiet(certRef, out);
1516         require_quiet(_FilterWithPolicy(q->q_match_policy, q->q_match_valid_on_date, certRef), out);
1517     }
1518
1519     if (q->q_match_valid_on_date && (q->q_class == identity_class() || q->q_class == cert_class())) {
1520         if (!certRef)
1521             certRef = CopyCertificateFromItem(q, item);
1522         require_quiet(certRef, out);
1523         require_quiet(_FilterWithDate(q->q_match_valid_on_date, certRef), out);
1524     }
1525
1526     if (q->q_match_trusted_only && (q->q_class == identity_class() || q->q_class == cert_class())) {
1527         if (!certRef)
1528             certRef = CopyCertificateFromItem(q, item);
1529         require_quiet(certRef, out);
1530         require_quiet(_FilterWithTrust(CFBooleanGetValue(q->q_match_trusted_only), certRef), out);
1531     }
1532
1533     /* Add future match checks here. */
1534     ok = true;
1535 out:
1536     CFReleaseSafe(certRef);
1537     return ok;
1538 }
1539
1540 /****************************************************************************
1541  **************** Beginning of Externally Callable Interface ****************
1542  ****************************************************************************/
1543
1544 static bool SecEntitlementError(CFErrorRef *error)
1545 {
1546 #if TARGET_OS_OSX
1547 #define SEC_ENTITLEMENT_WARNING CFSTR("com.apple.application-identifier nor com.apple.security.application-groups nor keychain-access-groups")
1548 #elif TARGET_OS_IOSMAC
1549 #define SEC_ENTITLEMENT_WARNING CFSTR("com.apple.developer.associated-application-identifier nor application-identifier nor com.apple.security.application-groups nor keychain-access-groups")
1550 #else
1551 #define SEC_ENTITLEMENT_WARNING CFSTR("application-identifier nor keychain-access-groups")
1552 #endif
1553
1554     return SecError(errSecMissingEntitlement, error, CFSTR("Client has neither %@ entitlements"), SEC_ENTITLEMENT_WARNING);
1555 }
1556
1557 static bool SecEntitlementErrorForExplicitAccessGroup(CFStringRef agrp, CFArrayRef clientGroups, CFErrorRef* error)
1558 {
1559     return SecError(errSecMissingEntitlement, error, CFSTR("Client explicitly specifies access group %@ but is only entitled for %@"), agrp, clientGroups);
1560 }
1561
1562 static CFStringRef CopyAccessGroupForRowID(sqlite_int64 rowID, CFStringRef itemClass)
1563 {
1564     __block CFStringRef accessGroup = NULL;
1565
1566     __block CFErrorRef error = NULL;
1567     bool ok = kc_with_dbt(false, &error, ^bool(SecDbConnectionRef dbt) {
1568         CFStringRef table = CFEqual(itemClass, kSecClassIdentity) ? kSecClassCertificate : itemClass;
1569         CFStringRef sql = CFStringCreateWithFormat(NULL, NULL, CFSTR("SELECT agrp FROM %@ WHERE rowid == %u"), table, (unsigned int)rowID);
1570         bool dbOk = SecDbWithSQL(dbt, sql, &error, ^bool(sqlite3_stmt *stmt) {
1571             bool rowOk = SecDbForEach(dbt, stmt, &error, ^bool(int row_index) {
1572                 accessGroup = CFStringCreateWithBytes(NULL, sqlite3_column_blob(stmt, 0), sqlite3_column_bytes(stmt, 0), kCFStringEncodingUTF8, false);
1573                 return accessGroup != NULL;
1574             });
1575
1576             return (bool)(rowOk && accessGroup != NULL);
1577         });
1578
1579         CFReleaseNull(sql);
1580         return (bool)(dbOk && accessGroup);
1581     });
1582
1583     if (ok) {
1584         return accessGroup;
1585     }
1586     else {
1587         CFReleaseNull(accessGroup);
1588         return NULL;
1589     }
1590 }
1591
1592 /* AUDIT[securityd](done):
1593    query (ok) is a caller provided dictionary, only its cf type has been checked.
1594  */
1595 static bool
1596 SecItemServerCopyMatching(CFDictionaryRef query, CFTypeRef *result,
1597     SecurityClient *client, CFErrorRef *error)
1598 {
1599     CFArrayRef accessGroups = CFRetainSafe(client->accessGroups);
1600
1601     CFIndex ag_count;
1602     if (!accessGroups || 0 == (ag_count = CFArrayGetCount(accessGroups))) {
1603         CFReleaseNull(accessGroups);
1604         return SecEntitlementError(error);
1605     }
1606
1607     SecSignpostStart(SecSignpostSecItemCopyMatching);
1608
1609     if (client->canAccessNetworkExtensionAccessGroups) {
1610         CFDataRef persistentRef = CFDictionaryGetValue(query, kSecValuePersistentRef);
1611         CFStringRef itemClass = NULL;
1612         sqlite_int64 itemRowID = 0;
1613         if (persistentRef && _SecItemParsePersistentRef(persistentRef, &itemClass, &itemRowID, NULL)) {
1614             CFStringRef accessGroup = CopyAccessGroupForRowID(itemRowID, itemClass);
1615             if (accessGroup && CFStringHasSuffix(accessGroup, kSecNetworkExtensionAccessGroupSuffix) && !CFArrayContainsValue(accessGroups, CFRangeMake(0, ag_count), accessGroup)) {
1616                 CFMutableArrayRef mutableAccessGroups = CFArrayCreateMutableCopy(NULL, 0, accessGroups);
1617                 CFArrayAppendValue(mutableAccessGroups, accessGroup);
1618                 CFReleaseNull(accessGroups);
1619                 accessGroups = mutableAccessGroups;
1620             }
1621             CFReleaseNull(accessGroup);
1622         }
1623     }
1624
1625     if (CFArrayContainsValue(accessGroups, CFRangeMake(0, ag_count), CFSTR("*"))) {
1626         /* Having the special accessGroup "*" allows access to all accessGroups. */
1627         CFReleaseNull(accessGroups);
1628     }
1629
1630     bool ok = false;
1631     Query *q = query_create_with_limit(query, client->musr, 1, error);
1632     if (q) {
1633         CFStringRef agrp = CFDictionaryGetValue(q->q_item, kSecAttrAccessGroup);
1634         if (agrp) {
1635             if (accessGroupsAllows(accessGroups, agrp, client)) {
1636                 const void *val = agrp;
1637                 CFReleaseNull(accessGroups);
1638                 accessGroups = CFArrayCreate(0, &val, 1, &kCFTypeArrayCallBacks);
1639             } else {
1640                 (void)SecEntitlementErrorForExplicitAccessGroup(agrp, accessGroups, error);
1641                 CFReleaseNull(accessGroups);
1642                 query_destroy(q, NULL);
1643                 return false;
1644             }
1645         } else {
1646 #if !TARGET_OS_OSX
1647             if (accessGroups != NULL) {
1648                 // On iOS, drop 'com.apple.token' AG from allowed accessGroups, to avoid inserting token elements into
1649                 // unsuspecting application's keychain. If the application on iOS wants to access token items, it needs
1650                 // explicitly specify kSecAttrAccessGroup=kSecAttrAccessGroupToken in its query.
1651                 CFMutableArrayRef mutableGroups =  CFArrayCreateMutableCopy(kCFAllocatorDefault, 0, accessGroups);
1652                 CFArrayRemoveAllValue(mutableGroups, kSecAttrAccessGroupToken);
1653                 CFReleaseNull(accessGroups);
1654                 accessGroups = mutableGroups;
1655             }
1656 #endif
1657         }
1658
1659 #if TARGET_OS_IPHONE
1660         if (q->q_sync_bubble && client->inMultiUser) {
1661             CFReleaseNull(q->q_musrView);
1662             q->q_musrView = SecMUSRCreateSyncBubbleUserUUID(q->q_sync_bubble);
1663         } else if (client->inMultiUser && client->isNetworkExtension) {
1664             CFReleaseNull(q->q_musrView);
1665             q->q_musrView = SecMUSRCreateBothUserAndSystemUUID(client->uid);
1666         } else if (q->q_system_keychain && client->inMultiUser) {
1667             CFReleaseNull(q->q_musrView);
1668             q->q_musrView = SecMUSRCopySystemKeychainUUID();
1669         } else {
1670             q->q_system_keychain = false;
1671         }
1672 #endif
1673
1674         query_set_caller_access_groups(q, accessGroups);
1675
1676         /* Sanity check the query. */
1677         if (q->q_system_keychain && !client->allowSystemKeychain) {
1678             ok = SecError(errSecMissingEntitlement, error, CFSTR("client doesn't have entitlement for system keychain"));
1679         } else if (q->q_sync_bubble && !client->allowSyncBubbleKeychain) {
1680             ok = SecError(errSecMissingEntitlement, error, CFSTR("client doesn't have entitlement for syncbubble keychain"));
1681         } else if (q->q_system_keychain && q->q_sync_bubble) {
1682             ok = SecError(errSecMissingEntitlement, error, CFSTR("can't do both system and syncbubble keychain"));
1683         } else if (q->q_use_item_list) {
1684             ok = SecError(errSecUseItemListUnsupported, error, CFSTR("use item list unsupported"));
1685         } else if (q->q_match_issuer && ((q->q_class != cert_class()) &&
1686                     (q->q_class != identity_class()))) {
1687             ok = SecError(errSecUnsupportedOperation, error, CFSTR("unsupported match attribute"));
1688         } else if (q->q_match_policy && ((q->q_class != cert_class()) &&
1689                     (q->q_class != identity_class()))) {
1690             ok = SecError(errSecUnsupportedOperation, error, CFSTR("unsupported kSecMatchPolicy attribute"));
1691         } else if (q->q_return_type != 0 && result == NULL) {
1692             ok = SecError(errSecReturnMissingPointer, error, CFSTR("missing pointer"));
1693         } else if (!q->q_error) {
1694             ok = kc_with_dbt(false, error, ^(SecDbConnectionRef dbt) {
1695                 return s3dl_copy_matching(dbt, q, result, accessGroups, error);
1696             });
1697         }
1698
1699         if (!query_destroy(q, error))
1700             ok = false;
1701     }
1702     CFReleaseNull(accessGroups);
1703
1704     SecSignpostStop(SecSignpostSecItemCopyMatching);
1705
1706         return ok;
1707 }
1708
1709 bool
1710 _SecItemCopyMatching(CFDictionaryRef query, SecurityClient *client, CFTypeRef *result, CFErrorRef *error) {
1711     return SecItemServerCopyMatching(query, result, client, error);
1712 }
1713
1714 #if TARGET_OS_IPHONE
1715 static bool
1716 SecItemSynchronizable(CFDictionaryRef query)
1717 {
1718     bool result = false;
1719     CFTypeRef value = CFDictionaryGetValue(query, kSecAttrSynchronizable);
1720     if (isBoolean(value))
1721         return CFBooleanGetValue(value);
1722     else if (isNumber(value)) {
1723         SInt32 number = 0;
1724         (void)CFNumberGetValue(value, kCFNumberSInt32Type, &number);
1725         result = !!number;
1726     }
1727
1728     return result;
1729 }
1730 #endif
1731
1732 static CFArrayRef
1733 SecurityClientCopyWritableAccessGroups(SecurityClient *client) {
1734     if (client == NULL || client->accessGroups == NULL) {
1735         return NULL;
1736     }
1737     CFIndex count = CFArrayGetCount(client->accessGroups);
1738     if (CFArrayContainsValue(client->accessGroups, CFRangeMake(0, count), kSecAttrAccessGroupToken)) {
1739         CFMutableArrayRef writableGroups = CFArrayCreateMutableCopy(kCFAllocatorDefault, 0, client->accessGroups);
1740         CFArrayRemoveAllValue(writableGroups, kSecAttrAccessGroupToken);
1741         return writableGroups;
1742     } else {
1743         return CFRetainSafe(client->accessGroups);
1744     }
1745 }
1746
1747
1748 /* AUDIT[securityd](done):
1749    attributes (ok) is a caller provided dictionary, only its cf type has
1750        been checked.
1751  */
1752 bool
1753 _SecItemAdd(CFDictionaryRef attributes, SecurityClient *client, CFTypeRef *result, CFErrorRef *error)
1754 {
1755     CFArrayRef accessGroups = SecurityClientCopyWritableAccessGroups(client);
1756
1757     bool ok = true;
1758     CFIndex ag_count;
1759     if (!accessGroups || 0 == (ag_count = CFArrayGetCount(accessGroups))) {
1760         CFReleaseNull(accessGroups);
1761         return SecEntitlementError(error);
1762     }
1763
1764     SecSignpostStart(SecSignpostSecItemAdd);
1765
1766     Query *q = query_create_with_limit(attributes, client->musr, 0, error);
1767     if (q) {
1768         /* Access group sanity checking. */
1769         CFStringRef agrp = (CFStringRef)CFDictionaryGetValue(attributes,
1770             kSecAttrAccessGroup);
1771
1772         /* Having the special accessGroup "*" allows access to all accessGroups. */
1773         if (CFArrayContainsValue(accessGroups, CFRangeMake(0, ag_count), CFSTR("*")))
1774             CFReleaseNull(accessGroups);
1775
1776         if (agrp) {
1777             /* The user specified an explicit access group, validate it. */
1778             if (!accessGroupsAllows(accessGroups, agrp, client))
1779                 ok = SecEntitlementErrorForExplicitAccessGroup(agrp, accessGroups, error);
1780         } else {
1781             agrp = (CFStringRef)CFArrayGetValueAtIndex(client->accessGroups, 0);
1782
1783             /* We are using an implicit access group, add it as if the user
1784                specified it as an attribute. */
1785             query_add_attribute(kSecAttrAccessGroup, agrp, q);
1786         }
1787
1788         if (SecPLShouldLogRegisteredEvent(CFSTR("SecItem"))) {
1789             CFDictionaryRef dict = CFDictionaryCreateForCFTypes(NULL, CFSTR("operation"), CFSTR("add"), CFSTR("AccessGroup"), agrp, NULL);
1790             if (dict) {
1791                 SecPLLogRegisteredEvent(CFSTR("SecItem"), dict);
1792                 CFRelease(dict);
1793             }
1794         }
1795 #if TARGET_OS_IPHONE
1796         if (q->q_system_keychain && client->inMultiUser) {
1797             CFReleaseNull(q->q_musrView);
1798             q->q_musrView = SecMUSRCopySystemKeychainUUID();
1799         } else {
1800             q->q_system_keychain = false;
1801         }
1802         query_add_attribute_with_desc(&v8musr, q->q_musrView, q);
1803 #endif
1804
1805         if (ok) {
1806             query_ensure_access_control(q, agrp);
1807
1808 #if OCTAGON
1809             void (^add_sync_callback)(bool, CFErrorRef) = CFDictionaryGetValue(attributes, CFSTR("f_ckkscallback"));
1810             if(add_sync_callback) {
1811                 // The existence of this callback indicates that we need a predictable UUID for this item.
1812                 q->q_uuid_from_primary_key = true;
1813                 q->q_add_sync_callback = add_sync_callback;
1814             }
1815 #endif
1816
1817             if (q->q_system_keychain && !client->allowSystemKeychain) {
1818                 ok = SecError(errSecMissingEntitlement, error, CFSTR("client doesn't have entitlement for system keychain"));
1819             } else if (q->q_sync_bubble && !client->allowSyncBubbleKeychain) {
1820                 ok = SecError(errSecMissingEntitlement, error, CFSTR("client doesn't have entitlement for syncbubble keychain"));
1821 #if TARGET_OS_IPHONE
1822             } else if (q->q_system_keychain && SecItemSynchronizable(attributes) && !client->inMultiUser) {
1823                 ok = SecError(errSecInvalidKey, error, CFSTR("Can't store system keychain and synchronizable"));
1824 #endif
1825             } else if (q->q_row_id || q->q_token_object_id) {
1826                 ok = SecError(errSecValuePersistentRefUnsupported, error, CFSTR("q_row_id"));  // TODO: better error string
1827             } else if (!q->q_error) {
1828                 ok = kc_with_dbt(true, error, ^(SecDbConnectionRef dbt){
1829                     return kc_transaction(dbt, error, ^{
1830                         query_pre_add(q, true);
1831                         return s3dl_query_add(dbt, q, result, error);
1832                     });
1833                 });
1834             }
1835         }
1836         ok = query_notify_and_destroy(q, ok, error);
1837     } else {
1838         ok = false;
1839     }
1840     CFReleaseNull(accessGroups);
1841
1842     SecSignpostStop(SecSignpostSecItemAdd);
1843
1844     return ok;
1845 }
1846
1847 /* AUDIT[securityd](done):
1848    query (ok) and attributesToUpdate (ok) are a caller provided dictionaries,
1849        only their cf types have been checked.
1850  */
1851 bool
1852 _SecItemUpdate(CFDictionaryRef query, CFDictionaryRef attributesToUpdate,
1853                SecurityClient *client, CFErrorRef *error)
1854 {
1855     CFArrayRef accessGroups = SecurityClientCopyWritableAccessGroups(client);
1856
1857     CFIndex ag_count;
1858     if (!accessGroups || 0 == (ag_count = CFArrayGetCount(accessGroups))) {
1859         CFReleaseNull(accessGroups);
1860         return SecEntitlementError(error);
1861     }
1862
1863     // Queries using implicit access groups which only find items that're inaccessible yield errSecItemNotFound,
1864     // but we can pre-emptively shut down queries which are clearly illegal
1865     CFTypeRef q_agrp = CFDictionaryGetValue(query, kSecAttrAccessGroup);
1866     if (q_agrp && !accessGroupsAllows(accessGroups, q_agrp, client)) {
1867         SecEntitlementErrorForExplicitAccessGroup(q_agrp, accessGroups, error);
1868         CFReleaseSafe(accessGroups);
1869         return false;
1870     }
1871
1872     SecSignpostStart(SecSignpostSecItemUpdate);
1873
1874     if (SecPLShouldLogRegisteredEvent(CFSTR("SecItem"))) {
1875         CFTypeRef agrp = CFArrayGetValueAtIndex(accessGroups, 0);
1876         CFDictionaryRef dict = CFDictionaryCreateForCFTypes(NULL, CFSTR("operation"), CFSTR("update"), CFSTR("AccessGroup"), agrp, NULL);
1877         if (dict) {
1878             SecPLLogRegisteredEvent(CFSTR("SecItem"), dict);
1879             CFRelease(dict);
1880         }
1881     }
1882
1883     if (CFArrayContainsValue(accessGroups, CFRangeMake(0, ag_count), CFSTR("*"))) {
1884         /* Having the special accessGroup "*" allows access to all accessGroups. */
1885         CFReleaseNull(accessGroups);
1886     }
1887
1888     bool ok = true;
1889     Query *q = query_create_with_limit(query, client->musr, kSecMatchUnlimited, error);
1890     if (!q) {
1891         ok = false;
1892     }
1893     if (ok) {
1894 #if TARGET_OS_IPHONE
1895         if (q->q_system_keychain && client->inMultiUser) {
1896             CFReleaseNull(q->q_musrView);
1897             q->q_musrView = SecMUSRCopySystemKeychainUUID();
1898         } else {
1899             q->q_system_keychain = false;
1900         }
1901 #endif
1902
1903         /* Sanity check the query. */
1904         query_set_caller_access_groups(q, accessGroups);
1905         if (q->q_system_keychain && !client->allowSystemKeychain) {
1906             ok = SecError(errSecMissingEntitlement, error, CFSTR("client doesn't have entitlement for system keychain"));
1907         } else if (q->q_sync_bubble && !client->allowSyncBubbleKeychain) {
1908             ok = SecError(errSecMissingEntitlement, error, CFSTR("client doesn't have entitlement for syncbubble keychain"));
1909 #if TARGET_OS_IPHONE
1910         } else if (q->q_system_keychain && SecItemSynchronizable(attributesToUpdate) && !client->inMultiUser) {
1911             ok = SecError(errSecInvalidKey, error, CFSTR("Can't update an system keychain item with synchronizable"));
1912 #endif
1913         } else if (q->q_use_item_list) {
1914             ok = SecError(errSecUseItemListUnsupported, error, CFSTR("use item list not supported"));
1915         } else if (q->q_return_type & kSecReturnDataMask) {
1916             /* Update doesn't return anything so don't ask for it. */
1917             ok = SecError(errSecReturnDataUnsupported, error, CFSTR("return data not supported by update"));
1918         } else if (q->q_return_type & kSecReturnAttributesMask) {
1919             ok = SecError(errSecReturnAttributesUnsupported, error, CFSTR("return attributes not supported by update"));
1920         } else if (q->q_return_type & kSecReturnRefMask) {
1921             ok = SecError(errSecReturnRefUnsupported, error, CFSTR("return ref not supported by update"));
1922         } else if (q->q_return_type & kSecReturnPersistentRefMask) {
1923             ok = SecError(errSecReturnPersistentRefUnsupported, error, CFSTR("return persistent ref not supported by update"));
1924         } else {
1925             /* Access group sanity checking. */
1926             CFStringRef agrp = (CFStringRef)CFDictionaryGetValue(attributesToUpdate,
1927                 kSecAttrAccessGroup);
1928             if (agrp) {
1929                 /* The user is attempting to modify the access group column,
1930                    validate it to make sure the new value is allowable. */
1931                 if (!accessGroupsAllows(accessGroups, agrp, client)) {
1932                     secerror("Cannot update keychain item to access group %@", agrp);
1933                     ok = SecEntitlementErrorForExplicitAccessGroup(agrp, accessGroups, error);
1934                 }
1935             }
1936         }
1937     }
1938     if (ok) {
1939         ok = kc_with_dbt(true, error, ^(SecDbConnectionRef dbt) {
1940             return kc_transaction(dbt, error, ^{
1941                 return s3dl_query_update(dbt, q, attributesToUpdate, accessGroups, error);
1942             });
1943         });
1944     }
1945     if (q) {
1946         ok = query_notify_and_destroy(q, ok, error);
1947     }
1948     CFReleaseNull(accessGroups);
1949
1950     SecSignpostStop(SecSignpostSecItemUpdate);
1951
1952     return ok;
1953 }
1954
1955
1956 /* AUDIT[securityd](done):
1957    query (ok) is a caller provided dictionary, only its cf type has been checked.
1958  */
1959 bool
1960 _SecItemDelete(CFDictionaryRef query, SecurityClient *client, CFErrorRef *error)
1961 {
1962     CFArrayRef accessGroups = SecurityClientCopyWritableAccessGroups(client);
1963
1964     CFIndex ag_count;
1965     if (!accessGroups || 0 == (ag_count = CFArrayGetCount(accessGroups))) {
1966         CFReleaseNull(accessGroups);
1967         return SecEntitlementError(error);
1968     }
1969
1970     CFTypeRef q_agrp = CFDictionaryGetValue(query, kSecAttrAccessGroup);
1971     if (q_agrp && !accessGroupsAllows(accessGroups, q_agrp, client)) {
1972         SecEntitlementErrorForExplicitAccessGroup(q_agrp, accessGroups, error);
1973         CFReleaseSafe(accessGroups);
1974         return false;
1975     }
1976
1977     SecSignpostStart(SecSignpostSecItemDelete);
1978
1979     if (SecPLShouldLogRegisteredEvent(CFSTR("SecItem"))) {
1980         CFTypeRef agrp = CFArrayGetValueAtIndex(accessGroups, 0);
1981         CFDictionaryRef dict = CFDictionaryCreateForCFTypes(NULL, CFSTR("operation"), CFSTR("delete"), CFSTR("AccessGroup"), agrp, NULL);
1982         if (dict) {
1983             SecPLLogRegisteredEvent(CFSTR("SecItem"), dict);
1984             CFRelease(dict);
1985         }
1986     }
1987
1988     if (CFArrayContainsValue(accessGroups, CFRangeMake(0, ag_count), CFSTR("*"))) {
1989         /* Having the special accessGroup "*" allows access to all accessGroups. */
1990         CFReleaseNull(accessGroups);
1991     }
1992
1993     Query *q = query_create_with_limit(query, client->musr, kSecMatchUnlimited, error);
1994     bool ok;
1995     if (q) {
1996 #if TARGET_OS_IPHONE
1997         if (q->q_system_keychain && client->inMultiUser) {
1998             CFReleaseNull(q->q_musrView);
1999             q->q_musrView = SecMUSRCopySystemKeychainUUID();
2000         } else {
2001             q->q_system_keychain = false;
2002         }
2003 #endif
2004
2005         query_set_caller_access_groups(q, accessGroups);
2006         /* Sanity check the query. */
2007         if (q->q_system_keychain && !client->allowSystemKeychain) {
2008             ok = SecError(errSecMissingEntitlement, error, CFSTR("client doesn't have entitlement for system keychain"));
2009         } else if (q->q_sync_bubble && !client->allowSyncBubbleKeychain) {
2010             ok = SecError(errSecMissingEntitlement, error, CFSTR("client doesn't have entitlement for syncbubble keychain"));
2011         } else if (q->q_limit != kSecMatchUnlimited) {
2012             ok = SecError(errSecMatchLimitUnsupported, error, CFSTR("match limit not supported by delete"));
2013         } else if (query_match_count(q) != 0) {
2014             ok = SecError(errSecItemMatchUnsupported, error, CFSTR("match not supported by delete"));
2015         } else if (q->q_ref) {
2016             ok = SecError(errSecValueRefUnsupported, error, CFSTR("value ref not supported by delete"));
2017         } else if (q->q_row_id && query_attr_count(q)) {
2018             ok = SecError(errSecItemIllegalQuery, error, CFSTR("rowid and other attributes are mutually exclusive"));
2019         } else if (q->q_token_object_id && query_attr_count(q) != 1) {
2020             ok = SecError(errSecItemIllegalQuery, error, CFSTR("token persistent ref and other attributes are mutually exclusive"));
2021         } else {
2022             ok = kc_with_dbt(true, error, ^(SecDbConnectionRef dbt) {
2023                 return kc_transaction(dbt, error, ^{
2024                     return s3dl_query_delete(dbt, q, accessGroups, error);
2025                 });
2026             });
2027         }
2028         ok = query_notify_and_destroy(q, ok, error);
2029     } else {
2030         ok = false;
2031     }
2032     CFReleaseNull(accessGroups);
2033
2034     SecSignpostStop(SecSignpostSecItemDelete);
2035
2036     return ok;
2037 }
2038
2039 static bool SecItemDeleteTokenItems(SecDbConnectionRef dbt, CFTypeRef classToDelete, CFTypeRef tokenID, CFArrayRef accessGroups, SecurityClient *client, CFErrorRef *error) {
2040     CFTypeRef keys[] = { kSecClass, kSecAttrTokenID };
2041     CFTypeRef values[] = { classToDelete, tokenID };
2042
2043     CFDictionaryRef query = CFDictionaryCreate(kCFAllocatorDefault, keys, values, 2, &kCFTypeDictionaryKeyCallBacks, &kCFTypeDictionaryValueCallBacks);
2044     Query *q = query_create_with_limit(query, client->musr, kSecMatchUnlimited, error);
2045     CFRelease(query);
2046     bool ok;
2047     if (q) {
2048         query_set_caller_access_groups(q, accessGroups);
2049         ok = s3dl_query_delete(dbt, q, accessGroups, error);
2050         ok = query_notify_and_destroy(q, ok, error);
2051     } else {
2052         ok = false;
2053     }
2054
2055     return ok;
2056 }
2057
2058 static bool SecItemAddTokenItem(SecDbConnectionRef dbt, CFDictionaryRef attributes, CFArrayRef accessGroups, SecurityClient *client, CFErrorRef *error) {
2059     bool ok = true;
2060     Query *q = query_create_with_limit(attributes, client->musr, 0, error);
2061     if (q) {
2062         CFStringRef agrp = kSecAttrAccessGroupToken;
2063         query_add_attribute(kSecAttrAccessGroup, agrp, q);
2064
2065         if (ok) {
2066             query_ensure_access_control(q, agrp);
2067             if (q->q_system_keychain && !client->allowSystemKeychain) {
2068                 ok = SecError(errSecMissingEntitlement, error, CFSTR("client doesn't have entitlement for system keychain"));
2069             } else if (q->q_sync_bubble && !client->allowSyncBubbleKeychain) {
2070                 ok = SecError(errSecMissingEntitlement, error, CFSTR("client doesn't have entitlement for syncbubble keychain"));
2071             } else if (q->q_row_id || q->q_token_object_id) {
2072                 ok = SecError(errSecValuePersistentRefUnsupported, error, CFSTR("q_row_id"));  // TODO: better error string
2073             } else if (!q->q_error) {
2074                 query_pre_add(q, true);
2075                 ok = s3dl_query_add(dbt, q, NULL, error);
2076             }
2077         }
2078         ok = query_notify_and_destroy(q, ok, error);
2079     } else {
2080         return false;
2081     }
2082     return ok;
2083 }
2084
2085 bool _SecItemUpdateTokenItems(CFStringRef tokenID, CFArrayRef items, SecurityClient *client, CFErrorRef *error) {
2086     bool ok = true;
2087     CFArrayRef accessGroups = client->accessGroups;
2088     CFIndex ag_count;
2089     if (!accessGroups || 0 == (ag_count = CFArrayGetCount(accessGroups))) {
2090         return SecEntitlementError(error);
2091     }
2092
2093     ok = kc_with_dbt(true, error, ^bool (SecDbConnectionRef dbt) {
2094         return kc_transaction(dbt, error, ^bool {
2095             if (items) {
2096                 const CFTypeRef classToDelete[] = { kSecClassGenericPassword, kSecClassInternetPassword, kSecClassCertificate, kSecClassKey };
2097                 for (size_t i = 0; i < sizeof(classToDelete) / sizeof(classToDelete[0]); ++i) {
2098                     SecItemDeleteTokenItems(dbt, classToDelete[i], tokenID, accessGroups, client, NULL);
2099                 }
2100
2101                 for (CFIndex i = 0; i < CFArrayGetCount(items); ++i) {
2102                     if (!SecItemAddTokenItem(dbt, CFArrayGetValueAtIndex(items, i), accessGroups, client, error))
2103                         return false;
2104                 }
2105                 return true;
2106             }
2107             else {
2108                 const CFTypeRef classToDelete[] = { kSecClassGenericPassword, kSecClassInternetPassword, kSecClassCertificate, kSecClassKey };
2109                 bool deleted = true;
2110                 for (size_t i = 0; i < sizeof(classToDelete) / sizeof(classToDelete[0]); ++i) {
2111                     if (!SecItemDeleteTokenItems(dbt, classToDelete[i], tokenID, accessGroups, client, error) && error && CFErrorGetCode(*error) != errSecItemNotFound) {
2112                         deleted = false;
2113                         break;
2114                     }
2115                     else if (error && *error) {
2116                         CFReleaseNull(*error);
2117                     }
2118                 }
2119                 return deleted;
2120             }
2121         });
2122     });
2123
2124     return ok;
2125 }
2126
2127 static bool deleteNonSysboundItemsForItemClass(SecDbConnectionRef dbt, SecDbClass const* class, CFErrorRef* error) {
2128     CFMutableDictionaryRef query = CFDictionaryCreateMutableForCFTypes(NULL);
2129     CFDictionaryAddValue(query, kSecMatchLimit, kSecMatchLimitAll);
2130
2131     __block CFErrorRef localError = NULL;
2132     SecDbQueryRef q = query_create(class, NULL, query, &localError);
2133     if (q == NULL) {    // illegal query or out of memory
2134         secerror("SecItemServerDeleteAll: aborting because failed to initialize Query: %@", localError);
2135         abort();
2136     }
2137     SecDbItemSelect(q, dbt, &localError, ^bool(const SecDbAttr *attr) {
2138         return (attr->flags & kSecDbInFlag) && !CFEqual(attr->name, CFSTR("data"));
2139     }, NULL, NULL, NULL,
2140     ^(SecDbItemRef item, bool *stop) {
2141         if (!SecItemIsSystemBound(item->attributes, class, false) &&
2142             !CFEqual(CFDictionaryGetValue(item->attributes, kSecAttrAccessGroup), CFSTR("com.apple.bluetooth")))
2143         {
2144             SecDbItemDelete(item, dbt, kCFBooleanFalse, &localError);
2145         }
2146     });
2147     query_destroy(q, &localError);
2148
2149     if (localError) {
2150         if (error) {
2151             CFReleaseNull(*error);
2152             *error = localError;
2153         } else {
2154             CFReleaseNull(localError);
2155         }
2156         return false;
2157     }
2158     return true;
2159 }
2160
2161 // Delete all the items except sysbound ones because horrible things happen if you do, like bluetooth devices unpairing
2162 static bool
2163 SecItemServerDeleteAll(CFErrorRef *error) {
2164     secerror("SecItemServerDeleteAll");
2165     return kc_with_dbt(true, error, ^bool (SecDbConnectionRef dbt) {
2166         return (kc_transaction(dbt, error, ^bool {
2167             bool ok = true;
2168             ok &= SecDbExec(dbt, CFSTR("DELETE FROM genp WHERE sync=1;"), error);
2169             ok &= SecDbExec(dbt, CFSTR("DELETE FROM inet WHERE sync=1;"), error);
2170             ok &= SecDbExec(dbt, CFSTR("DELETE FROM cert WHERE sync=1;"), error);
2171             ok &= SecDbExec(dbt, CFSTR("DELETE FROM keys WHERE sync=1;"), error);
2172
2173             ok &= deleteNonSysboundItemsForItemClass(dbt, genp_class(), error);
2174             ok &= deleteNonSysboundItemsForItemClass(dbt, inet_class(), error);
2175             ok &= deleteNonSysboundItemsForItemClass(dbt, cert_class(), error);
2176             ok &= deleteNonSysboundItemsForItemClass(dbt, keys_class(), error);
2177
2178             return ok;
2179         }) && SecDbExec(dbt, CFSTR("VACUUM;"), error));
2180     });
2181 }
2182
2183 bool
2184 _SecItemDeleteAll(CFErrorRef *error) {
2185     return SecItemServerDeleteAll(error);
2186 }
2187
2188 bool
2189 _SecItemServerDeleteAllWithAccessGroups(CFArrayRef accessGroups, SecurityClient *client, CFErrorRef *error)
2190 {
2191     __block bool ok = true;
2192     static dispatch_once_t onceToken;
2193     static CFSetRef illegalAccessGroups = NULL;
2194
2195     dispatch_once(&onceToken, ^{
2196         const CFStringRef values[] = {
2197             CFSTR("*"),
2198             CFSTR("apple"),
2199             CFSTR("com.apple.security.sos"),
2200             CFSTR("lockdown-identities"),
2201         };
2202         illegalAccessGroups = CFSetCreate(NULL, (const void **)values, sizeof(values)/sizeof(values[0]), &kCFTypeSetCallBacks);
2203     });
2204
2205     static CFTypeRef qclasses[] = {
2206         NULL,
2207         NULL,
2208         NULL,
2209         NULL
2210     };
2211     // strange construction needed for schema indirection
2212     static dispatch_once_t qclassesOnceToken;
2213     dispatch_once(&qclassesOnceToken, ^{
2214         qclasses[0] = inet_class();
2215         qclasses[1] = genp_class();
2216         qclasses[2] = keys_class();
2217         qclasses[3] = cert_class();
2218     });
2219
2220     require_action_quiet(isArray(accessGroups), fail,
2221                          ok = false;
2222                          SecCFCreateErrorWithFormat(kSecXPCErrorUnexpectedType, sSecXPCErrorDomain, NULL, error, NULL, CFSTR("accessGroups not CFArray, got %@"), accessGroups));
2223
2224     // TODO: whitelist instead? look for dev IDs like 7123498YQX.com.somedev.app
2225
2226     require_action(CFArrayGetCount(accessGroups) != 0, fail,
2227                    ok = false;
2228                    SecCFCreateErrorWithFormat(kSecXPCErrorUnexpectedType, sSecXPCErrorDomain, NULL, error, NULL, CFSTR("accessGroups e empty")));
2229
2230
2231     // Pre-check accessGroups for prohibited values
2232     CFArrayForEach(accessGroups, ^(const void *value) {
2233         CFStringRef agrp = (CFStringRef)value;
2234
2235         if (!isString(agrp)) {
2236             SecCFCreateErrorWithFormat(kSecXPCErrorUnexpectedType, sSecXPCErrorDomain, NULL, error, NULL,
2237                                        CFSTR("access not a string: %@"), agrp);
2238             ok &= false;
2239         } else if (CFSetContainsValue(illegalAccessGroups, agrp)) {
2240             SecCFCreateErrorWithFormat(kSecXPCErrorUnexpectedType, sSecXPCErrorDomain, NULL, error, NULL,
2241                                        CFSTR("illegal access group: %@"), accessGroups);
2242             ok &= false;
2243         }
2244     });
2245     require(ok,fail);
2246
2247     ok = kc_with_dbt(true, error, ^bool(SecDbConnectionRef dbt) {
2248         return kc_transaction(dbt, error, ^bool {
2249             CFErrorRef localError = NULL;
2250             bool ok1 = true;
2251             size_t n;
2252
2253             for (n = 0; n < sizeof(qclasses)/sizeof(qclasses[0]) && ok1; n++) {
2254                 Query *q;
2255
2256                 q = query_create(qclasses[n], client->musr, NULL, error);
2257                 require(q, fail2);
2258
2259                 (void)s3dl_query_delete(dbt, q, accessGroups, &localError);
2260             fail2:
2261                 query_destroy(q, error);
2262                 CFReleaseNull(localError);
2263             }
2264             return ok1;
2265         }) && SecDbExec(dbt, CFSTR("VACUUM"), error);
2266     });
2267
2268 fail:
2269     return ok;
2270 }
2271
2272
2273 // MARK: -
2274 // MARK: Shared web credentials
2275
2276 #if SHAREDWEBCREDENTIALS
2277
2278 /* constants */
2279 #define SEC_CONST_DECL(k,v) const CFStringRef k = CFSTR(v);
2280
2281 SEC_CONST_DECL (kSecSafariAccessGroup, "com.apple.cfnetwork");
2282 SEC_CONST_DECL (kSecSafariDefaultComment, "default");
2283 SEC_CONST_DECL (kSecSafariPasswordsNotSaved, "Passwords not saved");
2284 SEC_CONST_DECL (kSecSharedCredentialUrlScheme, "https://");
2285 SEC_CONST_DECL (kSecSharedWebCredentialsService, "webcredentials");
2286
2287 #if !TARGET_OS_SIMULATOR
2288 static SWCFlags
2289 _SecAppDomainApprovalStatus(CFStringRef appID, CFStringRef fqdn, CFErrorRef *error)
2290 {
2291     __block SWCFlags flags = kSWCFlags_None;
2292     OSStatus status;
2293
2294     secnotice("swc", "Application %@ is requesting approval for %@", appID, fqdn);
2295
2296     dispatch_semaphore_t semaphore = dispatch_semaphore_create(0);
2297     if (semaphore == NULL)
2298         return 0;
2299
2300     status = SWCCheckService(kSecSharedWebCredentialsService, appID, fqdn, ^void (OSStatus inStatus, SWCFlags inFlags, CFDictionaryRef inDetails)
2301     {
2302         if (inStatus == 0) {
2303             flags = inFlags;
2304         } else {
2305             secerror("SWCCheckService failed with %d", (int)inStatus);
2306         }
2307         dispatch_semaphore_signal(semaphore);
2308     });
2309
2310     if (status == 0) {
2311         dispatch_semaphore_wait(semaphore, DISPATCH_TIME_FOREVER);
2312     } else {
2313         secerror("SWCCheckService: failed to queue");
2314     }
2315     dispatch_release(semaphore);
2316
2317     if (error) {
2318         if (!(flags & kSWCFlag_SiteApproved)) {
2319             SecError(errSecAuthFailed, error, CFSTR("\"%@\" failed to approve \"%@\""), fqdn, appID);
2320         } else if (flags & kSWCFlag_UserDenied) {
2321             SecError(errSecAuthFailed, error, CFSTR("User denied access to \"%@\" by \"%@\""), fqdn, appID);
2322         }
2323     }
2324     return flags;
2325 }
2326
2327 static bool
2328 _SecEntitlementContainsDomainForService(CFArrayRef domains, CFStringRef domain, CFStringRef service)
2329 {
2330     bool result = false;
2331     CFIndex idx, count = (domains) ? CFArrayGetCount(domains) : (CFIndex) 0;
2332     if (!count || !domain || !service) {
2333         return result;
2334     }
2335     for (idx=0; idx < count; idx++) {
2336         CFStringRef str = (CFStringRef) CFArrayGetValueAtIndex(domains, idx);
2337         if (str && CFStringHasPrefix(str, kSecSharedWebCredentialsService)) {
2338             CFIndex prefix_len = CFStringGetLength(kSecSharedWebCredentialsService)+1;
2339             CFIndex substr_len = CFStringGetLength(str) - prefix_len;
2340             CFRange range = { prefix_len, substr_len };
2341             CFStringRef substr = CFStringCreateWithSubstring(kCFAllocatorDefault, str, range);
2342             if (substr && CFEqual(substr, domain)) {
2343                 result = true;
2344             }
2345             CFReleaseSafe(substr);
2346             if (result) {
2347                 break;
2348             }
2349         }
2350     }
2351     return result;
2352 }
2353 #endif /* !TARGET_OS_SIMULATOR */
2354
2355 static bool
2356 _SecAddNegativeWebCredential(SecurityClient *client, CFStringRef fqdn, CFStringRef appID, bool forSafari)
2357 {
2358 #if !TARGET_OS_SIMULATOR
2359     bool result = false;
2360     if (!fqdn) { return result; }
2361
2362     // update our database
2363     CFRetainSafe(appID);
2364     CFRetainSafe(fqdn);
2365     if (0 == SWCSetServiceFlags(kSecSharedWebCredentialsService, appID, fqdn, kSWCFlag_ExternalMask, kSWCFlag_UserDenied,
2366         ^void(OSStatus inStatus, SWCFlags inNewFlags){
2367             CFReleaseSafe(appID);
2368             CFReleaseSafe(fqdn);
2369         }))
2370     {
2371         result = true;
2372     }
2373     else // didn't queue the block
2374     {
2375         CFReleaseSafe(appID);
2376         CFReleaseSafe(fqdn);
2377     }
2378
2379     if (!forSafari) { return result; }
2380
2381     // below this point: create a negative Safari web credential item
2382
2383     CFMutableDictionaryRef attrs = CFDictionaryCreateMutable(kCFAllocatorDefault, 0, &kCFTypeDictionaryKeyCallBacks, &kCFTypeDictionaryValueCallBacks);
2384     if (!attrs) { return result; }
2385
2386     CFErrorRef error = NULL;
2387     CFStringRef accessGroup = CFSTR("*");
2388     SecurityClient swcclient = {
2389         .task = NULL,
2390         .accessGroups =  CFArrayCreate(kCFAllocatorDefault, (const void **)&accessGroup, 1, &kCFTypeArrayCallBacks),
2391         .allowSystemKeychain = false,
2392         .allowSyncBubbleKeychain = false,
2393         .isNetworkExtension = false,
2394         .musr = client->musr,
2395     };
2396
2397     CFDictionaryAddValue(attrs, kSecClass, kSecClassInternetPassword);
2398     CFDictionaryAddValue(attrs, kSecAttrAccessGroup, kSecSafariAccessGroup);
2399     CFDictionaryAddValue(attrs, kSecAttrAuthenticationType, kSecAttrAuthenticationTypeHTMLForm);
2400     CFDictionaryAddValue(attrs, kSecAttrProtocol, kSecAttrProtocolHTTPS);
2401     CFDictionaryAddValue(attrs, kSecAttrServer, fqdn);
2402     CFDictionaryAddValue(attrs, kSecAttrSynchronizable, kCFBooleanTrue);
2403
2404     (void)_SecItemDelete(attrs, &swcclient, &error);
2405     CFReleaseNull(error);
2406
2407     CFDictionaryAddValue(attrs, kSecAttrAccount, kSecSafariPasswordsNotSaved);
2408     CFDictionaryAddValue(attrs, kSecAttrComment, kSecSafariDefaultComment);
2409
2410     CFStringRef label = CFStringCreateWithFormat(kCFAllocatorDefault,
2411                                                  NULL, CFSTR("%@ (%@)"), fqdn, kSecSafariPasswordsNotSaved);
2412     if (label) {
2413         CFDictionaryAddValue(attrs, kSecAttrLabel, label);
2414         CFReleaseSafe(label);
2415     }
2416
2417     UInt8 space = ' ';
2418     CFDataRef data = CFDataCreate(kCFAllocatorDefault, &space, 1);
2419     if (data) {
2420         CFDictionarySetValue(attrs, kSecValueData, data);
2421         CFReleaseSafe(data);
2422     }
2423
2424     CFTypeRef addResult = NULL;
2425     result = _SecItemAdd(attrs, &swcclient, &addResult, &error);
2426
2427     CFReleaseSafe(addResult);
2428     CFReleaseSafe(error);
2429     CFReleaseSafe(attrs);
2430     CFReleaseSafe(swcclient.accessGroups);
2431
2432     return result;
2433 #else
2434     return true;
2435 #endif
2436 }
2437
2438 /* Specialized version of SecItemAdd for shared web credentials */
2439 bool
2440 _SecAddSharedWebCredential(CFDictionaryRef attributes,
2441                            SecurityClient *client,
2442                            const audit_token_t *clientAuditToken,
2443                            CFStringRef appID,
2444                            CFArrayRef domains,
2445                            CFTypeRef *result,
2446                            CFErrorRef *error)
2447 {
2448
2449     SecurityClient swcclient = {};
2450
2451     CFStringRef fqdn = CFRetainSafe(CFDictionaryGetValue(attributes, kSecAttrServer));
2452     CFStringRef account = CFDictionaryGetValue(attributes, kSecAttrAccount);
2453 #if TARGET_OS_IOS && !TARGET_OS_BRIDGE
2454     CFStringRef password = CFDictionaryGetValue(attributes, kSecSharedPassword);
2455 #else
2456     CFStringRef password = CFDictionaryGetValue(attributes, CFSTR("spwd"));
2457 #endif
2458     CFStringRef accessGroup = CFSTR("*");
2459     CFMutableDictionaryRef query = NULL, attrs = NULL;
2460     SInt32 port = -1;
2461     bool ok = false;
2462
2463     // check autofill enabled status
2464     if (!swca_autofill_enabled(clientAuditToken)) {
2465         SecError(errSecBadReq, error, CFSTR("Autofill is not enabled in Safari settings"));
2466         goto cleanup;
2467     }
2468
2469     // parse fqdn with CFURL here, since it could be specified as domain:port
2470     if (fqdn) {
2471         CFStringRef urlStr = CFStringCreateWithFormat(kCFAllocatorDefault, NULL, CFSTR("%@%@"), kSecSharedCredentialUrlScheme, fqdn);
2472         if (urlStr) {
2473             CFURLRef url = CFURLCreateWithString(kCFAllocatorDefault, urlStr, nil);
2474             if (url) {
2475                 CFStringRef hostname = CFURLCopyHostName(url);
2476                 if (hostname) {
2477                     CFReleaseSafe(fqdn);
2478                     fqdn = hostname;
2479                     port = CFURLGetPortNumber(url);
2480                 }
2481                 CFReleaseSafe(url);
2482             }
2483             CFReleaseSafe(urlStr);
2484         }
2485     }
2486
2487     if (!account) {
2488         SecError(errSecParam, error, CFSTR("No account provided"));
2489         goto cleanup;
2490     }
2491     if (!fqdn) {
2492         SecError(errSecParam, error, CFSTR("No domain provided"));
2493         goto cleanup;
2494     }
2495
2496 #if TARGET_OS_SIMULATOR
2497     secerror("app/site association entitlements not checked in Simulator");
2498 #else
2499     OSStatus status = errSecMissingEntitlement;
2500     // validate that fqdn is part of caller's shared credential domains entitlement
2501     if (!appID) {
2502         SecError(status, error, CFSTR("Missing application-identifier entitlement"));
2503         goto cleanup;
2504     }
2505     if (_SecEntitlementContainsDomainForService(domains, fqdn, kSecSharedWebCredentialsService)) {
2506         status = errSecSuccess;
2507     }
2508     if (errSecSuccess != status) {
2509         CFStringRef msg = CFStringCreateWithFormat(kCFAllocatorDefault, NULL,
2510                                                    CFSTR("%@ not found in %@ entitlement"), fqdn, kSecEntitlementAssociatedDomains);
2511         if (!msg) {
2512             msg = CFRetain(CFSTR("Requested domain not found in entitlement"));
2513         }
2514         SecError(status, error, CFSTR("%@"), msg);
2515         CFReleaseSafe(msg);
2516         goto cleanup;
2517     }
2518 #endif
2519
2520 #if TARGET_OS_SIMULATOR
2521     secerror("Ignoring app/site approval state in the Simulator.");
2522 #else
2523     // get approval status for this app/domain pair
2524     SWCFlags flags = _SecAppDomainApprovalStatus(appID, fqdn, error);
2525     if (!(flags & kSWCFlag_SiteApproved)) {
2526         goto cleanup;
2527     }
2528 #endif
2529
2530     // give ourselves access to see matching items for kSecSafariAccessGroup
2531     swcclient.task = NULL;
2532     swcclient.accessGroups =  CFArrayCreate(kCFAllocatorDefault, (const void **)&accessGroup, 1, &kCFTypeArrayCallBacks);
2533     swcclient.allowSystemKeychain = false;
2534     swcclient.musr = client->musr;
2535     swcclient.allowSystemKeychain = false;
2536     swcclient.allowSyncBubbleKeychain = false;
2537     swcclient.isNetworkExtension = false;
2538
2539
2540     // create lookup query
2541     query = CFDictionaryCreateMutable(kCFAllocatorDefault, 0, &kCFTypeDictionaryKeyCallBacks, &kCFTypeDictionaryValueCallBacks);
2542     if (!query) {
2543         SecError(errSecAllocate, error, CFSTR("Unable to create query dictionary"));
2544         goto cleanup;
2545     }
2546     CFDictionaryAddValue(query, kSecClass, kSecClassInternetPassword);
2547     CFDictionaryAddValue(query, kSecAttrAccessGroup, kSecSafariAccessGroup);
2548     CFDictionaryAddValue(query, kSecAttrAuthenticationType, kSecAttrAuthenticationTypeHTMLForm);
2549     CFDictionaryAddValue(query, kSecAttrServer, fqdn);
2550     CFDictionaryAddValue(query, kSecAttrSynchronizable, kCFBooleanTrue);
2551
2552     // check for presence of Safari's negative entry ('passwords not saved')
2553     CFDictionarySetValue(query, kSecAttrAccount, kSecSafariPasswordsNotSaved);
2554     ok = _SecItemCopyMatching(query, &swcclient, result, error);
2555     if(result) CFReleaseNull(*result);
2556     if (error) CFReleaseNull(*error);
2557     if (ok) {
2558         SecError(errSecDuplicateItem, error, CFSTR("Item already exists for this server"));
2559         goto cleanup;
2560     }
2561
2562     // now use the provided account (and optional port number, if one was present)
2563     CFDictionarySetValue(query, kSecAttrAccount, account);
2564     if (port < -1 || port > 0) {
2565         SInt16 portValueShort = (port & 0xFFFF);
2566         CFNumberRef portNumber = CFNumberCreate(NULL, kCFNumberSInt16Type, &portValueShort);
2567         CFDictionaryAddValue(query, kSecAttrPort, portNumber);
2568         CFReleaseSafe(portNumber);
2569     }
2570
2571     // look up existing password
2572     CFDictionaryAddValue(query, kSecReturnData, kCFBooleanTrue);
2573     bool matched = _SecItemCopyMatching(query, &swcclient, result, error);
2574     CFDictionaryRemoveValue(query, kSecReturnData);
2575     if (matched) {
2576         // found it, so this becomes either an "update password" or "delete password" operation
2577         bool update = (password != NULL);
2578         if (update) {
2579             attrs = CFDictionaryCreateMutable(kCFAllocatorDefault, 0, &kCFTypeDictionaryKeyCallBacks, &kCFTypeDictionaryValueCallBacks);
2580             CFDataRef credential = CFStringCreateExternalRepresentation(kCFAllocatorDefault, password, kCFStringEncodingUTF8, 0);
2581             CFDictionaryAddValue(attrs, kSecValueData, credential);
2582             bool samePassword = result && *result && CFEqual(*result, credential);
2583             CFReleaseSafe(credential);
2584             CFDictionaryAddValue(attrs, kSecAttrComment, kSecSafariDefaultComment);
2585
2586             ok = samePassword || swca_confirm_operation(swca_update_request_id, clientAuditToken, query, error,
2587                 ^void (CFStringRef confirm_fqdn) {
2588                     _SecAddNegativeWebCredential(client, confirm_fqdn, appID, false);
2589                 });
2590             if (ok) {
2591                 ok = _SecItemUpdate(query, attrs, &swcclient, error);
2592             }
2593         }
2594         else {
2595             // confirm the delete
2596             // (per rdar://16676288 we always prompt, even if there was prior user approval)
2597             ok = /*approved ||*/ swca_confirm_operation(swca_delete_request_id, clientAuditToken, query, error,
2598                 ^void (CFStringRef confirm_fqdn) {
2599                     _SecAddNegativeWebCredential(client, confirm_fqdn, appID, false);
2600                 });
2601             if (ok) {
2602                 ok = _SecItemDelete(query, &swcclient, error);
2603             }
2604         }
2605
2606         if(result) CFReleaseNull(*result);
2607         if(error) CFReleaseNull(*error);
2608
2609         goto cleanup;
2610     }
2611     if (result) CFReleaseNull(*result);
2612     if (error) CFReleaseNull(*error);
2613
2614     // password does not exist, so prepare to add it
2615     if (!password) {
2616         // a NULL password value removes the existing credential. Since we didn't find it, this is a no-op.
2617         ok = true;
2618         goto cleanup;
2619     }
2620     else {
2621         CFStringRef label = CFStringCreateWithFormat(kCFAllocatorDefault, NULL, CFSTR("%@ (%@)"), fqdn, account);
2622         if (label) {
2623             CFDictionaryAddValue(query, kSecAttrLabel, label);
2624             CFReleaseSafe(label);
2625         }
2626         // NOTE: we always expect to use HTTPS for web forms.
2627         CFDictionaryAddValue(query, kSecAttrProtocol, kSecAttrProtocolHTTPS);
2628
2629         CFDataRef credential = CFStringCreateExternalRepresentation(kCFAllocatorDefault, password, kCFStringEncodingUTF8, 0);
2630         CFDictionarySetValue(query, kSecValueData, credential);
2631         CFReleaseSafe(credential);
2632         CFDictionarySetValue(query, kSecAttrComment, kSecSafariDefaultComment);
2633
2634         CFReleaseSafe(swcclient.accessGroups);
2635         swcclient.accessGroups = CFArrayCreate(kCFAllocatorDefault, (const void **)&kSecSafariAccessGroup, 1, &kCFTypeArrayCallBacks);
2636
2637         // mark the item as created by this function
2638         const int32_t creator_value = 'swca';
2639         CFNumberRef creator = CFNumberCreate(kCFAllocatorDefault, kCFNumberSInt32Type, &creator_value);
2640         if (creator) {
2641             CFDictionarySetValue(query, kSecAttrCreator, creator);
2642             CFReleaseSafe(creator);
2643         }
2644
2645         // confirm the add
2646         ok = swca_confirm_operation(swca_add_request_id, clientAuditToken, query, error, ^void (CFStringRef confirm_fqdn) {
2647             _SecAddNegativeWebCredential(client, confirm_fqdn, appID, false);
2648         });
2649     }
2650     if (ok) {
2651         ok = _SecItemAdd(query, &swcclient, result, error);
2652     }
2653
2654 cleanup:
2655     CFReleaseSafe(attrs);
2656     CFReleaseSafe(query);
2657     CFReleaseSafe(swcclient.accessGroups);
2658     CFReleaseSafe(fqdn);
2659     return ok;
2660 }
2661
2662 /* Specialized version of SecItemCopyMatching for shared web credentials */
2663 bool
2664 _SecCopySharedWebCredential(CFDictionaryRef query,
2665                             SecurityClient *client,
2666                             const audit_token_t *clientAuditToken,
2667                             CFStringRef appID,
2668                             CFArrayRef domains,
2669                             CFTypeRef *result,
2670                             CFErrorRef *error)
2671 {
2672     CFMutableArrayRef credentials = NULL;
2673     CFMutableArrayRef foundItems = NULL;
2674     CFMutableArrayRef fqdns = NULL;
2675     CFStringRef fqdn = NULL;
2676     CFStringRef account = NULL;
2677     SInt32 port = -1;
2678     bool ok = false;
2679
2680     require_quiet(result, cleanup);
2681     credentials = CFArrayCreateMutable(kCFAllocatorDefault, 0, &kCFTypeArrayCallBacks);
2682     foundItems = CFArrayCreateMutable(kCFAllocatorDefault, 0, &kCFTypeArrayCallBacks);
2683     fqdns = CFArrayCreateMutable(kCFAllocatorDefault, 0, &kCFTypeArrayCallBacks);
2684
2685     // give ourselves access to see matching items for kSecSafariAccessGroup
2686     CFStringRef accessGroup = CFSTR("*");
2687     SecurityClient swcclient = {
2688         .task = NULL,
2689         .accessGroups =  CFArrayCreate(kCFAllocatorDefault, (const void **)&accessGroup, 1, &kCFTypeArrayCallBacks),
2690         .allowSystemKeychain = false,
2691         .allowSyncBubbleKeychain = false,
2692         .isNetworkExtension = false,
2693         .musr = client->musr,
2694     };
2695
2696     // On input, the query dictionary contains optional fqdn and account entries.
2697     fqdn = CFDictionaryGetValue(query, kSecAttrServer);
2698     account = CFDictionaryGetValue(query, kSecAttrAccount);
2699
2700     // Check autofill enabled status
2701     if (!swca_autofill_enabled(clientAuditToken)) {
2702         SecError(errSecBadReq, error, CFSTR("Autofill is not enabled in Safari settings"));
2703         goto cleanup;
2704     }
2705
2706     // Check fqdn; if NULL, add domains from caller's entitlement.
2707     if (fqdn) {
2708         CFArrayAppendValue(fqdns, fqdn);
2709     }
2710     else if (domains) {
2711         CFIndex idx, count = CFArrayGetCount(domains);
2712         for (idx=0; idx < count; idx++) {
2713             CFStringRef str = (CFStringRef) CFArrayGetValueAtIndex(domains, idx);
2714             // Parse the entry for our service label prefix
2715             if (str && CFStringHasPrefix(str, kSecSharedWebCredentialsService)) {
2716                 CFIndex prefix_len = CFStringGetLength(kSecSharedWebCredentialsService)+1;
2717                 CFIndex substr_len = CFStringGetLength(str) - prefix_len;
2718                 CFRange range = { prefix_len, substr_len };
2719                 fqdn = CFStringCreateWithSubstring(kCFAllocatorDefault, str, range);
2720                 if (fqdn) {
2721                     CFArrayAppendValue(fqdns, fqdn);
2722                     CFRelease(fqdn);
2723                 }
2724             }
2725         }
2726     }
2727     CFIndex count, idx;
2728
2729     count = CFArrayGetCount(fqdns);
2730     if (count < 1) {
2731         SecError(errSecParam, error, CFSTR("No domain provided"));
2732         goto cleanup;
2733     }
2734
2735     // Aggregate search results for each domain
2736     for (idx = 0; idx < count; idx++) {
2737         CFMutableArrayRef items = NULL;
2738         CFMutableDictionaryRef attrs = NULL;
2739         fqdn = (CFStringRef) CFArrayGetValueAtIndex(fqdns, idx);
2740         CFRetainSafe(fqdn);
2741         port = -1;
2742
2743         // Parse the fqdn for a possible port specifier.
2744         if (fqdn) {
2745             CFStringRef urlStr = CFStringCreateWithFormat(kCFAllocatorDefault, NULL, CFSTR("%@%@"), kSecSharedCredentialUrlScheme, fqdn);
2746             if (urlStr) {
2747                 CFURLRef url = CFURLCreateWithString(kCFAllocatorDefault, urlStr, nil);
2748                 if (url) {
2749                     CFStringRef hostname = CFURLCopyHostName(url);
2750                     if (hostname) {
2751                         CFReleaseSafe(fqdn);
2752                         fqdn = hostname;
2753                         port = CFURLGetPortNumber(url);
2754                     }
2755                     CFReleaseSafe(url);
2756                 }
2757                 CFReleaseSafe(urlStr);
2758             }
2759         }
2760
2761 #if TARGET_OS_SIMULATOR
2762         secerror("app/site association entitlements not checked in Simulator");
2763 #else
2764             OSStatus status = errSecMissingEntitlement;
2765         if (!appID) {
2766             SecError(status, error, CFSTR("Missing application-identifier entitlement"));
2767             CFReleaseSafe(fqdn);
2768             goto cleanup;
2769         }
2770         // validate that fqdn is part of caller's entitlement
2771         if (_SecEntitlementContainsDomainForService(domains, fqdn, kSecSharedWebCredentialsService)) {
2772             status = errSecSuccess;
2773         }
2774         if (errSecSuccess != status) {
2775             CFStringRef msg = CFStringCreateWithFormat(kCFAllocatorDefault, NULL,
2776                 CFSTR("%@ not found in %@ entitlement"), fqdn, kSecEntitlementAssociatedDomains);
2777             if (!msg) {
2778                 msg = CFRetain(CFSTR("Requested domain not found in entitlement"));
2779             }
2780             SecError(status, error, CFSTR("%@"), msg);
2781             CFReleaseSafe(msg);
2782             CFReleaseSafe(fqdn);
2783             goto cleanup;
2784         }
2785 #endif
2786
2787         attrs = CFDictionaryCreateMutable(kCFAllocatorDefault, 0, &kCFTypeDictionaryKeyCallBacks, &kCFTypeDictionaryValueCallBacks);
2788         if (!attrs) {
2789             SecError(errSecAllocate, error, CFSTR("Unable to create query dictionary"));
2790             CFReleaseSafe(fqdn);
2791             goto cleanup;
2792         }
2793         CFDictionaryAddValue(attrs, kSecClass, kSecClassInternetPassword);
2794         CFDictionaryAddValue(attrs, kSecAttrAccessGroup, kSecSafariAccessGroup);
2795         CFDictionaryAddValue(attrs, kSecAttrProtocol, kSecAttrProtocolHTTPS);
2796         CFDictionaryAddValue(attrs, kSecAttrAuthenticationType, kSecAttrAuthenticationTypeHTMLForm);
2797         CFDictionaryAddValue(attrs, kSecAttrServer, fqdn);
2798         if (account) {
2799             CFDictionaryAddValue(attrs, kSecAttrAccount, account);
2800         }
2801         if (port < -1 || port > 0) {
2802             SInt16 portValueShort = (port & 0xFFFF);
2803             CFNumberRef portNumber = CFNumberCreate(NULL, kCFNumberSInt16Type, &portValueShort);
2804             CFDictionaryAddValue(attrs, kSecAttrPort, portNumber);
2805             CFReleaseSafe(portNumber);
2806         }
2807         CFDictionaryAddValue(attrs, kSecAttrSynchronizable, kCFBooleanTrue);
2808         CFDictionaryAddValue(attrs, kSecMatchLimit, kSecMatchLimitAll);
2809         CFDictionaryAddValue(attrs, kSecReturnAttributes, kCFBooleanTrue);
2810         CFDictionaryAddValue(attrs, kSecReturnData, kCFBooleanTrue);
2811
2812         ok = _SecItemCopyMatching(attrs, &swcclient, (CFTypeRef*)&items, error);
2813         if (count > 1) {
2814             // ignore interim error since we have multiple domains to search
2815             CFReleaseNull(*error);
2816         }
2817         if (ok && items && CFGetTypeID(items) == CFArrayGetTypeID()) {
2818 #if TARGET_OS_SIMULATOR
2819             secerror("Ignoring app/site approval state in the Simulator.");
2820             bool approved = true;
2821 #else
2822             // get approval status for this app/domain pair
2823             SWCFlags flags = _SecAppDomainApprovalStatus(appID, fqdn, error);
2824             if (count > 1) {
2825                 // ignore interim error since we have multiple domains to check
2826                 CFReleaseNull(*error);
2827             }
2828             bool approved = (flags & kSWCFlag_SiteApproved);
2829 #endif
2830             if (approved) {
2831                 CFArrayAppendArray(foundItems, items, CFRangeMake(0, CFArrayGetCount(items)));
2832             }
2833         }
2834         CFReleaseSafe(items);
2835         CFReleaseSafe(attrs);
2836         CFReleaseSafe(fqdn);
2837     }
2838
2839 //  If matching credentials are found, the credentials provided to the completionHandler
2840 //  will be a CFArrayRef containing CFDictionaryRef entries. Each dictionary entry will
2841 //  contain the following pairs (see Security/SecItem.h):
2842 //  key: kSecAttrServer     value: CFStringRef (the website)
2843 //  key: kSecAttrAccount    value: CFStringRef (the account)
2844 //  key: kSecSharedPassword value: CFStringRef (the password)
2845 //  Optional keys:
2846 //  key: kSecAttrPort       value: CFNumberRef (the port number, if non-standard for https)
2847
2848     count = CFArrayGetCount(foundItems);
2849     for (idx = 0; idx < count; idx++) {
2850         CFDictionaryRef dict = (CFDictionaryRef) CFArrayGetValueAtIndex(foundItems, idx);
2851         CFMutableDictionaryRef newdict = CFDictionaryCreateMutable(kCFAllocatorDefault, 0, &kCFTypeDictionaryKeyCallBacks, &kCFTypeDictionaryValueCallBacks);
2852         if (newdict && dict && CFGetTypeID(dict) == CFDictionaryGetTypeID()) {
2853             CFStringRef srvr = CFDictionaryGetValue(dict, kSecAttrServer);
2854             CFStringRef acct = CFDictionaryGetValue(dict, kSecAttrAccount);
2855             CFNumberRef pnum = CFDictionaryGetValue(dict, kSecAttrPort);
2856             CFStringRef icmt = CFDictionaryGetValue(dict, kSecAttrComment);
2857             CFDataRef data = CFDictionaryGetValue(dict, kSecValueData);
2858             if (srvr) {
2859                 CFDictionaryAddValue(newdict, kSecAttrServer, srvr);
2860             }
2861             if (acct) {
2862                 CFDictionaryAddValue(newdict, kSecAttrAccount, acct);
2863             }
2864             if (pnum) {
2865                 SInt16 pval = -1;
2866                 if (CFNumberGetValue(pnum, kCFNumberSInt16Type, &pval) &&
2867                     (pval < -1 || pval > 0)) {
2868                     CFDictionaryAddValue(newdict, kSecAttrPort, pnum);
2869                 }
2870             }
2871             if (data) {
2872                 CFStringRef password = CFStringCreateFromExternalRepresentation(kCFAllocatorDefault, data, kCFStringEncodingUTF8);
2873                 if (password) {
2874                 #if TARGET_OS_IPHONE && !TARGET_OS_WATCH && !TARGET_OS_TV
2875                     CFDictionaryAddValue(newdict, kSecSharedPassword, password);
2876                 #else
2877                     CFDictionaryAddValue(newdict, CFSTR("spwd"), password);
2878                 #endif
2879                     CFReleaseSafe(password);
2880                 }
2881             }
2882
2883             if (acct && CFEqual(acct, kSecSafariPasswordsNotSaved)) {
2884                 // Do not add to credentials list!
2885                 secwarning("copySWC: Skipping \"%@\" item", kSecSafariPasswordsNotSaved);
2886             } else if (icmt && CFEqual(icmt, kSecSafariDefaultComment)) {
2887                 CFArrayInsertValueAtIndex(credentials, 0, newdict);
2888             } else {
2889                 CFArrayAppendValue(credentials, newdict);
2890             }
2891         }
2892         CFReleaseSafe(newdict);
2893     }
2894
2895     count = CFArrayGetCount(credentials);
2896     if (count) {
2897         ok = false;
2898         // create a new array of dictionaries (without the actual password) for picker UI
2899         CFMutableArrayRef items = CFArrayCreateMutable(kCFAllocatorDefault, 0, &kCFTypeArrayCallBacks);
2900         for (idx = 0; idx < count; idx++) {
2901             CFDictionaryRef dict = (CFDictionaryRef) CFArrayGetValueAtIndex(credentials, idx);
2902             CFMutableDictionaryRef newdict = CFDictionaryCreateMutableCopy(kCFAllocatorDefault, 0, dict);
2903         #if TARGET_OS_IPHONE && !TARGET_OS_WATCH && !TARGET_OS_TV
2904             CFDictionaryRemoveValue(newdict, kSecSharedPassword);
2905         #else
2906             CFDictionaryRemoveValue(newdict, CFSTR("spwd"));
2907         #endif
2908             CFArrayAppendValue(items, newdict);
2909             CFReleaseSafe(newdict);
2910         }
2911
2912         // prompt user to select one of the dictionary items
2913         CFDictionaryRef selected = swca_copy_selected_dictionary(swca_select_request_id,
2914                                                                  clientAuditToken, items, error);
2915         if (selected) {
2916             // find the matching item in our credentials array
2917             CFStringRef srvr = CFDictionaryGetValue(selected, kSecAttrServer);
2918             CFStringRef acct = CFDictionaryGetValue(selected, kSecAttrAccount);
2919             CFNumberRef pnum = CFDictionaryGetValue(selected, kSecAttrPort);
2920             for (idx = 0; idx < count; idx++) {
2921                 CFDictionaryRef dict = (CFDictionaryRef) CFArrayGetValueAtIndex(credentials, idx);
2922                 CFStringRef srvr1 = CFDictionaryGetValue(dict, kSecAttrServer);
2923                 CFStringRef acct1 = CFDictionaryGetValue(dict, kSecAttrAccount);
2924                 CFNumberRef pnum1 = CFDictionaryGetValue(dict, kSecAttrPort);
2925
2926                 if (!srvr || !srvr1 || !CFEqual(srvr, srvr1)) continue;
2927                 if (!acct || !acct1 || !CFEqual(acct, acct1)) continue;
2928                 if ((pnum && pnum1) && !CFEqual(pnum, pnum1)) continue;
2929
2930                 // we have a match!
2931                 CFReleaseSafe(selected);
2932                 CFRetainSafe(dict);
2933                 selected = dict;
2934                 ok = true;
2935                 break;
2936             }
2937         }
2938         CFReleaseSafe(items);
2939         CFArrayRemoveAllValues(credentials);
2940         if (selected && ok) {
2941 #if TARGET_OS_IOS && !TARGET_OS_BRIDGE && !TARGET_OS_SIMULATOR
2942             fqdn = CFDictionaryGetValue(selected, kSecAttrServer);
2943 #endif
2944             CFArrayAppendValue(credentials, selected);
2945         }
2946
2947         if (ok) {
2948 #if TARGET_OS_IOS && !TARGET_OS_BRIDGE && !TARGET_OS_SIMULATOR
2949             // register confirmation with database
2950             CFRetainSafe(appID);
2951             CFRetainSafe(fqdn);
2952             if (0 != SWCSetServiceFlags(kSecSharedWebCredentialsService,
2953                 appID, fqdn, kSWCFlag_ExternalMask, kSWCFlag_UserApproved,
2954                 ^void(OSStatus inStatus, SWCFlags inNewFlags){
2955                     CFReleaseSafe(appID);
2956                     CFReleaseSafe(fqdn);
2957                 }))
2958             {
2959                  // we didn't queue the block
2960                 CFReleaseSafe(appID);
2961                 CFReleaseSafe(fqdn);
2962             }
2963 #endif
2964         }
2965         CFReleaseSafe(selected);
2966     }
2967     else if (NULL == *error) {
2968         // found no items, and we haven't already filled in the error
2969         SecError(errSecItemNotFound, error, CFSTR("no matching items found"));
2970     }
2971
2972 cleanup:
2973     if (!ok) {
2974         CFArrayRemoveAllValues(credentials);
2975         CFReleaseNull(credentials);
2976     }
2977     CFReleaseSafe(foundItems);
2978     *result = credentials;
2979     CFReleaseSafe(swcclient.accessGroups);
2980     CFReleaseSafe(fqdns);
2981
2982     return ok;
2983 }
2984
2985 #endif /* SHAREDWEBCREDENTIALS */
2986
2987
2988 // MARK: -
2989 // MARK: Keychain backup
2990
2991 CF_RETURNS_RETAINED CFDataRef
2992 _SecServerKeychainCreateBackup(SecurityClient *client, CFDataRef keybag, CFDataRef passcode, bool emcs, CFErrorRef *error) {
2993     __block CFDataRef backup = NULL;
2994     kc_with_dbt(false, error, ^bool (SecDbConnectionRef dbt) {
2995
2996         LKABackupReportStart(!!keybag, !!passcode, emcs);
2997
2998         return kc_transaction_type(dbt, kSecDbNormalTransactionType, error, ^bool{
2999             secnotice("SecServerKeychainCreateBackup", "Performing backup from %s keybag%s", keybag ? "provided" : "device", emcs ? ", EMCS mode" : "");
3000
3001             if (keybag == NULL && passcode == NULL) {
3002 #if USE_KEYSTORE
3003                 backup = SecServerExportBackupableKeychain(dbt, client, KEYBAG_DEVICE, backup_keybag_handle, error);
3004 #else /* !USE_KEYSTORE */
3005                 (void)client;
3006                 SecError(errSecParam, error, CFSTR("Why are you doing this?"));
3007                 backup = NULL;
3008 #endif /* USE_KEYSTORE */
3009             } else {
3010                 backup = SecServerKeychainCreateBackup(dbt, client, keybag, passcode, emcs, error);
3011             }
3012             return (backup != NULL);
3013         });
3014     });
3015
3016     secnotice("SecServerKeychainCreateBackup", "Backup result: %s (%@)", backup ? "success" : "fail", error ? *error : NULL);
3017     LKABackupReportEnd(!!backup, error ? *error : NULL);
3018
3019     return backup;
3020 }
3021
3022 bool
3023 _SecServerKeychainRestore(CFDataRef backup, SecurityClient *client, CFDataRef keybag, CFDataRef passcode, CFErrorRef *error) {
3024     if (backup == NULL || keybag == NULL)
3025         return SecError(errSecParam, error, CFSTR("backup or keybag missing"));
3026
3027     __block bool ok = true;
3028     ok &= kc_with_dbt(true, error, ^bool (SecDbConnectionRef dbconn) {
3029         return SecServerKeychainRestore(dbconn, client, backup, keybag, passcode, error);
3030     });
3031
3032     if (ok) {
3033         SecKeychainChanged();
3034     }
3035
3036     return ok;
3037 }
3038
3039 CFStringRef
3040 _SecServerBackupCopyUUID(CFDataRef data, CFErrorRef *error)
3041 {
3042     CFStringRef uuid = NULL;
3043     CFDictionaryRef backup;
3044
3045     backup = CFPropertyListCreateWithData(kCFAllocatorDefault, data,
3046                                           kCFPropertyListImmutable, NULL,
3047                                           error);
3048     if (isDictionary(backup)) {
3049         uuid = SecServerBackupGetKeybagUUID(backup, error);
3050         if (uuid)
3051             CFRetain(uuid);
3052     }
3053     CFReleaseNull(backup);
3054
3055     return uuid;
3056 }
3057
3058
3059
3060 // MARK: -
3061 // MARK: SecItemDataSource
3062
3063 #if SECUREOBJECTSYNC
3064
3065 // Make sure to call this before any writes to the keychain, so that we fire
3066 // up the engines to monitor manifest changes.
3067 SOSDataSourceFactoryRef SecItemDataSourceFactoryGetDefault(void) {
3068     return SecItemDataSourceFactoryGetShared(kc_dbhandle(NULL));
3069 }
3070
3071 /* AUDIT[securityd]:
3072    args_in (ok) is a caller provided, CFDictionaryRef.
3073  */
3074
3075 CF_RETURNS_RETAINED CFArrayRef
3076 _SecServerKeychainSyncUpdateMessage(CFDictionaryRef updates, CFErrorRef *error) {
3077     // This never fails, trust us!
3078     return SOSCCHandleUpdateMessage(updates);
3079 }
3080
3081 //
3082 // Truthiness in the cloud backup/restore support.
3083 //
3084
3085 static CFDictionaryRef
3086 _SecServerCopyTruthInTheCloud(CFDataRef keybag, CFDataRef password,
3087     CFDictionaryRef backup, CFErrorRef *error)
3088 {
3089     SOSManifestRef mold = NULL, mnow = NULL, mdelete = NULL, madd = NULL;
3090     __block CFMutableDictionaryRef backup_new = NULL;
3091     keybag_handle_t bag_handle;
3092     if (!ks_open_keybag(keybag, password, &bag_handle, error))
3093         return backup_new;
3094
3095     // We need to have a datasource singleton for protection domain
3096     // kSecAttrAccessibleWhenUnlocked and keep a single shared engine
3097     // instance around which we create in the datasource constructor as well.
3098     SOSDataSourceFactoryRef dsf = SecItemDataSourceFactoryGetDefault();
3099     SOSDataSourceRef ds = SOSDataSourceFactoryCreateDataSource(dsf, kSecAttrAccessibleWhenUnlocked, error);
3100     if (ds) {
3101         backup_new = backup ? CFDictionaryCreateMutableCopy(kCFAllocatorDefault, 0, backup) : CFDictionaryCreateMutable(kCFAllocatorDefault, 0, &kCFTypeDictionaryKeyCallBacks, &kCFTypeDictionaryValueCallBacks);
3102         mold = SOSCreateManifestWithBackup(backup, error);
3103         SOSEngineRef engine = SOSDataSourceGetSharedEngine(ds, error);
3104         mnow = SOSEngineCopyManifest(engine, NULL);
3105         if (!mnow) {
3106             mnow = SOSDataSourceCopyManifestWithViewNameSet(ds, SOSViewsGetV0ViewSet(), error);
3107         }
3108         if (!mnow) {
3109             CFReleaseNull(backup_new);
3110             secerror("failed to obtain manifest for keychain: %@", error ? *error : NULL);
3111         } else {
3112             SOSManifestDiff(mold, mnow, &mdelete, &madd, error);
3113         }
3114
3115         // Delete everything from the new_backup that is no longer in the datasource according to the datasources manifest.
3116         SOSManifestForEach(mdelete, ^(CFDataRef digest_data, bool *stop) {
3117             CFStringRef deleted_item_key = CFDataCopyHexString(digest_data);
3118             CFDictionaryRemoveValue(backup_new, deleted_item_key);
3119             CFRelease(deleted_item_key);
3120         });
3121
3122         CFMutableArrayRef changes = CFArrayCreateMutableForCFTypes(kCFAllocatorDefault);
3123         SOSDataSourceForEachObject(ds, NULL, madd, error, ^void(CFDataRef digest, SOSObjectRef object, bool *stop) {
3124             CFErrorRef localError = NULL;
3125             CFDataRef digest_data = NULL;
3126             CFTypeRef value = NULL;
3127             if (!object) {
3128                 // Key in our manifest can't be found in db, remove it from our manifest
3129                 SOSChangesAppendDelete(changes, digest);
3130             } else if (!(digest_data = SOSObjectCopyDigest(ds, object, &localError))
3131                 || !(value = SOSObjectCopyBackup(ds, object, bag_handle, &localError))) {
3132                 if (SecErrorGetOSStatus(localError) == errSecDecode) {
3133                     // Ignore decode errors, pretend the objects aren't there
3134                     CFRelease(localError);
3135                     // Object undecodable, remove it from our manifest
3136                     SOSChangesAppendDelete(changes, digest);
3137                 } else {
3138                     // Stop iterating and propagate out all other errors.
3139                     *stop = true;
3140                     *error = localError;
3141                     CFReleaseNull(backup_new);
3142                 }
3143             } else {
3144                 // TODO: Should we skip tombstones here?
3145                 CFStringRef key = CFDataCopyHexString(digest_data);
3146                 CFDictionarySetValue(backup_new, key, value);
3147                 CFReleaseSafe(key);
3148             }
3149             CFReleaseSafe(digest_data);
3150             CFReleaseSafe(value);
3151         }) || CFReleaseNull(backup_new);
3152
3153         if (CFArrayGetCount(changes)) {
3154             if (!SOSEngineUpdateChanges(engine, kSOSDataSourceSOSTransaction, changes, error)) {
3155                 CFReleaseNull(backup_new);
3156             }
3157         }
3158         CFReleaseSafe(changes);
3159
3160         SOSDataSourceRelease(ds, error) || CFReleaseNull(backup_new);
3161     }
3162
3163     CFReleaseSafe(mold);
3164     CFReleaseSafe(mnow);
3165     CFReleaseSafe(madd);
3166     CFReleaseSafe(mdelete);
3167     ks_close_keybag(bag_handle, error) || CFReleaseNull(backup_new);
3168
3169     return backup_new;
3170 }
3171
3172 static bool
3173 _SecServerRestoreTruthInTheCloud(CFDataRef keybag, CFDataRef password, CFDictionaryRef backup_in, CFErrorRef *error) {
3174     __block bool ok = true;
3175     keybag_handle_t bag_handle;
3176     if (!ks_open_keybag(keybag, password, &bag_handle, error))
3177         return false;
3178
3179     SOSManifestRef mbackup = SOSCreateManifestWithBackup(backup_in, error);
3180     if (mbackup) {
3181         SOSDataSourceFactoryRef dsf = SecItemDataSourceFactoryGetDefault();
3182         SOSDataSourceRef ds = SOSDataSourceFactoryCreateDataSource(dsf, kSecAttrAccessibleWhenUnlocked, error);
3183         ok &= ds && SOSDataSourceWith(ds, error, ^(SOSTransactionRef txn, bool *commit) {
3184             SOSManifestRef mnow = SOSDataSourceCopyManifestWithViewNameSet(ds, SOSViewsGetV0BackupViewSet(), error);
3185             SOSManifestRef mdelete = NULL, madd = NULL;
3186             SOSManifestDiff(mnow, mbackup, &mdelete, &madd, error);
3187
3188             // Don't delete everything in datasource not in backup.
3189
3190             // Add items from the backup
3191             SOSManifestForEach(madd, ^void(CFDataRef e, bool *stop) {
3192                 CFDictionaryRef item = NULL;
3193                 CFStringRef sha1 = CFDataCopyHexString(e);
3194                 if (sha1) {
3195                     item = CFDictionaryGetValue(backup_in, sha1);
3196                     CFRelease(sha1);
3197                 }
3198                 if (item) {
3199                     CFErrorRef localError = NULL;
3200
3201                     if (!SOSObjectRestoreObject(ds, txn, bag_handle, item, &localError)) {
3202                         OSStatus status = SecErrorGetOSStatus(localError);
3203                         if (status == errSecDuplicateItem) {
3204                             // Log and ignore duplicate item errors during restore
3205                             secnotice("titc", "restore %@ not replacing existing item", item);
3206                         } else if (status == errSecDecode) {
3207                             // Log and ignore corrupted item errors during restore
3208                             secnotice("titc", "restore %@ skipping corrupted item %@", item, localError);
3209                         } else {
3210                             if (status == errSecInteractionNotAllowed)
3211                                 *stop = true;
3212                             // Propagate the first other error upwards (causing the restore to fail).
3213                             secerror("restore %@ failed %@", item, localError);
3214                             ok = false;
3215                             if (error && !*error) {
3216                                 *error = localError;
3217                                 localError = NULL;
3218                             }
3219                         }
3220                         CFReleaseSafe(localError);
3221                     }
3222                 }
3223             });
3224             ok &= SOSDataSourceRelease(ds, error);
3225             CFReleaseNull(mdelete);
3226             CFReleaseNull(madd);
3227             CFReleaseNull(mnow);
3228         });
3229         CFRelease(mbackup);
3230     }
3231
3232     ok &= ks_close_keybag(bag_handle, error);
3233
3234     return ok;
3235 }
3236
3237
3238 CF_RETURNS_RETAINED CFDictionaryRef
3239 _SecServerBackupSyncable(CFDictionaryRef backup, CFDataRef keybag, CFDataRef password, CFErrorRef *error) {
3240     require_action_quiet(isData(keybag), errOut, SecError(errSecParam, error, CFSTR("keybag %@ not a data"), keybag));
3241     require_action_quiet(!backup || isDictionary(backup), errOut, SecError(errSecParam, error, CFSTR("backup %@ not a dictionary"), backup));
3242     require_action_quiet(!password || isData(password), errOut, SecError(errSecParam, error, CFSTR("password %@ not a data"), password));
3243
3244     return _SecServerCopyTruthInTheCloud(keybag, password, backup, error);
3245
3246 errOut:
3247     return NULL;
3248 }
3249
3250 bool
3251 _SecServerRestoreSyncable(CFDictionaryRef backup, CFDataRef keybag, CFDataRef password, CFErrorRef *error) {
3252     bool ok;
3253     require_action_quiet(isData(keybag), errOut, ok = SecError(errSecParam, error, CFSTR("keybag %@ not a data"), keybag));
3254     require_action_quiet(isDictionary(backup), errOut, ok = SecError(errSecParam, error, CFSTR("backup %@ not a dictionary"), backup));
3255     if (password) {
3256
3257         require_action_quiet(isData(password), errOut, ok = SecError(errSecParam, error, CFSTR("password not a data")));
3258     }
3259
3260     ok = _SecServerRestoreTruthInTheCloud(keybag, password, backup, error);
3261
3262 errOut:
3263     return ok;
3264 }
3265 #endif /* SECUREOBJECTSYNC */
3266
3267 bool _SecServerRollKeysGlue(bool force, CFErrorRef *error) {
3268     return _SecServerRollKeys(force, NULL, error);
3269 }
3270
3271
3272 bool _SecServerRollKeys(bool force, SecurityClient *client, CFErrorRef *error) {
3273 #if USE_KEYSTORE
3274     uint32_t keystore_generation_status = 0;
3275     if (aks_generation(KEYBAG_DEVICE, generation_noop, &keystore_generation_status))
3276         return false;
3277     uint32_t current_generation = keystore_generation_status & generation_current;
3278
3279     return kc_with_dbt(true, error, ^(SecDbConnectionRef dbt) {
3280         bool up_to_date = s3dl_dbt_keys_current(dbt, current_generation, NULL);
3281
3282         if (force && !up_to_date) {
3283             up_to_date = s3dl_dbt_update_keys(dbt, client, error);
3284             if (up_to_date) {
3285                 secerror("Completed roll keys.");
3286                 up_to_date = s3dl_dbt_keys_current(dbt, current_generation, NULL);
3287             }
3288             if (!up_to_date)
3289                 secerror("Failed to roll keys.");
3290         }
3291         return up_to_date;
3292     });
3293 #else
3294     return true;
3295 #endif
3296 }
3297
3298 static bool
3299 InitialSyncItems(CFMutableArrayRef items, bool limitToCurrent, CFStringRef agrp, CFStringRef svce, const SecDbClass *qclass, CFErrorRef *error)
3300 {
3301     bool result = false;
3302     Query *q = NULL;
3303
3304     q = query_create(qclass, NULL, NULL, error);
3305     require(q, fail);
3306
3307     q->q_return_type = kSecReturnDataMask | kSecReturnAttributesMask;
3308     q->q_limit = kSecMatchUnlimited;
3309     q->q_keybag = KEYBAG_DEVICE;
3310
3311     query_add_attribute(kSecAttrAccessGroup, agrp, q);
3312     query_add_attribute(kSecAttrSynchronizable, kCFBooleanTrue, q);
3313     query_add_attribute(kSecAttrTombstone, kCFBooleanFalse, q);
3314     if (svce)
3315         query_add_attribute(kSecAttrService, svce, q);
3316
3317     result = kc_with_dbt(false, error, ^(SecDbConnectionRef dbt) {
3318         return kc_transaction(dbt, error, ^{
3319             CFErrorRef error2 = NULL;
3320
3321             SecDbItemSelect(q, dbt, &error2, NULL, ^bool(const SecDbAttr *attr) {
3322                 return CFDictionaryGetValue(q->q_item, attr->name);
3323             }, NULL, NULL, ^(SecDbItemRef item, bool *stop) {
3324                 CFErrorRef error3 = NULL;
3325                 secinfo("InitialSyncItems", "Copy item");
3326
3327                 CFMutableDictionaryRef attrs = SecDbItemCopyPListWithMask(item, kSecDbSyncFlag, &error3);
3328                 if (attrs) {
3329                     int match = true;
3330                     CFStringRef itemvwht = CFDictionaryGetValue(attrs, kSecAttrSyncViewHint);
3331                     /*
3332                      * Saying its a SOS viewhint is really not the right answer post Triangle
3333                      */
3334                     if (isString(itemvwht) && !SOSViewInSOSSystem(itemvwht)) {
3335                         match = false;
3336                     }
3337                     /*
3338                      * Here we encode how PCS stores identities so that we only copy the
3339                      * current identites for performance reasons.
3340                      */
3341                     if (limitToCurrent) {
3342                         enum { PCS_CURRENT_IDENTITY_OFFSET              = 0x10000 };
3343                         int32_t s32;
3344
3345                         CFNumberRef type = CFDictionaryGetValue(attrs, kSecAttrType);
3346                         if (!isNumber(type)) {
3347                             // still allow this case since its not a service identity ??
3348                         } else if (!CFNumberGetValue(type, kCFNumberSInt32Type, &s32)) {
3349                             match = false;
3350                         } else if ((s32 & PCS_CURRENT_IDENTITY_OFFSET) == 0) {
3351                             match = false;
3352                         }
3353                     }
3354                     if (match) {
3355                         CFDictionaryAddValue(attrs, kSecClass, SecDbItemGetClass(item)->name);
3356                         CFArrayAppendValue(items, attrs);
3357                     }
3358
3359                     CFReleaseNull(attrs);
3360                 }
3361                 CFReleaseNull(error3);
3362             });
3363             CFReleaseNull(error2);
3364
3365             return (bool)true;
3366         });
3367     });
3368
3369 fail:
3370     if (q)
3371         query_destroy(q, NULL);
3372     return result;
3373 }
3374
3375 CFArrayRef
3376 _SecServerCopyInitialSyncCredentials(uint32_t flags, CFErrorRef *error)
3377 {
3378     CFMutableArrayRef items = CFArrayCreateMutableForCFTypes(NULL);
3379
3380     if (flags & SecServerInitialSyncCredentialFlagTLK) {
3381         require_action(InitialSyncItems(items, false, CFSTR("com.apple.security.ckks"), NULL, inet_class(), error), fail,
3382                        secerror("failed to collect CKKS-inet keys: %@", error ? *error : NULL));
3383     }
3384     if (flags & SecServerInitialSyncCredentialFlagPCS) {
3385         bool onlyCurrent = !(flags & SecServerInitialSyncCredentialFlagPCSNonCurrent);
3386
3387         require_action(InitialSyncItems(items, false, CFSTR("com.apple.ProtectedCloudStorage"), NULL, genp_class(), error), fail,
3388                        secerror("failed to collect PCS-genp keys: %@", error ? *error : NULL));
3389         require_action(InitialSyncItems(items, onlyCurrent, CFSTR("com.apple.ProtectedCloudStorage"), NULL, inet_class(), error), fail,
3390                        secerror("failed to collect PCS-inet keys: %@", error ? *error : NULL));
3391     }
3392     if (flags & SecServerInitialSyncCredentialFlagBluetoothMigration) {
3393         require_action(InitialSyncItems(items, false, CFSTR("com.apple.nanoregistry.migration"), NULL, genp_class(), error), fail,
3394                        secerror("failed to collect com.apple.nanoregistry.migration-genp item: %@", error ? *error : NULL));
3395         require_action(InitialSyncItems(items, false, CFSTR("com.apple.nanoregistry.migration2"), NULL, genp_class(), error), fail,
3396                        secerror("failed to collect com.apple.nanoregistry.migration2-genp item: %@", error ? *error : NULL));
3397         require_action(InitialSyncItems(items, false, CFSTR("com.apple.bluetooth"), CFSTR("BluetoothLESync"), genp_class(), error), fail,
3398                        secerror("failed to collect com.apple.bluetooth-genp item: %@", error ? *error : NULL));
3399
3400     }
3401
3402 fail:
3403     return items;
3404 }
3405
3406 bool
3407 _SecServerImportInitialSyncCredentials(CFArrayRef array, CFErrorRef *error)
3408 {
3409     return kc_with_dbt(true, error, ^bool(SecDbConnectionRef dbt) {
3410         return kc_transaction(dbt, error, ^bool(void){
3411             CFIndex n, count = CFArrayGetCount(array);
3412
3413             secinfo("ImportInitialSyncItems", "Importing %d items", (int)count);
3414
3415             for (n = 0; n < count; n++) {
3416                 CFErrorRef cferror = NULL;
3417
3418                 CFDictionaryRef item = CFArrayGetValueAtIndex(array, n);
3419                 if (!isDictionary(item))
3420                     continue;
3421
3422                 CFStringRef className = CFDictionaryGetValue(item, kSecClass);
3423                 if (className == NULL) {
3424                     secinfo("ImportInitialSyncItems", "Item w/o class");
3425                     continue;
3426                 }
3427
3428                 const SecDbClass *cls = kc_class_with_name(className);
3429                 if (cls == NULL) {
3430                     secinfo("ImportInitialSyncItems", "Item with unknown class: %@", className);
3431                     continue;
3432                 }
3433
3434                 SecDbItemRef dbi = SecDbItemCreateWithAttributes(NULL, cls, item, KEYBAG_DEVICE, &cferror);
3435                 if (dbi == NULL) {
3436                     secinfo("ImportInitialSyncItems", "Item creation failed with: %@", cferror);
3437                     CFReleaseNull(cferror);
3438                     continue;
3439                 }
3440
3441                 if (!SecDbItemSetSyncable(dbi, true, &cferror)) {
3442                     secinfo("ImportInitialSyncItems", "Failed to set sync=1: %@ for item %@", cferror, dbi);
3443                     CFReleaseNull(cferror);
3444                     CFReleaseNull(dbi);
3445                     continue;
3446                 }
3447
3448                 if (!SecDbItemInsert(dbi, dbt, &cferror)) {
3449                     secinfo("ImportInitialSyncItems", "Item store failed with: %@: %@", cferror, dbi);
3450                     CFReleaseNull(cferror);
3451                 }
3452                 CFReleaseNull(dbi);
3453             }
3454             return true;
3455         });
3456     });
3457 }
3458
3459
3460
3461 #if TARGET_OS_IOS
3462
3463 /*
3464  * Sync bubble migration code
3465  */
3466
3467 struct SyncBubbleRule {
3468     CFStringRef attribute;
3469     CFTypeRef value;
3470 };
3471
3472 static bool
3473 TransmogrifyItemsToSyncBubble(SecurityClient *client, uid_t uid,
3474                               bool onlyDelete,
3475                               bool copyToo,
3476                               const SecDbClass *qclass,
3477                               struct SyncBubbleRule *items, CFIndex nItems,
3478                               CFErrorRef *error)
3479 {
3480     CFMutableDictionaryRef updateAttributes = NULL;
3481     CFDataRef syncBubbleView = NULL;
3482     CFDataRef activeUserView = NULL;
3483     bool res = false;
3484     Query *q = NULL;
3485     CFIndex n;
3486
3487     syncBubbleView = SecMUSRCreateSyncBubbleUserUUID(uid);
3488     require(syncBubbleView, fail);
3489
3490     activeUserView = SecMUSRCreateActiveUserUUID(uid);
3491     require(activeUserView, fail);
3492
3493
3494     if ((onlyDelete && !copyToo) || !onlyDelete) {
3495
3496         /*
3497          * Clean out items first
3498          */
3499
3500         secnotice("syncbubble", "cleaning out old items");
3501
3502         q = query_create(qclass, NULL, NULL, error);
3503         require(q, fail);
3504
3505         q->q_limit = kSecMatchUnlimited;
3506         q->q_keybag = device_keybag_handle;
3507
3508         for (n = 0; n < nItems; n++) {
3509             query_add_attribute(items[n].attribute, items[n].value, q);
3510         }
3511         q->q_musrView = CFRetain(syncBubbleView);
3512         require(q->q_musrView, fail);
3513
3514         kc_with_dbt(false, error, ^(SecDbConnectionRef dbt) {
3515             return kc_transaction(dbt, error, ^{
3516                 return s3dl_query_delete(dbt, q, NULL, error);
3517             });
3518         });
3519
3520         query_destroy(q, NULL);
3521         q = NULL;
3522     }
3523
3524
3525     if (onlyDelete || !copyToo) {
3526         secnotice("syncbubble", "skip migration of items");
3527     } else {
3528         /*
3529          * Copy over items from EMCS to sync bubble
3530          */
3531
3532         secnotice("syncbubble", "migrating sync bubble items");
3533
3534         q = query_create(qclass, NULL, NULL, error);
3535         require(q, fail);
3536
3537         q->q_return_type = kSecReturnDataMask | kSecReturnAttributesMask;
3538         q->q_limit = kSecMatchUnlimited;
3539         q->q_keybag = device_keybag_handle; /* XXX change to session key bag when it exists */
3540
3541         for (n = 0; n < nItems; n++) {
3542             query_add_or_attribute(items[n].attribute, items[n].value, q);
3543         }
3544         query_add_or_attribute(CFSTR("musr"), activeUserView, q);
3545         q->q_musrView = CFRetain(activeUserView);
3546
3547         updateAttributes = CFDictionaryCreateMutableForCFTypes(NULL);
3548         require(updateAttributes, fail);
3549
3550         CFDictionarySetValue(updateAttributes, CFSTR("musr"), syncBubbleView); /* XXX should use kSecAttrMultiUser */
3551
3552
3553         kc_with_dbt(true, error, ^(SecDbConnectionRef dbt) {
3554             return kc_transaction(dbt, error, ^{
3555                 CFErrorRef error2 = NULL;
3556
3557                 SecDbItemSelect(q, dbt, &error2, NULL, ^bool(const SecDbAttr *attr) {
3558                     return CFDictionaryGetValue(q->q_item, attr->name);
3559                 }, NULL, NULL, ^(SecDbItemRef item, bool *stop) {
3560                     CFErrorRef error3 = NULL;
3561                     secinfo("syncbubble", "migrating item");
3562
3563                     SecDbItemRef new_item = SecDbItemCopyWithUpdates(item, updateAttributes, NULL);
3564                     if (new_item == NULL)
3565                         return;
3566
3567                     SecDbItemClearRowId(new_item, NULL);
3568
3569                     if (!SecDbItemSetKeybag(new_item, device_keybag_handle, NULL)) {
3570                         CFRelease(new_item);
3571                         return;
3572                     }
3573
3574                     if (!SecDbItemInsert(new_item, dbt, &error3)) {
3575                         secnotice("syncbubble", "migration failed with %@ for item %@", error3, new_item);
3576                     }
3577                     CFRelease(new_item);
3578                     CFReleaseNull(error3);
3579                 });
3580                 CFReleaseNull(error2);
3581
3582                 return (bool)true;
3583             });
3584         });
3585     }
3586     res = true;
3587
3588 fail:
3589     CFReleaseNull(syncBubbleView);
3590     CFReleaseNull(activeUserView);
3591     CFReleaseNull(updateAttributes);
3592     if (q)
3593         query_destroy(q, NULL);
3594
3595     return res;
3596 }
3597
3598 static struct SyncBubbleRule PCSItems[] = {
3599     {
3600         .attribute = CFSTR("agrp"),
3601         .value = CFSTR("com.apple.ProtectedCloudStorage"),
3602     }
3603 };
3604 static struct SyncBubbleRule NSURLSesssiond[] = {
3605     {
3606         .attribute = CFSTR("agrp"),
3607         .value = CFSTR("com.apple.nsurlsessiond"),
3608     }
3609 };
3610 static struct SyncBubbleRule AccountsdItems[] = {
3611     {
3612         .attribute = CFSTR("svce"),
3613         .value = CFSTR("com.apple.account.AppleAccount.token"),
3614     },
3615     {
3616         .attribute = CFSTR("svce"),
3617         .value = CFSTR("com.apple.account.AppleAccount.password"),
3618     },
3619     {
3620         .attribute = CFSTR("svce"),
3621         .value = CFSTR("com.apple.account.AppleAccount.rpassword"),
3622     },
3623     {
3624         .attribute = CFSTR("svce"),
3625         .value = CFSTR("com.apple.account.idms.token"),
3626     },
3627     {
3628         .attribute = CFSTR("svce"),
3629         .value = CFSTR("com.apple.account.idms.continuation-key"),
3630     },
3631     {
3632         .attribute = CFSTR("svce"),
3633         .value = CFSTR("com.apple.account.CloudKit.token"),
3634     },
3635 };
3636
3637 static struct SyncBubbleRule MobileMailItems[] = {
3638     {
3639         .attribute = CFSTR("svce"),
3640         .value = CFSTR("com.apple.account.IMAP.password"),
3641     },
3642     {
3643         .attribute = CFSTR("svce"),
3644         .value = CFSTR("com.apple.account.SMTP.password"),
3645     },
3646     {
3647         .attribute = CFSTR("svce"),
3648         .value = CFSTR("com.apple.account.Exchange.password"),
3649     },
3650     {
3651         .attribute = CFSTR("svce"),
3652         .value = CFSTR("com.apple.account.Hotmail.password"),
3653     },
3654     {
3655         .attribute = CFSTR("svce"),
3656         .value = CFSTR("com.apple.account.Google.password"),
3657     },
3658     {
3659         .attribute = CFSTR("svce"),
3660         .value = CFSTR("com.apple.account.Google.oauth-token"),
3661     },
3662     {
3663         .attribute = CFSTR("svce"),
3664         .value = CFSTR("com.apple.account.Google.oath-refresh-token"),
3665     },
3666     {
3667         .attribute = CFSTR("svce"),
3668         .value = CFSTR("com.apple.account.Yahoo.password"),
3669     },
3670     {
3671         .attribute = CFSTR("svce"),
3672         .value = CFSTR("com.apple.account.Yahoo.oauth-token"),
3673     },
3674     {
3675         .attribute = CFSTR("svce"),
3676         .value = CFSTR("com.apple.account.Yahoo.oauth-token-nosync"),
3677     },
3678     {
3679         .attribute = CFSTR("svce"),
3680         .value = CFSTR("com.apple.account.Yahoo.oath-refresh-token"),
3681     },
3682     {
3683         .attribute = CFSTR("svce"),
3684         .value = CFSTR("com.apple.account.IMAPNotes.password"),
3685     },
3686     {
3687         .attribute = CFSTR("svce"),
3688         .value = CFSTR("com.apple.account.IMAPMail.password"),
3689     },
3690     {
3691         .attribute = CFSTR("svce"),
3692         .value = CFSTR("com.apple.account.126.password"),
3693     },
3694     {
3695         .attribute = CFSTR("svce"),
3696         .value = CFSTR("com.apple.account.163.password"),
3697     },
3698     {
3699         .attribute = CFSTR("svce"),
3700         .value = CFSTR("com.apple.account.aol.password"),
3701     },
3702 };
3703
3704 static bool
3705 ArrayContains(CFArrayRef array, CFStringRef service)
3706 {
3707     return CFArrayContainsValue(array, CFRangeMake(0, CFArrayGetCount(array)), service);
3708 }
3709
3710 bool
3711 _SecServerTransmogrifyToSyncBubble(CFArrayRef services, uid_t uid, SecurityClient *client, CFErrorRef *error)
3712 {
3713     bool copyCloudAuthToken = false;
3714     bool copyMobileMail = false;
3715     bool res = true;
3716     bool copyPCS = false;
3717     bool onlyDelete = false;
3718     bool copyNSURLSesssion = false;
3719
3720     if (!client->inMultiUser)
3721         return false;
3722
3723     secnotice("syncbubble", "migration for uid %d uid for services %@", (int)uid, services);
3724
3725 #if TARGET_OS_SIMULATOR
3726     // no delete in sim
3727 #elif TARGET_OS_IOS
3728     if (uid != (uid_t)client->activeUser)
3729         onlyDelete = true;
3730 #else
3731 #error "no sync bubble on other platforms"
3732 #endif
3733
3734     /*
3735      * First select that services to copy/delete
3736      */
3737
3738     if (ArrayContains(services, CFSTR("com.apple.bird.usermanager.sync"))
3739         || ArrayContains(services, CFSTR("com.apple.cloudphotod.sync"))
3740         || ArrayContains(services, CFSTR("com.apple.cloudphotod.syncstakeholder"))
3741         || ArrayContains(services, CFSTR("com.apple.cloudd.usermanager.sync")))
3742     {
3743         copyCloudAuthToken = true;
3744         copyPCS = true;
3745     }
3746
3747     if (ArrayContains(services, CFSTR("com.apple.nsurlsessiond.usermanager.sync")))
3748     {
3749         copyCloudAuthToken = true;
3750         copyNSURLSesssion = true;
3751     }
3752
3753     if (ArrayContains(services, CFSTR("com.apple.syncdefaultsd.usermanager.sync"))) {
3754         copyCloudAuthToken = true;
3755     }
3756     if (ArrayContains(services, CFSTR("com.apple.mailq.sync")) || ArrayContains(services, CFSTR("com.apple.mailq.sync.xpc"))) {
3757         copyCloudAuthToken = true;
3758         copyMobileMail = true;
3759         copyPCS = true;
3760     }
3761
3762     /*
3763      * The actually copy/delete the items selected
3764      */
3765
3766     res = TransmogrifyItemsToSyncBubble(client, uid, onlyDelete, copyPCS, inet_class(), PCSItems, sizeof(PCSItems)/sizeof(PCSItems[0]), error);
3767     require(res, fail);
3768     res = TransmogrifyItemsToSyncBubble(client, uid, onlyDelete, copyPCS, genp_class(), PCSItems, sizeof(PCSItems)/sizeof(PCSItems[0]), error);
3769     require(res, fail);
3770
3771     /* mail */
3772     res = TransmogrifyItemsToSyncBubble(client, uid, onlyDelete, copyMobileMail, genp_class(), MobileMailItems, sizeof(MobileMailItems)/sizeof(MobileMailItems[0]), error);
3773     require(res, fail);
3774
3775     /* accountsd */
3776     res = TransmogrifyItemsToSyncBubble(client, uid, onlyDelete, copyCloudAuthToken, genp_class(), AccountsdItems, sizeof(AccountsdItems)/sizeof(AccountsdItems[0]), error);
3777     require(res, fail);
3778
3779     /* nsurlsessiond */
3780     res = TransmogrifyItemsToSyncBubble(client, uid, onlyDelete, copyNSURLSesssion, inet_class(), NSURLSesssiond, sizeof(NSURLSesssiond)/sizeof(NSURLSesssiond[0]), error);
3781     require(res, fail);
3782
3783 fail:
3784     return res;
3785 }
3786
3787 /*
3788  * Migrate from user keychain to system keychain when switching to edu mode
3789  */
3790
3791 bool
3792 _SecServerTransmogrifyToSystemKeychain(SecurityClient *client, CFErrorRef *error)
3793 {
3794     __block bool ok = true;
3795
3796     /*
3797      * we are not in multi user yet, about to switch, otherwise we would
3798      * check that for client->inMultiuser here
3799      */
3800
3801     kc_with_dbt(true, error, ^(SecDbConnectionRef dbt) {
3802         return kc_transaction(dbt, error, ^{
3803             CFDataRef systemUUID = SecMUSRGetSystemKeychainUUID();
3804
3805             const SecDbSchema *newSchema = current_schema();
3806             SecDbClass const *const *kcClass;
3807
3808             for (kcClass = newSchema->classes; *kcClass != NULL; kcClass++) {
3809                 CFErrorRef localError = NULL;
3810                 Query *q = NULL;
3811
3812                 if (!((*kcClass)->itemclass)) {
3813                     continue;
3814                 }
3815
3816                 q = query_create(*kcClass, SecMUSRGetSingleUserKeychainUUID(), NULL, error);
3817                 if (q == NULL)
3818                     continue;
3819
3820                 ok &= SecDbItemSelect(q, dbt, error, ^bool(const SecDbAttr *attr) {
3821                     return (attr->flags & kSecDbInFlag) != 0;
3822                 }, ^bool(const SecDbAttr *attr) {
3823                     // No filtering please.
3824                     return false;
3825                 }, ^bool(CFMutableStringRef sql, bool *needWhere) {
3826                     SecDbAppendWhereOrAnd(sql, needWhere);
3827                     CFStringAppendFormat(sql, NULL, CFSTR("musr = ?"));
3828                     return true;
3829                 }, ^bool(sqlite3_stmt *stmt, int col) {
3830                     return SecDbBindObject(stmt, col++, SecMUSRGetSingleUserKeychainUUID(), error);
3831                 }, ^(SecDbItemRef item, bool *stop) {
3832                     CFErrorRef itemError = NULL;
3833
3834                     if (!SecDbItemSetValueWithName(item, kSecAttrMultiUser, systemUUID, &itemError)) {
3835                         secerror("item: %@ update musr to system failed: %@", item, itemError);
3836                         ok = false;
3837                         goto out;
3838                     }
3839
3840                     if (!SecDbItemDoUpdate(item, item, dbt, &itemError, ^bool (const SecDbAttr *attr) {
3841                         return attr->kind == kSecDbRowIdAttr;
3842                     })) {
3843                         secerror("item: %@ insert during UPDATE: %@", item, itemError);
3844                         ok = false;
3845                         goto out;
3846                     }
3847
3848                 out:
3849                     SecErrorPropagate(itemError, error);
3850                 });
3851
3852                 if (q)
3853                     query_destroy(q, &localError);
3854
3855             }
3856             return (bool)true;
3857         });
3858     });
3859
3860     return ok;
3861 }
3862
3863 /*
3864  * Delete account from local usage
3865  */
3866
3867 bool
3868 _SecServerDeleteMUSERViews(SecurityClient *client, uid_t uid, CFErrorRef *error)
3869 {
3870     return kc_with_dbt(true, error, ^(SecDbConnectionRef dbt) {
3871         CFDataRef musrView = NULL, syncBubbleView = NULL;
3872         bool ok = false;
3873
3874         syncBubbleView = SecMUSRCreateSyncBubbleUserUUID(uid);
3875         require(syncBubbleView, fail);
3876
3877         musrView = SecMUSRCreateActiveUserUUID(uid);
3878         require(musrView, fail);
3879
3880         require(ok = SecServerDeleteAllForUser(dbt, syncBubbleView, false, error), fail);
3881         require(ok = SecServerDeleteAllForUser(dbt, musrView, false, error), fail);
3882
3883     fail:
3884         CFReleaseNull(syncBubbleView);
3885         CFReleaseNull(musrView);
3886         return ok;
3887     });
3888 }
3889
3890
3891 #endif /* TARGET_OS_IOS */
3892
3893 CFArrayRef _SecItemCopyParentCertificates(CFDataRef normalizedIssuer, CFArrayRef accessGroups, CFErrorRef *error) {
3894     const void *keys[] = {
3895         kSecClass,
3896         kSecReturnData,
3897         kSecMatchLimit,
3898         kSecAttrSubject
3899     },
3900     *values[] = {
3901         kSecClassCertificate,
3902         kCFBooleanTrue,
3903         kSecMatchLimitAll,
3904         normalizedIssuer
3905     };
3906     CFDictionaryRef query = CFDictionaryCreate(NULL, keys, values, 4,
3907                                                NULL, NULL);
3908     CFTypeRef results = NULL;
3909     SecurityClient client = {
3910         .task = NULL,
3911         .accessGroups = accessGroups,
3912         .allowSystemKeychain = true,
3913         .allowSyncBubbleKeychain = false,
3914         .isNetworkExtension = false,
3915     };
3916
3917     (void)_SecItemCopyMatching(query, &client, &results, error);
3918     CFRelease(query);
3919     return results;
3920 }
3921
3922 bool _SecItemCertificateExists(CFDataRef normalizedIssuer, CFDataRef serialNumber, CFArrayRef accessGroups, CFErrorRef *error) {
3923     const void *keys[] = {
3924         kSecClass,
3925         kSecMatchLimit,
3926         kSecAttrIssuer,
3927         kSecAttrSerialNumber
3928     },
3929     *values[] = {
3930         kSecClassCertificate,
3931         kSecMatchLimitOne,
3932         normalizedIssuer,
3933         serialNumber
3934     };
3935     SecurityClient client = {
3936         .task = NULL,
3937         .accessGroups = accessGroups,
3938         .allowSystemKeychain = true,
3939         .allowSyncBubbleKeychain = false,
3940         .isNetworkExtension = false,
3941     };
3942     CFDictionaryRef query = CFDictionaryCreate(NULL, keys, values, 4, NULL, NULL);
3943     CFTypeRef results = NULL;
3944     bool ok = _SecItemCopyMatching(query, &client, &results, error);
3945     CFReleaseSafe(query);
3946     CFReleaseSafe(results);
3947     if (!ok) {
3948         return false;
3949     }
3950     return true;
3951 }