keychain/securityd/SecItemServer.c

   1 /*
   2  * Copyright (c) 2006-2017 Apple Inc. All Rights Reserved.
   3  *
   4  * @APPLE_LICENSE_HEADER_START@
   5  *
   6  * This file contains Original Code and/or Modifications of Original Code
   7  * as defined in and that are subject to the Apple Public Source License
   8  * Version 2.0 (the 'License'). You may not use this file except in
   9  * compliance with the License. Please obtain a copy of the License at
  10  * http://www.opensource.apple.com/apsl/ and read it before using this
  11  * file.
  12  *
  13  * The Original Code and all software distributed under the License are
  14  * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  15  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  16  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
  17  * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
  18  * Please see the License for the specific language governing rights and
  19  * limitations under the License.
  20  *
  21  * @APPLE_LICENSE_HEADER_END@
  22  */
  23
  24 /*
  25  * SecItemServer.c - CoreFoundation-based constants and functions for
  26     access to Security items (certificates, keys, identities, and
  27     passwords.)
  28  */
  29
  30 #if TARGET_DARWINOS
  31 #undef OCTAGON
  32 #undef SECUREOBJECTSYNC
  33 #undef SHAREDWEBCREDENTIALS
  34 #endif
  35
  36 #include "keychain/securityd/SecItemServer.h"
  37
  38 #include <notify.h>
  39 #include "keychain/securityd/SecItemDataSource.h"
  40 #include "keychain/securityd/SecItemDb.h"
  41 #include "keychain/securityd/SecItemSchema.h"
  42 #include <utilities/SecDb.h>
  43 #include <utilities/SecDbInternal.h>
  44 #import <utilities/SecCoreAnalytics.h>
  45 #include "keychain/securityd/SecDbKeychainItem.h"
  46 #include "keychain/securityd/SOSCloudCircleServer.h"
  47 #include <Security/SecBasePriv.h>
  48 #include <Security/SecItemPriv.h>
  49 #include <Security/SecItemInternal.h>
  50 #include "keychain/securityd/SecDbBackupManager.h"
  51 #import "keychain/SecureObjectSync/SOSChangeTracker.h"
  52 #include "keychain/SecureObjectSync/SOSDigestVector.h"
  53 #include "keychain/SecureObjectSync/SOSEngine.h"
  54 #include <Security/SecureObjectSync/SOSViews.h>
  55 #include <Security/SecTrustPriv.h>
  56 #include <Security/SecTrustInternal.h>
  57 #include <Security/SecCertificatePriv.h>
  58 #include <Security/SecEntitlements.h>
  59 #include <Security/SecSignpost.h>
  60
  61 #include <keychain/ckks/CKKS.h>
  62 #import "keychain/ot/OT.h"
  63 #import "keychain/ot/OTConstants.h"
  64 #import "keychain/escrowrequest/EscrowRequestServerHelpers.h"
  65
  66 #if USE_KEYSTORE
  67
  68 #if __has_include(<MobileKeyBag/MobileKeyBag.h>)
  69 #include <MobileKeyBag/MobileKeyBag.h>
  70 #else
  71 #include "OSX/utilities/SecAKSWrappers.h"
  72 #endif
  73
  74 #if __has_include(<Kernel/IOKit/crypto/AppleKeyStoreDefs.h>)
  75 #include <Kernel/IOKit/crypto/AppleKeyStoreDefs.h>
  76 #endif
  77
  78 #include <IOKit/IOReturn.h>
  79
  80 #endif
  81
  82 #if TARGET_OS_IPHONE && !TARGET_OS_SIMULATOR
  83 #include <sys/time.h>
  84 #include <unistd.h>
  85 #endif
  86
  87 #include <utilities/array_size.h>
  88 #include <utilities/SecFileLocations.h>
  89 #include <utilities/SecTrace.h>
  90 #include <utilities/SecXPCError.h>
  91 #include <utilities/sec_action.h>
  92 #include <Security/SecuritydXPC.h>
  93 #include "swcagent_client.h"
  94 #include "SecPLWrappers.h"
  95 #include "SecItemServer+SWC.h"
  96 #include <ipc/server_entitlement_helpers.h>
  97
  98 #include <os/variant_private.h>
  99
 100
 101 #include "Analytics/Clients/LocalKeychainAnalytics.h"
 102
 103 /* Changed the name of the keychain changed notification, for testing */
 104 static const char *g_keychain_changed_notification = kSecServerKeychainChangedNotification;
 105
 106 void SecItemServerSetKeychainChangedNotification(const char *notification_name)
 107 {
 108     g_keychain_changed_notification = notification_name;
 109 }
 110
 111 void SecKeychainChanged() {
 112     static dispatch_once_t once;
 113     static sec_action_t action;
 114
 115     dispatch_once(&once, ^{
 116         action = sec_action_create("SecKeychainChanged", 1);
 117         sec_action_set_handler(action, ^{
 118             uint32_t result = notify_post(g_keychain_changed_notification);
 119             if (result == NOTIFY_STATUS_OK)
 120                 secnotice("item", "Sent %s", g_keychain_changed_notification);
 121             else
 122                 secerror("notify_post %s returned: %" PRIu32, g_keychain_changed_notification, result);
 123         });
 124     });
 125
 126     sec_action_perform(action);
 127 }
 128
 129 /* Return the current database version in *version. */
 130 bool SecKeychainDbGetVersion(SecDbConnectionRef dbt, int *version, CFErrorRef *error)
 131 {
 132     __block bool ok = true;
 133     __block CFErrorRef localError = NULL;
 134     __block bool found = false;
 135
 136     /*
 137      * First check for the version table itself
 138      */
 139
 140     ok &= SecDbPrepare(dbt, CFSTR("SELECT name FROM sqlite_master WHERE type='table' AND name='tversion'"), &localError, ^(sqlite3_stmt *stmt) {
 141         ok = SecDbStep(dbt, stmt, NULL, ^(bool *stop) {
 142             found = true;
 143             *stop = true;
 144         });
 145     });
 146     require_action(ok, out, SecDbError(SQLITE_CORRUPT, error, CFSTR("Failed to read sqlite_master table: %@"), localError));
 147     if (!found) {
 148         secnotice("upgr", "no tversion table, will setup a new database: %@", localError);
 149         *version = 0;
 150         goto out;
 151     }
 152
 153     /*
 154      * Now build up major.minor
 155      */
 156
 157     ok &= SecDbPrepare(dbt, CFSTR("SELECT version FROM tversion"), &localError, ^(sqlite3_stmt *stmt) {
 158         ok = SecDbStep(dbt, stmt, NULL, ^(bool *stop) {
 159             *version = sqlite3_column_int(stmt, 0);
 160             if (*version)
 161                 *stop = true;
 162         });
 163     });
 164     if (ok && (*version & 0xffff) >= 9) {
 165         ok &= SecDbPrepare(dbt, CFSTR("SELECT minor FROM tversion WHERE version = ?"), &localError, ^(sqlite3_stmt *stmt) {
 166             ok = SecDbBindInt(stmt, 1, *version, &localError) &&
 167             SecDbStep(dbt, stmt, NULL, ^(bool *stop) {
 168                 int64_t minor = sqlite3_column_int64(stmt, 0);
 169                 *version |= ((minor & 0xff) << 8) | ((minor & 0xff0000) << 8);
 170                 *stop = true;
 171             });
 172         });
 173         ok = true;
 174     }
 175 out:
 176     secnotice("upgr", "database version is: 0x%08x : %d : %@", *version, ok, localError);
 177     secnotice("upgr", "UID: %d  EUID: %d", getuid(), geteuid());
 178     CFReleaseSafe(localError);
 179
 180
 181     return ok;
 182 }
 183
 184 static bool
 185 isClassD(SecDbItemRef item)
 186 {
 187     CFTypeRef accessible = SecDbItemGetCachedValueWithName(item, kSecAttrAccessible);
 188
 189     if (CFEqualSafe(accessible, kSecAttrAccessibleAlwaysPrivate) || CFEqualSafe(accessible, kSecAttrAccessibleAlwaysThisDeviceOnlyPrivate))
 190         return true;
 191     return false;
 192 }
 193
 194 #if TARGET_OS_IPHONE && !TARGET_OS_SIMULATOR
 195
 196 static int64_t
 197 measureDuration(struct timeval *start)
 198 {
 199     struct timeval stop;
 200     int64_t duration;
 201
 202     gettimeofday(&stop, NULL);
 203
 204     duration = (stop.tv_sec-start->tv_sec) * 1000;
 205     duration += (stop.tv_usec / 1000) - (start->tv_usec / 1000);
 206
 207     return SecBucket2Significant(duration);
 208 }
 209
 210 static void
 211 measureUpgradePhase1(struct timeval *start, bool success, int64_t itemsMigrated)
 212 {
 213     int64_t duration = measureDuration(start);
 214
 215     if (success) {
 216         SecCoreAnalyticsSendValue(CFSTR("com.apple.keychain.phase1.migrated-items-success"), itemsMigrated);
 217         SecCoreAnalyticsSendValue(CFSTR("com.apple.keychain.phase1.migrated-time-success"), duration);
 218     } else {
 219         SecCoreAnalyticsSendValue(CFSTR("com.apple.keychain.phase1.migrated-items-fail"), itemsMigrated);
 220         SecCoreAnalyticsSendValue(CFSTR("com.apple.keychain.phase1.migrated-time-fail"), duration);
 221     }
 222 }
 223
 224 static void
 225 measureUpgradePhase2(struct timeval *start, int64_t itemsMigrated)
 226 {
 227     int64_t duration = measureDuration(start);
 228
 229     SecCoreAnalyticsSendValue(CFSTR("com.apple.keychain.phase2.migrated-items"), itemsMigrated);
 230     SecCoreAnalyticsSendValue(CFSTR("com.apple.keychain.phase2.migrated-time"), duration);
 231 }
 232 #endif /* TARGET_OS_IPHONE && !TARGET_OS_SIMULATOR */
 233
 234 static bool DBClassesAreEqual(const SecDbClass* class1, const SecDbClass* class2)
 235 {
 236     if (CFEqual(class1->name, class2->name) && class1->itemclass == class2->itemclass) {
 237         int attrIndex = 0;
 238         const SecDbAttr* class1Attr = class1->attrs[attrIndex];
 239         const SecDbAttr* class2Attr = class2->attrs[attrIndex];
 240
 241         while (class1Attr && class2Attr) {
 242             if (CFEqual(class1Attr->name, class2Attr->name) && class1Attr->kind == class2Attr->kind && class1Attr->flags == class2Attr->flags && class1Attr->copyValue == class2Attr->copyValue && class1Attr->setValue == class2Attr->setValue) {
 243                 attrIndex++;
 244                 class1Attr = class1->attrs[attrIndex];
 245                 class2Attr = class2->attrs[attrIndex];
 246             }
 247             else {
 248                 return false;
 249             }
 250         }
 251
 252         // if everything has checked out to this point, and we've hit the end of both class's attr list, then they're equal
 253         if (class1Attr == NULL && class2Attr == NULL) {
 254             return true;
 255         }
 256     }
 257
 258     return false;
 259 }
 260
 261 static bool ShouldRenameTable(const SecDbClass* class1, const SecDbClass* class2, int oldTableVersion)
 262 {
 263     return oldTableVersion < 10 || !DBClassesAreEqual(class1, class2);
 264 }
 265
 266 #define SCHEMA_VERSION(schema) ((((schema)->minorVersion) << 8) | ((schema)->majorVersion))
 267 #define VERSION_MAJOR(version)  ((version)        & 0xff)
 268 #define VERSION_MINOR(version) (((version) >>  8) & 0xff)
 269 #define VERSION_NEW(version)    ((version)        & 0xffff)
 270 #define VERSION_OLD(version)   (((version) >> 16) & 0xffff)
 271
 272 // Goes through all tables represented by old_schema and tries to migrate all items from them into new (current version) tables.
 273 static bool UpgradeSchemaPhase1(SecDbConnectionRef dbt, const SecDbSchema *oldSchema, CFErrorRef *error)
 274 {
 275     int oldVersion = SCHEMA_VERSION(oldSchema);
 276     const SecDbSchema *newSchema = current_schema();
 277     int newVersion = SCHEMA_VERSION(newSchema);
 278     __block bool ok = true;
 279     SecDbClass const *const *oldClass;
 280     SecDbClass const *const *newClass;
 281     SecDbQueryRef query = NULL;
 282     CFMutableStringRef sql = NULL;
 283     SecDbClass* renamedOldClass = NULL;
 284 #if TARGET_OS_IPHONE && !TARGET_OS_SIMULATOR
 285     __block int64_t itemsMigrated = 0;
 286     struct timeval start;
 287
 288     gettimeofday(&start, NULL);
 289 #endif
 290
 291     // Rename existing tables to names derived from old schema names
 292     sql = CFStringCreateMutable(NULL, 0);
 293     bool oldClassDone = false;
 294     CFMutableArrayRef classIndexesForNewTables = CFArrayCreateMutable(NULL, 0, NULL);
 295     int classIndex = 0;
 296     for (oldClass = oldSchema->classes, newClass = newSchema->classes;
 297          *newClass != NULL; classIndex++, oldClass++, newClass++) {
 298         if (!oldClassDone) {
 299             oldClassDone |= (*oldClass) == NULL; // Check if the new schema has more tables than the old
 300         }
 301
 302         if (!oldClassDone && !CFEqual((*oldClass)->name, (*newClass)->name) && ShouldRenameTable(*oldClass, *newClass, oldSchema->majorVersion)) {
 303             CFStringAppendFormat(sql, NULL, CFSTR("ALTER TABLE %@ RENAME TO %@_old;"), (*newClass)->name, (*oldClass)->name);
 304             CFArrayAppendValue(classIndexesForNewTables, (void*)(long)classIndex);
 305
 306         } else if (!oldClassDone && !DBClassesAreEqual(*oldClass, *newClass)) {
 307             CFStringAppendFormat(sql, NULL, CFSTR("ALTER TABLE %@ RENAME TO %@_old;"), (*newClass)->name, (*oldClass)->name);
 308             CFArrayAppendValue(classIndexesForNewTables, (void*)(long)classIndex);
 309         }
 310
 311         if(oldClassDone && *newClass) {
 312             // These should be no-ops, unless you're upgrading a previously-upgraded database with an invalid version number
 313             CFStringAppendFormat(sql, NULL, CFSTR("DROP TABLE IF EXISTS %@;"), (*newClass)->name);
 314
 315             if (classIndexesForNewTables) {
 316                 CFArrayAppendValue(classIndexesForNewTables, (void*)(long)classIndex);
 317             }
 318         }
 319     }
 320
 321     if(CFStringGetLength(sql) > 0) {
 322         require_action_quiet(ok &= SecDbExec(dbt, sql, error), out,
 323                              LKAReportKeychainUpgradeOutcomeWithError(oldVersion, newVersion, LKAKeychainUpgradeOutcomePhase1AlterTables, error ? *error : NULL));
 324     }
 325     CFReleaseNull(sql);
 326
 327     // Drop indices that that new schemas will use
 328     sql = CFStringCreateMutable(NULL, 0);
 329     for (newClass = newSchema->classes; *newClass != NULL; newClass++) {
 330         SecDbForEachAttrWithMask((*newClass), desc, kSecDbIndexFlag | kSecDbInFlag) {
 331             CFStringAppendFormat(sql, 0, CFSTR("DROP INDEX IF EXISTS %@%@;"), (*newClass)->name, desc->name);
 332             if (desc->kind == kSecDbSyncAttr) {
 333                 CFStringAppendFormat(sql, 0, CFSTR("DROP INDEX IF EXISTS %@%@0;"), (*newClass)->name, desc->name);
 334             }
 335         }
 336     }
 337     require_action_quiet(ok &= SecDbExec(dbt, sql, error), out,
 338                          LKAReportKeychainUpgradeOutcomeWithError(oldVersion, newVersion, LKAKeychainUpgradeOutcomePhase1DropIndices, error ? *error : NULL));
 339     CFReleaseNull(sql);
 340
 341     // Create tables for new schema.
 342     require_action_quiet(ok &= SecItemDbCreateSchema(dbt, newSchema, classIndexesForNewTables, false, error), out,
 343                          LKAReportKeychainUpgradeOutcomeWithError(oldVersion, newVersion, LKAKeychainUpgradeOutcomePhase1CreateSchema, error ? *error : NULL));
 344     // Go through all classes of current schema to transfer all items to new tables.
 345     for (oldClass = oldSchema->classes, newClass = newSchema->classes;
 346          *oldClass != NULL && *newClass != NULL; oldClass++, newClass++) {
 347
 348         if (CFEqual((*oldClass)->name, (*newClass)->name) && DBClassesAreEqual(*oldClass, *newClass)) {
 349             continue;
 350         }
 351
 352         secnotice("upgr", "Upgrading table %@", (*oldClass)->name);
 353
 354         // Create a new 'old' class with a new 'old' name.
 355         int count = 0;
 356         SecDbForEachAttr(*oldClass, attr) {
 357             count++;
 358         }
 359         if(renamedOldClass) {
 360             CFReleaseNull(renamedOldClass->name);
 361             free(renamedOldClass);
 362         }
 363         renamedOldClass = (SecDbClass*) malloc(sizeof(SecDbClass) + sizeof(SecDbAttr*)*(count+1));
 364         renamedOldClass->name = CFStringCreateWithFormat(NULL, NULL, CFSTR("%@_old"), (*oldClass)->name);
 365         renamedOldClass->itemclass = (*oldClass)->itemclass;
 366         for(; count >= 0; count--) {
 367             renamedOldClass->attrs[count] = (*oldClass)->attrs[count];
 368         }
 369
 370         // SecDbItemSelect only works for item classes.
 371         if((*oldClass)->itemclass) {
 372             // Prepare query to iterate through all items in cur_class.
 373             if (query != NULL)
 374                 query_destroy(query, NULL);
 375             require_quiet(query = query_create(renamedOldClass, SecMUSRGetAllViews(), NULL, NULL, error), out);
 376
 377             ok &= SecDbItemSelect(query, dbt, error, ^bool(const SecDbAttr *attr) {
 378                 // We are interested in all attributes which are physically present in the DB.
 379                 return (attr->flags & kSecDbInFlag) != 0;
 380             }, ^bool(const SecDbAttr *attr) {
 381                 // No filtering please.
 382                 return false;
 383             }, NULL, NULL, ^(SecDbItemRef item, bool *stop) {
 384                 CFErrorRef localError = NULL;
 385
 386     #if TARGET_OS_IPHONE && !TARGET_OS_SIMULATOR
 387                 itemsMigrated++;
 388     #endif
 389                 // Switch item to the new class.
 390                 item->class = *newClass;
 391
 392                 if (isClassD(item)) {
 393                     // Decrypt the item.
 394                     ok &= SecDbItemEnsureDecrypted(item, true, &localError);
 395                     require_quiet(ok, out);
 396
 397                     // Delete SHA1 field from the item, so that it is newly recalculated before storing
 398                     // the item into the new table.
 399                     require_quiet(ok &= SecDbItemSetValue(item, SecDbClassAttrWithKind(item->class, kSecDbSHA1Attr, error),
 400                                                               kCFNull, error), out);
 401                 } else {
 402                     // Leave item encrypted, do not ever try to decrypt it since it will fail.
 403                     item->_edataState = kSecDbItemAlwaysEncrypted;
 404                 }
 405                 // Drop items with kSecAttrAccessGroupToken, as these items should not be there at all. Since agrp attribute
 406                 // is always stored as cleartext in the DB column, we can always rely on this attribute being present in item->attributes.
 407                 // <rdar://problem/33401870>
 408                 if (CFEqualSafe(SecDbItemGetCachedValueWithName(item, kSecAttrAccessGroup), kSecAttrAccessGroupToken) &&
 409                     SecDbItemGetCachedValueWithName(item, kSecAttrTokenID) == NULL) {
 410                     secnotice("upgr", "dropping item during schema upgrade due to agrp=com.apple.token: %@", item);
 411                 } else {
 412                     // Insert new item into the new table.
 413                     if (!SecDbItemInsert(item, dbt, false, &localError)) {
 414                         secerror("item: %@ insert during upgrade: %@", item, localError);
 415                         ok = false;
 416                     }
 417                 }
 418
 419             out:
 420                 if (localError) {
 421                     OSStatus status = SecErrorGetOSStatus(localError);
 422
 423                     switch (status) {
 424                         // continue to upgrade and don't propagate errors for insert failures
 425                         // that are typical of a single item failure
 426                         case errSecDecode:
 427                         case errSecDuplicateItem:
 428                             ok = true;
 429                             break;
 430                         case errSecInteractionNotAllowed:
 431                         case errSecAuthNeeded:
 432                             ok = true;
 433                             break;
 434                         // This does not mean the keychain is hosed, we just can't use it right now
 435 #if USE_KEYSTORE
 436                         case kAKSReturnNotReady:
 437                         case kAKSReturnTimeout:
 438 #endif
 439                         case errSecNotAvailable:
 440                             secnotice("upgr", "Bailing in phase 1 because AKS is unavailable: %@", localError);
 441                             // FALLTHROUGH
 442                         default:
 443                             ok &= CFErrorPropagate(CFRetainSafe(localError), error);
 444                             break;
 445                     }
 446                     CFReleaseSafe(localError);
 447                 }
 448
 449                 *stop = !ok;
 450
 451             });
 452
 453             require_action_quiet(ok, out,
 454                                  LKAReportKeychainUpgradeOutcomeWithError(oldVersion, newVersion, LKAKeychainUpgradeOutcomePhase1Items, error ? *error : NULL));
 455         } else {
 456             // This table does not contain secdb items, and must be transferred without using SecDbItemSelect.
 457             // For now, this code does not support removing or renaming any columns, or adding any new non-null columns.
 458             CFReleaseNull(sql);
 459             sql = CFStringCreateMutable(NULL, 0);
 460             bool comma = false;
 461
 462             CFMutableStringRef columns = CFStringCreateMutable(NULL, 0);
 463
 464             SecDbForEachAttr(renamedOldClass, attr) {
 465                 if(comma) {
 466                     CFStringAppendFormat(columns, NULL, CFSTR(","));
 467                 }
 468                 CFStringAppendFormat(columns, NULL, CFSTR("%@"), attr->name);
 469                 comma = true;
 470             }
 471
 472             CFStringAppendFormat(sql, NULL, CFSTR("INSERT OR REPLACE INTO %@ (%@) SELECT %@ FROM %@;"), (*newClass)->name, columns, columns, renamedOldClass->name);
 473
 474             CFReleaseNull(columns);
 475             require_action_quiet(ok &= SecDbExec(dbt, sql, error), out,
 476                                  LKAReportKeychainUpgradeOutcomeWithError(oldVersion, newVersion, LKAKeychainUpgradeOutcomePhase1NonItems, error ? *error : NULL));
 477         }
 478     }
 479
 480     // Remove old tables from the DB.
 481     CFReleaseNull(sql);
 482     sql = CFStringCreateMutable(NULL, 0);
 483     for (oldClass = oldSchema->classes, newClass = newSchema->classes;
 484          *oldClass != NULL && *newClass != NULL; oldClass++, newClass++) {
 485         if (CFEqual((*oldClass)->name, (*newClass)->name) && DBClassesAreEqual(*oldClass, *newClass)) {
 486             continue;
 487         }
 488
 489         CFStringAppendFormat(sql, NULL, CFSTR("DROP TABLE %@_old;"), (*oldClass)->name);
 490     }
 491
 492     if(CFStringGetLength(sql) > 0) {
 493         require_action_quiet(ok &= SecDbExec(dbt, sql, error), out,
 494                              LKAReportKeychainUpgradeOutcomeWithError(oldVersion, newVersion, LKAKeychainUpgradeOutcomePhase1DropOld, error ? *error : NULL));
 495     }
 496 out:
 497 #if TARGET_OS_IPHONE && !TARGET_OS_SIMULATOR
 498     measureUpgradePhase1(&start, ok, SecBucket2Significant(itemsMigrated));
 499 #endif
 500
 501     if (query != NULL) {
 502         query_destroy(query, NULL);
 503     }
 504     CFReleaseSafe(sql);
 505     CFReleaseNull(classIndexesForNewTables);
 506     if(renamedOldClass) {
 507         CFReleaseNull(renamedOldClass->name);
 508         free(renamedOldClass);
 509     }
 510     return ok;
 511 }
 512
 513 __thread SecDbConnectionRef threadDbt = NULL;
 514
 515 // Goes through all tables represented by old_schema and tries to migrate all items from them into new (current version) tables.
 516 static bool UpgradeItemPhase2(SecDbConnectionRef inDbt, bool *inProgress, int oldVersion, CFErrorRef *error) {
 517     SecDbConnectionRef oldDbt = threadDbt;
 518     threadDbt = inDbt;
 519     __block bool ok = true;
 520     SecDbQueryRef query = NULL;
 521 #if TARGET_OS_IPHONE && !TARGET_OS_SIMULATOR
 522     __block int64_t itemsMigrated = 0;
 523     struct timeval start;
 524
 525     gettimeofday(&start, NULL);
 526 #endif
 527
 528 #if SECDB_BACKUPS_ENABLED
 529     CFErrorRef backuperror = NULL;
 530     if (!SecDbBackupCreateOrLoadBackupInfrastructure(&backuperror)) {
 531         secerror("upgr: failed to create backup infrastructure: %@", backuperror);
 532     } else {
 533         secnotice("upgr", "setup backup infrastructure");
 534     }
 535 #else
 536     secnotice("upgr", "skipping backup setup for this platform");
 537 #endif
 538
 539     // Go through all classes in new schema
 540     const SecDbSchema *newSchema = current_schema();
 541     int newVersion = SCHEMA_VERSION(newSchema);
 542     for (const SecDbClass *const *class = newSchema->classes; *class != NULL && !*inProgress; class++) {
 543         if(!((*class)->itemclass)) {
 544             //Don't try to decrypt non-item 'classes'
 545             continue;
 546         }
 547
 548         const SecDbAttr *pdmn = SecDbClassAttrWithKind(*class, kSecDbAccessAttr, error);
 549         if (pdmn == nil) {
 550             continue;
 551         }
 552
 553         // Prepare query to go through all non-DK|DKU items
 554         if (query != NULL) {
 555             query_destroy(query, NULL);
 556         }
 557         require_action_quiet(query = query_create(*class, SecMUSRGetAllViews(), NULL, NULL, error), out, ok = false);
 558         ok &= SecDbItemSelect(query, threadDbt, error, NULL, ^bool(const SecDbAttr *attr) {
 559             // No simple per-attribute filtering.
 560             return false;
 561         }, ^bool(CFMutableStringRef sql, bool *needWhere) {
 562             // Select only non-D-class items
 563             SecDbAppendWhereOrAnd(sql, needWhere);
 564             CFStringAppendFormat(sql, NULL, CFSTR("NOT %@ IN (?,?)"), pdmn->name);
 565             return true;
 566         }, ^bool(sqlite3_stmt *stmt, int col) {
 567             return SecDbBindObject(stmt, col++, kSecAttrAccessibleAlwaysPrivate, error) &&
 568             SecDbBindObject(stmt, col++, kSecAttrAccessibleAlwaysThisDeviceOnlyPrivate, error);
 569         }, ^(SecDbItemRef item, bool *stop) {
 570             CFErrorRef localError = NULL;
 571
 572 #if TARGET_OS_IPHONE && !TARGET_OS_SIMULATOR
 573             itemsMigrated++;
 574 #endif
 575
 576             // Decrypt the item.
 577             if (SecDbItemEnsureDecrypted(item, true, &localError)) {
 578
 579                 // Delete SHA1 field from the item, so that it is newly recalculated before storing
 580                 // the item into the new table.
 581                 require_quiet(ok = SecDbItemSetValue(item, SecDbClassAttrWithKind(item->class, kSecDbSHA1Attr, error),
 582                                                      kCFNull, &localError), out);
 583                 // Drop items with kSecAttrAccessGroupToken, as these items should not be there at all. Since agrp attribute
 584                 // is always stored as cleartext in the DB column, we can always rely on this attribute being present in item->attributes.
 585                 // <rdar://problem/33401870>
 586                 if (CFEqualSafe(SecDbItemGetCachedValueWithName(item, kSecAttrAccessGroup), kSecAttrAccessGroupToken) &&
 587                     SecDbItemGetCachedValueWithName(item, kSecAttrTokenID) == NULL) {
 588                     secnotice("upgr", "dropping item during item upgrade due to agrp=com.apple.token: %@", item);
 589                     ok = SecDbItemDelete(item, threadDbt, kCFBooleanFalse, false, &localError);
 590                 } else {
 591                     // Replace item with the new value in the table; this will cause the item to be decoded and recoded back,
 592                     // incl. recalculation of item's hash.
 593                     ok = SecDbItemUpdate(item, item, threadDbt, false, query->q_uuid_from_primary_key, &localError);
 594                 }
 595             }
 596
 597             if (localError) {
 598                 CFIndex status = CFErrorGetCode(localError);
 599
 600                 switch (status) {
 601                     case errSecDecode: {
 602                         // Items producing errSecDecode are silently dropped - they are not decodable and lost forever.
 603                         // make sure we use a local error so that this error is not proppaged upward and cause a
 604                         // migration failure.
 605                         CFErrorRef deleteError = NULL;
 606                         (void)SecDbItemDelete(item, threadDbt, false, false, &deleteError);
 607                         CFReleaseNull(deleteError);
 608                         ok = true;
 609                         break;
 610                     }
 611                     case errSecInteractionNotAllowed:
 612                         // If we are still not able to decrypt the item because the class key is not released yet,
 613                         // remember that DB still needs phase2 migration to be run next time a connection is made.  Also
 614                         // stop iterating next items, it would be just waste of time because the whole iteration will be run
 615                         // next time when this phase2 will be rerun.
 616                         LKAReportKeychainUpgradeOutcome(oldVersion, newVersion, LKAKeychainUpgradeOutcomeLocked);
 617                         *inProgress = true;
 618                         *stop = true;
 619                         ok = true;
 620                         break;
 621                     case errSecAuthNeeded:
 622                         // errSecAuthNeeded means that it is an ACL-based item which requires authentication (or at least
 623                         // ACM context, which we do not have).
 624                         ok = true;
 625                         break;
 626                     case SQLITE_CONSTRAINT:         // yeah...
 627                         if (!CFEqual(kSecDbErrorDomain, CFErrorGetDomain(localError))) {
 628                             secerror("Received SQLITE_CONSTRAINT with wrong error domain. Huh? Item: %@, error: %@", item, localError);
 629                             break;
 630                         }
 631                     case errSecDuplicateItem:
 632                         // continue to upgrade and don't propagate errors for insert failures
 633                         // that are typical of a single item failure
 634                         secnotice("upgr", "Ignoring duplicate item: %@", item);
 635                         secdebug("upgr", "Duplicate item error: %@", localError);
 636                         ok = true;
 637                         break;
 638 #if USE_KEYSTORE
 639                     case kAKSReturnNotReady:
 640                     case kAKSReturnTimeout:
 641 #endif
 642                     case errSecNotAvailable:
 643                         *inProgress = true;     // We're not done, call me again later!
 644                         secnotice("upgr", "Bailing in phase 2 because AKS is unavailable: %@", localError);
 645                         // FALLTHROUGH
 646                     default:
 647                         //  Other errors should abort the migration completely.
 648                         ok = CFErrorPropagate(CFRetainSafe(localError), error);
 649                         break;
 650                 }
 651             }
 652
 653         out:
 654             CFReleaseSafe(localError);
 655             *stop = *stop || !ok;
 656
 657         });
 658         require_action(ok, out, LKAReportKeychainUpgradeOutcomeWithError(oldVersion, newVersion, LKAKeychainUpgradeOutcomePhase2, error ? *error : NULL));
 659     }
 660
 661 #if TARGET_OS_IPHONE && !TARGET_OS_SIMULATOR
 662     measureUpgradePhase2(&start, SecBucket2Significant(itemsMigrated));
 663 #endif
 664
 665 out:
 666     if (query != NULL)
 667         query_destroy(query, NULL);
 668
 669     threadDbt = oldDbt;
 670     return ok;
 671 }
 672
 673 // There's no data-driven approach for this. Let's think about it more if it gets unwieldy
 674 static void performCustomIndexProcessing(SecDbConnectionRef dbt) {
 675     CFErrorRef cfErr = NULL;
 676     require(SecDbExec(dbt, CFSTR("DROP INDEX IF EXISTS genpagrp; DROP INDEX IF EXISTS genpsync;"), &cfErr), errhandler);
 677     require(SecDbExec(dbt, CFSTR("DROP INDEX IF EXISTS inetagrp; DROP INDEX IF EXISTS inetsync;"), &cfErr), errhandler);
 678     require(SecDbExec(dbt, CFSTR("DROP INDEX IF EXISTS certagrp; DROP INDEX IF EXISTS certsync;"), &cfErr), errhandler);
 679     require(SecDbExec(dbt, CFSTR("DROP INDEX IF EXISTS keysagrp; DROP INDEX IF EXISTS keyssync;"), &cfErr), errhandler);
 680
 681     require(SecDbExec(dbt, CFSTR("DROP INDEX IF EXISTS genpsync0;"), &cfErr), errhandler);
 682     require(SecDbExec(dbt, CFSTR("DROP INDEX IF EXISTS inetsync0;"), &cfErr), errhandler);
 683     require(SecDbExec(dbt, CFSTR("DROP INDEX IF EXISTS certsync0;"), &cfErr), errhandler);
 684     require(SecDbExec(dbt, CFSTR("DROP INDEX IF EXISTS keyssync0;"), &cfErr), errhandler);
 685
 686     require(SecDbExec(dbt, CFSTR("DROP INDEX IF EXISTS genpmusr;"), &cfErr), errhandler);
 687     require(SecDbExec(dbt, CFSTR("DROP INDEX IF EXISTS inetmusr;"), &cfErr), errhandler);
 688     require(SecDbExec(dbt, CFSTR("DROP INDEX IF EXISTS certmusr;"), &cfErr), errhandler);
 689     require(SecDbExec(dbt, CFSTR("DROP INDEX IF EXISTS keysmusr;"), &cfErr), errhandler);
 690     require(SecDbExec(dbt, CFSTR("DROP INDEX IF EXISTS item_backupmusr;"), &cfErr), errhandler);
 691     require(SecDbExec(dbt, CFSTR("DROP INDEX IF EXISTS backup_keybagmusr;"), &cfErr), errhandler);
 692     require(SecDbExec(dbt, CFSTR("DROP INDEX IF EXISTS backup_keyarchivemusr;"), &cfErr), errhandler);
 693     require(SecDbExec(dbt, CFSTR("DROP INDEX IF EXISTS archived_key_backupmusr;"), &cfErr), errhandler);
 694
 695     require(SecDbExec(dbt, CFSTR("CREATE INDEX IF NOT EXISTS agrp_musr_tomb_svce ON genp(agrp, musr, tomb, svce);"), &cfErr), errhandler);
 696     require(SecDbExec(dbt, CFSTR("CREATE INDEX IF NOT EXISTS agrp_musr_tomb_srvr ON inet(agrp, musr, tomb, srvr);"), &cfErr), errhandler);
 697     require(SecDbExec(dbt, CFSTR("CREATE INDEX IF NOT EXISTS agrp_musr_tomb_subj ON cert(agrp, musr, tomb, subj);"), &cfErr), errhandler);
 698     require(SecDbExec(dbt, CFSTR("CREATE INDEX IF NOT EXISTS agrp_musr_tomb_atag ON keys(agrp, musr, tomb, atag);"), &cfErr), errhandler);
 699
 700     secnotice("upgr", "processed custom indexes (now or in the past)");
 701     CFReleaseNull(cfErr);   // Should be nil but belt and suspenders
 702     return;
 703
 704 errhandler:
 705     secerror("upgr: failed to process custom indexes: %@", cfErr);
 706     CFReleaseNull(cfErr);
 707 }
 708
 709 static bool SecKeychainDbUpgradeFromVersion(SecDbConnectionRef dbt, int version, bool *inProgress, CFErrorRef *error) {
 710     __block bool didPhase2 = false;
 711     __block bool ok = true;
 712     __block CFErrorRef localError = NULL;
 713
 714     if (error)
 715         *error = NULL;
 716
 717     const SecDbSchema *newSchema = current_schema();
 718     int newVersion = SCHEMA_VERSION(newSchema);
 719     bool skipped_upgrade = false;
 720
 721     // If DB schema is the one we want, we are done.
 722     require_action_quiet(SCHEMA_VERSION(newSchema) != version, out, skipped_upgrade = true);
 723
 724     // Check if the schema of the database on disk is the same major, but newer version then what we have
 725     // in code, lets just skip this since a newer version of the OS have upgrade it. Since its the same
 726     // major, its a promise that it will be compatible.
 727     if (newSchema->majorVersion == VERSION_MAJOR(version) && newSchema->minorVersion < VERSION_MINOR(version)) {
 728         secnotice("upgr", "skipping upgrade since minor is newer");
 729         goto out;
 730     }
 731
 732     ok &= SecDbTransaction(dbt, kSecDbExclusiveTransactionType, &localError, ^(bool *commit) {
 733         CFStringRef sql = NULL;
 734         bool didPhase1 = false;
 735
 736         // Get version again once we start a transaction, someone else might change the migration state.
 737         int version2 = 0;
 738         require_quiet(ok = SecKeychainDbGetVersion(dbt, &version2, &localError), out);
 739         // Check if someone has raced us to the migration of the database
 740         require_action(version == version2, out, CFReleaseNull(localError); ok = true);
 741
 742         require_quiet(SCHEMA_VERSION(newSchema) != version2, out);
 743
 744         // If this is empty database, just create table according to schema and be done with it.
 745         require_action_quiet(version2 != 0, out, ok = SecItemDbCreateSchema(dbt, newSchema, NULL, true, &localError);
 746                              performCustomIndexProcessing(dbt);
 747                              LKAReportKeychainUpgradeOutcomeWithError(version2, newVersion, LKAKeychainUpgradeOutcomeNewDb, localError));
 748
 749         int oldVersion = VERSION_OLD(version2);
 750         version2 = VERSION_NEW(version2);
 751
 752         require_action_quiet(version2 == SCHEMA_VERSION(newSchema) || oldVersion == 0, out,
 753                              ok = SecDbError(SQLITE_CORRUPT, &localError,
 754                                              CFSTR("Half migrated but obsolete DB found: found 0x%x(0x%x) but 0x%x is needed"),
 755                                              version2, oldVersion, SCHEMA_VERSION(newSchema));
 756                              LKAReportKeychainUpgradeOutcome(version2, newVersion, LKAKeychainUpgradeOutcomeObsoleteDb));
 757
 758         // Check whether we have both old and new tables in the DB.
 759         if (oldVersion == 0) {
 760             // Pure old-schema migration attempt, with full blown table renames etc (a.k.a. phase1)
 761             oldVersion = version2;
 762             version2 = SCHEMA_VERSION(newSchema);
 763
 764             // Find schema for old database.
 765             const SecDbSchema *oldSchema = NULL;
 766             for (const SecDbSchema * const *pschema = all_schemas(); *pschema; ++pschema) {
 767                 if (SCHEMA_VERSION((*pschema)) == oldVersion) {
 768                     oldSchema = *pschema;
 769                     break;
 770                 }
 771             }
 772
 773             // If we are attempting to upgrade from a version for which we have no schema, fail.
 774             require_action_quiet(oldSchema != NULL, out,
 775                                  ok = SecDbError(SQLITE_CORRUPT, &localError, CFSTR("no schema for version: 0x%x"), oldVersion);
 776                                  secerror("no schema for version 0x%x", oldVersion);
 777                                  LKAReportKeychainUpgradeOutcome(version2, newVersion, LKAKeychainUpgradeOutcomeNoSchema));
 778
 779             secnotice("upgr", "Upgrading from version 0x%x to 0x%x", oldVersion, SCHEMA_VERSION(newSchema));
 780             SecSignpostStart(SecSignpostUpgradePhase1);
 781             require_action(ok = UpgradeSchemaPhase1(dbt, oldSchema, &localError), out, secerror("upgrade: Upgrade phase1 failed: %@", localError));
 782             SecSignpostStop(SecSignpostUpgradePhase1);
 783
 784             didPhase1 = true;
 785         }
 786
 787         {
 788             CFErrorRef phase2Error = NULL;
 789
 790             SecSignpostStart(SecSignpostUpgradePhase2);
 791
 792             // Lets try to go through non-D-class items in new tables and apply decode/encode on them
 793             // If this fails the error will be ignored after doing a phase1 since but not in the second
 794             // time when we are doing phase2.
 795             ok = UpgradeItemPhase2(dbt, inProgress, version2, &phase2Error);
 796             if (!ok) {
 797                 if (didPhase1) {
 798                     *inProgress = true;
 799                     ok = true;
 800                     CFReleaseNull(phase2Error);
 801                 } else {
 802                     SecErrorPropagate(phase2Error, &localError);
 803                 }
 804             }
 805             require_action(ok, out, secerror("upgrade: Upgrade phase2 (%d) failed: %@", didPhase1, localError));
 806
 807             if (!*inProgress) {
 808                 // If either migration path we did reported that the migration was complete, signalize that
 809                 // in the version database by cleaning oldVersion (which is stored in upper halfword of the version)
 810                 secnotice("upgr", "Done upgrading from version 0x%x to 0x%x", oldVersion, SCHEMA_VERSION(newSchema));
 811                 oldVersion = 0;
 812
 813                 didPhase2 = true;
 814                 SecSignpostStop(SecSignpostUpgradePhase2);
 815             }
 816         }
 817
 818         // Update database version table.
 819         uint32_t major = (VERSION_MAJOR(version2)) | (VERSION_MAJOR(oldVersion) << 16);
 820         uint32_t minor = (VERSION_MINOR(version2)) | (VERSION_MINOR(oldVersion) << 16);
 821         secnotice("upgr", "Upgrading saving version major 0x%x minor 0x%x", major, minor);
 822         sql = CFStringCreateWithFormat(NULL, NULL, CFSTR("UPDATE tversion SET version='%d', minor='%d'"),
 823                                        major, minor);
 824         require_action_quiet(ok = SecDbExec(dbt, sql, &localError), out, secerror("upgrade: Setting version failed: %@", localError));
 825
 826     out:
 827         if (!ok) {
 828             secerror("upgrade: SecDB upgrade failed: %@", localError);
 829         }
 830         CFReleaseSafe(sql);
 831         *commit = ok;
 832     });
 833
 834     if (ok && didPhase2) {
 835 #if TARGET_OS_IPHONE && !TARGET_OS_SIMULATOR
 836         SecCoreAnalyticsSendValue(CFSTR("com.apple.keychain.migration-success"), 1);
 837 #endif
 838     }
 839
 840 out:
 841     if (!ok || localError) {
 842         // TODO: This logic should be inverted to a do-not-corrupt-unless default, <rdar://problem/29771874>
 843         /*
 844          * We assume that database is corrupt at this point, but we need to
 845          * check if the error we got isn't severe enough to mark the database as corrupt.
 846          * In those cases we opt out of corrupting the database.
 847          */
 848         bool markedCorrupt = true;
 849
 850         if (ok) {
 851             secwarning("upgrade: error has been set but status is true");
 852             ok = false;
 853         }
 854         secerror("upgrade: error occurred, considering marking database as corrupt: %@", localError);
 855         if (localError) {
 856             CFStringRef domain = CFErrorGetDomain(localError);
 857             CFIndex code = CFErrorGetCode(localError);
 858
 859             if ((CFEqualSafe(domain, kSecDbErrorDomain) &&
 860                 ((code & 0xff) == SQLITE_LOCKED || (code & 0xff) == SQLITE_BUSY || (code & 0xff) == SQLITE_FULL)) ||
 861 #if USE_KEYSTORE
 862                 code == kAKSReturnNotReady || code == kAKSReturnTimeout ||
 863 #endif
 864                 code == errSecNotAvailable)
 865             {
 866                 secerror("upgrade: not marking keychain database corrupt for error: %@", localError);
 867                 markedCorrupt = false;
 868                 CFReleaseNull(localError);
 869             } else {
 870                 secerror("upgrade: unable to complete upgrade, marking DB as corrupt: %@", localError);
 871             }
 872         } else {
 873             secerror("upgrade: unable to complete upgrade and no error object returned, marking DB as corrupt");
 874         }
 875         if (markedCorrupt) {
 876             secerror("upgrade: marking database as corrupt");
 877             SecDbCorrupt(dbt, localError);
 878 #if TARGET_OS_IPHONE && !TARGET_OS_SIMULATOR
 879             SecCoreAnalyticsSendValue(CFSTR("com.apple.keychain.migration-failure"), 1);
 880 #endif
 881         }
 882     } else {
 883         // Things seemed to go okay!
 884         if (didPhase2) {
 885             LKAReportKeychainUpgradeOutcome(version, newVersion, LKAKeychainUpgradeOutcomeSuccess);
 886         }
 887
 888         //If we're done here, we should opportunistically re-add all indices (just in case)
 889         if(skipped_upgrade || didPhase2) {
 890             // Create indices, ignoring all errors
 891             performCustomIndexProcessing(dbt);
 892             for (SecDbClass const* const* newClass = newSchema->classes; *newClass; ++newClass) {
 893                 SecDbForEachAttrWithMask((*newClass), desc, kSecDbIndexFlag | kSecDbInFlag) {
 894                     CFStringRef sql = NULL;
 895                     CFErrorRef classLocalError = NULL;
 896                     bool localOk = true;
 897                     sql = CFStringCreateWithFormat(NULL, NULL, CFSTR("CREATE INDEX IF NOT EXISTS %@%@ ON %@(%@);"), (*newClass)->name, desc->name, (*newClass)->name, desc->name);
 898                     localOk &= SecDbExec(dbt, sql, &classLocalError);
 899                     CFReleaseNull(sql);
 900
 901                     if(!localOk) {
 902                         secerror("upgrade: unable to opportunistically create index (%@,%@): %@", (*newClass)->name, desc->name, classLocalError);
 903                     }
 904                     CFReleaseNull(classLocalError);
 905                 }
 906             }
 907         }
 908     }
 909     if (localError) {
 910         if (error) {
 911             *error = (CFErrorRef)CFRetain(localError);
 912         }
 913         CFReleaseNull(localError);
 914     }
 915
 916     return ok;
 917 }
 918
 919 static bool accessGroupIsNetworkExtensionAndClientIsEntitled(CFStringRef accessGroup, SecurityClient* client)
 920 {
 921     return client && client->canAccessNetworkExtensionAccessGroups && accessGroup && CFStringHasSuffix(accessGroup, kSecNetworkExtensionAccessGroupSuffix);
 922 }
 923
 924 /* AUDIT[securityd](done):
 925    accessGroup (ok) is a caller provided, non NULL CFTypeRef.
 926
 927    Return true iff accessGroup is allowable according to accessGroups.
 928  */
 929 bool accessGroupsAllows(CFArrayRef accessGroups, CFStringRef accessGroup, SecurityClient* client) {
 930     /* NULL accessGroups is wildcard. */
 931     if (!accessGroups)
 932         return true;
 933     /* Make sure we have a string. */
 934     if (!isString(accessGroup))
 935         return false;
 936
 937     /* Having the special accessGroup "*" allows access to all accessGroups. */
 938     CFRange range = { 0, CFArrayGetCount(accessGroups) };
 939     if (range.length &&
 940         (CFArrayContainsValue(accessGroups, range, accessGroup) ||
 941          CFArrayContainsValue(accessGroups, range, CFSTR("*")) ||
 942          accessGroupIsNetworkExtensionAndClientIsEntitled(accessGroup, client)))
 943         return true;
 944
 945     return false;
 946 }
 947
 948 bool itemInAccessGroup(CFDictionaryRef item, CFArrayRef accessGroups) {
 949     return accessGroupsAllows(accessGroups,
 950                               CFDictionaryGetValue(item, kSecAttrAccessGroup), NULL);
 951 }
 952
 953
 954 static CF_RETURNS_RETAINED CFDataRef SecServerExportBackupableKeychain(SecDbConnectionRef dbt,
 955     SecurityClient *client,
 956     keybag_handle_t src_keybag, keybag_handle_t dest_keybag, CFErrorRef *error) {
 957     CFDataRef data_out = NULL;
 958
 959     SecSignpostStart(SecSignpostBackupKeychainBackupable);
 960
 961     /* Export everything except the items for which SecItemIsSystemBound()
 962        returns true. */
 963     CFDictionaryRef keychain = SecServerCopyKeychainPlist(dbt, client,
 964         src_keybag, dest_keybag, kSecBackupableItemFilter,
 965         error);
 966     if (keychain) {
 967         data_out = CFPropertyListCreateData(kCFAllocatorDefault, keychain,
 968                                              kCFPropertyListBinaryFormat_v1_0,
 969                                              0, error);
 970         CFRelease(keychain);
 971     }
 972     SecSignpostStop(SecSignpostBackupKeychainBackupable);
 973
 974     return data_out;
 975 }
 976
 977 static bool SecServerImportBackupableKeychain(SecDbConnectionRef dbt,
 978                                               SecurityClient *client,
 979                                               keybag_handle_t src_keybag,
 980                                               keybag_handle_t dest_keybag,
 981                                               CFDataRef data,
 982                                               CFErrorRef *error)
 983 {
 984     return kc_transaction(dbt, error, ^{
 985         bool ok = false;
 986         CFDictionaryRef keychain;
 987
 988         SecSignpostStart(SecSignpostRestoreKeychainBackupable);
 989
 990         keychain = CFPropertyListCreateWithData(kCFAllocatorDefault, data,
 991                                                 kCFPropertyListImmutable, NULL,
 992                                                 error);
 993         if (keychain) {
 994             if (isDictionary(keychain)) {
 995                 ok = SecServerImportKeychainInPlist(dbt,
 996                                                     client,
 997                                                     src_keybag,
 998                                                     dest_keybag,
 999                                                     keychain,
1000                                                     kSecBackupableItemFilter,
1001                                                     false,  // Restoring backup should not remove stuff that got into the keychain before us
1002                                                     error);
1003             } else {
1004                 ok = SecError(errSecParam, error, CFSTR("import: keychain is not a dictionary"));
1005             }
1006             CFRelease(keychain);
1007         }
1008
1009         SecSignpostStop(SecSignpostRestoreKeychainBackupable);
1010
1011         return ok;
1012     });
1013 }
1014
1015 #if USE_KEYSTORE
1016 /*
1017  * Similar to ks_open_keybag, but goes through MKB interface
1018  */
1019 static bool mkb_open_keybag(CFDataRef keybag, CFDataRef password, MKBKeyBagHandleRef *handle, bool emcs, CFErrorRef *error) {
1020     kern_return_t rc;
1021     MKBKeyBagHandleRef mkbhandle = NULL;
1022
1023     rc = MKBKeyBagCreateWithData(keybag, &mkbhandle);
1024     if (rc != kMobileKeyBagSuccess) {
1025         return SecKernError(rc, error, CFSTR("MKBKeyBagCreateWithData failed: %d"), rc);
1026     }
1027
1028     if (!emcs) {
1029         rc = MKBKeyBagUnlock(mkbhandle, password);
1030         if (rc != kMobileKeyBagSuccess) {
1031             CFRelease(mkbhandle);
1032             return SecKernError(rc, error, CFSTR("failed to unlock bag: %d"), rc);
1033         }
1034     } else {
1035         secnotice("keychainbackup", "skipping keybag unlock for EMCS");
1036     }
1037
1038     *handle = mkbhandle;
1039
1040     return true;
1041 }
1042 #endif
1043
1044
1045 static CFDataRef SecServerKeychainCreateBackup(SecDbConnectionRef dbt, SecurityClient *client, CFDataRef keybag,
1046     CFDataRef password, bool emcs, CFErrorRef *error) {
1047     CFDataRef backup = NULL;
1048     keybag_handle_t backup_keybag;
1049
1050     SecSignpostStart(SecSignpostBackupOpenKeybag);
1051
1052 #if USE_KEYSTORE
1053     MKBKeyBagHandleRef mkbhandle = NULL;
1054     require(mkb_open_keybag(keybag, password, &mkbhandle, emcs, error), out);
1055
1056     require_noerr(MKBKeyBagGetAKSHandle(mkbhandle, &backup_keybag), out);
1057
1058 #else
1059     backup_keybag = KEYBAG_NONE;
1060 #endif
1061     SecSignpostStop(SecSignpostBackupOpenKeybag);
1062     SecSignpostStart(SecSignpostBackupKeychain);
1063
1064     /* Export from system keybag to backup keybag. */
1065     backup = SecServerExportBackupableKeychain(dbt, client, KEYBAG_DEVICE, backup_keybag, error);
1066
1067 #if USE_KEYSTORE
1068 out:
1069     SecSignpostStop(SecSignpostBackupOpenKeybag);
1070
1071     if (mkbhandle)
1072         CFRelease(mkbhandle);
1073 #endif
1074     return backup;
1075 }
1076
1077 static bool SecServerKeychainRestore(SecDbConnectionRef dbt,
1078                                      SecurityClient *client,
1079                                      CFDataRef backup,
1080                                      CFDataRef keybag,
1081                                      CFDataRef password,
1082                                      CFErrorRef *error)
1083 {
1084     bool ok = false;
1085     keybag_handle_t backup_keybag;
1086
1087     secnotice("SecServerKeychainRestore", "Restoring keychain backup");
1088
1089     SecSignpostStart(SecSignpostRestoreOpenKeybag);
1090 #if USE_KEYSTORE
1091     MKBKeyBagHandleRef mkbhandle = NULL;
1092     require(mkb_open_keybag(keybag, password, &mkbhandle, false, error), out);
1093
1094     require_noerr(MKBKeyBagGetAKSHandle(mkbhandle, &backup_keybag), out);
1095 #else
1096     backup_keybag = KEYBAG_NONE;
1097 #endif
1098     SecSignpostStop(SecSignpostRestoreOpenKeybag);
1099     SecSignpostStart(SecSignpostRestoreKeychain);
1100
1101     /* Import from backup keybag to system keybag. */
1102     require(SecServerImportBackupableKeychain(dbt, client, backup_keybag, KEYBAG_DEVICE, backup, error), out);
1103
1104     ok = true;
1105 out:
1106     SecSignpostStop(SecSignpostRestoreKeychain);
1107 #if USE_KEYSTORE
1108     if (mkbhandle)
1109         CFRelease(mkbhandle);
1110 #endif
1111
1112     if (ok) {
1113         secnotice("SecServerKeychainRestore", "Restore completed successfully");
1114     } else {
1115         secwarning("SecServerKeychainRestore: Restore failed with: %@", error ? *error : NULL);
1116     }
1117
1118     return ok;
1119 }
1120
1121 // MARK - External SPI support code.
1122
1123 CFStringRef __SecKeychainCopyPath(void) {
1124     CFStringRef kcRelPath = NULL;
1125
1126     if (os_variant_is_recovery("securityd")) {
1127         kcRelPath = CFSTR("keychain-recovery-2.db");
1128     } else if (use_hwaes()) {
1129         kcRelPath = CFSTR("keychain-2.db");
1130     } else {
1131         kcRelPath = CFSTR("keychain-2-debug.db");
1132     }
1133
1134     CFStringRef kcPath = NULL;
1135     CFURLRef kcURL = SecCopyURLForFileInKeychainDirectory(kcRelPath);
1136     if (kcURL) {
1137         kcPath = CFURLCopyFileSystemPath(kcURL, kCFURLPOSIXPathStyle);
1138         CFRelease(kcURL);
1139     }
1140     return kcPath;
1141 }
1142
1143 // MARK; -
1144 // MARK: kc_dbhandle init and reset
1145
1146 SecDbRef SecKeychainDbCreate(CFStringRef path, CFErrorRef* error) {
1147     __block CFErrorRef localerror = NULL;
1148
1149     SecDbRef kc = SecDbCreate(path, 0600, true, true, true, true, kSecDbMaxIdleHandles,
1150         ^bool (SecDbRef db, SecDbConnectionRef dbconn, bool didCreate, bool *callMeAgainForNextConnection, CFErrorRef *createError)
1151     {
1152         // Upgrade from version 0 means create the schema in empty db.
1153         int version = 0;
1154         bool ok = true;
1155         if (!didCreate)
1156             ok = SecKeychainDbGetVersion(dbconn, &version, createError);
1157
1158         ok = ok && SecKeychainDbUpgradeFromVersion(dbconn, version, callMeAgainForNextConnection, createError);
1159         if (!ok)
1160             secerror("Upgrade %sfailed: %@", didCreate ? "from v0 " : "", createError ? *createError : NULL);
1161
1162         localerror = createError ? *createError : NULL;
1163
1164         if(ok) {
1165             // This block might get called many, many times due to callMeAgainForNextConnection.
1166             // When we no longer want to be called, we believe we're done. Begin the rest of initialization.
1167             if( !callMeAgainForNextConnection || !(*callMeAgainForNextConnection)) {
1168                 SecKeychainDbInitialize(db);
1169             }
1170         }
1171
1172         return ok;
1173     });
1174
1175     if (kc) {
1176         SecDbSetCorruptionReset(kc, ^{
1177             SecDbResetMetadataKeys();
1178         });
1179     }
1180
1181     if(error) {
1182         *error = localerror;
1183     }
1184
1185     return kc;
1186 }
1187
1188 SecDbRef SecKeychainDbInitialize(SecDbRef db) {
1189
1190 #if OCTAGON
1191     // This needs to be async, otherwise we get hangs between securityd, cloudd, and apsd
1192     dispatch_async(dispatch_get_global_queue(DISPATCH_QUEUE_PRIORITY_DEFAULT, 0), ^{
1193         if(OctagonIsEnabled() && OctagonShouldPerformInitialization()) {
1194             OctagonInitialize();
1195         }
1196
1197         if(SecCKKSIsEnabled()) {
1198             SecCKKSInitialize(db);
1199         }
1200     });
1201
1202     if(EscrowRequestServerIsEnabled()) {
1203         dispatch_async(dispatch_get_global_queue(DISPATCH_QUEUE_PRIORITY_DEFAULT, 0), ^{
1204             EscrowRequestServerInitialize();
1205         });
1206     }
1207 #endif
1208
1209     return db;
1210 }
1211
1212 static SecDbRef _kc_dbhandle = NULL;
1213 static dispatch_queue_t _kc_dbhandle_dispatch = NULL;
1214 static dispatch_once_t _kc_dbhandle_dispatch_onceToken = 0;
1215 static dispatch_queue_t get_kc_dbhandle_dispatch() {
1216     dispatch_once(&_kc_dbhandle_dispatch_onceToken, ^{
1217         _kc_dbhandle_dispatch = dispatch_queue_create("sec_kc_dbhandle", DISPATCH_QUEUE_SERIAL);
1218     });
1219
1220     return _kc_dbhandle_dispatch;
1221 }
1222
1223 static bool kc_dbhandle_init(CFErrorRef* error) {
1224     SecDbRef oldHandle = _kc_dbhandle;
1225     _kc_dbhandle = NULL;
1226     CFStringRef dbPath = __SecKeychainCopyPath();
1227     if (dbPath) {
1228         _kc_dbhandle = SecKeychainDbCreate(dbPath, error);
1229         CFRelease(dbPath);
1230     } else {
1231         secerror("no keychain path available");
1232     }
1233     if (oldHandle) {
1234         secerror("replaced %@ with %@", oldHandle, _kc_dbhandle);
1235         CFRelease(oldHandle);
1236     }
1237     // Having a dbhandle means we succeeded.
1238     return !!_kc_dbhandle;
1239 }
1240
1241 static SecDbRef kc_dbhandle(CFErrorRef* error)
1242 {
1243     dispatch_sync(get_kc_dbhandle_dispatch(), ^{
1244         if(_kc_dbhandle == NULL) {
1245             _SecDbServerSetup();
1246             kc_dbhandle_init(error);
1247         }
1248     });
1249     return _kc_dbhandle;
1250 }
1251
1252 /* whitebox testing only, and I really hope you call DbReset soon */
1253 void SecKeychainDbForceClose(void)
1254 {
1255     dispatch_sync(get_kc_dbhandle_dispatch(), ^{
1256         if(_kc_dbhandle) {
1257             SecDbForceClose(_kc_dbhandle);
1258         }
1259     });
1260 }
1261
1262 /* For whitebox testing only */
1263 void SecKeychainDbReset(dispatch_block_t inbetween)
1264 {
1265     dispatch_sync(get_kc_dbhandle_dispatch(), ^{
1266         CFReleaseNull(_kc_dbhandle);
1267         SecDbResetMetadataKeys();
1268         SecDbResetBackupManager();
1269         if (inbetween)
1270             inbetween();
1271     });
1272 }
1273
1274 static bool kc_acquire_dbt(bool writeAndRead, SecDbConnectionRef* dbconn, CFErrorRef *error) {
1275     SecDbRef db = kc_dbhandle(error);
1276     if (db == NULL) {
1277         if(error && !(*error)) {
1278             SecError(errSecDataNotAvailable, error, CFSTR("failed to get a db handle"));
1279         }
1280         return NULL;
1281     }
1282
1283     return SecDbConnectionAcquireRefMigrationSafe(db, !writeAndRead, dbconn, error);
1284 }
1285
1286 /* Return a per thread dbt handle for the keychain.  If create is true create
1287  the database if it does not yet exist.  If it is false, just return an
1288  error if it fails to auto-create. */
1289 bool kc_with_dbt(bool writeAndRead, CFErrorRef *error, bool (^perform)(SecDbConnectionRef dbt))
1290 {
1291     return kc_with_custom_db(writeAndRead, true, NULL, error, perform);
1292 }
1293
1294 bool kc_with_dbt_non_item_tables(bool writeAndRead, CFErrorRef* error, bool (^perform)(SecDbConnectionRef dbt))
1295 {
1296     return kc_with_custom_db(writeAndRead, false, NULL, error, perform);
1297 }
1298
1299 bool kc_with_custom_db(bool writeAndRead, bool usesItemTables, SecDbRef db, CFErrorRef *error, bool (^perform)(SecDbConnectionRef dbt))
1300 {
1301     if (db && db != kc_dbhandle(error)) {
1302         __block bool result = false;
1303         if (writeAndRead) {
1304             SecDbPerformWrite(db, error, ^(SecDbConnectionRef dbconn) {
1305                 result = perform(dbconn);
1306             });
1307         }
1308         else {
1309             SecDbPerformRead(db, error, ^(SecDbConnectionRef dbconn) {
1310                 result = perform(dbconn);
1311             });
1312         }
1313         return result;
1314     }
1315
1316     if(threadDbt) {
1317         // The kc_with_dbt upthread will clean this up when it's done.
1318         return perform(threadDbt);
1319     }
1320
1321 #if SECUREOBJECTSYNC
1322     if (writeAndRead && usesItemTables) {
1323         SecItemDataSourceFactoryGetDefault();
1324     }
1325 #endif
1326
1327     bool ok = false;
1328     if (kc_acquire_dbt(writeAndRead, &threadDbt, error)) {
1329         ok = perform(threadDbt);
1330         SecDbConnectionRelease(threadDbt);
1331         threadDbt = NULL;
1332     }
1333     return ok;
1334 }
1335
1336 static bool
1337 items_matching_issuer_parent(SecDbConnectionRef dbt, CFArrayRef accessGroups, CFDataRef musrView,
1338                              CFDataRef issuer, CFArrayRef issuers, int recurse)
1339 {
1340     Query *q;
1341     CFArrayRef results = NULL;
1342     CFIndex i, count;
1343     bool found = false;
1344
1345     if (CFArrayContainsValue(issuers, CFRangeMake(0, CFArrayGetCount(issuers)), issuer))
1346         return true;
1347
1348     /* XXX make musr supported */
1349     const void *keys[] = { kSecClass, kSecReturnRef, kSecAttrSubject };
1350     const void *vals[] = { kSecClassCertificate, kCFBooleanTrue, issuer };
1351     CFDictionaryRef query = CFDictionaryCreate(kCFAllocatorDefault, keys, vals, array_size(keys), NULL, NULL);
1352
1353     if (!query)
1354         return false;
1355
1356     CFErrorRef localError = NULL;
1357     q = query_create_with_limit(query, musrView, kSecMatchUnlimited, NULL, &localError);
1358     CFRelease(query);
1359     if (q) {
1360         s3dl_copy_matching(dbt, q, (CFTypeRef*)&results, accessGroups, &localError);
1361         query_destroy(q, &localError);
1362     }
1363     if (localError) {
1364         secerror("items matching issuer parent: %@", localError);
1365         CFReleaseNull(localError);
1366         return false;
1367     }
1368
1369     count = CFArrayGetCount(results);
1370     for (i = 0; (i < count) && !found; i++) {
1371         CFDictionaryRef cert_dict = (CFDictionaryRef)CFArrayGetValueAtIndex(results, i);
1372         CFDataRef cert_issuer = CFDictionaryGetValue(cert_dict, kSecAttrIssuer);
1373         if (CFEqual(cert_issuer, issuer))
1374             continue;
1375         if (recurse-- > 0)
1376             found = items_matching_issuer_parent(dbt, accessGroups, musrView, cert_issuer, issuers, recurse);
1377     }
1378     CFReleaseSafe(results);
1379
1380     return found;
1381 }
1382
1383 static bool
1384 _FilterWithPolicy(SecPolicyRef policy, CFDateRef date, SecCertificateRef cert)
1385 {
1386     CFDictionaryRef props = NULL;
1387     CFArrayRef keychains = NULL;
1388     CFArrayRef anchors = NULL;
1389     CFArrayRef certs = NULL;
1390     CFArrayRef chain = NULL;
1391     SecTrustRef trust = NULL;
1392
1393     SecTrustResultType  trustResult;
1394     Boolean needChain = false;
1395     __block bool ok = false;
1396
1397     if (!policy || !cert) return false;
1398
1399     certs = CFArrayCreate(NULL, (const void **)&cert, (CFIndex)1, &kCFTypeArrayCallBacks);
1400     require_noerr_quiet(SecTrustCreateWithCertificates(certs, policy, &trust), cleanup);
1401
1402     /* Set evaluation date, if specified (otherwise current date is implied) */
1403     if (date && (CFGetTypeID(date) == CFDateGetTypeID())) {
1404         require_noerr_quiet(SecTrustSetVerifyDate(trust, date), cleanup);
1405     }
1406
1407     /* Check whether this is the X509 Basic policy, which means chain building */
1408     props = SecPolicyCopyProperties(policy);
1409     if (props) {
1410         CFTypeRef oid = (CFTypeRef) CFDictionaryGetValue(props, kSecPolicyOid);
1411         if (oid && (CFEqual(oid, kSecPolicyAppleX509Basic) ||
1412                     CFEqual(oid, kSecPolicyAppleRevocation))) {
1413             needChain = true;
1414         }
1415     }
1416
1417     if (!needChain) {
1418         require_noerr_quiet(SecTrustEvaluateLeafOnly(trust, &trustResult), cleanup);
1419     } else {
1420         require_noerr_quiet(SecTrustEvaluate(trust, &trustResult), cleanup);
1421     }
1422
1423     require_quiet((trustResult == kSecTrustResultProceed ||
1424                          trustResult == kSecTrustResultUnspecified ||
1425                          trustResult == kSecTrustResultRecoverableTrustFailure), cleanup);
1426
1427     ok = true;
1428 #if TARGET_OS_IPHONE
1429     CFArrayRef properties = SecTrustCopyProperties(trust);
1430 #else
1431     CFArrayRef properties = SecTrustCopyProperties_ios(trust);
1432 #endif
1433     if (properties) {
1434         CFArrayForEach(properties, ^(const void *property) {
1435             CFDictionaryForEach((CFDictionaryRef)property, ^(const void *key, const void *value) {
1436                 if (CFEqual((CFTypeRef)key, kSecPropertyKeyType) && CFEqual((CFTypeRef)value, kSecPropertyTypeError))
1437                     ok = false;
1438             });
1439         });
1440         CFRelease(properties);
1441     }
1442
1443 cleanup:
1444     if(props) CFRelease(props);
1445     if(chain) CFRelease(chain);
1446     if(anchors) CFRelease(anchors);
1447     if(keychains) CFRelease(keychains);
1448     if(certs) CFRelease(certs);
1449     if(trust) CFRelease(trust);
1450
1451     return ok;
1452 }
1453
1454 static bool
1455 _FilterWithDate(CFDateRef validOnDate, SecCertificateRef cert)
1456 {
1457     if (!validOnDate || !cert) return false;
1458
1459     CFAbsoluteTime at, nb, na;
1460     at = CFDateGetAbsoluteTime((CFDateRef)validOnDate);
1461
1462     bool ok = true;
1463     nb = SecCertificateNotValidBefore(cert);
1464     na = SecCertificateNotValidAfter(cert);
1465
1466     if (nb == 0 || na == 0 || nb == na) {
1467         ok = false;
1468         secnotice("FilterWithDate", "certificate cannot operate");
1469     }
1470     else if (at < nb) {
1471         ok = false;
1472         secnotice("FilterWithDate", "certificate is not valid yet");
1473     }
1474     else if (at > na) {
1475         ok = false;
1476         secnotice("FilterWithDate", "certificate expired");
1477     }
1478
1479     return ok;
1480 }
1481
1482 static bool
1483 _FilterWithTrust(Boolean trustedOnly, SecCertificateRef cert)
1484 {
1485     if (!cert) return false;
1486     if (!trustedOnly) return true;
1487
1488     bool ok = false;
1489     CFArrayRef certArray = CFArrayCreate(NULL, (const void**)&cert, 1, &kCFTypeArrayCallBacks);
1490     SecTrustRef trust = NULL;
1491     SecPolicyRef policy = SecPolicyCreateBasicX509();
1492     require_quiet(policy, out);
1493
1494     require_noerr_quiet(SecTrustCreateWithCertificates(certArray, policy, &trust), out);
1495     SecTrustResultType trustResult;
1496     require_noerr_quiet(SecTrustEvaluate(trust, &trustResult), out);
1497
1498     require_quiet((trustResult == kSecTrustResultProceed ||
1499                    trustResult == kSecTrustResultUnspecified), out);
1500     ok = true;
1501 out:
1502     CFReleaseSafe(trust);
1503     CFReleaseSafe(policy);
1504     CFReleaseSafe(certArray);
1505     return ok;
1506 }
1507
1508 static SecCertificateRef
1509 CopyCertificateFromItem(Query *q, CFDictionaryRef item) {
1510     SecCertificateRef certRef = NULL;
1511     CFDictionaryRef itemValue = NULL;
1512
1513     CFTypeRef tokenID = NULL;
1514     CFDataRef certData = NULL;
1515     if (q->q_class == identity_class()) {
1516         certData = CFDictionaryGetValue(item, kSecAttrIdentityCertificateData);
1517         tokenID = CFDictionaryGetValue(item, kSecAttrIdentityCertificateTokenID);
1518     } else if (q->q_class == cert_class()) {
1519         certData = CFDictionaryGetValue(item, kSecValueData);
1520         tokenID = CFDictionaryGetValue(item, kSecAttrTokenID);
1521     }
1522
1523     require_quiet(certData, out);
1524     if (tokenID != NULL) {
1525         CFErrorRef error = NULL;
1526         itemValue = SecTokenItemValueCopy(certData, &error);
1527         require_action_quiet(itemValue, out, { secerror("function SecTokenItemValueCopy failed with: %@", error); CFReleaseSafe(error); });
1528         CFDataRef tokenCertData = CFDictionaryGetValue(itemValue, kSecTokenValueDataKey);
1529         require_action_quiet(tokenCertData, out, { secerror("token item doesn't contain token value data");});
1530         certRef = SecCertificateCreateWithData(kCFAllocatorDefault, tokenCertData);
1531     }
1532     else
1533         certRef = SecCertificateCreateWithData(kCFAllocatorDefault, certData);
1534
1535 out:
1536     CFReleaseNull(itemValue);
1537     return certRef;
1538 }
1539
1540 bool match_item(SecDbConnectionRef dbt, Query *q, CFArrayRef accessGroups, CFDictionaryRef item)
1541 {
1542     bool ok = false;
1543     SecCertificateRef certRef = NULL;
1544     if (q->q_match_issuer) {
1545         CFDataRef issuer = CFDictionaryGetValue(item, kSecAttrIssuer);
1546         if (!items_matching_issuer_parent(dbt, accessGroups, q->q_musrView, issuer, q->q_match_issuer, 10 /*max depth*/))
1547             return ok;
1548     }
1549
1550     if (q->q_match_policy && (q->q_class == identity_class() || q->q_class == cert_class())) {
1551         if (!certRef)
1552             certRef = CopyCertificateFromItem(q, item);
1553         require_quiet(certRef, out);
1554         require_quiet(_FilterWithPolicy(q->q_match_policy, q->q_match_valid_on_date, certRef), out);
1555     }
1556
1557     if (q->q_match_valid_on_date && (q->q_class == identity_class() || q->q_class == cert_class())) {
1558         if (!certRef)
1559             certRef = CopyCertificateFromItem(q, item);
1560         require_quiet(certRef, out);
1561         require_quiet(_FilterWithDate(q->q_match_valid_on_date, certRef), out);
1562     }
1563
1564     if (q->q_match_trusted_only && (q->q_class == identity_class() || q->q_class == cert_class())) {
1565         if (!certRef)
1566             certRef = CopyCertificateFromItem(q, item);
1567         require_quiet(certRef, out);
1568         require_quiet(_FilterWithTrust(CFBooleanGetValue(q->q_match_trusted_only), certRef), out);
1569     }
1570
1571     /* Add future match checks here. */
1572     ok = true;
1573 out:
1574     CFReleaseSafe(certRef);
1575     return ok;
1576 }
1577
1578 //Mark: -
1579
1580 void deleteCorruptedItemAsync(SecDbConnectionRef dbt, CFStringRef tablename, sqlite_int64 rowid)
1581 {
1582     // should really get db from dbt, but I don't know much much we should poke holes thought the boundaries.
1583     SecDbRef db = kc_dbhandle(NULL);
1584     if (db == NULL) {
1585         return;
1586     }
1587
1588     CFRetain(db);
1589     CFRetain(tablename);
1590
1591     dispatch_async(dispatch_get_global_queue(QOS_CLASS_DEFAULT, 0), ^{
1592         __block CFErrorRef localErr = NULL;
1593         kc_with_custom_db(true, true, db, &localErr, ^bool(SecDbConnectionRef dbt) {
1594             CFStringRef sql = CFStringCreateWithFormat(kCFAllocatorDefault, NULL, CFSTR("DELETE FROM %@ WHERE rowid=%lli"), tablename, rowid);
1595             __block bool ok = true;
1596             ok &= SecDbPrepare(dbt, sql, &localErr, ^(sqlite3_stmt *stmt) {
1597                 ok &= SecDbStep(dbt, stmt, &localErr, NULL);
1598             });
1599
1600             if (!ok || localErr) {
1601                 secerror("Failed to delete corrupt item, %@ row %lli: %@", tablename, rowid, localErr);
1602             } else {
1603                 secnotice("item", "Deleted corrupt rowid %lli from table %@", rowid, tablename);
1604             }
1605             CFReleaseNull(localErr);
1606             CFReleaseNull(sql);
1607             return true;
1608         });
1609
1610         CFRelease(tablename); // can't be a CFReleaseNull() because of scope
1611         CFRelease(db);
1612     });
1613
1614 }
1615
1616
1617
1618 /****************************************************************************
1619  **************** Beginning of Externally Callable Interface ****************
1620  ****************************************************************************/
1621
1622 static bool SecEntitlementError(CFErrorRef *error)
1623 {
1624 #if TARGET_OS_OSX
1625 #define SEC_ENTITLEMENT_WARNING CFSTR("com.apple.application-identifier nor com.apple.security.application-groups nor keychain-access-groups")
1626 #elif TARGET_OS_MACCATALYST
1627 #define SEC_ENTITLEMENT_WARNING CFSTR("com.apple.developer.associated-application-identifier nor application-identifier nor com.apple.security.application-groups nor keychain-access-groups")
1628 #else
1629 #define SEC_ENTITLEMENT_WARNING CFSTR("application-identifier nor keychain-access-groups")
1630 #endif
1631
1632     return SecError(errSecMissingEntitlement, error, CFSTR("Client has neither %@ entitlements"), SEC_ENTITLEMENT_WARNING);
1633 }
1634
1635 static bool SecEntitlementErrorForExplicitAccessGroup(CFStringRef agrp, CFArrayRef clientGroups, CFErrorRef* error)
1636 {
1637     return SecError(errSecMissingEntitlement, error, CFSTR("Client explicitly specifies access group %@ but is only entitled for %@"), agrp, clientGroups);
1638 }
1639
1640 static CFStringRef CopyAccessGroupForRowID(sqlite_int64 rowID, CFStringRef itemClass)
1641 {
1642     __block CFStringRef accessGroup = NULL;
1643
1644     __block CFErrorRef error = NULL;
1645     bool ok = kc_with_dbt(false, &error, ^bool(SecDbConnectionRef dbt) {
1646         CFStringRef table = CFEqual(itemClass, kSecClassIdentity) ? kSecClassCertificate : itemClass;
1647         CFStringRef sql = CFStringCreateWithFormat(NULL, NULL, CFSTR("SELECT agrp FROM %@ WHERE rowid == %u"), table, (unsigned int)rowID);
1648         bool dbOk = SecDbWithSQL(dbt, sql, &error, ^bool(sqlite3_stmt *stmt) {
1649             bool rowOk = SecDbForEach(dbt, stmt, &error, ^bool(int row_index) {
1650                 accessGroup = CFStringCreateWithBytes(NULL, sqlite3_column_blob(stmt, 0), sqlite3_column_bytes(stmt, 0), kCFStringEncodingUTF8, false);
1651                 return accessGroup != NULL;
1652             });
1653
1654             return (bool)(rowOk && accessGroup != NULL);
1655         });
1656
1657         CFReleaseNull(sql);
1658         return (bool)(dbOk && accessGroup);
1659     });
1660
1661     if (ok) {
1662         return accessGroup;
1663     }
1664     else {
1665         CFReleaseNull(accessGroup);
1666         return NULL;
1667     }
1668 }
1669
1670 // Expand this, rdar://problem/59297616
1671 // Known attributes which are not API/SPI should not be permitted in queries
1672 static bool nonAPIAttributesInDictionary(CFDictionaryRef attrs) {
1673     return CFDictionaryContainsKey(attrs, kSecAttrAppClipItem);
1674 }
1675
1676 static bool appClipHasSaneAccessGroups(SecurityClient* client) {
1677     if (!client || !client->applicationIdentifier || !client->accessGroups) {
1678         secerror("item: no app clip client or attributes not set, cannot verify restrictions");
1679         return false;
1680     }
1681
1682     CFIndex count = CFArrayGetCount(client->accessGroups);
1683     if (count == 1 && CFEqualSafe(client->applicationIdentifier, CFArrayGetValueAtIndex(client->accessGroups, 0))) {
1684         return true;
1685     }
1686
1687     // sigh, alright is the _other_ access group the application identifier?
1688     if (count == 2) {
1689         CFIndex tokenIdx = CFArrayGetFirstIndexOfValue(client->accessGroups, CFRangeMake(0, count), kSecAttrAccessGroupToken);
1690         if (tokenIdx != kCFNotFound) {
1691             return CFEqualSafe(client->applicationIdentifier, CFArrayGetValueAtIndex(client->accessGroups, tokenIdx == 0 ? 1 : 0));
1692         }
1693     }
1694
1695     return false;
1696 }
1697
1698 static bool
1699 SecItemServerCopyMatching(CFDictionaryRef query, CFTypeRef *result,
1700     SecurityClient *client, CFErrorRef *error)
1701 {
1702     if (nonAPIAttributesInDictionary(query)) {
1703         SecError(errSecParam, error, CFSTR("Non-API attributes present in query"));
1704         return false;
1705     }
1706
1707     CFArrayRef accessGroups = CFRetainSafe(client->accessGroups);
1708
1709     CFIndex ag_count;
1710     if (!accessGroups || 0 == (ag_count = CFArrayGetCount(accessGroups))) {
1711         CFReleaseNull(accessGroups);
1712         return SecEntitlementError(error);
1713     }
1714
1715     SecSignpostStart(SecSignpostSecItemCopyMatching);
1716
1717     if (client->canAccessNetworkExtensionAccessGroups) {
1718         CFDataRef persistentRef = CFDictionaryGetValue(query, kSecValuePersistentRef);
1719         CFStringRef itemClass = NULL;
1720         sqlite_int64 itemRowID = 0;
1721         if (persistentRef && _SecItemParsePersistentRef(persistentRef, &itemClass, &itemRowID, NULL)) {
1722             CFStringRef accessGroup = CopyAccessGroupForRowID(itemRowID, itemClass);
1723             if (accessGroup && CFStringHasSuffix(accessGroup, kSecNetworkExtensionAccessGroupSuffix) && !CFArrayContainsValue(accessGroups, CFRangeMake(0, ag_count), accessGroup)) {
1724                 CFMutableArrayRef mutableAccessGroups = CFArrayCreateMutableCopy(NULL, 0, accessGroups);
1725                 CFArrayAppendValue(mutableAccessGroups, accessGroup);
1726                 CFReleaseNull(accessGroups);
1727                 accessGroups = mutableAccessGroups;
1728             }
1729             CFReleaseNull(accessGroup);
1730         }
1731     }
1732
1733     if (CFArrayContainsValue(accessGroups, CFRangeMake(0, ag_count), CFSTR("*"))) {
1734         /* Having the special accessGroup "*" allows access to all accessGroups. */
1735         CFReleaseNull(accessGroups);
1736     }
1737
1738     bool ok = false;
1739     Query *q = query_create_with_limit(query, client->musr, 1, client, error);
1740     if (q) {
1741         CFStringRef agrp = CFDictionaryGetValue(q->q_item, kSecAttrAccessGroup);
1742         if (agrp) {
1743             if (accessGroupsAllows(accessGroups, agrp, client)) {
1744                 const void *val = agrp;
1745                 CFReleaseNull(accessGroups);
1746                 accessGroups = CFArrayCreate(0, &val, 1, &kCFTypeArrayCallBacks);
1747             } else {
1748                 (void)SecEntitlementErrorForExplicitAccessGroup(agrp, accessGroups, error);
1749                 CFReleaseNull(accessGroups);
1750                 query_destroy(q, NULL);
1751                 return false;
1752             }
1753         }
1754
1755 #if TARGET_OS_IPHONE
1756         if (q->q_sync_bubble && client->inMultiUser) {
1757             CFReleaseNull(q->q_musrView);
1758             q->q_musrView = SecMUSRCreateSyncBubbleUserUUID(q->q_sync_bubble);
1759         } else if (client->inMultiUser && client->isNetworkExtension) {
1760             CFReleaseNull(q->q_musrView);
1761             q->q_musrView = SecMUSRCreateBothUserAndSystemUUID(client->uid);
1762         } else if (q->q_system_keychain && client->inMultiUser) {
1763             CFReleaseNull(q->q_musrView);
1764             q->q_musrView = SecMUSRCopySystemKeychainUUID();
1765         } else {
1766             q->q_system_keychain = false;
1767         }
1768 #endif
1769
1770         query_set_caller_access_groups(q, accessGroups);
1771         if (client->isAppClip && !appClipHasSaneAccessGroups(client)) {
1772             ok = SecError(errSecRestrictedAPI, error, CFSTR("App clips are not permitted to use access groups other than application identifier"));
1773         } else if (q->q_system_keychain && !client->allowSystemKeychain) {
1774             ok = SecError(errSecMissingEntitlement, error, CFSTR("client doesn't have entitlement for system keychain"));
1775         } else if (q->q_sync_bubble && !client->allowSyncBubbleKeychain) {
1776             ok = SecError(errSecMissingEntitlement, error, CFSTR("client doesn't have entitlement for syncbubble keychain"));
1777         } else if (q->q_system_keychain && q->q_sync_bubble) {
1778             ok = SecError(errSecMissingEntitlement, error, CFSTR("can't do both system and syncbubble keychain"));
1779         } else if (q->q_use_item_list) {
1780             ok = SecError(errSecUseItemListUnsupported, error, CFSTR("use item list unsupported"));
1781         } else if (q->q_match_issuer && ((q->q_class != cert_class()) &&
1782                     (q->q_class != identity_class()))) {
1783             ok = SecError(errSecUnsupportedOperation, error, CFSTR("unsupported match attribute"));
1784         } else if (q->q_match_policy && ((q->q_class != cert_class()) &&
1785                     (q->q_class != identity_class()))) {
1786             ok = SecError(errSecUnsupportedOperation, error, CFSTR("unsupported kSecMatchPolicy attribute"));
1787         } else if (q->q_return_type != 0 && result == NULL) {
1788             ok = SecError(errSecReturnMissingPointer, error, CFSTR("missing pointer"));
1789         } else if (!q->q_error) {
1790             ok = kc_with_dbt(false, error, ^(SecDbConnectionRef dbt) {
1791                 return s3dl_copy_matching(dbt, q, result, accessGroups, error);
1792             });
1793         }
1794
1795         if (!query_destroy(q, error)) {
1796             ok = false;
1797         }
1798     }
1799     CFReleaseNull(accessGroups);
1800
1801     SecSignpostStop(SecSignpostSecItemCopyMatching);
1802
1803         return ok;
1804 }
1805
1806 bool
1807 _SecItemCopyMatching(CFDictionaryRef query, SecurityClient *client, CFTypeRef *result, CFErrorRef *error) {
1808     return SecItemServerCopyMatching(query, result, client, error);
1809 }
1810
1811 #if TARGET_OS_IPHONE
1812 static bool
1813 SecItemSynchronizable(CFDictionaryRef query)
1814 {
1815     bool result = false;
1816     CFTypeRef value = CFDictionaryGetValue(query, kSecAttrSynchronizable);
1817     if (isBoolean(value))
1818         return CFBooleanGetValue(value);
1819     else if (isNumber(value)) {
1820         SInt32 number = 0;
1821         (void)CFNumberGetValue(value, kCFNumberSInt32Type, &number);
1822         result = !!number;
1823     }
1824
1825     return result;
1826 }
1827 #endif
1828
1829 static CFArrayRef
1830 SecurityClientCopyWritableAccessGroups(SecurityClient *client) {
1831     if (client == NULL || client->accessGroups == NULL) {
1832         return NULL;
1833     }
1834     CFIndex count = CFArrayGetCount(client->accessGroups);
1835     if (CFArrayContainsValue(client->accessGroups, CFRangeMake(0, count), kSecAttrAccessGroupToken)) {
1836         CFMutableArrayRef writableGroups = CFArrayCreateMutableCopy(kCFAllocatorDefault, 0, client->accessGroups);
1837         CFArrayRemoveAllValue(writableGroups, kSecAttrAccessGroupToken);
1838         return writableGroups;
1839     } else {
1840         return CFRetainSafe(client->accessGroups);
1841     }
1842 }
1843
1844 bool
1845 _SecItemAdd(CFDictionaryRef attributes, SecurityClient *client, CFTypeRef *result, CFErrorRef *error)
1846 {
1847     if (nonAPIAttributesInDictionary(attributes)) {
1848         SecError(errSecParam, error, CFSTR("Non-API attributes present"));
1849         return false;
1850     }
1851
1852     CFArrayRef accessGroups = SecurityClientCopyWritableAccessGroups(client);
1853
1854     bool ok = true;
1855     CFIndex ag_count;
1856     if (!accessGroups || 0 == (ag_count = CFArrayGetCount(accessGroups))) {
1857         CFReleaseNull(accessGroups);
1858         return SecEntitlementError(error);
1859     }
1860
1861     SecSignpostStart(SecSignpostSecItemAdd);
1862
1863     Query *q = query_create_with_limit(attributes, client->musr, 0, client, error);
1864     if (q) {
1865         /* Access group sanity checking. */
1866         CFStringRef agrp = (CFStringRef)CFDictionaryGetValue(attributes,
1867             kSecAttrAccessGroup);
1868
1869         /* Having the special accessGroup "*" allows access to all accessGroups. */
1870         if (CFArrayContainsValue(accessGroups, CFRangeMake(0, ag_count), CFSTR("*"))) {
1871             CFReleaseNull(accessGroups);
1872         }
1873
1874         if (agrp) {
1875             /* The user specified an explicit access group, validate it. */
1876             if (!accessGroupsAllows(accessGroups, agrp, client)) {
1877                 ok = SecEntitlementErrorForExplicitAccessGroup(agrp, accessGroups, error);
1878             }
1879         } else {
1880             agrp = (CFStringRef)CFArrayGetValueAtIndex(client->accessGroups, 0);
1881
1882             /* We are using an implicit access group, add it as if the user
1883                specified it as an attribute. */
1884             query_add_attribute(kSecAttrAccessGroup, agrp, q);
1885         }
1886
1887         if (SecPLShouldLogRegisteredEvent(CFSTR("SecItem"))) {
1888             CFDictionaryRef dict = CFDictionaryCreateForCFTypes(NULL, CFSTR("operation"), CFSTR("add"), CFSTR("AccessGroup"), agrp, NULL);
1889             if (dict) {
1890                 SecPLLogRegisteredEvent(CFSTR("SecItem"), dict);
1891                 CFRelease(dict);
1892             }
1893         }
1894 #if TARGET_OS_IPHONE
1895         if (q->q_system_keychain && client->inMultiUser) {
1896             CFReleaseNull(q->q_musrView);
1897             q->q_musrView = SecMUSRCopySystemKeychainUUID();
1898         } else {
1899             q->q_system_keychain = false;
1900         }
1901         query_add_attribute_with_desc(&v8musr, q->q_musrView, q);
1902 #endif
1903
1904         if (ok) {
1905             query_ensure_access_control(q, agrp);
1906
1907 #if OCTAGON
1908             void (^add_sync_callback)(bool, CFErrorRef) = CFDictionaryGetValue(attributes, CFSTR("f_ckkscallback"));
1909             if(add_sync_callback) {
1910                 // The existence of this callback indicates that we need a predictable UUID for this item.
1911                 q->q_uuid_from_primary_key = true;
1912                 q->q_add_sync_callback = add_sync_callback;
1913             }
1914 #endif
1915             if (client->isAppClip && !appClipHasSaneAccessGroups(client)) {
1916                 ok = SecError(errSecRestrictedAPI, error, CFSTR("App clips are not permitted to use access groups other than application identifier"));
1917             } else if (client->isAppClip && CFEqualSafe(CFDictionaryGetValue(attributes, kSecAttrSynchronizable), kCFBooleanTrue)) {
1918                 ok = SecError(errSecRestrictedAPI, error, CFSTR("App clips are not permitted to add synchronizable items to the keychain"));
1919             } else if (q->q_system_keychain && !client->allowSystemKeychain) {
1920                 ok = SecError(errSecMissingEntitlement, error, CFSTR("client doesn't have entitlement for system keychain"));
1921             } else if (q->q_sync_bubble && !client->allowSyncBubbleKeychain) {
1922                 ok = SecError(errSecMissingEntitlement, error, CFSTR("client doesn't have entitlement for syncbubble keychain"));
1923 #if TARGET_OS_IPHONE
1924             } else if (q->q_system_keychain && SecItemSynchronizable(attributes) && !client->inMultiUser) {
1925                 ok = SecError(errSecInvalidKey, error, CFSTR("Can't store system keychain and synchronizable"));
1926 #endif
1927             } else if (q->q_row_id || q->q_token_object_id) {
1928                 ok = SecError(errSecValuePersistentRefUnsupported, error, CFSTR("q_row_id"));  // TODO: better error string
1929             } else if (!q->q_error) {
1930                 ok = kc_with_dbt(true, error, ^(SecDbConnectionRef dbt){
1931                     return kc_transaction(dbt, error, ^{
1932                         query_pre_add(q, true);
1933                         return s3dl_query_add(dbt, q, result, error);
1934                     });
1935                 });
1936             }
1937         }
1938         ok = query_notify_and_destroy(q, ok, error);
1939     } else {
1940         ok = false;
1941     }
1942     CFReleaseNull(accessGroups);
1943
1944     SecSignpostStop(SecSignpostSecItemAdd);
1945
1946     return ok;
1947 }
1948
1949 bool
1950 _SecItemUpdate(CFDictionaryRef query, CFDictionaryRef attributesToUpdate,
1951                SecurityClient *client, CFErrorRef *error)
1952 {
1953     if (nonAPIAttributesInDictionary(query) || nonAPIAttributesInDictionary(attributesToUpdate)) {
1954         SecError(errSecParam, error, CFSTR("Non-API attributes present"));
1955         return false;
1956     }
1957
1958     CFArrayRef accessGroups = SecurityClientCopyWritableAccessGroups(client);
1959
1960     CFIndex ag_count;
1961     if (!accessGroups || 0 == (ag_count = CFArrayGetCount(accessGroups))) {
1962         CFReleaseNull(accessGroups);
1963         return SecEntitlementError(error);
1964     }
1965
1966     // Queries using implicit access groups which only find items that're inaccessible yield errSecItemNotFound,
1967     // but we can pre-emptively shut down queries which are clearly illegal
1968     CFTypeRef q_agrp = CFDictionaryGetValue(query, kSecAttrAccessGroup);
1969     if (q_agrp && !accessGroupsAllows(accessGroups, q_agrp, client)) {
1970         SecEntitlementErrorForExplicitAccessGroup(q_agrp, accessGroups, error);
1971         CFReleaseSafe(accessGroups);
1972         return false;
1973     }
1974
1975     SecSignpostStart(SecSignpostSecItemUpdate);
1976
1977     if (SecPLShouldLogRegisteredEvent(CFSTR("SecItem"))) {
1978         CFTypeRef agrp = CFArrayGetValueAtIndex(accessGroups, 0);
1979         CFDictionaryRef dict = CFDictionaryCreateForCFTypes(NULL, CFSTR("operation"), CFSTR("update"), CFSTR("AccessGroup"), agrp, NULL);
1980         if (dict) {
1981             SecPLLogRegisteredEvent(CFSTR("SecItem"), dict);
1982             CFRelease(dict);
1983         }
1984     }
1985
1986     if (CFArrayContainsValue(accessGroups, CFRangeMake(0, ag_count), CFSTR("*"))) {
1987         /* Having the special accessGroup "*" allows access to all accessGroups. */
1988         CFReleaseNull(accessGroups);
1989     }
1990
1991     bool ok = true;
1992     Query *q = query_create_with_limit(query, client->musr, kSecMatchUnlimited, client, error);
1993     if (!q) {
1994         ok = false;
1995     }
1996     if (ok) {
1997 #if TARGET_OS_IPHONE
1998         if (q->q_system_keychain && client->inMultiUser) {
1999             CFReleaseNull(q->q_musrView);
2000             q->q_musrView = SecMUSRCopySystemKeychainUUID();
2001         } else {
2002             q->q_system_keychain = false;
2003         }
2004 #endif
2005         query_set_caller_access_groups(q, accessGroups);
2006         if (client->isAppClip && !appClipHasSaneAccessGroups(client)) {
2007             ok = SecError(errSecRestrictedAPI, error, CFSTR("App clips are not permitted to use access groups other than application identifier"));
2008         } else if (client->isAppClip && (CFEqualSafe(CFDictionaryGetValue(query, kSecAttrSynchronizable), kCFBooleanTrue) ||
2009             CFEqualSafe(CFDictionaryGetValue(attributesToUpdate, kSecAttrSynchronizable), kCFBooleanTrue)))
2010         {
2011             ok = SecError(errSecRestrictedAPI, error, CFSTR("App clips are not permitted to make items synchronizable"));
2012         } else if (q->q_system_keychain && !client->allowSystemKeychain) {
2013             ok = SecError(errSecMissingEntitlement, error, CFSTR("client doesn't have entitlement for system keychain"));
2014         } else if (q->q_sync_bubble && !client->allowSyncBubbleKeychain) {
2015             ok = SecError(errSecMissingEntitlement, error, CFSTR("client doesn't have entitlement for syncbubble keychain"));
2016 #if TARGET_OS_IPHONE
2017         } else if (q->q_system_keychain && SecItemSynchronizable(attributesToUpdate) && !client->inMultiUser) {
2018             ok = SecError(errSecInvalidKey, error, CFSTR("Can't update an system keychain item with synchronizable"));
2019 #endif
2020         } else if (q->q_use_item_list) {
2021             ok = SecError(errSecUseItemListUnsupported, error, CFSTR("use item list not supported"));
2022         } else if (q->q_return_type & kSecReturnDataMask) {
2023             /* Update doesn't return anything so don't ask for it. */
2024             ok = SecError(errSecReturnDataUnsupported, error, CFSTR("return data not supported by update"));
2025         } else if (q->q_return_type & kSecReturnAttributesMask) {
2026             ok = SecError(errSecReturnAttributesUnsupported, error, CFSTR("return attributes not supported by update"));
2027         } else if (q->q_return_type & kSecReturnRefMask) {
2028             ok = SecError(errSecReturnRefUnsupported, error, CFSTR("return ref not supported by update"));
2029         } else if (q->q_return_type & kSecReturnPersistentRefMask) {
2030             ok = SecError(errSecReturnPersistentRefUnsupported, error, CFSTR("return persistent ref not supported by update"));
2031         } else {
2032             /* Access group sanity checking. */
2033             CFStringRef agrp = (CFStringRef)CFDictionaryGetValue(attributesToUpdate,
2034                 kSecAttrAccessGroup);
2035             if (agrp) {
2036                 /* The user is attempting to modify the access group column,
2037                    validate it to make sure the new value is allowable. */
2038                 if (!accessGroupsAllows(accessGroups, agrp, client)) {
2039                     secerror("Cannot update keychain item to access group %@", agrp);
2040                     ok = SecEntitlementErrorForExplicitAccessGroup(agrp, accessGroups, error);
2041                 }
2042             }
2043         }
2044     }
2045     if (ok) {
2046         ok = kc_with_dbt(true, error, ^(SecDbConnectionRef dbt) {
2047             return kc_transaction(dbt, error, ^{
2048                 return s3dl_query_update(dbt, q, attributesToUpdate, accessGroups, error);
2049             });
2050         });
2051     }
2052     if (q) {
2053         ok = query_notify_and_destroy(q, ok, error);
2054     }
2055     CFReleaseNull(accessGroups);
2056
2057     SecSignpostStop(SecSignpostSecItemUpdate);
2058
2059     return ok;
2060 }
2061
2062 bool
2063 _SecItemDelete(CFDictionaryRef query, SecurityClient *client, CFErrorRef *error)
2064 {
2065     if (nonAPIAttributesInDictionary(query)) {
2066         SecError(errSecParam, error, CFSTR("Non-API attributes present"));
2067         return false;
2068     }
2069
2070     CFArrayRef accessGroups = SecurityClientCopyWritableAccessGroups(client);
2071
2072     CFIndex ag_count;
2073     if (!accessGroups || 0 == (ag_count = CFArrayGetCount(accessGroups))) {
2074         CFReleaseNull(accessGroups);
2075         return SecEntitlementError(error);
2076     }
2077
2078     CFTypeRef q_agrp = CFDictionaryGetValue(query, kSecAttrAccessGroup);
2079     if (q_agrp && !accessGroupsAllows(accessGroups, q_agrp, client)) {
2080         SecEntitlementErrorForExplicitAccessGroup(q_agrp, accessGroups, error);
2081         CFReleaseSafe(accessGroups);
2082         return false;
2083     }
2084
2085     SecSignpostStart(SecSignpostSecItemDelete);
2086
2087     if (SecPLShouldLogRegisteredEvent(CFSTR("SecItem"))) {
2088         CFTypeRef agrp = CFArrayGetValueAtIndex(accessGroups, 0);
2089         CFDictionaryRef dict = CFDictionaryCreateForCFTypes(NULL, CFSTR("operation"), CFSTR("delete"), CFSTR("AccessGroup"), agrp, NULL);
2090         if (dict) {
2091             SecPLLogRegisteredEvent(CFSTR("SecItem"), dict);
2092             CFRelease(dict);
2093         }
2094     }
2095
2096     if (CFArrayContainsValue(accessGroups, CFRangeMake(0, ag_count), CFSTR("*"))) {
2097         /* Having the special accessGroup "*" allows access to all accessGroups. */
2098         CFReleaseNull(accessGroups);
2099     }
2100
2101     Query *q = query_create_with_limit(query, client->musr, kSecMatchUnlimited, client, error);
2102     bool ok;
2103     if (q) {
2104 #if TARGET_OS_IPHONE
2105         if (q->q_system_keychain && client->inMultiUser) {
2106             CFReleaseNull(q->q_musrView);
2107             q->q_musrView = SecMUSRCopySystemKeychainUUID();
2108         } else {
2109             q->q_system_keychain = false;
2110         }
2111 #endif
2112
2113         query_set_caller_access_groups(q, accessGroups);
2114         if (client->isAppClip && !appClipHasSaneAccessGroups(client)) {
2115             ok = SecError(errSecRestrictedAPI, error, CFSTR("App clips are not permitted to use access groups other than application identifier"));
2116         } else if (q->q_system_keychain && !client->allowSystemKeychain) {
2117             ok = SecError(errSecMissingEntitlement, error, CFSTR("client doesn't have entitlement for system keychain"));
2118         } else if (q->q_sync_bubble && !client->allowSyncBubbleKeychain) {
2119             ok = SecError(errSecMissingEntitlement, error, CFSTR("client doesn't have entitlement for syncbubble keychain"));
2120         } else if (q->q_limit != kSecMatchUnlimited) {
2121             ok = SecError(errSecMatchLimitUnsupported, error, CFSTR("match limit not supported by delete"));
2122         } else if (query_match_count(q) != 0) {
2123             ok = SecError(errSecItemMatchUnsupported, error, CFSTR("match not supported by delete"));
2124         } else if (q->q_ref) {
2125             ok = SecError(errSecValueRefUnsupported, error, CFSTR("value ref not supported by delete"));
2126         } else if (q->q_row_id && query_attr_count(q)) {
2127             ok = SecError(errSecItemIllegalQuery, error, CFSTR("rowid and other attributes are mutually exclusive"));
2128         } else if (q->q_token_object_id && query_attr_count(q) != 1) {
2129             ok = SecError(errSecItemIllegalQuery, error, CFSTR("token persistent ref and other attributes are mutually exclusive"));
2130         } else {
2131             ok = kc_with_dbt(true, error, ^(SecDbConnectionRef dbt) {
2132                 return kc_transaction(dbt, error, ^{
2133                     return s3dl_query_delete(dbt, q, accessGroups, error);
2134                 });
2135             });
2136         }
2137         ok = query_notify_and_destroy(q, ok, error);
2138     } else {
2139         ok = false;
2140     }
2141     CFReleaseNull(accessGroups);
2142
2143     SecSignpostStop(SecSignpostSecItemDelete);
2144
2145     return ok;
2146 }
2147
2148 static bool SecItemDeleteTokenItems(SecDbConnectionRef dbt, CFTypeRef classToDelete, CFTypeRef tokenID, CFArrayRef accessGroups, SecurityClient *client, CFErrorRef *error) {
2149     CFDictionaryRef query = CFDictionaryCreateForCFTypes(kCFAllocatorDefault, kSecClass, classToDelete, kSecAttrTokenID, tokenID, NULL);
2150     Query *q = query_create_with_limit(query, client->musr, kSecMatchUnlimited, client, error);
2151     CFRelease(query);
2152     bool ok;
2153     if (q) {
2154         query_set_caller_access_groups(q, accessGroups);
2155         ok = s3dl_query_delete(dbt, q, accessGroups, error);
2156         ok = query_notify_and_destroy(q, ok, error);
2157     } else {
2158         ok = false;
2159     }
2160
2161     return ok;
2162 }
2163
2164 static bool SecItemAddTokenItemToAccessGroups(SecDbConnectionRef dbt, CFDictionaryRef attributes, CFArrayRef accessGroups, SecurityClient *client, CFErrorRef *error) {
2165     Query *q;
2166     bool ok = false;
2167     for (CFIndex i = 0; i < CFArrayGetCount(accessGroups); ++i) {
2168         require_quiet(q = query_create_with_limit(attributes, client->musr, 0, client, error), out);
2169         CFStringRef agrp = CFArrayGetValueAtIndex(accessGroups, i);
2170         query_add_attribute(kSecAttrAccessGroup, agrp, q);
2171         query_ensure_access_control(q, agrp);
2172         bool added = false;
2173         if (!q->q_error) {
2174             query_pre_add(q, true);
2175             added = s3dl_query_add(dbt, q, NULL, error);
2176         }
2177         require_quiet(query_notify_and_destroy(q, added, error), out);
2178     }
2179     ok = true;
2180 out:
2181     return ok;
2182 }
2183
2184 bool _SecItemUpdateTokenItemsForAccessGroups(CFStringRef tokenID, CFArrayRef accessGroups, CFArrayRef items, SecurityClient *client, CFErrorRef *error) {
2185     // This is SPI for CTK only, don't even listen to app clips.
2186     if (client->isAppClip) {
2187         return SecError(errSecRestrictedAPI, error, CFSTR("App Clips may not call this API"));
2188     }
2189
2190     return kc_with_dbt(true, error, ^bool (SecDbConnectionRef dbt) {
2191         return kc_transaction(dbt, error, ^bool {
2192             bool ok = false;
2193             CFErrorRef localError = NULL;
2194             const CFTypeRef classToDelete[] = { kSecClassGenericPassword, kSecClassInternetPassword, kSecClassCertificate, kSecClassKey };
2195             for (size_t i = 0; i < sizeof(classToDelete) / sizeof(classToDelete[0]); ++i) {
2196                 if (!SecItemDeleteTokenItems(dbt, classToDelete[i], tokenID, accessGroups, client, &localError)) {
2197                     require_action_quiet(CFErrorGetCode(localError) == errSecItemNotFound, out, CFErrorPropagate(localError, error));
2198                     CFReleaseNull(localError);
2199                 }
2200             }
2201
2202             if (items) {
2203                 for (CFIndex i = 0; i < CFArrayGetCount(items); ++i) {
2204                     require_quiet(SecItemAddTokenItemToAccessGroups(dbt, CFArrayGetValueAtIndex(items, i), accessGroups, client, error), out);
2205                 }
2206             }
2207
2208             ok = true;
2209         out:
2210             return ok;
2211         });
2212     });
2213 }
2214
2215 static bool deleteNonSysboundItemsForItemClass(SecDbConnectionRef dbt, SecDbClass const* class, CFErrorRef* error) {
2216     CFMutableDictionaryRef query = CFDictionaryCreateMutableForCFTypes(NULL);
2217     CFDictionaryAddValue(query, kSecMatchLimit, kSecMatchLimitAll);
2218
2219     __block CFErrorRef localError = NULL;
2220     SecDbQueryRef q = query_create(class, NULL, query, NULL, &localError);
2221     if (q == NULL) {    // illegal query or out of memory
2222         secerror("SecItemServerDeleteAll: aborting because failed to initialize Query: %@", localError);
2223         abort();
2224     }
2225     SecDbItemSelect(q, dbt, &localError, ^bool(const SecDbAttr *attr) {
2226         return (attr->flags & kSecDbInFlag) && !CFEqual(attr->name, CFSTR("data"));
2227     }, NULL, NULL, NULL,
2228     ^(SecDbItemRef item, bool *stop) {
2229         if (!SecItemIsSystemBound(item->attributes, class, false) &&
2230             !CFEqual(CFDictionaryGetValue(item->attributes, kSecAttrAccessGroup), CFSTR("com.apple.bluetooth")))
2231         {
2232             SecDbItemDelete(item, dbt, kCFBooleanFalse, false, &localError);
2233         }
2234     });
2235     query_destroy(q, &localError);
2236
2237     if (localError) {
2238         if (error) {
2239             CFReleaseNull(*error);
2240             *error = localError;
2241         } else {
2242             CFReleaseNull(localError);
2243         }
2244         return false;
2245     }
2246     return true;
2247 }
2248
2249 // Delete all the items except sysbound ones because horrible things happen if you do, like bluetooth devices unpairing
2250 static bool
2251 SecItemServerDeleteAll(CFErrorRef *error) {
2252     secerror("SecItemServerDeleteAll");
2253     return kc_with_dbt(true, error, ^bool (SecDbConnectionRef dbt) {
2254         return (kc_transaction(dbt, error, ^bool {
2255             bool ok = true;
2256             ok &= SecDbExec(dbt, CFSTR("DELETE FROM genp WHERE sync=1;"), error);
2257             ok &= SecDbExec(dbt, CFSTR("DELETE FROM inet WHERE sync=1;"), error);
2258             ok &= SecDbExec(dbt, CFSTR("DELETE FROM cert WHERE sync=1;"), error);
2259             ok &= SecDbExec(dbt, CFSTR("DELETE FROM keys WHERE sync=1;"), error);
2260
2261             ok &= deleteNonSysboundItemsForItemClass(dbt, genp_class(), error);
2262             ok &= deleteNonSysboundItemsForItemClass(dbt, inet_class(), error);
2263             ok &= deleteNonSysboundItemsForItemClass(dbt, cert_class(), error);
2264             ok &= deleteNonSysboundItemsForItemClass(dbt, keys_class(), error);
2265
2266             return ok;
2267         }) && SecDbExec(dbt, CFSTR("VACUUM;"), error));
2268     });
2269 }
2270
2271 bool
2272 _SecItemDeleteAll(CFErrorRef *error) {
2273     return SecItemServerDeleteAll(error);
2274 }
2275
2276 bool
2277 _SecItemServerDeleteAllWithAccessGroups(CFArrayRef accessGroups, SecurityClient *client, CFErrorRef *error)
2278 {
2279     __block bool ok = true;
2280     static dispatch_once_t onceToken;
2281     static CFSetRef illegalAccessGroups = NULL;
2282
2283     dispatch_once(&onceToken, ^{
2284         const CFStringRef values[] = {
2285             CFSTR("*"),
2286             CFSTR("apple"),
2287             CFSTR("com.apple.security.sos"),
2288             CFSTR("lockdown-identities"),
2289         };
2290         illegalAccessGroups = CFSetCreate(NULL, (const void **)values, sizeof(values)/sizeof(values[0]), &kCFTypeSetCallBacks);
2291     });
2292
2293     static CFTypeRef qclasses[] = {
2294         NULL,
2295         NULL,
2296         NULL,
2297         NULL
2298     };
2299     // strange construction needed for schema indirection
2300     static dispatch_once_t qclassesOnceToken;
2301     dispatch_once(&qclassesOnceToken, ^{
2302         qclasses[0] = inet_class();
2303         qclasses[1] = genp_class();
2304         qclasses[2] = keys_class();
2305         qclasses[3] = cert_class();
2306     });
2307
2308     require_action_quiet(isArray(accessGroups), fail,
2309                          ok = false;
2310                          SecCFCreateErrorWithFormat(kSecXPCErrorUnexpectedType, sSecXPCErrorDomain, NULL, error, NULL, CFSTR("accessGroups not CFArray, got %@"), accessGroups));
2311
2312     // TODO: whitelist instead? look for dev IDs like 7123498YQX.com.somedev.app
2313
2314     require_action(CFArrayGetCount(accessGroups) != 0, fail,
2315                    ok = false;
2316                    SecCFCreateErrorWithFormat(kSecXPCErrorUnexpectedType, sSecXPCErrorDomain, NULL, error, NULL, CFSTR("accessGroups e empty")));
2317
2318
2319     // Pre-check accessGroups for prohibited values
2320     CFArrayForEach(accessGroups, ^(const void *value) {
2321         CFStringRef agrp = (CFStringRef)value;
2322
2323         if (!isString(agrp)) {
2324             SecCFCreateErrorWithFormat(kSecXPCErrorUnexpectedType, sSecXPCErrorDomain, NULL, error, NULL,
2325                                        CFSTR("access not a string: %@"), agrp);
2326             ok &= false;
2327         } else if (CFSetContainsValue(illegalAccessGroups, agrp)) {
2328             SecCFCreateErrorWithFormat(kSecXPCErrorUnexpectedType, sSecXPCErrorDomain, NULL, error, NULL,
2329                                        CFSTR("illegal access group: %@"), accessGroups);
2330             ok &= false;
2331         }
2332     });
2333     require(ok,fail);
2334
2335     ok = kc_with_dbt(true, error, ^bool(SecDbConnectionRef dbt) {
2336         return kc_transaction(dbt, error, ^bool {
2337             CFErrorRef localError = NULL;
2338             bool ok1 = true;
2339             size_t n;
2340
2341             for (n = 0; n < sizeof(qclasses)/sizeof(qclasses[0]) && ok1; n++) {
2342                 Query *q;
2343
2344                 q = query_create(qclasses[n], client->musr, NULL, client, error);
2345                 require(q, fail2);
2346
2347                 (void)s3dl_query_delete(dbt, q, accessGroups, &localError);
2348             fail2:
2349                 query_destroy(q, error);
2350                 CFReleaseNull(localError);
2351             }
2352             return ok1;
2353         }) && SecDbExec(dbt, CFSTR("VACUUM"), error);
2354     });
2355
2356 fail:
2357     return ok;
2358 }
2359
2360
2361 // MARK: -
2362 // MARK: Shared web credentials
2363
2364 #if SHAREDWEBCREDENTIALS
2365
2366 // OSX now has SWC enabled, but cannot link SharedWebCredentials framework: rdar://59958701
2367 #if TARGET_OS_IOS && !TARGET_OS_BRIDGE && !TARGET_OS_WATCH && !TARGET_OS_TV
2368
2369 /* constants */
2370 #define SEC_CONST_DECL(k,v) const CFStringRef k = CFSTR(v);
2371
2372 SEC_CONST_DECL (kSecSafariAccessGroup, "com.apple.cfnetwork");
2373 SEC_CONST_DECL (kSecSafariDefaultComment, "default");
2374 SEC_CONST_DECL (kSecSafariPasswordsNotSaved, "Passwords not saved");
2375
2376 static bool
2377 _SecAddNegativeWebCredential(SecurityClient *client, CFStringRef fqdn, CFStringRef appID, bool forSafari)
2378 {
2379 #if !TARGET_OS_SIMULATOR
2380     bool result = false;
2381     if (!fqdn) { return result; }
2382
2383     // update our database
2384         _SecSetAppDomainApprovalStatus(appID, fqdn, kCFBooleanFalse);
2385
2386     if (!forSafari) { return result; }
2387
2388     // below this point: create a negative Safari web credential item
2389
2390     CFMutableDictionaryRef attrs = CFDictionaryCreateMutable(kCFAllocatorDefault, 0, &kCFTypeDictionaryKeyCallBacks, &kCFTypeDictionaryValueCallBacks);
2391     if (!attrs) { return result; }
2392
2393     CFErrorRef error = NULL;
2394     CFStringRef accessGroup = CFSTR("*");
2395     SecurityClient swcclient = {
2396         .task = NULL,
2397         .accessGroups =  CFArrayCreate(kCFAllocatorDefault, (const void **)&accessGroup, 1, &kCFTypeArrayCallBacks),
2398         .allowSystemKeychain = false,
2399         .allowSyncBubbleKeychain = false,
2400         .isNetworkExtension = false,
2401         .musr = client->musr,
2402     };
2403
2404     CFDictionaryAddValue(attrs, kSecClass, kSecClassInternetPassword);
2405     CFDictionaryAddValue(attrs, kSecAttrAccessGroup, kSecSafariAccessGroup);
2406     CFDictionaryAddValue(attrs, kSecAttrAuthenticationType, kSecAttrAuthenticationTypeHTMLForm);
2407     CFDictionaryAddValue(attrs, kSecAttrProtocol, kSecAttrProtocolHTTPS);
2408     CFDictionaryAddValue(attrs, kSecAttrServer, fqdn);
2409     CFDictionaryAddValue(attrs, kSecAttrSynchronizable, kCFBooleanTrue);
2410
2411     (void)_SecItemDelete(attrs, &swcclient, &error);
2412     CFReleaseNull(error);
2413
2414     CFDictionaryAddValue(attrs, kSecAttrAccount, kSecSafariPasswordsNotSaved);
2415     CFDictionaryAddValue(attrs, kSecAttrComment, kSecSafariDefaultComment);
2416
2417     CFStringRef label = CFStringCreateWithFormat(kCFAllocatorDefault,
2418                                                  NULL, CFSTR("%@ (%@)"), fqdn, kSecSafariPasswordsNotSaved);
2419     if (label) {
2420         CFDictionaryAddValue(attrs, kSecAttrLabel, label);
2421         CFReleaseSafe(label);
2422     }
2423
2424     UInt8 space = ' ';
2425     CFDataRef data = CFDataCreate(kCFAllocatorDefault, &space, 1);
2426     if (data) {
2427         CFDictionarySetValue(attrs, kSecValueData, data);
2428         CFReleaseSafe(data);
2429     }
2430
2431     CFTypeRef addResult = NULL;
2432     result = _SecItemAdd(attrs, &swcclient, &addResult, &error);
2433
2434     CFReleaseSafe(addResult);
2435     CFReleaseSafe(error);
2436     CFReleaseSafe(attrs);
2437     CFReleaseSafe(swcclient.accessGroups);
2438
2439     return result;
2440 #else
2441     return true;
2442 #endif
2443 }
2444
2445 /* Specialized version of SecItemAdd for shared web credentials */
2446 bool
2447 _SecAddSharedWebCredential(CFDictionaryRef attributes,
2448                            SecurityClient *client,
2449                            const audit_token_t *clientAuditToken,
2450                            CFStringRef appID,
2451                            CFArrayRef domains,
2452                            CFTypeRef *result,
2453                            CFErrorRef *error)
2454 {
2455
2456     SecurityClient swcclient = {};
2457
2458     CFStringRef fqdn = CFRetainSafe(CFDictionaryGetValue(attributes, kSecAttrServer));
2459     CFStringRef account = CFDictionaryGetValue(attributes, kSecAttrAccount);
2460 #if TARGET_OS_IOS && !TARGET_OS_BRIDGE
2461     CFStringRef password = CFDictionaryGetValue(attributes, kSecSharedPassword);
2462 #else
2463     CFStringRef password = CFDictionaryGetValue(attributes, CFSTR("spwd"));
2464 #endif
2465     CFStringRef accessGroup = CFSTR("*");
2466     CFMutableDictionaryRef query = NULL, attrs = NULL;
2467     SInt32 port = -1;
2468     bool ok = false;
2469
2470     // check autofill enabled status
2471     if (!swca_autofill_enabled(clientAuditToken)) {
2472         SecError(errSecBadReq, error, CFSTR("Autofill is not enabled in Safari settings"));
2473         goto cleanup;
2474     }
2475
2476     // parse fqdn with CFURL here, since it could be specified as domain:port
2477     if (fqdn) {
2478                 CFTypeRef fqdnObject = _SecCopyFQDNObjectFromString(fqdn);
2479                 if (fqdnObject) {
2480                         CFReleaseSafe(fqdn);
2481                         fqdn = _SecGetFQDNFromFQDNObject(fqdnObject, &port);
2482                         CFRetainSafe(fqdn);
2483                         CFReleaseSafe(fqdnObject);
2484                 }
2485     }
2486
2487     if (!account) {
2488         SecError(errSecParam, error, CFSTR("No account provided"));
2489         goto cleanup;
2490     }
2491     if (!fqdn) {
2492         SecError(errSecParam, error, CFSTR("No domain provided"));
2493         goto cleanup;
2494     }
2495
2496 #if TARGET_OS_SIMULATOR
2497     secerror("app/site association entitlements not checked in Simulator");
2498 #else
2499     OSStatus status = errSecMissingEntitlement;
2500     // validate that fqdn is part of caller's shared credential domains entitlement
2501     if (!appID) {
2502         SecError(status, error, CFSTR("Missing application-identifier entitlement"));
2503         goto cleanup;
2504     }
2505     if (_SecEntitlementContainsDomainForService(domains, fqdn, port)) {
2506         status = errSecSuccess;
2507     }
2508     if (errSecSuccess != status) {
2509         CFStringRef msg = CFStringCreateWithFormat(kCFAllocatorDefault, NULL,
2510                                                    CFSTR("%@ not found in %@ entitlement"), fqdn, kSecEntitlementAssociatedDomains);
2511         if (!msg) {
2512             msg = CFRetain(CFSTR("Requested domain not found in entitlement"));
2513         }
2514         SecError(status, error, CFSTR("%@"), msg);
2515         CFReleaseSafe(msg);
2516         goto cleanup;
2517     }
2518 #endif
2519
2520 #if TARGET_OS_SIMULATOR
2521     secerror("Ignoring app/site approval state in the Simulator.");
2522 #else
2523     // get approval status for this app/domain pair
2524     SecSWCFlags flags = _SecAppDomainApprovalStatus(appID, fqdn, error);
2525     if (!(flags & kSecSWCFlag_SiteApproved)) {
2526         goto cleanup;
2527     }
2528 #endif
2529
2530     // give ourselves access to see matching items for kSecSafariAccessGroup
2531     swcclient.task = NULL;
2532     swcclient.accessGroups =  CFArrayCreate(kCFAllocatorDefault, (const void **)&accessGroup, 1, &kCFTypeArrayCallBacks);
2533     swcclient.allowSystemKeychain = false;
2534     swcclient.musr = client->musr;
2535     swcclient.allowSystemKeychain = false;
2536     swcclient.allowSyncBubbleKeychain = false;
2537     swcclient.isNetworkExtension = false;
2538
2539
2540     // create lookup query
2541     query = CFDictionaryCreateMutable(kCFAllocatorDefault, 0, &kCFTypeDictionaryKeyCallBacks, &kCFTypeDictionaryValueCallBacks);
2542     if (!query) {
2543         SecError(errSecAllocate, error, CFSTR("Unable to create query dictionary"));
2544         goto cleanup;
2545     }
2546     CFDictionaryAddValue(query, kSecClass, kSecClassInternetPassword);
2547     CFDictionaryAddValue(query, kSecAttrAccessGroup, kSecSafariAccessGroup);
2548     CFDictionaryAddValue(query, kSecAttrAuthenticationType, kSecAttrAuthenticationTypeHTMLForm);
2549     CFDictionaryAddValue(query, kSecAttrServer, fqdn);
2550     CFDictionaryAddValue(query, kSecAttrSynchronizable, kCFBooleanTrue);
2551
2552     // check for presence of Safari's negative entry ('passwords not saved')
2553     CFDictionarySetValue(query, kSecAttrAccount, kSecSafariPasswordsNotSaved);
2554     ok = _SecItemCopyMatching(query, &swcclient, result, error);
2555     if(result) CFReleaseNull(*result);
2556     if (error) CFReleaseNull(*error);
2557     if (ok) {
2558         SecError(errSecDuplicateItem, error, CFSTR("Item already exists for this server"));
2559         goto cleanup;
2560     }
2561
2562     // now use the provided account (and optional port number, if one was present)
2563     CFDictionarySetValue(query, kSecAttrAccount, account);
2564     if (port < -1 || port > 0) {
2565         SInt16 portValueShort = (port & 0xFFFF);
2566         CFNumberRef portNumber = CFNumberCreate(NULL, kCFNumberSInt16Type, &portValueShort);
2567         CFDictionaryAddValue(query, kSecAttrPort, portNumber);
2568         CFReleaseSafe(portNumber);
2569     }
2570
2571     // look up existing password
2572     CFDictionaryAddValue(query, kSecReturnData, kCFBooleanTrue);
2573     bool matched = _SecItemCopyMatching(query, &swcclient, result, error);
2574     CFDictionaryRemoveValue(query, kSecReturnData);
2575     if (matched) {
2576         // found it, so this becomes either an "update password" or "delete password" operation
2577         bool update = (password != NULL);
2578         if (update) {
2579             attrs = CFDictionaryCreateMutable(kCFAllocatorDefault, 0, &kCFTypeDictionaryKeyCallBacks, &kCFTypeDictionaryValueCallBacks);
2580             CFDataRef credential = CFStringCreateExternalRepresentation(kCFAllocatorDefault, password, kCFStringEncodingUTF8, 0);
2581             CFDictionaryAddValue(attrs, kSecValueData, credential);
2582             bool samePassword = result && *result && CFEqual(*result, credential);
2583             CFReleaseSafe(credential);
2584             CFDictionaryAddValue(attrs, kSecAttrComment, kSecSafariDefaultComment);
2585
2586             ok = samePassword || swca_confirm_operation(swca_update_request_id, clientAuditToken, query, error,
2587                 ^void (CFStringRef confirm_fqdn) {
2588                     _SecAddNegativeWebCredential(client, confirm_fqdn, appID, false);
2589                 });
2590             if (ok) {
2591                 ok = _SecItemUpdate(query, attrs, &swcclient, error);
2592             }
2593         }
2594         else {
2595             // confirm the delete
2596             // (per rdar://16676288 we always prompt, even if there was prior user approval)
2597             ok = /*approved ||*/ swca_confirm_operation(swca_delete_request_id, clientAuditToken, query, error,
2598                 ^void (CFStringRef confirm_fqdn) {
2599                     _SecAddNegativeWebCredential(client, confirm_fqdn, appID, false);
2600                 });
2601             if (ok) {
2602                 ok = _SecItemDelete(query, &swcclient, error);
2603             }
2604         }
2605
2606         if(result) CFReleaseNull(*result);
2607         if(error) CFReleaseNull(*error);
2608
2609         goto cleanup;
2610     }
2611     if (result) CFReleaseNull(*result);
2612     if (error) CFReleaseNull(*error);
2613
2614     // password does not exist, so prepare to add it
2615     if (!password) {
2616         // a NULL password value removes the existing credential. Since we didn't find it, this is a no-op.
2617         ok = true;
2618         goto cleanup;
2619     }
2620     else {
2621         CFStringRef label = CFStringCreateWithFormat(kCFAllocatorDefault, NULL, CFSTR("%@ (%@)"), fqdn, account);
2622         if (label) {
2623             CFDictionaryAddValue(query, kSecAttrLabel, label);
2624             CFReleaseSafe(label);
2625         }
2626         // NOTE: we always expect to use HTTPS for web forms.
2627         CFDictionaryAddValue(query, kSecAttrProtocol, kSecAttrProtocolHTTPS);
2628
2629         CFDataRef credential = CFStringCreateExternalRepresentation(kCFAllocatorDefault, password, kCFStringEncodingUTF8, 0);
2630         CFDictionarySetValue(query, kSecValueData, credential);
2631         CFReleaseSafe(credential);
2632         CFDictionarySetValue(query, kSecAttrComment, kSecSafariDefaultComment);
2633
2634         CFReleaseSafe(swcclient.accessGroups);
2635         swcclient.accessGroups = CFArrayCreate(kCFAllocatorDefault, (const void **)&kSecSafariAccessGroup, 1, &kCFTypeArrayCallBacks);
2636
2637         // mark the item as created by this function
2638         const int32_t creator_value = 'swca';
2639         CFNumberRef creator = CFNumberCreate(kCFAllocatorDefault, kCFNumberSInt32Type, &creator_value);
2640         if (creator) {
2641             CFDictionarySetValue(query, kSecAttrCreator, creator);
2642             CFReleaseSafe(creator);
2643         }
2644
2645         // confirm the add
2646         ok = swca_confirm_operation(swca_add_request_id, clientAuditToken, query, error, ^void (CFStringRef confirm_fqdn) {
2647             _SecAddNegativeWebCredential(client, confirm_fqdn, appID, false);
2648         });
2649     }
2650     if (ok) {
2651         ok = _SecItemAdd(query, &swcclient, result, error);
2652     }
2653
2654 cleanup:
2655     CFReleaseSafe(attrs);
2656     CFReleaseSafe(query);
2657     CFReleaseSafe(swcclient.accessGroups);
2658     CFReleaseSafe(fqdn);
2659     return ok;
2660 }
2661
2662 /* Specialized version of SecItemCopyMatching for shared web credentials */
2663 bool
2664 _SecCopySharedWebCredential(CFDictionaryRef query,
2665                             SecurityClient *client,
2666                             const audit_token_t *clientAuditToken,
2667                             CFStringRef appID,
2668                             CFArrayRef domains,
2669                             CFTypeRef *result,
2670                             CFErrorRef *error)
2671 {
2672     CFMutableArrayRef credentials = NULL;
2673     CFMutableArrayRef foundItems = NULL;
2674     CFMutableArrayRef fqdns = NULL;
2675     CFStringRef account = NULL;
2676     bool ok = false;
2677
2678     require_quiet(result, cleanup);
2679     credentials = CFArrayCreateMutable(kCFAllocatorDefault, 0, &kCFTypeArrayCallBacks);
2680     foundItems = CFArrayCreateMutable(kCFAllocatorDefault, 0, &kCFTypeArrayCallBacks);
2681     fqdns = CFArrayCreateMutable(kCFAllocatorDefault, 0, &kCFTypeArrayCallBacks);
2682
2683     // give ourselves access to see matching items for kSecSafariAccessGroup
2684     CFStringRef accessGroup = CFSTR("*");
2685     SecurityClient swcclient = {
2686         .task = NULL,
2687         .accessGroups =  CFArrayCreate(kCFAllocatorDefault, (const void **)&accessGroup, 1, &kCFTypeArrayCallBacks),
2688         .allowSystemKeychain = false,
2689         .allowSyncBubbleKeychain = false,
2690         .isNetworkExtension = false,
2691         .musr = client->musr,
2692     };
2693
2694     // On input, the query dictionary contains optional fqdn and account entries.
2695     account = CFDictionaryGetValue(query, kSecAttrAccount);
2696
2697     // Check autofill enabled status
2698     if (!swca_autofill_enabled(clientAuditToken)) {
2699         SecError(errSecBadReq, error, CFSTR("Autofill is not enabled in Safari settings"));
2700         goto cleanup;
2701     }
2702
2703     // Check fqdn; if NULL, add domains from caller's entitlement.
2704     {
2705         CFStringRef fqdn = CFDictionaryGetValue(query, kSecAttrServer);
2706         if (fqdn) {
2707             CFTypeRef fqdnObject = _SecCopyFQDNObjectFromString(fqdn);
2708             if (fqdnObject) {
2709                 CFArrayAppendValue(fqdns, fqdnObject);
2710                 CFReleaseSafe(fqdnObject);
2711             }
2712         }
2713         else if (domains) {
2714             CFIndex idx, count = CFArrayGetCount(domains);
2715             for (idx=0; idx < count; idx++) {
2716                 CFStringRef str = (CFStringRef) CFArrayGetValueAtIndex(domains, idx);
2717                 // Parse the entry for our service label prefix
2718                 CFTypeRef fqdnObject = _SecCopyFQDNObjectFromString(str);
2719                 if (fqdnObject) {
2720                     CFArrayAppendValue(fqdns, fqdnObject);
2721                     CFReleaseSafe(fqdnObject);
2722                 }
2723             }
2724         }
2725     }
2726     CFIndex count, idx;
2727
2728     count = CFArrayGetCount(fqdns);
2729     if (count < 1) {
2730         SecError(errSecParam, error, CFSTR("No domain provided"));
2731         goto cleanup;
2732     }
2733
2734     // Aggregate search results for each domain
2735     for (idx = 0; idx < count; idx++) {
2736         CFMutableArrayRef items = NULL;
2737         CFMutableDictionaryRef attrs = NULL;
2738
2739         CFTypeRef fqdnObject = CFArrayGetValueAtIndex(fqdns, idx);
2740         SInt32 port = -1;
2741         CFStringRef fqdn = _SecGetFQDNFromFQDNObject(fqdnObject, &port);
2742
2743 #if TARGET_OS_SIMULATOR
2744         secerror("app/site association entitlements not checked in Simulator");
2745 #else
2746             OSStatus status = errSecMissingEntitlement;
2747         if (!appID) {
2748             SecError(status, error, CFSTR("Missing application-identifier entitlement"));
2749             goto cleanup;
2750         }
2751         // validate that fqdn is part of caller's entitlement
2752         if (_SecEntitlementContainsDomainForService(domains, fqdn, port)) {
2753             status = errSecSuccess;
2754         }
2755         if (errSecSuccess != status) {
2756             CFStringRef msg = CFStringCreateWithFormat(kCFAllocatorDefault, NULL,
2757                 CFSTR("%@ not found in %@ entitlement"), fqdn, kSecEntitlementAssociatedDomains);
2758             if (!msg) {
2759                 msg = CFRetain(CFSTR("Requested domain not found in entitlement"));
2760             }
2761             SecError(status, error, CFSTR("%@"), msg);
2762             CFReleaseSafe(msg);
2763             goto cleanup;
2764         }
2765 #endif
2766
2767         attrs = CFDictionaryCreateMutable(kCFAllocatorDefault, 0, &kCFTypeDictionaryKeyCallBacks, &kCFTypeDictionaryValueCallBacks);
2768         if (!attrs) {
2769             SecError(errSecAllocate, error, CFSTR("Unable to create query dictionary"));
2770             goto cleanup;
2771         }
2772         CFDictionaryAddValue(attrs, kSecClass, kSecClassInternetPassword);
2773         CFDictionaryAddValue(attrs, kSecAttrAccessGroup, kSecSafariAccessGroup);
2774         CFDictionaryAddValue(attrs, kSecAttrProtocol, kSecAttrProtocolHTTPS);
2775         CFDictionaryAddValue(attrs, kSecAttrAuthenticationType, kSecAttrAuthenticationTypeHTMLForm);
2776         CFDictionaryAddValue(attrs, kSecAttrServer, fqdn);
2777         if (account) {
2778             CFDictionaryAddValue(attrs, kSecAttrAccount, account);
2779         }
2780         if (port < -1 || port > 0) {
2781             SInt16 portValueShort = (port & 0xFFFF);
2782             CFNumberRef portNumber = CFNumberCreate(NULL, kCFNumberSInt16Type, &portValueShort);
2783             CFDictionaryAddValue(attrs, kSecAttrPort, portNumber);
2784             CFReleaseSafe(portNumber);
2785         }
2786         CFDictionaryAddValue(attrs, kSecAttrSynchronizable, kCFBooleanTrue);
2787         CFDictionaryAddValue(attrs, kSecMatchLimit, kSecMatchLimitAll);
2788         CFDictionaryAddValue(attrs, kSecReturnAttributes, kCFBooleanTrue);
2789         CFDictionaryAddValue(attrs, kSecReturnData, kCFBooleanTrue);
2790
2791         ok = _SecItemCopyMatching(attrs, &swcclient, (CFTypeRef*)&items, error);
2792         if (count > 1) {
2793             // ignore interim error since we have multiple domains to search
2794             CFReleaseNull(*error);
2795         }
2796         if (ok && items && CFGetTypeID(items) == CFArrayGetTypeID()) {
2797 #if TARGET_OS_SIMULATOR
2798             secerror("Ignoring app/site approval state in the Simulator.");
2799             bool approved = true;
2800 #else
2801             // get approval status for this app/domain pair
2802             SecSWCFlags flags = _SecAppDomainApprovalStatus(appID, fqdn, error);
2803             if (count > 1) {
2804                 // ignore interim error since we have multiple domains to check
2805                 CFReleaseNull(*error);
2806             }
2807             bool approved = (flags & kSecSWCFlag_SiteApproved);
2808 #endif
2809             if (approved) {
2810                 CFArrayAppendArray(foundItems, items, CFRangeMake(0, CFArrayGetCount(items)));
2811             }
2812         }
2813         CFReleaseSafe(items);
2814         CFReleaseSafe(attrs);
2815     }
2816
2817 //  If matching credentials are found, the credentials provided to the completionHandler
2818 //  will be a CFArrayRef containing CFDictionaryRef entries. Each dictionary entry will
2819 //  contain the following pairs (see Security/SecItem.h):
2820 //  key: kSecAttrServer     value: CFStringRef (the website)
2821 //  key: kSecAttrAccount    value: CFStringRef (the account)
2822 //  key: kSecSharedPassword value: CFStringRef (the password)
2823 //  Optional keys:
2824 //  key: kSecAttrPort       value: CFNumberRef (the port number, if non-standard for https)
2825
2826     count = CFArrayGetCount(foundItems);
2827     for (idx = 0; idx < count; idx++) {
2828         CFDictionaryRef dict = (CFDictionaryRef) CFArrayGetValueAtIndex(foundItems, idx);
2829         CFMutableDictionaryRef newdict = CFDictionaryCreateMutable(kCFAllocatorDefault, 0, &kCFTypeDictionaryKeyCallBacks, &kCFTypeDictionaryValueCallBacks);
2830         if (newdict && dict && CFGetTypeID(dict) == CFDictionaryGetTypeID()) {
2831             CFStringRef srvr = CFDictionaryGetValue(dict, kSecAttrServer);
2832             CFStringRef acct = CFDictionaryGetValue(dict, kSecAttrAccount);
2833             CFNumberRef pnum = CFDictionaryGetValue(dict, kSecAttrPort);
2834             CFStringRef icmt = CFDictionaryGetValue(dict, kSecAttrComment);
2835             CFDataRef data = CFDictionaryGetValue(dict, kSecValueData);
2836             if (srvr) {
2837                 CFDictionaryAddValue(newdict, kSecAttrServer, srvr);
2838             }
2839             if (acct) {
2840                 CFDictionaryAddValue(newdict, kSecAttrAccount, acct);
2841             }
2842             if (pnum) {
2843                 SInt16 pval = -1;
2844                 if (CFNumberGetValue(pnum, kCFNumberSInt16Type, &pval) &&
2845                     (pval < -1 || pval > 0)) {
2846                     CFDictionaryAddValue(newdict, kSecAttrPort, pnum);
2847                 }
2848             }
2849             if (data) {
2850                 CFStringRef password = CFStringCreateFromExternalRepresentation(kCFAllocatorDefault, data, kCFStringEncodingUTF8);
2851                 if (password) {
2852                 #if TARGET_OS_IPHONE && !TARGET_OS_WATCH && !TARGET_OS_TV
2853                     CFDictionaryAddValue(newdict, kSecSharedPassword, password);
2854                 #else
2855                     CFDictionaryAddValue(newdict, CFSTR("spwd"), password);
2856                 #endif
2857                     CFReleaseSafe(password);
2858                 }
2859             }
2860
2861             if (acct && CFEqual(acct, kSecSafariPasswordsNotSaved)) {
2862                 // Do not add to credentials list!
2863                 secwarning("copySWC: Skipping \"%@\" item", kSecSafariPasswordsNotSaved);
2864             } else if (icmt && CFEqual(icmt, kSecSafariDefaultComment)) {
2865                 CFArrayInsertValueAtIndex(credentials, 0, newdict);
2866             } else {
2867                 CFArrayAppendValue(credentials, newdict);
2868             }
2869         }
2870         CFReleaseSafe(newdict);
2871     }
2872
2873     count = CFArrayGetCount(credentials);
2874     if (count) {
2875         ok = false;
2876         // create a new array of dictionaries (without the actual password) for picker UI
2877         CFMutableArrayRef items = CFArrayCreateMutable(kCFAllocatorDefault, 0, &kCFTypeArrayCallBacks);
2878         for (idx = 0; idx < count; idx++) {
2879             CFDictionaryRef dict = (CFDictionaryRef) CFArrayGetValueAtIndex(credentials, idx);
2880             CFMutableDictionaryRef newdict = CFDictionaryCreateMutableCopy(kCFAllocatorDefault, 0, dict);
2881         #if TARGET_OS_IPHONE && !TARGET_OS_WATCH && !TARGET_OS_TV
2882             CFDictionaryRemoveValue(newdict, kSecSharedPassword);
2883         #else
2884             CFDictionaryRemoveValue(newdict, CFSTR("spwd"));
2885         #endif
2886             CFArrayAppendValue(items, newdict);
2887             CFReleaseSafe(newdict);
2888         }
2889
2890         // prompt user to select one of the dictionary items
2891         CFDictionaryRef selected = swca_copy_selected_dictionary(swca_select_request_id,
2892                                                                  clientAuditToken, items, error);
2893         if (selected) {
2894             // find the matching item in our credentials array
2895             CFStringRef srvr = CFDictionaryGetValue(selected, kSecAttrServer);
2896             CFStringRef acct = CFDictionaryGetValue(selected, kSecAttrAccount);
2897             CFNumberRef pnum = CFDictionaryGetValue(selected, kSecAttrPort);
2898             for (idx = 0; idx < count; idx++) {
2899                 CFDictionaryRef dict = (CFDictionaryRef) CFArrayGetValueAtIndex(credentials, idx);
2900                 CFStringRef srvr1 = CFDictionaryGetValue(dict, kSecAttrServer);
2901                 CFStringRef acct1 = CFDictionaryGetValue(dict, kSecAttrAccount);
2902                 CFNumberRef pnum1 = CFDictionaryGetValue(dict, kSecAttrPort);
2903
2904                 if (!srvr || !srvr1 || !CFEqual(srvr, srvr1)) continue;
2905                 if (!acct || !acct1 || !CFEqual(acct, acct1)) continue;
2906                 if ((pnum && pnum1) && !CFEqual(pnum, pnum1)) continue;
2907
2908                 // we have a match!
2909                 CFReleaseSafe(selected);
2910                 CFRetainSafe(dict);
2911                 selected = dict;
2912                 ok = true;
2913                 break;
2914             }
2915         }
2916         CFReleaseSafe(items);
2917         CFArrayRemoveAllValues(credentials);
2918         if (selected && ok) {
2919 #if TARGET_OS_IOS && !TARGET_OS_BRIDGE && !TARGET_OS_SIMULATOR
2920                         // register confirmation with database
2921             CFStringRef fqdn = CFDictionaryGetValue(selected, kSecAttrServer);
2922             _SecSetAppDomainApprovalStatus(appID, fqdn, kCFBooleanTrue);
2923 #endif
2924             CFArrayAppendValue(credentials, selected);
2925         }
2926
2927         CFReleaseSafe(selected);
2928     }
2929     else if (NULL == *error) {
2930         // found no items, and we haven't already filled in the error
2931         SecError(errSecItemNotFound, error, CFSTR("no matching items found"));
2932     }
2933
2934 cleanup:
2935     if (!ok) {
2936         CFArrayRemoveAllValues(credentials);
2937         CFReleaseNull(credentials);
2938     }
2939     CFReleaseSafe(foundItems);
2940     *result = credentials;
2941     CFReleaseSafe(swcclient.accessGroups);
2942     CFReleaseSafe(fqdns);
2943
2944     return ok;
2945 }
2946
2947 #else /* !(TARGET_OS_IOS && !TARGET_OS_BRIDGE && !TARGET_OS_WATCH && !TARGET_OS_TV) */
2948
2949 bool _SecAddSharedWebCredential(CFDictionaryRef attributes, SecurityClient *client, const audit_token_t *clientAuditToken, CFStringRef appID, CFArrayRef domains, CFTypeRef *result, CFErrorRef *error) {
2950     if (error) {
2951         SecError(errSecUnimplemented, error, CFSTR("_SecAddSharedWebCredential not supported on this platform"));
2952     }
2953     return false;
2954 }
2955
2956 bool _SecCopySharedWebCredential(CFDictionaryRef query, SecurityClient *client, const audit_token_t *clientAuditToken, CFStringRef appID, CFArrayRef domains, CFTypeRef *result, CFErrorRef *error) {
2957     if (error) {
2958         SecError(errSecUnimplemented, error, CFSTR("_SecCopySharedWebCredential not supported on this platform"));
2959     }
2960     return false;
2961 }
2962
2963 #endif /* !(TARGET_OS_IOS && !TARGET_OS_BRIDGE && !TARGET_OS_WATCH && !TARGET_OS_TV) */
2964 #endif /* SHAREDWEBCREDENTIALS */
2965
2966
2967 // MARK: -
2968 // MARK: Keychain backup
2969
2970 CF_RETURNS_RETAINED CFDataRef
2971 _SecServerKeychainCreateBackup(SecurityClient *client, CFDataRef keybag, CFDataRef passcode, bool emcs, CFErrorRef *error) {
2972     __block CFDataRef backup = NULL;
2973     kc_with_dbt(false, error, ^bool (SecDbConnectionRef dbt) {
2974
2975         LKABackupReportStart(!!keybag, !!passcode, emcs);
2976
2977         return kc_transaction_type(dbt, kSecDbNormalTransactionType, error, ^bool{
2978             secnotice("SecServerKeychainCreateBackup", "Performing backup from %s keybag%s", keybag ? "provided" : "device", emcs ? ", EMCS mode" : "");
2979
2980             if (keybag == NULL && passcode == NULL) {
2981 #if USE_KEYSTORE
2982                 backup = SecServerExportBackupableKeychain(dbt, client, KEYBAG_DEVICE, backup_keybag_handle, error);
2983 #else /* !USE_KEYSTORE */
2984                 (void)client;
2985                 SecError(errSecParam, error, CFSTR("Why are you doing this?"));
2986                 backup = NULL;
2987 #endif /* USE_KEYSTORE */
2988             } else {
2989                 backup = SecServerKeychainCreateBackup(dbt, client, keybag, passcode, emcs, error);
2990             }
2991             return (backup != NULL);
2992         });
2993     });
2994
2995     secnotice("SecServerKeychainCreateBackup", "Backup result: %s (%@)", backup ? "success" : "fail", error ? *error : NULL);
2996     LKABackupReportEnd(!!backup, error ? *error : NULL);
2997
2998     return backup;
2999 }
3000
3001 bool
3002 _SecServerKeychainRestore(CFDataRef backup, SecurityClient *client, CFDataRef keybag, CFDataRef passcode, CFErrorRef *error) {
3003     if (backup == NULL || keybag == NULL)
3004         return SecError(errSecParam, error, CFSTR("backup or keybag missing"));
3005
3006     __block bool ok = true;
3007     ok &= kc_with_dbt(true, error, ^bool (SecDbConnectionRef dbconn) {
3008         return SecServerKeychainRestore(dbconn, client, backup, keybag, passcode, error);
3009     });
3010
3011     if (ok) {
3012         SecKeychainChanged();
3013     }
3014
3015     return ok;
3016 }
3017
3018 CFStringRef
3019 _SecServerBackupCopyUUID(CFDataRef data, CFErrorRef *error)
3020 {
3021     CFStringRef uuid = NULL;
3022     CFDictionaryRef backup;
3023
3024     backup = CFPropertyListCreateWithData(kCFAllocatorDefault, data,
3025                                           kCFPropertyListImmutable, NULL,
3026                                           error);
3027     if (isDictionary(backup)) {
3028         uuid = SecServerBackupGetKeybagUUID(backup, error);
3029         if (uuid)
3030             CFRetain(uuid);
3031     }
3032     CFReleaseNull(backup);
3033
3034     return uuid;
3035 }
3036
3037
3038
3039 // MARK: -
3040 // MARK: SecItemDataSource
3041
3042 #if SECUREOBJECTSYNC
3043
3044 // Make sure to call this before any writes to the keychain, so that we fire
3045 // up the engines to monitor manifest changes.
3046 SOSDataSourceFactoryRef SecItemDataSourceFactoryGetDefault(void) {
3047     return SecItemDataSourceFactoryGetShared(kc_dbhandle(NULL));
3048 }
3049
3050 /* AUDIT[securityd]:
3051    args_in (ok) is a caller provided, CFDictionaryRef.
3052  */
3053
3054 CF_RETURNS_RETAINED CFArrayRef
3055 _SecServerKeychainSyncUpdateMessage(CFDictionaryRef updates, CFErrorRef *error) {
3056     // This never fails, trust us!
3057     return SOSCCHandleUpdateMessage(updates);
3058 }
3059
3060 //
3061 // Truthiness in the cloud backup/restore support.
3062 //
3063
3064 static CFDictionaryRef
3065 _SecServerCopyTruthInTheCloud(CFDataRef keybag, CFDataRef password,
3066     CFDictionaryRef backup, CFErrorRef *error)
3067 {
3068     SOSManifestRef mold = NULL, mnow = NULL, mdelete = NULL, madd = NULL;
3069     __block CFMutableDictionaryRef backup_new = NULL;
3070     keybag_handle_t bag_handle;
3071     if (!ks_open_keybag(keybag, password, &bag_handle, error))
3072         return backup_new;
3073
3074     // We need to have a datasource singleton for protection domain
3075     // kSecAttrAccessibleWhenUnlocked and keep a single shared engine
3076     // instance around which we create in the datasource constructor as well.
3077     SOSDataSourceFactoryRef dsf = SecItemDataSourceFactoryGetDefault();
3078     SOSDataSourceRef ds = SOSDataSourceFactoryCreateDataSource(dsf, kSecAttrAccessibleWhenUnlocked, error);
3079     if (ds) {
3080         backup_new = backup ? CFDictionaryCreateMutableCopy(kCFAllocatorDefault, 0, backup) : CFDictionaryCreateMutable(kCFAllocatorDefault, 0, &kCFTypeDictionaryKeyCallBacks, &kCFTypeDictionaryValueCallBacks);
3081         mold = SOSCreateManifestWithBackup(backup, error);
3082         SOSEngineRef engine = SOSDataSourceGetSharedEngine(ds, error);
3083         mnow = SOSEngineCopyManifest(engine, NULL);
3084         if (!mnow) {
3085             mnow = SOSDataSourceCopyManifestWithViewNameSet(ds, SOSViewsGetV0ViewSet(), error);
3086         }
3087         if (!mnow) {
3088             CFReleaseNull(backup_new);
3089             secerror("failed to obtain manifest for keychain: %@", error ? *error : NULL);
3090         } else {
3091             SOSManifestDiff(mold, mnow, &mdelete, &madd, error);
3092         }
3093
3094         // Delete everything from the new_backup that is no longer in the datasource according to the datasources manifest.
3095         SOSManifestForEach(mdelete, ^(CFDataRef digest_data, bool *stop) {
3096             CFStringRef deleted_item_key = CFDataCopyHexString(digest_data);
3097             CFDictionaryRemoveValue(backup_new, deleted_item_key);
3098             CFRelease(deleted_item_key);
3099         });
3100
3101         CFMutableArrayRef changes = CFArrayCreateMutableForCFTypes(kCFAllocatorDefault);
3102         SOSDataSourceForEachObject(ds, NULL, madd, error, ^void(CFDataRef digest, SOSObjectRef object, bool *stop) {
3103             CFErrorRef localError = NULL;
3104             CFDataRef digest_data = NULL;
3105             CFTypeRef value = NULL;
3106             if (!object) {
3107                 // Key in our manifest can't be found in db, remove it from our manifest
3108                 SOSChangesAppendDelete(changes, digest);
3109             } else if (!(digest_data = SOSObjectCopyDigest(ds, object, &localError))
3110                 || !(value = SOSObjectCopyBackup(ds, object, bag_handle, &localError))) {
3111                 if (SecErrorGetOSStatus(localError) == errSecDecode) {
3112                     // Ignore decode errors, pretend the objects aren't there
3113                     CFRelease(localError);
3114                     // Object undecodable, remove it from our manifest
3115                     SOSChangesAppendDelete(changes, digest);
3116                 } else {
3117                     // Stop iterating and propagate out all other errors.
3118                     *stop = true;
3119                     *error = localError;
3120                     CFReleaseNull(backup_new);
3121                 }
3122             } else {
3123                 // TODO: Should we skip tombstones here?
3124                 CFStringRef key = CFDataCopyHexString(digest_data);
3125                 CFDictionarySetValue(backup_new, key, value);
3126                 CFReleaseSafe(key);
3127             }
3128             CFReleaseSafe(digest_data);
3129             CFReleaseSafe(value);
3130         }) || CFReleaseNull(backup_new);
3131
3132         if (CFArrayGetCount(changes)) {
3133             if (!SOSEngineUpdateChanges(engine, kSOSDataSourceSOSTransaction, changes, error)) {
3134                 CFReleaseNull(backup_new);
3135             }
3136         }
3137         CFReleaseSafe(changes);
3138
3139         SOSDataSourceRelease(ds, error) || CFReleaseNull(backup_new);
3140     }
3141
3142     CFReleaseSafe(mold);
3143     CFReleaseSafe(mnow);
3144     CFReleaseSafe(madd);
3145     CFReleaseSafe(mdelete);
3146     ks_close_keybag(bag_handle, error) || CFReleaseNull(backup_new);
3147
3148     return backup_new;
3149 }
3150
3151 static bool
3152 _SecServerRestoreTruthInTheCloud(CFDataRef keybag, CFDataRef password, CFDictionaryRef backup_in, CFErrorRef *error) {
3153     __block bool ok = true;
3154     keybag_handle_t bag_handle;
3155     if (!ks_open_keybag(keybag, password, &bag_handle, error))
3156         return false;
3157
3158     SOSManifestRef mbackup = SOSCreateManifestWithBackup(backup_in, error);
3159     if (mbackup) {
3160         SOSDataSourceFactoryRef dsf = SecItemDataSourceFactoryGetDefault();
3161         SOSDataSourceRef ds = SOSDataSourceFactoryCreateDataSource(dsf, kSecAttrAccessibleWhenUnlocked, error);
3162         ok &= ds && SOSDataSourceWith(ds, error, ^(SOSTransactionRef txn, bool *commit) {
3163             SOSManifestRef mnow = SOSDataSourceCopyManifestWithViewNameSet(ds, SOSViewsGetV0BackupViewSet(), error);
3164             SOSManifestRef mdelete = NULL, madd = NULL;
3165             SOSManifestDiff(mnow, mbackup, &mdelete, &madd, error);
3166
3167             // Don't delete everything in datasource not in backup.
3168
3169             // Add items from the backup
3170             SOSManifestForEach(madd, ^void(CFDataRef e, bool *stop) {
3171                 CFDictionaryRef item = NULL;
3172                 CFStringRef sha1 = CFDataCopyHexString(e);
3173                 if (sha1) {
3174                     item = CFDictionaryGetValue(backup_in, sha1);
3175                     CFRelease(sha1);
3176                 }
3177                 if (item) {
3178                     CFErrorRef localError = NULL;
3179
3180                     if (!SOSObjectRestoreObject(ds, txn, bag_handle, item, &localError)) {
3181                         OSStatus status = SecErrorGetOSStatus(localError);
3182                         if (status == errSecDuplicateItem) {
3183                             // Log and ignore duplicate item errors during restore
3184                             secnotice("titc", "restore %@ not replacing existing item", item);
3185                         } else if (status == errSecDecode) {
3186                             // Log and ignore corrupted item errors during restore
3187                             secnotice("titc", "restore %@ skipping corrupted item %@", item, localError);
3188                         } else {
3189                             if (status == errSecInteractionNotAllowed)
3190                                 *stop = true;
3191                             // Propagate the first other error upwards (causing the restore to fail).
3192                             secerror("restore %@ failed %@", item, localError);
3193                             ok = false;
3194                             if (error && !*error) {
3195                                 *error = localError;
3196                                 localError = NULL;
3197                             }
3198                         }
3199                         CFReleaseSafe(localError);
3200                     }
3201                 }
3202             });
3203             ok &= SOSDataSourceRelease(ds, error);
3204             CFReleaseNull(mdelete);
3205             CFReleaseNull(madd);
3206             CFReleaseNull(mnow);
3207         });
3208         CFRelease(mbackup);
3209     }
3210
3211     ok &= ks_close_keybag(bag_handle, error);
3212
3213     return ok;
3214 }
3215
3216
3217 CF_RETURNS_RETAINED CFDictionaryRef
3218 _SecServerBackupSyncable(CFDictionaryRef backup, CFDataRef keybag, CFDataRef password, CFErrorRef *error) {
3219     require_action_quiet(isData(keybag), errOut, SecError(errSecParam, error, CFSTR("keybag %@ not a data"), keybag));
3220     require_action_quiet(!backup || isDictionary(backup), errOut, SecError(errSecParam, error, CFSTR("backup %@ not a dictionary"), backup));
3221     require_action_quiet(!password || isData(password), errOut, SecError(errSecParam, error, CFSTR("password %@ not a data"), password));
3222
3223     return _SecServerCopyTruthInTheCloud(keybag, password, backup, error);
3224
3225 errOut:
3226     return NULL;
3227 }
3228
3229 bool
3230 _SecServerRestoreSyncable(CFDictionaryRef backup, CFDataRef keybag, CFDataRef password, CFErrorRef *error) {
3231     bool ok;
3232     require_action_quiet(isData(keybag), errOut, ok = SecError(errSecParam, error, CFSTR("keybag %@ not a data"), keybag));
3233     require_action_quiet(isDictionary(backup), errOut, ok = SecError(errSecParam, error, CFSTR("backup %@ not a dictionary"), backup));
3234     if (password) {
3235
3236         require_action_quiet(isData(password), errOut, ok = SecError(errSecParam, error, CFSTR("password not a data")));
3237     }
3238
3239     ok = _SecServerRestoreTruthInTheCloud(keybag, password, backup, error);
3240
3241 errOut:
3242     return ok;
3243 }
3244
3245 #else /* SECUREOBJECTSYNC */
3246
3247 SOSDataSourceFactoryRef SecItemDataSourceFactoryGetDefault(void) {
3248     return NULL;
3249 }
3250
3251 #endif /* SECUREOBJECTSYNC */
3252
3253 bool _SecServerRollKeysGlue(bool force, CFErrorRef *error) {
3254     return _SecServerRollKeys(force, NULL, error);
3255 }
3256
3257
3258 bool _SecServerRollKeys(bool force, SecurityClient *client, CFErrorRef *error) {
3259 #if USE_KEYSTORE
3260     uint32_t keystore_generation_status = 0;
3261     if (aks_generation(KEYBAG_DEVICE, generation_noop, &keystore_generation_status))
3262         return false;
3263     uint32_t current_generation = keystore_generation_status & generation_current;
3264
3265     return kc_with_dbt(true, error, ^(SecDbConnectionRef dbt) {
3266         bool up_to_date = s3dl_dbt_keys_current(dbt, current_generation, NULL);
3267
3268         if (force && !up_to_date) {
3269             up_to_date = s3dl_dbt_update_keys(dbt, client, error);
3270             if (up_to_date) {
3271                 secerror("Completed roll keys.");
3272                 up_to_date = s3dl_dbt_keys_current(dbt, current_generation, NULL);
3273             }
3274             if (!up_to_date)
3275                 secerror("Failed to roll keys.");
3276         }
3277         return up_to_date;
3278     });
3279 #else
3280     return true;
3281 #endif
3282 }
3283
3284 static bool
3285 InitialSyncItems(CFMutableArrayRef items, bool limitToCurrent, CFStringRef agrp, CFStringRef svce, const SecDbClass *qclass, CFErrorRef *error)
3286 {
3287     bool result = false;
3288     Query *q = NULL;
3289
3290     q = query_create(qclass, NULL, NULL, NULL, error);
3291     require(q, fail);
3292
3293     q->q_return_type = kSecReturnDataMask | kSecReturnAttributesMask;
3294     q->q_limit = kSecMatchUnlimited;
3295     q->q_keybag = KEYBAG_DEVICE;
3296
3297     query_add_attribute(kSecAttrAccessGroup, agrp, q);
3298     query_add_attribute(kSecAttrSynchronizable, kCFBooleanTrue, q);
3299     query_add_attribute(kSecAttrTombstone, kCFBooleanFalse, q);
3300     if (svce)
3301         query_add_attribute(kSecAttrService, svce, q);
3302
3303     result = kc_with_dbt(false, error, ^(SecDbConnectionRef dbt) {
3304         return kc_transaction(dbt, error, ^{
3305             CFErrorRef error2 = NULL;
3306
3307             SecDbItemSelect(q, dbt, &error2, NULL, ^bool(const SecDbAttr *attr) {
3308                 return CFDictionaryGetValue(q->q_item, attr->name);
3309             }, NULL, NULL, ^(SecDbItemRef item, bool *stop) {
3310                 CFErrorRef error3 = NULL;
3311                 secinfo("InitialSyncItems", "Copy item");
3312
3313                 CFMutableDictionaryRef attrs = SecDbItemCopyPListWithMask(item, kSecDbSyncFlag, &error3);
3314                 if (attrs) {
3315                     int match = true;
3316                     CFStringRef itemvwht = CFDictionaryGetValue(attrs, kSecAttrSyncViewHint);
3317                     /*
3318                      * Saying its a SOS viewhint is really not the right answer post Triangle
3319                      */
3320                     if (isString(itemvwht) && !SOSViewInSOSSystem(itemvwht)) {
3321                         match = false;
3322                     }
3323                     /*
3324                      * Here we encode how PCS stores identities so that we only copy the
3325                      * current identites for performance reasons.
3326                      */
3327                     if (limitToCurrent) {
3328                         enum { PCS_CURRENT_IDENTITY_OFFSET              = 0x10000 };
3329                         int32_t s32;
3330
3331                         CFNumberRef type = CFDictionaryGetValue(attrs, kSecAttrType);
3332                         if (!isNumber(type)) {
3333                             // still allow this case since its not a service identity ??
3334                         } else if (!CFNumberGetValue(type, kCFNumberSInt32Type, &s32)) {
3335                             match = false;
3336                         } else if ((s32 & PCS_CURRENT_IDENTITY_OFFSET) == 0) {
3337                             match = false;
3338                         }
3339                     }
3340                     if (match) {
3341                         CFDictionaryAddValue(attrs, kSecClass, SecDbItemGetClass(item)->name);
3342                         CFArrayAppendValue(items, attrs);
3343                     }
3344
3345                     CFReleaseNull(attrs);
3346                 }
3347                 CFReleaseNull(error3);
3348             });
3349             CFReleaseNull(error2);
3350
3351             return (bool)true;
3352         });
3353     });
3354
3355 fail:
3356     if (q)
3357         query_destroy(q, NULL);
3358     return result;
3359 }
3360
3361 CFArrayRef
3362 _SecServerCopyInitialSyncCredentials(uint32_t flags, CFErrorRef *error)
3363 {
3364     CFMutableArrayRef items = CFArrayCreateMutableForCFTypes(NULL);
3365
3366     if (flags & SecServerInitialSyncCredentialFlagTLK) {
3367         require_action(InitialSyncItems(items, false, CFSTR("com.apple.security.ckks"), NULL, inet_class(), error), fail,
3368                        secerror("failed to collect CKKS-inet keys: %@", error ? *error : NULL));
3369     }
3370     if (flags & SecServerInitialSyncCredentialFlagPCS) {
3371         bool onlyCurrent = !(flags & SecServerInitialSyncCredentialFlagPCSNonCurrent);
3372
3373         require_action(InitialSyncItems(items, false, CFSTR("com.apple.ProtectedCloudStorage"), NULL, genp_class(), error), fail,
3374                        secerror("failed to collect PCS-genp keys: %@", error ? *error : NULL));
3375         require_action(InitialSyncItems(items, onlyCurrent, CFSTR("com.apple.ProtectedCloudStorage"), NULL, inet_class(), error), fail,
3376                        secerror("failed to collect PCS-inet keys: %@", error ? *error : NULL));
3377     }
3378     if (flags & SecServerInitialSyncCredentialFlagBluetoothMigration) {
3379         require_action(InitialSyncItems(items, false, CFSTR("com.apple.nanoregistry.migration"), NULL, genp_class(), error), fail,
3380                        secerror("failed to collect com.apple.nanoregistry.migration-genp item: %@", error ? *error : NULL));
3381         require_action(InitialSyncItems(items, false, CFSTR("com.apple.nanoregistry.migration2"), NULL, genp_class(), error), fail,
3382                        secerror("failed to collect com.apple.nanoregistry.migration2-genp item: %@", error ? *error : NULL));
3383         require_action(InitialSyncItems(items, false, CFSTR("com.apple.bluetooth"), CFSTR("BluetoothLESync"), genp_class(), error), fail,
3384                        secerror("failed to collect com.apple.bluetooth-genp item: %@", error ? *error : NULL));
3385
3386     }
3387
3388 fail:
3389     return items;
3390 }
3391
3392 bool
3393 _SecServerImportInitialSyncCredentials(CFArrayRef array, CFErrorRef *error)
3394 {
3395     return kc_with_dbt(true, error, ^bool(SecDbConnectionRef dbt) {
3396         return kc_transaction(dbt, error, ^bool(void){
3397             CFIndex n, count = CFArrayGetCount(array);
3398
3399             secinfo("ImportInitialSyncItems", "Importing %d items", (int)count);
3400
3401             for (n = 0; n < count; n++) {
3402                 CFErrorRef cferror = NULL;
3403
3404                 CFDictionaryRef item = CFArrayGetValueAtIndex(array, n);
3405                 if (!isDictionary(item))
3406                     continue;
3407
3408                 CFStringRef className = CFDictionaryGetValue(item, kSecClass);
3409                 if (className == NULL) {
3410                     secinfo("ImportInitialSyncItems", "Item w/o class");
3411                     continue;
3412                 }
3413
3414                 const SecDbClass *cls = kc_class_with_name(className);
3415                 if (cls == NULL) {
3416                     secinfo("ImportInitialSyncItems", "Item with unknown class: %@", className);
3417                     continue;
3418                 }
3419
3420                 SecDbItemRef dbi = SecDbItemCreateWithAttributes(NULL, cls, item, KEYBAG_DEVICE, &cferror);
3421                 if (dbi == NULL) {
3422                     secinfo("ImportInitialSyncItems", "Item creation failed with: %@", cferror);
3423                     CFReleaseNull(cferror);
3424                     continue;
3425                 }
3426
3427                 if (!SecDbItemSetSyncable(dbi, true, &cferror)) {
3428                     secinfo("ImportInitialSyncItems", "Failed to set sync=1: %@ for item %@", cferror, dbi);
3429                     CFReleaseNull(cferror);
3430                     CFReleaseNull(dbi);
3431                     continue;
3432                 }
3433
3434                 if (!SecDbItemInsert(dbi, dbt, false, &cferror)) {
3435                     secinfo("ImportInitialSyncItems", "Item store failed with: %@: %@", cferror, dbi);
3436                     CFReleaseNull(cferror);
3437                 }
3438                 CFReleaseNull(dbi);
3439             }
3440             return true;
3441         });
3442     });
3443 }
3444
3445
3446
3447 #if TARGET_OS_IOS
3448
3449 /*
3450  * Sync bubble migration code
3451  */
3452
3453 struct SyncBubbleRule {
3454     CFStringRef attribute;
3455     CFTypeRef value;
3456 };
3457
3458 static bool
3459 TransmogrifyItemsToSyncBubble(SecurityClient *client, uid_t uid,
3460                               bool onlyDelete,
3461                               bool copyToo,
3462                               const SecDbClass *qclass,
3463                               struct SyncBubbleRule *items, CFIndex nItems,
3464                               CFErrorRef *error)
3465 {
3466     CFMutableDictionaryRef updateAttributes = NULL;
3467     CFDataRef syncBubbleView = NULL;
3468     CFDataRef activeUserView = NULL;
3469     bool res = false;
3470     Query *q = NULL;
3471     CFIndex n;
3472
3473     syncBubbleView = SecMUSRCreateSyncBubbleUserUUID(uid);
3474     require(syncBubbleView, fail);
3475
3476     activeUserView = SecMUSRCreateActiveUserUUID(uid);
3477     require(activeUserView, fail);
3478
3479
3480     if ((onlyDelete && !copyToo) || !onlyDelete) {
3481
3482         /*
3483          * Clean out items first
3484          */
3485
3486         secnotice("syncbubble", "cleaning out old items");
3487
3488         q = query_create(qclass, NULL, NULL, client, error);
3489         require(q, fail);
3490
3491         q->q_limit = kSecMatchUnlimited;
3492         q->q_keybag = device_keybag_handle;
3493
3494         for (n = 0; n < nItems; n++) {
3495             query_add_attribute(items[n].attribute, items[n].value, q);
3496         }
3497         q->q_musrView = CFRetain(syncBubbleView);
3498         require(q->q_musrView, fail);
3499
3500         kc_with_dbt(false, error, ^(SecDbConnectionRef dbt) {
3501             return kc_transaction(dbt, error, ^{
3502                 return s3dl_query_delete(dbt, q, NULL, error);
3503             });
3504         });
3505
3506         query_destroy(q, NULL);
3507         q = NULL;
3508     }
3509
3510
3511     if (onlyDelete || !copyToo) {
3512         secnotice("syncbubble", "skip migration of items");
3513     } else {
3514         /*
3515          * Copy over items from EMCS to sync bubble
3516          */
3517
3518         secnotice("syncbubble", "migrating sync bubble items");
3519
3520         q = query_create(qclass, NULL, NULL, client, error);
3521         require(q, fail);
3522
3523         q->q_return_type = kSecReturnDataMask | kSecReturnAttributesMask;
3524         q->q_limit = kSecMatchUnlimited;
3525         q->q_keybag = device_keybag_handle; /* XXX change to session key bag when it exists */
3526
3527         for (n = 0; n < nItems; n++) {
3528             query_add_or_attribute(items[n].attribute, items[n].value, q);
3529         }
3530         query_add_or_attribute(CFSTR("musr"), activeUserView, q);
3531         q->q_musrView = CFRetain(activeUserView);
3532
3533         updateAttributes = CFDictionaryCreateMutableForCFTypes(NULL);
3534         require(updateAttributes, fail);
3535
3536         CFDictionarySetValue(updateAttributes, CFSTR("musr"), syncBubbleView); /* XXX should use kSecAttrMultiUser */
3537
3538
3539         kc_with_dbt(true, error, ^(SecDbConnectionRef dbt) {
3540             return kc_transaction(dbt, error, ^{
3541                 CFErrorRef error2 = NULL;
3542
3543                 SecDbItemSelect(q, dbt, &error2, NULL, ^bool(const SecDbAttr *attr) {
3544                     return CFDictionaryGetValue(q->q_item, attr->name);
3545                 }, NULL, NULL, ^(SecDbItemRef item, bool *stop) {
3546                     CFErrorRef error3 = NULL;
3547                     secinfo("syncbubble", "migrating item");
3548
3549                     SecDbItemRef new_item = SecDbItemCopyWithUpdates(item, updateAttributes, NULL);
3550                     if (new_item == NULL)
3551                         return;
3552
3553                     SecDbItemClearRowId(new_item, NULL);
3554
3555                     if (!SecDbItemSetKeybag(new_item, device_keybag_handle, NULL)) {
3556                         CFRelease(new_item);
3557                         return;
3558                     }
3559
3560                     if (!SecDbItemInsert(new_item, dbt, false, &error3)) {
3561                         secnotice("syncbubble", "migration failed with %@ for item %@", error3, new_item);
3562                     }
3563                     CFRelease(new_item);
3564                     CFReleaseNull(error3);
3565                 });
3566                 CFReleaseNull(error2);
3567
3568                 return (bool)true;
3569             });
3570         });
3571     }
3572     res = true;
3573
3574 fail:
3575     CFReleaseNull(syncBubbleView);
3576     CFReleaseNull(activeUserView);
3577     CFReleaseNull(updateAttributes);
3578     if (q)
3579         query_destroy(q, NULL);
3580
3581     return res;
3582 }
3583
3584 static struct SyncBubbleRule PCSItems[] = {
3585     {
3586         .attribute = CFSTR("agrp"),
3587         .value = CFSTR("com.apple.ProtectedCloudStorage"),
3588     }
3589 };
3590 static struct SyncBubbleRule NSURLSesssiond[] = {
3591     {
3592         .attribute = CFSTR("agrp"),
3593         .value = CFSTR("com.apple.nsurlsessiond"),
3594     }
3595 };
3596 static struct SyncBubbleRule AccountsdItems[] = {
3597     {
3598         .attribute = CFSTR("svce"),
3599         .value = CFSTR("com.apple.account.AppleAccount.token"),
3600     },
3601     {
3602         .attribute = CFSTR("svce"),
3603         .value = CFSTR("com.apple.account.AppleAccount.password"),
3604     },
3605     {
3606         .attribute = CFSTR("svce"),
3607         .value = CFSTR("com.apple.account.AppleAccount.rpassword"),
3608     },
3609     {
3610         .attribute = CFSTR("svce"),
3611         .value = CFSTR("com.apple.account.idms.token"),
3612     },
3613     {
3614         .attribute = CFSTR("svce"),
3615         .value = CFSTR("com.apple.account.idms.continuation-key"),
3616     },
3617     {
3618         .attribute = CFSTR("svce"),
3619         .value = CFSTR("com.apple.account.CloudKit.token"),
3620     },
3621 };
3622
3623 static struct SyncBubbleRule MobileMailItems[] = {
3624     {
3625         .attribute = CFSTR("svce"),
3626         .value = CFSTR("com.apple.account.IMAP.password"),
3627     },
3628     {
3629         .attribute = CFSTR("svce"),
3630         .value = CFSTR("com.apple.account.SMTP.password"),
3631     },
3632     {
3633         .attribute = CFSTR("svce"),
3634         .value = CFSTR("com.apple.account.Exchange.password"),
3635     },
3636     {
3637         .attribute = CFSTR("svce"),
3638         .value = CFSTR("com.apple.account.Hotmail.password"),
3639     },
3640     {
3641         .attribute = CFSTR("svce"),
3642         .value = CFSTR("com.apple.account.Google.password"),
3643     },
3644     {
3645         .attribute = CFSTR("svce"),
3646         .value = CFSTR("com.apple.account.Google.oauth-token"),
3647     },
3648     {
3649         .attribute = CFSTR("svce"),
3650         .value = CFSTR("com.apple.account.Google.oath-refresh-token"),
3651     },
3652     {
3653         .attribute = CFSTR("svce"),
3654         .value = CFSTR("com.apple.account.Yahoo.password"),
3655     },
3656     {
3657         .attribute = CFSTR("svce"),
3658         .value = CFSTR("com.apple.account.Yahoo.oauth-token"),
3659     },
3660     {
3661         .attribute = CFSTR("svce"),
3662         .value = CFSTR("com.apple.account.Yahoo.oauth-token-nosync"),
3663     },
3664     {
3665         .attribute = CFSTR("svce"),
3666         .value = CFSTR("com.apple.account.Yahoo.oath-refresh-token"),
3667     },
3668     {
3669         .attribute = CFSTR("svce"),
3670         .value = CFSTR("com.apple.account.IMAPNotes.password"),
3671     },
3672     {
3673         .attribute = CFSTR("svce"),
3674         .value = CFSTR("com.apple.account.IMAPMail.password"),
3675     },
3676     {
3677         .attribute = CFSTR("svce"),
3678         .value = CFSTR("com.apple.account.126.password"),
3679     },
3680     {
3681         .attribute = CFSTR("svce"),
3682         .value = CFSTR("com.apple.account.163.password"),
3683     },
3684     {
3685         .attribute = CFSTR("svce"),
3686         .value = CFSTR("com.apple.account.aol.password"),
3687     },
3688 };
3689
3690 static bool
3691 ArrayContains(CFArrayRef array, CFStringRef service)
3692 {
3693     return CFArrayContainsValue(array, CFRangeMake(0, CFArrayGetCount(array)), service);
3694 }
3695
3696 bool
3697 _SecServerTransmogrifyToSyncBubble(CFArrayRef services, uid_t uid, SecurityClient *client, CFErrorRef *error)
3698 {
3699     bool copyCloudAuthToken = false;
3700     bool copyMobileMail = false;
3701     bool res = true;
3702     bool copyPCS = false;
3703     bool onlyDelete = false;
3704     bool copyNSURLSesssion = false;
3705
3706     if (!client->inMultiUser)
3707         return false;
3708
3709     secnotice("syncbubble", "migration for uid %d uid for services %@", (int)uid, services);
3710
3711 #if TARGET_OS_SIMULATOR
3712     // no delete in sim
3713 #elif TARGET_OS_IOS
3714     if (uid != (uid_t)client->activeUser)
3715         onlyDelete = true;
3716 #else
3717 #error "no sync bubble on other platforms"
3718 #endif
3719
3720     /*
3721      * First select that services to copy/delete
3722      */
3723
3724     if (ArrayContains(services, CFSTR("com.apple.bird.usermanager.sync"))
3725         || ArrayContains(services, CFSTR("com.apple.cloudphotod.sync"))
3726         || ArrayContains(services, CFSTR("com.apple.cloudphotod.syncstakeholder"))
3727         || ArrayContains(services, CFSTR("com.apple.cloudd.usermanager.sync")))
3728     {
3729         copyCloudAuthToken = true;
3730         copyPCS = true;
3731     }
3732
3733     if (ArrayContains(services, CFSTR("com.apple.nsurlsessiond.usermanager.sync")))
3734     {
3735         copyCloudAuthToken = true;
3736         copyNSURLSesssion = true;
3737     }
3738
3739     if (ArrayContains(services, CFSTR("com.apple.syncdefaultsd.usermanager.sync"))) {
3740         copyCloudAuthToken = true;
3741     }
3742     if (ArrayContains(services, CFSTR("com.apple.mailq.sync")) || ArrayContains(services, CFSTR("com.apple.mailq.sync.xpc"))) {
3743         copyCloudAuthToken = true;
3744         copyMobileMail = true;
3745         copyPCS = true;
3746     }
3747
3748     /*
3749      * The actually copy/delete the items selected
3750      */
3751
3752     res = TransmogrifyItemsToSyncBubble(client, uid, onlyDelete, copyPCS, inet_class(), PCSItems, sizeof(PCSItems)/sizeof(PCSItems[0]), error);
3753     require(res, fail);
3754     res = TransmogrifyItemsToSyncBubble(client, uid, onlyDelete, copyPCS, genp_class(), PCSItems, sizeof(PCSItems)/sizeof(PCSItems[0]), error);
3755     require(res, fail);
3756
3757     /* mail */
3758     res = TransmogrifyItemsToSyncBubble(client, uid, onlyDelete, copyMobileMail, genp_class(), MobileMailItems, sizeof(MobileMailItems)/sizeof(MobileMailItems[0]), error);
3759     require(res, fail);
3760
3761     /* accountsd */
3762     res = TransmogrifyItemsToSyncBubble(client, uid, onlyDelete, copyCloudAuthToken, genp_class(), AccountsdItems, sizeof(AccountsdItems)/sizeof(AccountsdItems[0]), error);
3763     require(res, fail);
3764
3765     /* nsurlsessiond */
3766     res = TransmogrifyItemsToSyncBubble(client, uid, onlyDelete, copyNSURLSesssion, inet_class(), NSURLSesssiond, sizeof(NSURLSesssiond)/sizeof(NSURLSesssiond[0]), error);
3767     require(res, fail);
3768
3769 fail:
3770     return res;
3771 }
3772
3773 /*
3774  * Migrate from user keychain to system keychain when switching to edu mode
3775  */
3776
3777 bool
3778 _SecServerTransmogrifyToSystemKeychain(SecurityClient *client, CFErrorRef *error)
3779 {
3780     __block bool ok = true;
3781
3782     /*
3783      * we are not in multi user yet, about to switch, otherwise we would
3784      * check that for client->inMultiuser here
3785      */
3786
3787     kc_with_dbt(true, error, ^(SecDbConnectionRef dbt) {
3788         return kc_transaction(dbt, error, ^{
3789             CFDataRef systemUUID = SecMUSRGetSystemKeychainUUID();
3790
3791             const SecDbSchema *newSchema = current_schema();
3792             SecDbClass const *const *kcClass;
3793
3794             for (kcClass = newSchema->classes; *kcClass != NULL; kcClass++) {
3795                 CFErrorRef localError = NULL;
3796                 Query *q = NULL;
3797
3798                 if (!((*kcClass)->itemclass)) {
3799                     continue;
3800                 }
3801
3802                 q = query_create(*kcClass, SecMUSRGetSingleUserKeychainUUID(), NULL, client, error);
3803                 if (q == NULL)
3804                     continue;
3805
3806                 ok &= SecDbItemSelect(q, dbt, error, ^bool(const SecDbAttr *attr) {
3807                     return (attr->flags & kSecDbInFlag) != 0;
3808                 }, ^bool(const SecDbAttr *attr) {
3809                     // No filtering please.
3810                     return false;
3811                 }, ^bool(CFMutableStringRef sql, bool *needWhere) {
3812                     SecDbAppendWhereOrAnd(sql, needWhere);
3813                     CFStringAppendFormat(sql, NULL, CFSTR("musr = ?"));
3814                     return true;
3815                 }, ^bool(sqlite3_stmt *stmt, int col) {
3816                     return SecDbBindObject(stmt, col++, SecMUSRGetSingleUserKeychainUUID(), error);
3817                 }, ^(SecDbItemRef item, bool *stop) {
3818                     CFErrorRef itemError = NULL;
3819
3820                     if (!SecDbItemSetValueWithName(item, kSecAttrMultiUser, systemUUID, &itemError)) {
3821                         secerror("item: %@ update musr to system failed: %@", item, itemError);
3822                         ok = false;
3823                         goto out;
3824                     }
3825
3826                     if (!SecDbItemDoUpdate(item, item, dbt, &itemError, ^bool (const SecDbAttr *attr) {
3827                         return attr->kind == kSecDbRowIdAttr;
3828                     })) {
3829                         secerror("item: %@ insert during UPDATE: %@", item, itemError);
3830                         ok = false;
3831                         goto out;
3832                     }
3833
3834                 out:
3835                     SecErrorPropagate(itemError, error);
3836                 });
3837
3838                 if (q)
3839                     query_destroy(q, &localError);
3840
3841             }
3842             return (bool)true;
3843         });
3844     });
3845
3846     return ok;
3847 }
3848
3849 /*
3850  * Delete account from local usage
3851  */
3852
3853 bool
3854 _SecServerDeleteMUSERViews(SecurityClient *client, uid_t uid, CFErrorRef *error)
3855 {
3856     return kc_with_dbt(true, error, ^(SecDbConnectionRef dbt) {
3857         CFDataRef musrView = NULL, syncBubbleView = NULL;
3858         bool ok = false;
3859
3860         syncBubbleView = SecMUSRCreateSyncBubbleUserUUID(uid);
3861         require(syncBubbleView, fail);
3862
3863         musrView = SecMUSRCreateActiveUserUUID(uid);
3864         require(musrView, fail);
3865
3866         require(ok = SecServerDeleteAllForUser(dbt, syncBubbleView, false, error), fail);
3867         require(ok = SecServerDeleteAllForUser(dbt, musrView, false, error), fail);
3868
3869     fail:
3870         CFReleaseNull(syncBubbleView);
3871         CFReleaseNull(musrView);
3872         return ok;
3873     });
3874 }
3875
3876
3877 #endif /* TARGET_OS_IOS */
3878
3879 CFArrayRef _SecItemCopyParentCertificates(CFDataRef normalizedIssuer, CFArrayRef accessGroups, CFErrorRef *error) {
3880     const void *keys[] = {
3881         kSecClass,
3882         kSecReturnData,
3883         kSecMatchLimit,
3884         kSecAttrSubject
3885     },
3886     *values[] = {
3887         kSecClassCertificate,
3888         kCFBooleanTrue,
3889         kSecMatchLimitAll,
3890         normalizedIssuer
3891     };
3892     CFDictionaryRef query = CFDictionaryCreate(NULL, keys, values, 4,
3893                                                NULL, NULL);
3894     CFTypeRef results = NULL;
3895     SecurityClient client = {
3896         .task = NULL,
3897         .accessGroups = accessGroups,
3898         .allowSystemKeychain = true,
3899         .allowSyncBubbleKeychain = false,
3900         .isNetworkExtension = false,
3901     };
3902
3903     (void)_SecItemCopyMatching(query, &client, &results, error);
3904     CFRelease(query);
3905     return results;
3906 }
3907
3908 bool _SecItemCertificateExists(CFDataRef normalizedIssuer, CFDataRef serialNumber, CFArrayRef accessGroups, CFErrorRef *error) {
3909     const void *keys[] = {
3910         kSecClass,
3911         kSecMatchLimit,
3912         kSecAttrIssuer,
3913         kSecAttrSerialNumber
3914     },
3915     *values[] = {
3916         kSecClassCertificate,
3917         kSecMatchLimitOne,
3918         normalizedIssuer,
3919         serialNumber
3920     };
3921     SecurityClient client = {
3922         .task = NULL,
3923         .accessGroups = accessGroups,
3924         .allowSystemKeychain = true,
3925         .allowSyncBubbleKeychain = false,
3926         .isNetworkExtension = false,
3927     };
3928     CFDictionaryRef query = CFDictionaryCreate(NULL, keys, values, 4, NULL, NULL);
3929     CFTypeRef results = NULL;
3930     bool ok = _SecItemCopyMatching(query, &client, &results, error);
3931     CFReleaseSafe(query);
3932     CFReleaseSafe(results);
3933     if (!ok) {
3934         return false;
3935     }
3936     return true;
3937 }