OSX/sec/securityd/SecItemDb.c

   1 /*
   2  * Copyright (c) 2006-2014 Apple Inc. All Rights Reserved.
   3  *
   4  * @APPLE_LICENSE_HEADER_START@
   5  *
   6  * This file contains Original Code and/or Modifications of Original Code
   7  * as defined in and that are subject to the Apple Public Source License
   8  * Version 2.0 (the 'License'). You may not use this file except in
   9  * compliance with the License. Please obtain a copy of the License at
  10  * http://www.opensource.apple.com/apsl/ and read it before using this
  11  * file.
  12  *
  13  * The Original Code and all software distributed under the License are
  14  * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  15  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  16  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
  17  * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
  18  * Please see the License for the specific language governing rights and
  19  * limitations under the License.
  20  *
  21  * @APPLE_LICENSE_HEADER_END@
  22  */
  23
  24 /*
  25  *  SecItemDb.c - CoreFoundation-based constants and functions for
  26     access to Security items (certificates, keys, identities, and
  27     passwords.)
  28  */
  29
  30 #include <securityd/SecItemDb.h>
  31 #include <utilities/SecAKSWrappers.h>
  32
  33 #include <securityd/SecDbKeychainItem.h>
  34 #include <securityd/SecItemSchema.h>
  35 #include <securityd/SecItemServer.h>
  36 #include <Security/SecAccessControlPriv.h>
  37 #include <Security/SecBasePriv.h>
  38 #include <Security/SecItem.h>
  39 #include <Security/SecSignpost.h>
  40 #include <Security/SecItemPriv.h>
  41 #include <Security/SecItemInternal.h>
  42 #include <securityd/SOSCloudCircleServer.h>
  43 #include <utilities/array_size.h>
  44 #include <utilities/SecIOFormat.h>
  45 #include <utilities/SecCFCCWrappers.h>
  46 #include <SecAccessControlPriv.h>
  47 #include <uuid/uuid.h>
  48 #include "sec_action.h"
  49
  50 #include "keychain/ckks/CKKS.h"
  51
  52 #define kSecBackupKeybagUUIDKey CFSTR("keybag-uuid")
  53
  54 const SecDbAttr *SecDbAttrWithKey(const SecDbClass *c,
  55                                   CFTypeRef key,
  56                                   CFErrorRef *error) {
  57     /* Special case: identites can have all attributes of either cert
  58      or keys. */
  59     if (c == identity_class()) {
  60         const SecDbAttr *desc;
  61         if (!(desc = SecDbAttrWithKey(cert_class(), key, 0)))
  62             desc = SecDbAttrWithKey(keys_class(), key, error);
  63         return desc;
  64     }
  65
  66     if (isString(key)) {
  67         SecDbForEachAttr(c, a) {
  68             if (CFEqual(a->name, key))
  69                 return a;
  70         }
  71         if (CFEqual(kSecAttrNoLegacy, key)) {
  72             return NULL; /* results in no ops for this attribute */
  73         }
  74     }
  75
  76     SecError(errSecNoSuchAttr, error, CFSTR("attribute %@ not found in class %@"), key, c->name);
  77
  78     return NULL;
  79 }
  80
  81 bool kc_transaction(SecDbConnectionRef dbt, CFErrorRef *error, bool(^perform)(void)) {
  82     return kc_transaction_type(dbt, kSecDbExclusiveTransactionType, error, perform);
  83 }
  84
  85 bool kc_transaction_type(SecDbConnectionRef dbt, SecDbTransactionType type, CFErrorRef *error, bool(^perform)(void)) {
  86     __block bool ok = true;
  87     return ok && SecDbTransaction(dbt, type, error, ^(bool *commit) {
  88         ok = *commit = perform();
  89     });
  90 }
  91
  92 static CFStringRef SecDbGetKindSQL(SecDbAttrKind kind) {
  93     switch (kind) {
  94         case kSecDbBlobAttr:
  95         case kSecDbDataAttr:
  96         case kSecDbUUIDAttr:
  97         case kSecDbSHA1Attr:
  98         case kSecDbPrimaryKeyAttr:
  99         case kSecDbEncryptedDataAttr:
 100             return CFSTR("BLOB");
 101         case kSecDbAccessAttr:
 102         case kSecDbStringAttr:
 103             return CFSTR("TEXT");
 104         case kSecDbNumberAttr:
 105         case kSecDbSyncAttr:
 106         case kSecDbTombAttr:
 107             return CFSTR("INTEGER");
 108         case kSecDbDateAttr:
 109         case kSecDbCreationDateAttr:
 110         case kSecDbModificationDateAttr:
 111             return CFSTR("REAL");
 112         case kSecDbRowIdAttr:
 113             return CFSTR("INTEGER PRIMARY KEY AUTOINCREMENT");
 114         case kSecDbAccessControlAttr:
 115         case kSecDbUTombAttr:
 116             /* This attribute does not exist in the DB. */
 117             return NULL;
 118     }
 119 }
 120
 121 static void SecDbAppendUnique(CFMutableStringRef sql, CFStringRef value, bool *haveUnique) {
 122     assert(haveUnique);
 123     if (!*haveUnique)
 124         CFStringAppend(sql, CFSTR("UNIQUE("));
 125
 126     SecDbAppendElement(sql, value, haveUnique);
 127 }
 128
 129 static void SecDbAppendCreateTableWithClass(CFMutableStringRef sql, const SecDbClass *c) {
 130     CFStringAppendFormat(sql, 0, CFSTR("CREATE TABLE %@("), c->name);
 131     SecDbForEachAttrWithMask(c,desc,kSecDbInFlag) {
 132         CFStringAppendFormat(sql, 0, CFSTR("%@ %@"), desc->name, SecDbGetKindSQL(desc->kind));
 133         if (desc->flags & kSecDbNotNullFlag)
 134             CFStringAppend(sql, CFSTR(" NOT NULL"));
 135         if (desc->flags & kSecDbDefault0Flag)
 136             CFStringAppend(sql, CFSTR(" DEFAULT 0"));
 137         if (desc->flags & kSecDbDefaultEmptyFlag)
 138             CFStringAppend(sql, CFSTR(" DEFAULT ''"));
 139         CFStringAppend(sql, CFSTR(","));
 140     }
 141
 142     bool haveUnique = false;
 143     SecDbForEachAttrWithMask(c,desc,kSecDbPrimaryKeyFlag | kSecDbInFlag) {
 144         SecDbAppendUnique(sql, desc->name, &haveUnique);
 145     }
 146     if (haveUnique)
 147         CFStringAppend(sql, CFSTR(")"));
 148
 149     CFStringAppend(sql, CFSTR(");"));
 150
 151     // Create indices
 152     SecDbForEachAttrWithMask(c,desc, kSecDbIndexFlag | kSecDbInFlag) {
 153         if (desc->kind == kSecDbSyncAttr) {
 154             CFStringAppendFormat(sql, 0, CFSTR("CREATE INDEX %@%@0 ON %@(%@) WHERE %@=0;"), c->name, desc->name, c->name, desc->name, desc->name);
 155         } else {
 156             CFStringAppendFormat(sql, 0, CFSTR("CREATE INDEX %@%@ ON %@(%@);"), c->name, desc->name, c->name, desc->name);
 157         }
 158     }
 159 }
 160
 161 static void SecDbAppendDropTableWithClass(CFMutableStringRef sql, const SecDbClass *c) {
 162     CFStringAppendFormat(sql, 0, CFSTR("DROP TABLE %@;"), c->name);
 163 }
 164
 165 static CFDataRef SecPersistentRefCreateWithItem(SecDbItemRef item, CFErrorRef *error) {
 166     sqlite3_int64 row_id = SecDbItemGetRowId(item, error);
 167     if (row_id)
 168         return _SecItemCreatePersistentRef(SecDbItemGetClass(item)->name, row_id, item->attributes);
 169     return NULL;
 170 }
 171
 172 bool SecItemDbCreateSchema(SecDbConnectionRef dbt, const SecDbSchema *schema, CFArrayRef classIndexesForNewTables, bool includeVersion, CFErrorRef *error)
 173 {
 174     __block bool ok = true;
 175     CFMutableStringRef sql = CFStringCreateMutable(kCFAllocatorDefault, 0);
 176
 177     if (classIndexesForNewTables) {
 178         CFArrayForEach(classIndexesForNewTables, ^(const void* index) {
 179             const SecDbClass* class = schema->classes[(int)index];
 180             SecDbAppendCreateTableWithClass(sql, class);
 181         });
 182     }
 183     else {
 184         for (const SecDbClass * const *pclass = schema->classes; *pclass; ++pclass) {
 185             SecDbAppendCreateTableWithClass(sql, *pclass);
 186         }
 187     }
 188
 189     if (includeVersion) {
 190         CFStringAppendFormat(sql, NULL, CFSTR("INSERT INTO tversion(version,minor) VALUES(%d, %d);"),
 191                              schema->majorVersion, schema->minorVersion);
 192     }
 193     CFStringPerformWithCString(sql, ^(const char *sql_string) {
 194         ok = SecDbErrorWithDb(sqlite3_exec(SecDbHandle(dbt), sql_string, NULL, NULL, NULL),
 195                               SecDbHandle(dbt), error, CFSTR("sqlite3_exec: %s"), sql_string);
 196     });
 197     CFReleaseSafe(sql);
 198     return ok;
 199 }
 200
 201 bool SecItemDbDeleteSchema(SecDbConnectionRef dbt, const SecDbSchema *schema, CFErrorRef *error)
 202 {
 203     __block bool ok = true;
 204     CFMutableStringRef sql = CFStringCreateMutable(kCFAllocatorDefault, 0);
 205     for (const SecDbClass * const *pclass = schema->classes; *pclass; ++pclass) {
 206         SecDbAppendDropTableWithClass(sql, *pclass);
 207     }
 208     CFStringPerformWithCString(sql, ^(const char *sql_string) {
 209         ok = SecDbErrorWithDb(sqlite3_exec(SecDbHandle(dbt), sql_string, NULL, NULL, NULL),
 210                               SecDbHandle(dbt), error, CFSTR("sqlite3_exec: %s"), sql_string);
 211     });
 212     CFReleaseSafe(sql);
 213     return ok;
 214 }
 215
 216 CFTypeRef SecDbItemCopyResult(SecDbItemRef item, ReturnTypeMask return_type, CFErrorRef *error) {
 217     CFTypeRef a_result;
 218
 219         if (return_type == 0) {
 220                 /* Caller isn't interested in any results at all. */
 221                 a_result = kCFNull;
 222         } else if (return_type == kSecReturnDataMask) {
 223         a_result = SecDbItemGetCachedValueWithName(item, kSecValueData);
 224         if (a_result) {
 225             CFRetainSafe(a_result);
 226         } else {
 227             a_result = CFDataCreate(kCFAllocatorDefault, NULL, 0);
 228         }
 229         } else if (return_type == kSecReturnPersistentRefMask) {
 230                 a_result = SecPersistentRefCreateWithItem(item, error);
 231         } else {
 232         CFMutableDictionaryRef dict = CFDictionaryCreateMutableForCFTypes(CFGetAllocator(item));
 233                 /* We need to return more than one value. */
 234         if (return_type & kSecReturnRefMask) {
 235             CFDictionarySetValue(dict, kSecClass, SecDbItemGetClass(item)->name);
 236         }
 237         CFOptionFlags mask = (((return_type & kSecReturnDataMask || return_type & kSecReturnRefMask) ? kSecDbReturnDataFlag : 0) |
 238                               ((return_type & kSecReturnAttributesMask || return_type & kSecReturnRefMask) ? kSecDbReturnAttrFlag : 0));
 239         SecDbForEachAttr(SecDbItemGetClass(item), desc) {
 240             if ((desc->flags & mask) != 0) {
 241                 CFTypeRef value = SecDbItemGetValue(item, desc, error);
 242                 if (value && !CFEqual(kCFNull, value)) {
 243                     CFDictionarySetValue(dict, desc->name, value);
 244                 } else if (value == NULL) {
 245                     CFReleaseNull(dict);
 246                     break;
 247                 }
 248             }
 249         }
 250         CFDictionaryRemoveValue(dict, kSecAttrUUID);
 251
 252                 if (return_type & kSecReturnPersistentRefMask) {
 253             CFDataRef pref = SecPersistentRefCreateWithItem(item, error);
 254                         CFDictionarySetValue(dict, kSecValuePersistentRef, pref);
 255             CFReleaseSafe(pref);
 256                 }
 257
 258                 a_result = dict;
 259         }
 260
 261         return a_result;
 262 }
 263
 264 /* AUDIT[securityd](done):
 265  attributes (ok) is a caller provided dictionary, only its cf type has
 266  been checked.
 267  */
 268 bool
 269 s3dl_query_add(SecDbConnectionRef dbt, Query *q, CFTypeRef *result, CFErrorRef *error)
 270 {
 271     if (query_match_count(q) != 0)
 272         return errSecItemMatchUnsupported;
 273
 274     /* Add requires a class to be specified unless we are adding a ref. */
 275     if (q->q_use_item_list)
 276         return errSecUseItemListUnsupported;
 277
 278     /* Actual work here. */
 279     SecDbItemRef item = SecDbItemCreateWithAttributes(kCFAllocatorDefault, q->q_class, q->q_item, KEYBAG_DEVICE, error);
 280     if (!item)
 281         return false;
 282     if (SecDbItemIsTombstone(item))
 283         SecDbItemSetValue(item, &v7utomb, q->q_use_tomb ? q->q_use_tomb : kCFBooleanTrue, NULL);
 284
 285     bool ok = true;
 286     if (q->q_data)
 287         ok = SecDbItemSetValueWithName(item, CFSTR("v_Data"), q->q_data, error);
 288     if (q->q_row_id)
 289         ok = SecDbItemSetRowId(item, q->q_row_id, error);
 290     if (q->q_musrView)
 291         ok = SecDbItemSetValueWithName(item, CFSTR("musr"), q->q_musrView, error);
 292     SecDbItemSetCredHandle(item, q->q_use_cred_handle);
 293
 294 #if OCTAGON
 295     if(SecCKKSIsEnabled() && !SecCKKSTestDisableAutomaticUUID()) {
 296         s3dl_item_make_new_uuid(item, q->q_uuid_from_primary_key, error);
 297
 298         if(q->q_add_sync_callback) {
 299             CFTypeRef uuid = SecDbItemGetValue(item, &v10itemuuid, error);
 300             if(uuid) {
 301                 CKKSRegisterSyncStatusCallback(uuid, q->q_add_sync_callback);
 302             } else {
 303                 secerror("Couldn't fetch UUID from item; can't call callback");
 304             }
 305         }
 306     }
 307 #endif
 308
 309     if (ok)
 310         ok = SecDbItemInsert(item, dbt, error);
 311
 312     if (ok) {
 313         if (result && q->q_return_type) {
 314             *result = SecDbItemCopyResult(item, q->q_return_type, error);
 315         }
 316     }
 317     if (!ok && error && *error) {
 318         if (CFEqual(CFErrorGetDomain(*error), kSecDbErrorDomain) && CFErrorGetCode(*error) == SQLITE_CONSTRAINT) {
 319             CFReleaseNull(*error);
 320             SecError(errSecDuplicateItem, error, CFSTR("duplicate item %@"), item);
 321         } else if (CFEqual(CFErrorGetDomain(*error), kSecErrorDomain) && CFErrorGetCode(*error) == errSecDecode) { //handle situation when item have pdmn=akpu but passcode is not set
 322             CFTypeRef value = SecDbItemGetValue(item, SecDbClassAttrWithKind(item->class, kSecDbAccessAttr, error), error);
 323             if (value && CFEqual(value, kSecAttrAccessibleWhenPasscodeSetThisDeviceOnly)) {
 324                 CFReleaseNull(*error);
 325                 SecError(errSecAuthFailed, error, CFSTR("authentication failed"));
 326             }
 327         }
 328     }
 329
 330     if (ok) {
 331         q->q_changed = true;
 332         if (SecDbItemIsSyncable(item))
 333             q->q_sync_changed = true;
 334     }
 335
 336     secdebug("dbitem", "inserting item %@%s%@", item, ok ? "" : "failed: ", ok || error == NULL ? (CFErrorRef)CFSTR("") : *error);
 337
 338     CFRelease(item);
 339
 340         return ok;
 341 }
 342
 343 bool s3dl_item_make_new_uuid(SecDbItemRef item, bool uuid_from_primary_key, CFErrorRef* error) {
 344     if(!item) {
 345         return false;
 346     }
 347
 348     // Set the item UUID.
 349     CFUUIDRef uuid = NULL;
 350     // Were we asked to make the UUID static?
 351     if (uuid_from_primary_key) {
 352         // This UUID isn't used in any security mechanism, so we can
 353         // just use the first bits of the SHA256 hash.
 354         CFDataRef pkhash = SecDbKeychainItemCopySHA256PrimaryKey(item, error);
 355         if(CFDataGetLength(pkhash) >= 16) {
 356             UInt8 uuidBytes[16];
 357             CFRange range = CFRangeMake(0, 16);
 358             CFDataGetBytes(pkhash, range, uuidBytes);
 359
 360             uuid = CFUUIDCreateWithBytes(NULL,
 361                                          uuidBytes[ 0],
 362                                          uuidBytes[ 1],
 363                                          uuidBytes[ 2],
 364                                          uuidBytes[ 3],
 365                                          uuidBytes[ 4],
 366                                          uuidBytes[ 5],
 367                                          uuidBytes[ 6],
 368                                          uuidBytes[ 7],
 369                                          uuidBytes[ 8],
 370                                          uuidBytes[ 9],
 371                                          uuidBytes[10],
 372                                          uuidBytes[11],
 373                                          uuidBytes[12],
 374                                          uuidBytes[13],
 375                                          uuidBytes[14],
 376                                          uuidBytes[15]);
 377         }
 378         CFReleaseNull(pkhash);
 379     }
 380     if(uuid == NULL) {
 381         uuid = CFUUIDCreate(NULL);
 382     }
 383     SecDbItemSetValueWithName(item, kSecAttrUUID, uuid, error);
 384     CFReleaseNull(uuid);
 385     return true;
 386 }
 387
 388 typedef void (*s3dl_handle_row)(sqlite3_stmt *stmt, void *context);
 389
 390 static CFDataRef
 391 s3dl_copy_data_from_col(sqlite3_stmt *stmt, int col, CFErrorRef *error) {
 392     return CFDataCreateWithBytesNoCopy(0, sqlite3_column_blob(stmt, col),
 393                                        sqlite3_column_bytes(stmt, col),
 394                                        kCFAllocatorNull);
 395 }
 396
 397 static bool
 398 s3dl_item_from_col(sqlite3_stmt *stmt, Query *q, int col, CFArrayRef accessGroups,
 399                    CFMutableDictionaryRef *item, SecAccessControlRef *access_control, keyclass_t* keyclass, CFErrorRef *error) {
 400     CFDataRef edata = NULL;
 401     bool ok = false;
 402     require(edata = s3dl_copy_data_from_col(stmt, col, error), out);
 403     ok = s3dl_item_from_data(edata, q, accessGroups, item, access_control, keyclass, error);
 404
 405 out:
 406     CFReleaseSafe(edata);
 407     return ok;
 408 }
 409
 410 struct s3dl_query_ctx {
 411     Query *q;
 412     CFArrayRef accessGroups;
 413     SecDbConnectionRef dbt;
 414     CFTypeRef result;
 415     int found;
 416 };
 417
 418 /* Return whatever the caller requested based on the value of q->q_return_type.
 419  keys and values must be 3 larger than attr_count in size to accomadate the
 420  optional data, class and persistent ref results.  This is so we can use
 421  the CFDictionaryCreate() api here rather than appending to a
 422  mutable dictionary. */
 423 static CF_RETURNS_RETAINED CFTypeRef
 424 handle_result(Query *q,
 425               CFMutableDictionaryRef item,
 426               sqlite_int64 rowid)
 427 {
 428     CFTypeRef a_result;
 429     CFDataRef data;
 430     data = CFDictionaryGetValue(item, kSecValueData);
 431         if (q->q_return_type == 0) {
 432                 /* Caller isn't interested in any results at all. */
 433                 a_result = kCFNull;
 434         } else if (q->q_return_type == kSecReturnDataMask) {
 435         if (data) {
 436             a_result = data;
 437             CFRetain(a_result);
 438         } else {
 439             a_result = CFDataCreate(kCFAllocatorDefault, NULL, 0);
 440         }
 441         } else if (q->q_return_type == kSecReturnPersistentRefMask) {
 442                 a_result = _SecItemCreatePersistentRef(q->q_class->name, rowid, item);
 443         } else {
 444                 /* We need to return more than one value. */
 445         if (q->q_return_type & kSecReturnRefMask) {
 446             CFDictionarySetValue(item, kSecClass, q->q_class->name);
 447         } else if ((q->q_return_type & kSecReturnAttributesMask)) {
 448             if (!(q->q_return_type & kSecReturnDataMask)) {
 449                 CFDictionaryRemoveValue(item, kSecValueData);
 450             }
 451
 452             // Add any attributes which are supposed to be returned, are not present in the decrypted blob,
 453             // and have a way to generate themselves.
 454             SecDbItemRef itemRef = NULL;
 455             SecDbForEachAttrWithMask(q->q_class, attr, kSecDbReturnAttrFlag) {
 456                 if(!CFDictionaryGetValue(item, attr->name) && attr->copyValue) {
 457                     CFErrorRef cferror = NULL;
 458                     if(!itemRef) {
 459                         itemRef = SecDbItemCreateWithAttributes(NULL, q->q_class, item, KEYBAG_DEVICE, &cferror);
 460                     }
 461                     if(!cferror && itemRef) {
 462                         if (attr->kind != kSecDbSHA1Attr || (q->q_return_type & kSecReturnDataMask)) { // we'll skip returning the sha1 attribute unless the client has also asked us to return data, because without data our sha1 could be invalid
 463                             CFTypeRef attrValue = attr->copyValue(itemRef, attr, &cferror);
 464                             if (!cferror && attrValue) {
 465                                 CFDictionarySetValue(item, attr->name, attrValue);
 466                             }
 467                             CFReleaseNull(attrValue);
 468                         }
 469                     }
 470                     CFReleaseNull(cferror);
 471                 }
 472             }
 473             CFReleaseNull(itemRef);
 474
 475             CFDictionaryRemoveValue(item, kSecAttrUUID);
 476         } else {
 477             CFRetainSafe(data);
 478             CFDictionaryRemoveAllValues(item);
 479             if ((q->q_return_type & kSecReturnDataMask) && data) {
 480                 CFDictionarySetValue(item, kSecValueData, data);
 481             }
 482             CFReleaseSafe(data);
 483         }
 484                 if (q->q_return_type & kSecReturnPersistentRefMask) {
 485             CFDataRef pref = _SecItemCreatePersistentRef(q->q_class->name, rowid, item);
 486                         CFDictionarySetValue(item, kSecValuePersistentRef, pref);
 487             CFRelease(pref);
 488                 }
 489
 490                 a_result = item;
 491         CFRetain(item);
 492         }
 493
 494         return a_result;
 495 }
 496
 497 static void s3dl_merge_into_dict(const void *key, const void *value, void *context) {
 498     CFDictionarySetValue(context, key, value);
 499 }
 500
 501 static bool checkTokenObjectID(CFDataRef token_object_id, CFDataRef value_data) {
 502     bool equalOID = false;
 503     CFDictionaryRef itemValue = SecTokenItemValueCopy(value_data, NULL);
 504     require_quiet(itemValue, out);
 505     CFDataRef oID = CFDictionaryGetValue(itemValue, kSecTokenValueObjectIDKey);
 506     equalOID = CFEqualSafe(token_object_id, oID);
 507     CFRelease(itemValue);
 508 out:
 509     return equalOID;
 510 }
 511
 512 static void s3dl_query_row(sqlite3_stmt *stmt, void *context) {
 513     struct s3dl_query_ctx *c = context;
 514     Query *q = c->q;
 515     ReturnTypeMask saved_mask = q->q_return_type;
 516
 517     sqlite_int64 rowid = sqlite3_column_int64(stmt, 0);
 518     CFMutableDictionaryRef item = NULL;
 519     bool ok;
 520
 521 decode:
 522     ok = s3dl_item_from_col(stmt, q, 1, c->accessGroups, &item, NULL, NULL, &q->q_error);
 523     if (!ok) {
 524         OSStatus status = SecErrorGetOSStatus(q->q_error);
 525         // errSecDecode means the item is corrupted, stash it for delete.
 526         if (status == errSecDecode) {
 527             secwarning("ignoring corrupt %@,rowid=%" PRId64 " %@", q->q_class->name, rowid, q->q_error);
 528             {
 529                 CFDataRef edata = s3dl_copy_data_from_col(stmt, 1, NULL);
 530                 CFMutableStringRef edatastring =  CFStringCreateMutable(kCFAllocatorDefault, 0);
 531                 if(edatastring) {
 532                     CFStringAppendEncryptedData(edatastring, edata);
 533                     secnotice("item", "corrupted edata=%@", edatastring);
 534                 }
 535                 CFReleaseSafe(edata);
 536                 CFReleaseSafe(edatastring);
 537             }
 538             CFReleaseNull(q->q_error);
 539         } else if (status == errSecAuthNeeded) {
 540             secwarning("Authentication is needed for %@,rowid=%" PRId64 " (%" PRIdOSStatus "): %@", q->q_class->name, rowid, status, q->q_error);
 541         } else if (status == errSecInteractionNotAllowed) {
 542             static dispatch_once_t kclockedtoken;
 543             static sec_action_t kclockedaction;
 544             dispatch_once(&kclockedtoken, ^{
 545                 kclockedaction = sec_action_create("ratelimiterdisabledlogevent", 1);
 546                 sec_action_set_handler(kclockedaction, ^{
 547                     secerror("decode item failed, keychain is locked (%d)", (int)errSecInteractionNotAllowed);
 548                 });
 549             });
 550             sec_action_perform(kclockedaction);
 551         } else {
 552             secerror("decode %@,rowid=%" PRId64 " failed (%" PRIdOSStatus "): %@", q->q_class->name, rowid, status, q->q_error);
 553         }
 554         // q->q_error will be released appropriately by a call to query_error
 555         return;
 556     }
 557
 558     if (!item)
 559         goto out;
 560
 561     if (CFDictionaryContainsKey(item, kSecAttrTokenID) && (q->q_return_type & kSecReturnDataMask) == 0) {
 562         // For token-based items, to get really meaningful set of attributes we must provide also data field, so augment mask
 563         // and restart item decoding cycle.
 564         q->q_return_type |= kSecReturnDataMask;
 565         CFReleaseNull(item);
 566         goto decode;
 567     }
 568
 569     if (q->q_token_object_id != NULL && !checkTokenObjectID(q->q_token_object_id, CFDictionaryGetValue(item, kSecValueData)))
 570         goto out;
 571
 572     if (q->q_class == identity_class()) {
 573         // TODO: Use col 2 for key rowid and use both rowids in persistent ref.
 574
 575         CFMutableDictionaryRef key;
 576         /* TODO : if there is a errSecDecode error here, we should cleanup */
 577         if (!s3dl_item_from_col(stmt, q, 3, c->accessGroups, &key, NULL, NULL, &q->q_error) || !key)
 578             goto out;
 579
 580         CFDataRef certData = CFDictionaryGetValue(item, kSecValueData);
 581         if (certData) {
 582             CFDictionarySetValue(key, kSecAttrIdentityCertificateData, certData);
 583             CFDictionaryRemoveValue(item, kSecValueData);
 584         }
 585
 586         CFDataRef certTokenID = CFDictionaryGetValue(item, kSecAttrTokenID);
 587         if (certTokenID) {
 588             CFDictionarySetValue(key, kSecAttrIdentityCertificateTokenID, certTokenID);
 589             CFDictionaryRemoveValue(item, kSecAttrTokenID);
 590         }
 591         CFDictionaryApplyFunction(item, s3dl_merge_into_dict, key);
 592         CFRelease(item);
 593         item = key;
 594     }
 595
 596     if (!match_item(c->dbt, q, c->accessGroups, item))
 597         goto out;
 598
 599     CFTypeRef a_result = handle_result(q, item, rowid);
 600     if (a_result) {
 601         if (a_result == kCFNull) {
 602             /* Caller wasn't interested in a result, but we still
 603              count this row as found. */
 604             CFRelease(a_result);  // Help shut up clang
 605         } else if (q->q_limit == 1) {
 606             c->result = a_result;
 607         } else {
 608             CFArrayAppendValue((CFMutableArrayRef)c->result, a_result);
 609             CFRelease(a_result);
 610         }
 611         c->found++;
 612     }
 613
 614 out:
 615     q->q_return_type = saved_mask;
 616     CFReleaseSafe(item);
 617 }
 618
 619 static void
 620 SecDbAppendWhereROWID(CFMutableStringRef sql,
 621                       CFStringRef col, sqlite_int64 row_id,
 622                       bool *needWhere) {
 623     if (row_id > 0) {
 624         SecDbAppendWhereOrAnd(sql, needWhere);
 625         CFStringAppendFormat(sql, NULL, CFSTR("%@=%lld"), col, row_id);
 626     }
 627 }
 628
 629 static void
 630 SecDbAppendWhereAttrs(CFMutableStringRef sql, const Query *q, bool *needWhere) {
 631     CFIndex ix, attr_count = query_attr_count(q);
 632     for (ix = 0; ix < attr_count; ++ix) {
 633         SecDbAppendWhereOrAndEquals(sql, query_attr_at(q, ix).key, needWhere);
 634     }
 635 }
 636
 637 static void
 638 SecDbAppendWhereAccessGroups(CFMutableStringRef sql,
 639                              CFStringRef col,
 640                              CFArrayRef accessGroups,
 641                              bool *needWhere) {
 642     CFIndex ix, ag_count;
 643     if (!accessGroups || 0 == (ag_count = CFArrayGetCount(accessGroups))) {
 644         return;
 645     }
 646
 647     SecDbAppendWhereOrAnd(sql, needWhere);
 648     CFStringAppend(sql, col);
 649     CFStringAppend(sql, CFSTR(" IN (?"));
 650     for (ix = 1; ix < ag_count; ++ix) {
 651         CFStringAppend(sql, CFSTR(",?"));
 652     }
 653     CFStringAppend(sql, CFSTR(")"));
 654 }
 655
 656 static bool
 657 isQueryOverAllMUSRViews(CFTypeRef musrView)
 658 {
 659     return SecMUSRIsViewAllViews(musrView);
 660 }
 661
 662 static bool
 663 isQueryOverSingleUserView(CFTypeRef musrView)
 664 {
 665     return isNull(musrView);
 666 }
 667
 668 #if TARGET_OS_IPHONE
 669 static bool
 670 isQueryOverBothUserAndSystem(CFTypeRef musrView, uid_t *uid)
 671 {
 672     return SecMUSRGetBothUserAndSystemUUID(musrView, uid);
 673 }
 674 #endif
 675
 676 static void
 677 SecDbAppendWhereMusr(CFMutableStringRef sql,
 678                      const Query *q,
 679                      bool *needWhere)
 680 {
 681     SecDbAppendWhereOrAnd(sql, needWhere);
 682
 683 #if TARGET_OS_IPHONE
 684     if (isQueryOverBothUserAndSystem(q->q_musrView, NULL)) {
 685         CFStringAppend(sql, CFSTR("(musr = ? OR musr = ?)"));
 686     } else
 687 #endif
 688     if (isQueryOverAllMUSRViews(q->q_musrView)) {
 689             /* query over all items, regardless of view */
 690     } else if (isQueryOverSingleUserView(q->q_musrView)) {
 691         CFStringAppend(sql, CFSTR("musr = ?"));
 692     } else {
 693         CFStringAppend(sql, CFSTR("musr = ?"));
 694     }
 695 }
 696
 697 static void SecDbAppendWhereClause(CFMutableStringRef sql, const Query *q,
 698                                    CFArrayRef accessGroups) {
 699     bool needWhere = true;
 700     SecDbAppendWhereROWID(sql, CFSTR("ROWID"), q->q_row_id, &needWhere);
 701     SecDbAppendWhereAttrs(sql, q, &needWhere);
 702     SecDbAppendWhereMusr(sql, q, &needWhere);
 703     SecDbAppendWhereAccessGroups(sql, CFSTR("agrp"), accessGroups, &needWhere);
 704 }
 705
 706 static void SecDbAppendLimit(CFMutableStringRef sql, CFIndex limit) {
 707     if (limit != kSecMatchUnlimited)
 708         CFStringAppendFormat(sql, NULL, CFSTR(" LIMIT %" PRIdCFIndex), limit);
 709 }
 710
 711 static CFStringRef s3dl_create_select_sql(Query *q, CFArrayRef accessGroups) {
 712     CFMutableStringRef sql = CFStringCreateMutable(NULL, 0);
 713         if (q->q_class == identity_class()) {
 714         CFStringAppendFormat(sql, NULL, CFSTR("SELECT crowid, %@"
 715                                               ", rowid,data FROM "
 716                                               "(SELECT cert.rowid AS crowid, cert.labl AS labl,"
 717                                               " cert.issr AS issr, cert.slnr AS slnr, cert.skid AS skid,"
 718                                               " keys.*,cert.data AS %@"
 719                                               " FROM keys, cert"
 720                                               " WHERE keys.priv == 1 AND cert.pkhh == keys.klbl"),
 721                              kSecAttrIdentityCertificateData, kSecAttrIdentityCertificateData);
 722         SecDbAppendWhereAccessGroups(sql, CFSTR("cert.agrp"), accessGroups, 0);
 723         /* The next 3 SecDbAppendWhere calls are in the same order as in
 724          SecDbAppendWhereClause().  This makes sqlBindWhereClause() work,
 725          as long as we do an extra sqlBindAccessGroups first. */
 726         SecDbAppendWhereROWID(sql, CFSTR("crowid"), q->q_row_id, 0);
 727         CFStringAppend(sql, CFSTR(")"));
 728         bool needWhere = true;
 729         SecDbAppendWhereAttrs(sql, q, &needWhere);
 730         SecDbAppendWhereMusr(sql, q, &needWhere);
 731         SecDbAppendWhereAccessGroups(sql, CFSTR("agrp"), accessGroups, &needWhere);
 732         } else {
 733         CFStringAppend(sql, CFSTR("SELECT rowid, data FROM "));
 734                 CFStringAppend(sql, q->q_class->name);
 735         SecDbAppendWhereClause(sql, q, accessGroups);
 736     }
 737     //do not append limit for all queries which needs filtering
 738     if (q->q_match_issuer == NULL && q->q_match_policy == NULL && q->q_match_valid_on_date == NULL && q->q_match_trusted_only == NULL && q->q_token_object_id == NULL) {
 739         SecDbAppendLimit(sql, q->q_limit);
 740     }
 741
 742     return sql;
 743 }
 744
 745 static bool sqlBindMusr(sqlite3_stmt *stmt, const Query *q, int *pParam, CFErrorRef *error) {
 746     int param = *pParam;
 747     bool result = true;
 748 #if TARGET_OS_IPHONE
 749     uid_t uid;
 750
 751     if (isQueryOverBothUserAndSystem(q->q_musrView, &uid)) {
 752         /* network extensions are special and get to query both user and system views */
 753         CFDataRef systemUUID = SecMUSRGetSystemKeychainUUID();
 754         result = SecDbBindObject(stmt, param++, systemUUID, error);
 755         if (result) {
 756             CFDataRef activeUser = SecMUSRCreateActiveUserUUID(uid);
 757             result = SecDbBindObject(stmt, param++, activeUser, error);
 758             CFReleaseNull(activeUser);
 759         }
 760     } else
 761 #endif
 762     if (isQueryOverAllMUSRViews(q->q_musrView)) {
 763         /* query over all items, regardless of view */
 764     } else if (isQueryOverSingleUserView(q->q_musrView)) {
 765         CFDataRef singleUUID = SecMUSRGetSingleUserKeychainUUID();
 766         result = SecDbBindObject(stmt, param++, singleUUID, error);
 767     } else {
 768         result = SecDbBindObject(stmt, param++, q->q_musrView, error);
 769     }
 770
 771     *pParam = param;
 772     return result;
 773 }
 774
 775
 776 static bool sqlBindAccessGroups(sqlite3_stmt *stmt, CFArrayRef accessGroups,
 777                                 int *pParam, CFErrorRef *error) {
 778     bool result = true;
 779     int param = *pParam;
 780     CFIndex ix, count = accessGroups ? CFArrayGetCount(accessGroups) : 0;
 781     for (ix = 0; ix < count; ++ix) {
 782         result = SecDbBindObject(stmt, param++,
 783                                  CFArrayGetValueAtIndex(accessGroups, ix),
 784                                  error);
 785         if (!result)
 786             break;
 787     }
 788     *pParam = param;
 789     return result;
 790 }
 791
 792 static bool sqlBindWhereClause(sqlite3_stmt *stmt, const Query *q,
 793                                CFArrayRef accessGroups, int *pParam, CFErrorRef *error) {
 794     bool result = true;
 795     int param = *pParam;
 796     CFIndex ix, attr_count = query_attr_count(q);
 797     for (ix = 0; ix < attr_count; ++ix) {
 798         result = SecDbBindObject(stmt, param++, query_attr_at(q, ix).value, error);
 799         if (!result)
 800             break;
 801         }
 802
 803     if (result) {
 804         result = sqlBindMusr(stmt, q, &param, error);
 805     }
 806
 807     /* Bind the access group to the sql. */
 808     if (result) {
 809         result = sqlBindAccessGroups(stmt, accessGroups, &param, error);
 810     }
 811
 812     *pParam = param;
 813     return result;
 814 }
 815
 816 bool SecDbItemQuery(SecDbQueryRef query, CFArrayRef accessGroups, SecDbConnectionRef dbconn, CFErrorRef *error,
 817                     void (^handle_row)(SecDbItemRef item, bool *stop)) {
 818     __block bool ok = true;
 819     /* Sanity check the query. */
 820     if (query->q_ref)
 821         return SecError(errSecValueRefUnsupported, error, CFSTR("value ref not supported by queries"));
 822
 823     bool (^return_attr)(const SecDbAttr *attr) = ^bool (const SecDbAttr * attr) {
 824         // The attributes here must match field list hardcoded in s3dl_select_sql used below, which is
 825         // "rowid, data"
 826         return attr->kind == kSecDbRowIdAttr || attr->kind == kSecDbEncryptedDataAttr;
 827     };
 828
 829     CFStringRef sql = s3dl_create_select_sql(query, accessGroups);
 830     ok = sql;
 831     if (sql) {
 832         ok &= SecDbPrepare(dbconn, sql, error, ^(sqlite3_stmt *stmt) {
 833             /* Bind the values being searched for to the SELECT statement. */
 834             int param = 1;
 835             if (query->q_class == identity_class()) {
 836                 /* Bind the access groups to cert.agrp. */
 837                 ok &= sqlBindAccessGroups(stmt, accessGroups, &param, error);
 838             }
 839             if (ok)
 840                 ok &= sqlBindWhereClause(stmt, query, accessGroups, &param, error);
 841             if (ok) {
 842                 SecDbStep(dbconn, stmt, error, ^(bool *stop) {
 843                     SecDbItemRef itemFromStatement = SecDbItemCreateWithStatement(kCFAllocatorDefault, query->q_class, stmt, query->q_keybag, error, return_attr);
 844                     if (itemFromStatement) {
 845                         CFTransferRetained(itemFromStatement->credHandle, query->q_use_cred_handle);
 846                         if (match_item(dbconn, query, accessGroups, itemFromStatement->attributes))
 847                             handle_row(itemFromStatement, stop);
 848                         CFReleaseNull(itemFromStatement);
 849                     } else {
 850                         secerror("failed to create item from stmt: %@", error ? *error : (CFErrorRef)"no error");
 851                         if (error) {
 852                             CFReleaseNull(*error);
 853                         }
 854                         //*stop = true;
 855                         //ok = false;
 856                     }
 857                 });
 858             }
 859         });
 860         CFRelease(sql);
 861     }
 862
 863     return ok;
 864 }
 865
 866 static bool
 867 s3dl_query(s3dl_handle_row handle_row,
 868            void *context, CFErrorRef *error)
 869 {
 870     struct s3dl_query_ctx *c = context;
 871     SecDbConnectionRef dbt = c->dbt;
 872     Query *q = c->q;
 873     CFArrayRef accessGroups = c->accessGroups;
 874
 875     /* Sanity check the query. */
 876     if (q->q_ref)
 877         return SecError(errSecValueRefUnsupported, error, CFSTR("value ref not supported by queries"));
 878
 879         /* Actual work here. */
 880     if (q->q_limit == 1) {
 881         c->result = NULL;
 882     } else {
 883         c->result = CFArrayCreateMutable(NULL, 0, &kCFTypeArrayCallBacks);
 884     }
 885     CFStringRef sql = s3dl_create_select_sql(q, accessGroups);
 886     bool ok = SecDbWithSQL(dbt, sql, error, ^(sqlite3_stmt *stmt) {
 887         bool sql_ok = true;
 888         /* Bind the values being searched for to the SELECT statement. */
 889         int param = 1;
 890         if (q->q_class == identity_class()) {
 891             /* Bind the access groups to cert.agrp. */
 892             sql_ok = sqlBindAccessGroups(stmt, accessGroups, &param, error);
 893         }
 894         if (sql_ok)
 895             sql_ok = sqlBindWhereClause(stmt, q, accessGroups, &param, error);
 896         if (sql_ok) {
 897             SecDbForEach(dbt, stmt, error, ^bool (int row_index) {
 898                 handle_row(stmt, context);
 899
 900                 bool needs_auth = q->q_error && CFErrorGetCode(q->q_error) == errSecAuthNeeded;
 901                 if (q->q_skip_acl_items && needs_auth)
 902                     // Skip items needing authentication if we are told to do so.
 903                     CFReleaseNull(q->q_error);
 904
 905                 bool stop = q->q_limit != kSecMatchUnlimited && c->found >= q->q_limit;
 906                 stop = stop || (q->q_error && !needs_auth);
 907                 return !stop;
 908             });
 909         }
 910         return sql_ok;
 911     });
 912
 913     CFRelease(sql);
 914
 915     // First get the error from the query, since errSecDuplicateItem from an
 916     // update query should superceed the errSecItemNotFound below.
 917     if (!query_error(q, error))
 918         ok = false;
 919     if (ok && c->found == 0) {
 920         ok = SecError(errSecItemNotFound, error, CFSTR("no matching items found"));
 921         if (q->q_spindump_on_failure) {
 922             __security_stackshotreport(CFSTR("ItemNotFound"), __sec_exception_code_LostInMist);
 923         }
 924     }
 925
 926     return ok;
 927 }
 928
 929 bool
 930 s3dl_copy_matching(SecDbConnectionRef dbt, Query *q, CFTypeRef *result,
 931                    CFArrayRef accessGroups, CFErrorRef *error)
 932 {
 933     struct s3dl_query_ctx ctx = {
 934         .q = q, .accessGroups = accessGroups, .dbt = dbt,
 935     };
 936     if (q->q_row_id && query_attr_count(q))
 937         return SecError(errSecItemIllegalQuery, error,
 938                         CFSTR("attributes to query illegal; both row_id and other attributes can't be searched at the same time"));
 939     if (q->q_token_object_id && query_attr_count(q) != 1)
 940         return SecError(errSecItemIllegalQuery, error,
 941                         CFSTR("attributes to query illegal; both token persitent ref and other attributes can't be searched at the same time"));
 942
 943     // Only copy things that aren't tombstones unless the client explicitly asks otherwise.
 944     if (!CFDictionaryContainsKey(q->q_item, kSecAttrTombstone))
 945         query_add_attribute(kSecAttrTombstone, kCFBooleanFalse, q);
 946     bool ok = s3dl_query(s3dl_query_row, &ctx, error);
 947     if (ok && result)
 948         *result = ctx.result;
 949     else
 950         CFReleaseSafe(ctx.result);
 951
 952     return ok;
 953 }
 954
 955 typedef void (^s3dl_item_digest_callback)(CFDataRef persistantReference, CFDataRef encryptedData);
 956
 957 struct s3dl_digest_ctx {
 958     Query *q;
 959     SecDbConnectionRef dbt;
 960     s3dl_item_digest_callback item_callback;
 961 };
 962
 963 static void s3dl_query_row_digest(sqlite3_stmt *stmt, void *context) {
 964     struct s3dl_query_ctx *c = context;
 965     Query *q = c->q;
 966
 967     sqlite_int64 rowid = sqlite3_column_int64(stmt, 0);
 968     CFDataRef edata = s3dl_copy_data_from_col(stmt, 1, NULL);
 969     CFDataRef persistant_reference = _SecItemCreatePersistentRef(q->q_class->name, rowid, NULL);
 970     CFDataRef digest = NULL;
 971
 972     if (edata) {
 973         digest = CFDataCopySHA256Digest(edata, NULL);
 974     }
 975
 976     if (digest && persistant_reference) {
 977         CFDictionaryRef item = CFDictionaryCreateForCFTypes(NULL,
 978                                                             kSecValuePersistentRef, persistant_reference,
 979                                                             kSecValueData, digest,
 980                                                             NULL);
 981         if (item)
 982             CFArrayAppendValue((CFMutableArrayRef)c->result, item);
 983         CFReleaseNull(item);
 984         c->found++;
 985     } else {
 986         secinfo("item", "rowid %lu in %@ failed to create pref/digest", (unsigned long)rowid, q->q_class->name);
 987     }
 988     CFReleaseNull(digest);
 989     CFReleaseNull(edata);
 990     CFReleaseNull(persistant_reference);
 991 }
 992
 993
 994 bool
 995 s3dl_copy_digest(SecDbConnectionRef dbt, Query *q, CFArrayRef *result, CFArrayRef accessGroups, CFErrorRef *error)
 996 {
 997     struct s3dl_query_ctx ctx = {
 998         .q = q, .dbt = dbt, .accessGroups = accessGroups,
 999     };
1000     // Force to always return an array
1001     q->q_limit = kSecMatchUnlimited;
1002     // This interface only queries live data
1003     query_add_attribute(kSecAttrTombstone, kCFBooleanFalse, q);
1004     bool ok = s3dl_query(s3dl_query_row_digest, &ctx, error);
1005     if (ok && result)
1006         *result = (CFArrayRef)ctx.result;
1007     else
1008         CFReleaseSafe(ctx.result);
1009
1010     return ok;
1011 }
1012
1013 /* First remove key from q->q_pairs if it's present, then add the attribute again. */
1014 static void query_set_attribute_with_desc(const SecDbAttr *desc, const void *value, Query *q) {
1015     if (CFDictionaryContainsKey(q->q_item, desc->name)) {
1016         CFIndex ix;
1017         for (ix = 0; ix < q->q_attr_end; ++ix) {
1018             if (CFEqual(desc->name, q->q_pairs[ix].key)) {
1019                 CFReleaseSafe(q->q_pairs[ix].value);
1020                 --q->q_attr_end;
1021                 for (; ix < q->q_attr_end; ++ix) {
1022                     q->q_pairs[ix] = q->q_pairs[ix + 1];
1023                 }
1024                 CFDictionaryRemoveValue(q->q_item, desc->name);
1025                 break;
1026             }
1027         }
1028     }
1029     query_add_attribute_with_desc(desc, value, q);
1030 }
1031
1032 /* Update modification_date if needed. */
1033 static void query_pre_update(Query *q) {
1034     SecDbForEachAttr(q->q_class, desc) {
1035         if (desc->kind == kSecDbModificationDateAttr) {
1036             CFDateRef now = CFDateCreate(0, CFAbsoluteTimeGetCurrent());
1037             query_set_attribute_with_desc(desc, now, q);
1038             CFReleaseSafe(now);
1039         }
1040     }
1041 }
1042
1043 /* Make sure all attributes that are marked as not_null have a value.  If
1044  force_date is false, only set mdat and cdat if they aren't already set. */
1045 void query_pre_add(Query *q, bool force_date) {
1046     CFDateRef now = CFDateCreate(0, CFAbsoluteTimeGetCurrent());
1047     SecDbForEachAttrWithMask(q->q_class, desc, kSecDbInFlag) {
1048         if (desc->kind == kSecDbCreationDateAttr ||
1049             desc->kind == kSecDbModificationDateAttr) {
1050             if (force_date) {
1051                 query_set_attribute_with_desc(desc, now, q);
1052             } else if (!CFDictionaryContainsKey(q->q_item, desc->name)) {
1053                 query_add_attribute_with_desc(desc, now, q);
1054             }
1055         } else if ((desc->flags & kSecDbNotNullFlag) &&
1056                    !CFDictionaryContainsKey(q->q_item, desc->name)) {
1057             CFTypeRef value = NULL;
1058             if (desc->flags & kSecDbDefault0Flag) {
1059                 if (desc->kind == kSecDbDateAttr)
1060                     value = CFDateCreate(kCFAllocatorDefault, 0.0);
1061                 else {
1062                     SInt32 vzero = 0;
1063                     value = CFNumberCreate(0, kCFNumberSInt32Type, &vzero);
1064                 }
1065             } else if (desc->flags & kSecDbDefaultEmptyFlag) {
1066                 if (desc->kind == kSecDbDataAttr || desc->kind == kSecDbUUIDAttr)
1067                     value = CFDataCreate(kCFAllocatorDefault, NULL, 0);
1068                 else {
1069                     value = CFSTR("");
1070                     CFRetain(value);
1071                 }
1072             }
1073             if (value) {
1074                 /* Safe to use query_add_attribute here since the attr wasn't
1075                  set yet. */
1076                 query_add_attribute_with_desc(desc, value, q);
1077                 CFRelease(value);
1078             }
1079         }
1080     }
1081     CFReleaseSafe(now);
1082 }
1083
1084 // Return a tri state value false->never make a tombstone, true->always make a
1085 // tombstone, NULL->make a tombstone, but delete it if the tombstone itself is not currently being synced.
1086 static CFBooleanRef s3dl_should_make_tombstone(Query *q, bool item_is_syncable, SecDbItemRef item) {
1087     if (q->q_use_tomb)
1088         return q->q_use_tomb;
1089     else if (item_is_syncable && !SecDbItemIsTombstone(item))
1090         return NULL;
1091     else
1092         return kCFBooleanFalse;
1093 }
1094 /* AUDIT[securityd](done):
1095  attributesToUpdate (ok) is a caller provided dictionary,
1096  only its cf types have been checked.
1097  */
1098 bool
1099 s3dl_query_update(SecDbConnectionRef dbt, Query *q,
1100                   CFDictionaryRef attributesToUpdate, CFArrayRef accessGroups, CFErrorRef *error)
1101 {
1102     /* Sanity check the query. */
1103     if (query_match_count(q) != 0)
1104         return SecError(errSecItemMatchUnsupported, error, CFSTR("match not supported in attributes to update"));
1105     if (q->q_ref)
1106         return SecError(errSecValueRefUnsupported, error, CFSTR("value ref not supported in attributes to update"));
1107     if (q->q_row_id && query_attr_count(q))
1108         return SecError(errSecItemIllegalQuery, error, CFSTR("attributes to update illegal; both row_id and other attributes can't be updated at the same time"));
1109     if (q->q_token_object_id && query_attr_count(q) != 1)
1110         return SecError(errSecItemIllegalQuery, error, CFSTR("attributes to update illegal; both token persistent ref and other attributes can't be updated at the same time"));
1111
1112     __block bool result = true;
1113     Query *u = query_create(q->q_class, NULL, attributesToUpdate, error);
1114     if (u == NULL) return false;
1115     require_action_quiet(query_update_parse(u, attributesToUpdate, error), errOut, result = false);
1116     query_pre_update(u);
1117     result &= SecDbTransaction(dbt, kSecDbExclusiveTransactionType, error, ^(bool *commit) {
1118         // Make sure we only update real items, not tombstones, unless the client explicitly asks otherwise.
1119         if (!CFDictionaryContainsKey(q->q_item, kSecAttrTombstone))
1120             query_add_attribute(kSecAttrTombstone, kCFBooleanFalse, q);
1121         result &= SecDbItemQuery(q, accessGroups, dbt, error, ^(SecDbItemRef item, bool *stop) {
1122             // We always need to know the error here.
1123             CFErrorRef localError = NULL;
1124             if (q->q_token_object_id) {
1125                 const SecDbAttr *valueDataAttr = SecDbClassAttrWithKind(item->class, kSecDbDataAttr, NULL);
1126                 CFDataRef valueData = SecDbItemGetValue(item, valueDataAttr, NULL);
1127                 if (q->q_token_object_id != NULL && !checkTokenObjectID(q->q_token_object_id, valueData))
1128                     return;
1129             }
1130             // Cache the storedSHA1 digest so we use the one from the db not the recomputed one for notifications.
1131             const SecDbAttr *sha1attr = SecDbClassAttrWithKind(item->class, kSecDbSHA1Attr, NULL);
1132             CFDataRef storedSHA1 = CFRetainSafe(SecDbItemGetValue(item, sha1attr, NULL));
1133             SecDbItemRef new_item = SecDbItemCopyWithUpdates(item, u->q_item, &localError);
1134             SecDbItemSetValue(item, sha1attr, storedSHA1, NULL);
1135             CFReleaseSafe(storedSHA1);
1136             if (SecErrorGetOSStatus(localError) == errSecDecode) {
1137                 // We just ignore this, and treat as if item is not found.
1138                 secwarning("deleting corrupt %@,rowid=%" PRId64 " %@", q->q_class->name, SecDbItemGetRowId(item, NULL), localError);
1139                 CFReleaseNull(localError);
1140                 if (!SecDbItemDelete(item, dbt, false, &localError)) {
1141                     secerror("failed to delete corrupt %@,rowid=%" PRId64 " %@", q->q_class->name, SecDbItemGetRowId(item, NULL), localError);
1142                     CFReleaseNull(localError);
1143                 }
1144                 CFReleaseNull(new_item);
1145                 return;
1146             }
1147             if (new_item != NULL && u->q_access_control != NULL)
1148                 SecDbItemSetAccessControl(new_item, u->q_access_control, &localError);
1149             result = SecErrorPropagate(localError, error) && new_item;
1150             if (new_item) {
1151                 bool item_is_sync = SecDbItemIsSyncable(item);
1152                 result = SecDbItemUpdate(item, new_item, dbt, s3dl_should_make_tombstone(q, item_is_sync, item), q->q_uuid_from_primary_key, error);
1153                 if (result) {
1154                     q->q_changed = true;
1155                     if (item_is_sync || SecDbItemIsSyncable(new_item))
1156                         q->q_sync_changed = true;
1157                 }
1158                 CFRelease(new_item);
1159             }
1160         });
1161         if (!result)
1162             *commit = false;
1163     });
1164     if (result && !q->q_changed)
1165         result = SecError(errSecItemNotFound, error, CFSTR("No items updated"));
1166 errOut:
1167     if (!query_destroy(u, error))
1168         result = false;
1169     return result;
1170 }
1171
1172 static bool SecDbItemNeedAuth(SecDbItemRef item, CFErrorRef *error)
1173 {
1174     CFErrorRef localError = NULL;
1175     if (!SecDbItemEnsureDecrypted(item, true, &localError) && localError && CFErrorGetCode(localError) == errSecAuthNeeded) {
1176         if (error)
1177         *error = localError;
1178         return true;
1179     }
1180
1181     CFReleaseSafe(localError);
1182     return false;
1183 }
1184
1185 bool
1186 s3dl_query_delete(SecDbConnectionRef dbt, Query *q, CFArrayRef accessGroups, CFErrorRef *error)
1187 {
1188     __block bool ok = true;
1189     __block bool needAuth = false;
1190     // Only delete things that aren't tombstones, unless the client explicitly asks otherwise.
1191     if (!CFDictionaryContainsKey(q->q_item, kSecAttrTombstone))
1192         query_add_attribute(kSecAttrTombstone, kCFBooleanFalse, q);
1193     ok &= SecDbItemSelect(q, dbt, error, NULL, ^bool(const SecDbAttr *attr) {
1194         return false;
1195     },^bool(CFMutableStringRef sql, bool *needWhere) {
1196         SecDbAppendWhereClause(sql, q, accessGroups);
1197         return true;
1198     },^bool(sqlite3_stmt * stmt, int col) {
1199         return sqlBindWhereClause(stmt, q, accessGroups, &col, error);
1200     }, ^(SecDbItemRef item, bool *stop) {
1201         // Check if item for token persitence ref
1202         if (q->q_token_object_id) {
1203             const SecDbAttr *valueDataAttr = SecDbClassAttrWithKind(item->class, kSecDbDataAttr, NULL);
1204             CFDataRef valueData = SecDbItemGetValue(item, valueDataAttr, NULL);
1205             if (q->q_token_object_id != NULL && !checkTokenObjectID(q->q_token_object_id, valueData))
1206                 return;
1207         }
1208         // Check if item need to be authenticated by LocalAuthentication
1209         item->cryptoOp = kAKSKeyOpDelete;
1210         if (SecDbItemNeedAuth(item, error)) {
1211             needAuth = true;
1212             return;
1213         }
1214         // Cache the storedSHA1 digest so we use the one from the db not the recomputed one for notifications.
1215         const SecDbAttr *sha1attr = SecDbClassAttrWithKind(item->class, kSecDbSHA1Attr, NULL);
1216         CFDataRef storedSHA1 = CFRetainSafe(SecDbItemGetValue(item, sha1attr, NULL));
1217         bool item_is_sync = SecDbItemIsSyncable(item);
1218         SecDbItemSetValue(item, sha1attr, storedSHA1, NULL);
1219         CFReleaseSafe(storedSHA1);
1220         ok = SecDbItemDelete(item, dbt, s3dl_should_make_tombstone(q, item_is_sync, item), error);
1221         if (ok) {
1222             q->q_changed = true;
1223             if (item_is_sync)
1224                 q->q_sync_changed = true;
1225         }
1226     });
1227     if (ok && !q->q_changed && !needAuth) {
1228         ok = SecError(errSecItemNotFound, error, CFSTR("Delete failed to delete anything"));
1229     }
1230     return ok && !needAuth;
1231 }
1232
1233 static bool
1234 matchAnyString(CFStringRef needle, CFStringRef *haystack)
1235 {
1236     while (*haystack) {
1237         if (CFEqual(needle, *haystack))
1238             return true;
1239         haystack++;
1240     }
1241     return false;
1242 }
1243
1244 /* Return true iff the item in question should not be backed up, nor restored,
1245  but when restoring a backup the original version of the item should be
1246  added back to the keychain again after the restore completes. */
1247 static bool SecItemIsSystemBound(CFDictionaryRef item, const SecDbClass *cls, bool multiUser) {
1248     CFNumberRef sysb =  CFDictionaryGetValue(item, kSecAttrSysBound);
1249     if (isNumber(sysb)) {
1250         int32_t num = 0;
1251         if (!CFNumberGetValue(sysb, kCFNumberSInt32Type, &num))
1252             return false;
1253         if (num == kSecSecAttrSysBoundNot) {
1254             return false;
1255         } else if (num == kSecSecAttrSysBoundPreserveDuringRestore) {
1256             return true;
1257         }
1258         return true;
1259     }
1260
1261     CFStringRef agrp = CFDictionaryGetValue(item, kSecAttrAccessGroup);
1262     if (!isString(agrp))
1263         return false;
1264
1265     if (CFEqualSafe(agrp, kSOSInternalAccessGroup)) {
1266         secdebug("backup", "found sysbound item: %@", item);
1267         return true;
1268     }
1269
1270     if (CFEqual(agrp, CFSTR("lockdown-identities"))) {
1271         secdebug("backup", "found sys_bound item: %@", item);
1272         return true;
1273     }
1274
1275     if (CFEqual(agrp, CFSTR("apple")) && cls == genp_class()) {
1276         CFStringRef service = CFDictionaryGetValue(item, kSecAttrService);
1277         CFStringRef account = CFDictionaryGetValue(item, kSecAttrAccount);
1278
1279         if (isString(service) && isString(account)) {
1280             static CFStringRef mcAccounts[] = {
1281                 CFSTR("Public"),
1282                 CFSTR("Private"),
1283                 NULL,
1284             };
1285
1286             if (CFEqual(service, CFSTR("com.apple.managedconfiguration"))
1287                 && matchAnyString(account, mcAccounts))
1288             {
1289                 secdebug("backup", "found sys_bound item: %@", item);
1290                 return true;
1291             }
1292         }
1293     }
1294
1295     if (multiUser && CFEqual(agrp, CFSTR("com.apple.apsd")) && cls == genp_class()) {
1296         static CFStringRef pushServices[] = {
1297             CFSTR("push.apple.com"),
1298             CFSTR("push.apple.com,PerAppToken.v0"),
1299             NULL
1300         };
1301         CFStringRef service = CFDictionaryGetValue(item, kSecAttrService);
1302
1303         if (isString(service) && matchAnyString(service, pushServices)) {
1304             secdebug("backup", "found sys_bound item: %@", item);
1305             return true;
1306         }
1307     }
1308
1309     if (multiUser && CFEqual(agrp, CFSTR("appleaccount")) && cls == genp_class()) {
1310         static CFStringRef accountServices[] = {
1311             CFSTR("com.apple.appleaccount.fmf.token"), /* temporary tokens while accout is being setup */
1312             CFSTR("com.apple.appleaccount.fmf.apptoken"),
1313             CFSTR("com.apple.appleaccount.fmip.siritoken"),
1314             CFSTR("com.apple.appleaccount.cloudkit.token"),
1315             NULL
1316         };
1317         CFStringRef service = CFDictionaryGetValue(item, kSecAttrService);
1318
1319         if (isString(service) && matchAnyString(service, accountServices)) {
1320             secdebug("backup", "found exact sys_bound item: %@", item);
1321             return true;
1322         }
1323     }
1324
1325     if (multiUser && CFEqual(agrp, CFSTR("apple")) && cls == genp_class()) {
1326         static CFStringRef accountServices[] = {
1327             /* accounts, remove with rdar://37595482 */
1328             CFSTR("com.apple.account.AppleAccount.token"),
1329             CFSTR("com.apple.account.AppleAccount.password"),
1330             CFSTR("com.apple.account.AppleAccount.rpassword"),
1331             CFSTR("com.apple.account.idms.token"),
1332             CFSTR("com.apple.account.idms.heartbeat-token"),
1333             CFSTR("com.apple.account.idms.continuation-key"),
1334             CFSTR("com.apple.account.CloudKit.token"),
1335             CFSTR("com.apple.account.IdentityServices.password"), /* accountsd for ids */
1336             CFSTR("com.apple.account.IdentityServices.rpassword"),
1337             CFSTR("com.apple.account.IdentityServices.token"),
1338             /* IDS stuff */
1339             CFSTR("BackupIDSAccountToken"),
1340             CFSTR("com.apple.ids"),
1341             CFSTR("ids"),
1342             CFSTR("IDS"),
1343             NULL
1344         };
1345         CFStringRef service = CFDictionaryGetValue(item, kSecAttrService);
1346
1347         if (isString(service) && matchAnyString(service, accountServices)) {
1348             secdebug("backup", "found exact sys_bound item: %@", item);
1349             return true;
1350         }
1351         if (isString(service) && CFEqual(service, CFSTR("com.apple.facetime"))) {
1352             CFStringRef account = CFDictionaryGetValue(item, kSecAttrAccount);
1353             if (isString(account) && CFEqual(account, CFSTR("registrationV1"))) {
1354                 secdebug("backup", "found exact sys_bound item: %@", item);
1355                 return true;
1356             }
1357         }
1358     }
1359
1360     /* accounts, remove with rdar://37595482 */
1361     if (multiUser && CFEqual(agrp, CFSTR("com.apple.ind")) && cls == genp_class()) {
1362         CFStringRef service = CFDictionaryGetValue(item, kSecAttrService);
1363         if (isString(service) && CFEqual(service, CFSTR("com.apple.ind.registration"))) {
1364             secdebug("backup", "found exact sys_bound item: %@", item);
1365             return true;
1366         }
1367     }
1368
1369     if (multiUser && CFEqual(agrp, CFSTR("ichat")) && cls == genp_class()) {
1370         static CFStringRef accountServices[] = {
1371             CFSTR("ids"),
1372             NULL
1373         };
1374         CFStringRef service = CFDictionaryGetValue(item, kSecAttrService);
1375
1376         if (isString(service) && matchAnyString(service, accountServices)) {
1377             secdebug("backup", "found exact sys_bound item: %@", item);
1378             return true;
1379         }
1380     }
1381
1382     if (multiUser && CFEqual(agrp, CFSTR("ichat")) && cls == keys_class()) {
1383         static CFStringRef exactMatchingLabel[] = {
1384             CFSTR("iMessage Encryption Key"),
1385             CFSTR("iMessage Signing Key"),
1386         };
1387         CFStringRef label = CFDictionaryGetValue(item, kSecAttrLabel);
1388         if (isString(label)) {
1389             if (matchAnyString(label, exactMatchingLabel)) {
1390                 secdebug("backup", "found exact sys_bound item: %@", item);
1391                 return true;
1392             }
1393         }
1394     }
1395
1396
1397     secdebug("backup", "found non sys_bound item: %@", item);
1398     return false;
1399 }
1400
1401 /* Delete all items from the current keychain.  If this is not an in
1402  place upgrade we don't delete items in the 'lockdown-identities'
1403  access group, this ensures that an import or restore of a backup
1404  will never overwrite an existing activation record. */
1405 static bool SecServerDeleteAll(SecDbConnectionRef dbt, CFErrorRef *error) {
1406     secwarning("SecServerDeleteAll");
1407
1408     return kc_transaction(dbt, error, ^{
1409
1410         bool ok = (SecDbExec(dbt, CFSTR("DELETE from genp;"), error) &&
1411                    SecDbExec(dbt, CFSTR("DELETE from inet;"), error) &&
1412                    SecDbExec(dbt, CFSTR("DELETE from cert;"), error) &&
1413                    SecDbExec(dbt, CFSTR("DELETE from keys;"), error));
1414         return ok;
1415     });
1416 }
1417
1418 #if TARGET_OS_IPHONE
1419
1420 static bool DeleteAllFromTableForMUSRView(SecDbConnectionRef dbt,
1421                                           CFStringRef sql,
1422                                           CFDataRef musr,
1423                                           bool keepU,
1424                                           CFErrorRef *error)
1425 {
1426     sqlite3_stmt *stmt = NULL;
1427     CFStringRef sql2 = NULL;
1428     bool ok = false;
1429
1430     if (keepU) {
1431         sql2 = CFStringCreateWithFormat(NULL, NULL, CFSTR("%@ AND pdmn NOT IN ('aku','akpu','cku','dku')"), sql);
1432     } else {
1433         sql2 = CFRetain(sql);
1434     }
1435     require(sql2, fail);
1436
1437     stmt = SecDbCopyStmt(dbt, sql2, NULL, error);
1438     require(stmt, fail);
1439
1440     ok = SecDbBindObject(stmt, 1, musr, error);
1441     require(ok, fail);
1442
1443     ok = SecDbStep(dbt, stmt, error, ^(bool *stop) { });
1444     require(ok, fail);
1445
1446 fail:
1447     if (stmt) {
1448         ok = SecDbFinalize(stmt, error);
1449     }
1450     if (!ok)
1451         secwarning("DeleteAllFromTableForMUSRView failed for %@ for musr: %@: %@", sql2, musr, error ? *error : NULL);
1452
1453     CFReleaseNull(sql2);
1454
1455     return ok;
1456 }
1457
1458 bool SecServerDeleteAllForUser(SecDbConnectionRef dbt, CFDataRef musrView, bool keepU, CFErrorRef *error) {
1459     secwarning("SecServerDeleteAllForUser for user: %@ keepU %s", musrView, keepU ? "yes" : "no");
1460
1461     return kc_transaction(dbt, error, ^{
1462         bool ok;
1463
1464         ok = (DeleteAllFromTableForMUSRView(dbt, CFSTR("DELETE FROM genp WHERE musr = ?"), musrView, keepU, error) &&
1465               DeleteAllFromTableForMUSRView(dbt, CFSTR("DELETE FROM inet WHERE musr = ?"), musrView, keepU, error) &&
1466               DeleteAllFromTableForMUSRView(dbt, CFSTR("DELETE FROM cert WHERE musr = ?"), musrView, keepU, error) &&
1467               DeleteAllFromTableForMUSRView(dbt, CFSTR("DELETE FROM keys WHERE musr = ?"), musrView, keepU, error));
1468
1469         return ok;
1470     });
1471 }
1472 #endif
1473
1474
1475 struct s3dl_export_row_ctx {
1476     struct s3dl_query_ctx qc;
1477     keybag_handle_t dest_keybag;
1478     enum SecItemFilter filter;
1479     bool multiUser;
1480 };
1481
1482 static void s3dl_export_row(sqlite3_stmt *stmt, void *context) {
1483     struct s3dl_export_row_ctx *c = context;
1484     Query *q = c->qc.q;
1485     SecAccessControlRef access_control = NULL;
1486     CFErrorRef localError = NULL;
1487
1488     /* Skip akpu items when backing up, those are intentionally lost across restores. The same applies to SEP-based keys */
1489     bool skip_akpu_or_token = c->filter == kSecBackupableItemFilter;
1490
1491     sqlite_int64 rowid = sqlite3_column_int64(stmt, 0);
1492     CFMutableDictionaryRef allAttributes = NULL;
1493     CFMutableDictionaryRef metadataAttributes = NULL;
1494     CFMutableDictionaryRef secretStuff = CFDictionaryCreateMutable(NULL, 0, &kCFTypeDictionaryKeyCallBacks, &kCFTypeDictionaryValueCallBacks);
1495     keyclass_t keyclass = 0;
1496     bool ok = s3dl_item_from_col(stmt, q, 1, c->qc.accessGroups, &allAttributes, &access_control, &keyclass, &localError);
1497
1498     if (ok) {
1499         metadataAttributes = CFDictionaryCreateMutableCopy(NULL, 0, allAttributes);
1500         SecDbForEachAttrWithMask(q->q_class, desc, kSecDbReturnDataFlag) {
1501             CFTypeRef value = CFDictionaryGetValue(metadataAttributes, desc->name);
1502             if (value) {
1503                 CFDictionarySetValue(secretStuff, desc->name, value);
1504                 CFDictionaryRemoveValue(metadataAttributes, desc->name);
1505             }
1506         }
1507     }
1508
1509     bool is_akpu = access_control ? CFEqualSafe(SecAccessControlGetProtection(access_control),
1510                                                 kSecAttrAccessibleWhenPasscodeSetThisDeviceOnly) : keyclass == key_class_akpu;
1511     bool is_token = (ok && allAttributes != NULL) ? CFDictionaryContainsKey(allAttributes, kSecAttrTokenID) : false;
1512
1513     if (ok && allAttributes && !(skip_akpu_or_token && (is_akpu || is_token))) {
1514         /* Only export sysbound items if do_sys_bound is true, only export non sysbound items otherwise. */
1515         bool do_sys_bound = c->filter == kSecSysBoundItemFilter;
1516         if (c->filter == kSecNoItemFilter ||
1517             SecItemIsSystemBound(allAttributes, q->q_class, c->multiUser) == do_sys_bound) {
1518             /* Re-encode the item. */
1519             secdebug("item", "export rowid %llu item: %@", rowid, allAttributes);
1520             /* The code below could be moved into handle_row. */
1521             CFDataRef pref = _SecItemCreatePersistentRef(q->q_class->name, rowid, allAttributes);
1522             if (pref) {
1523                 if (c->dest_keybag != KEYBAG_NONE) {
1524                     CFMutableDictionaryRef auth_attribs = CFDictionaryCreateMutable(NULL, 0, &kCFTypeDictionaryKeyCallBacks, &kCFTypeDictionaryValueCallBacks);
1525                     SecDbForEachAttrWithMask(q->q_class, desc, kSecDbInAuthenticatedDataFlag) {
1526                         CFTypeRef value = CFDictionaryGetValue(metadataAttributes, desc->name);
1527                         if(value) {
1528                             CFDictionaryAddValue(auth_attribs, desc->name, value);
1529                             CFDictionaryRemoveValue(metadataAttributes, desc->name);
1530                         }
1531                     }
1532
1533                     /* Encode and encrypt the item to the specified keybag. */
1534                     CFDataRef edata = NULL;
1535                     bool encrypted = ks_encrypt_data(c->dest_keybag, access_control, q->q_use_cred_handle, secretStuff, metadataAttributes, auth_attribs, &edata, false, &q->q_error);
1536                     CFDictionaryRemoveAllValues(allAttributes);
1537                     CFRelease(auth_attribs);
1538                     if (encrypted) {
1539                         CFDictionarySetValue(allAttributes, kSecValueData, edata);
1540                         CFReleaseSafe(edata);
1541                     } else {
1542                         seccritical("ks_encrypt_data %@,rowid=%" PRId64 ": failed: %@", q->q_class->name, rowid, q->q_error);
1543                         CFReleaseNull(q->q_error);
1544                     }
1545                 }
1546                 if (CFDictionaryGetCount(allAttributes)) {
1547                     CFDictionarySetValue(allAttributes, kSecValuePersistentRef, pref);
1548                     CFArrayAppendValue((CFMutableArrayRef)c->qc.result, allAttributes);
1549                     c->qc.found++;
1550                 }
1551                 CFReleaseSafe(pref);
1552             }
1553         }
1554     } else {
1555         OSStatus status = SecErrorGetOSStatus(localError);
1556
1557         if (status == errSecInteractionNotAllowed && is_akpu && skip_akpu_or_token) {
1558             // We expect akpu items to be inaccessible when the device is locked.
1559             CFReleaseNull(localError);
1560         } else {
1561             /* This happens a lot when trying to migrate keychain before first unlock, so only a notice */
1562             /* If the error is "corrupted item" then we just ignore it, otherwise we save it in the query */
1563             secinfo("item","Could not export item for rowid %llu: %@", rowid, localError);
1564
1565             if(status == errSecDecode) {
1566                 CFReleaseNull(localError);
1567             } else {
1568                 CFReleaseSafe(q->q_error);
1569                 q->q_error=localError;
1570             }
1571         }
1572     }
1573     CFReleaseNull(access_control);
1574     CFReleaseNull(allAttributes);
1575     CFReleaseNull(metadataAttributes);
1576     CFReleaseNull(secretStuff);
1577 }
1578
1579 static CFStringRef
1580 SecCreateKeybagUUID(keybag_handle_t keybag)
1581 {
1582 #if !TARGET_HAS_KEYSTORE
1583     return NULL;
1584 #else
1585     char uuidstr[37];
1586     uuid_t uuid;
1587     if (aks_get_bag_uuid(keybag, uuid) != KERN_SUCCESS)
1588         return NULL;
1589     uuid_unparse_lower(uuid, uuidstr);
1590     return CFStringCreateWithCString(NULL, uuidstr, kCFStringEncodingUTF8);
1591 #endif
1592 }
1593
1594
1595 CFDictionaryRef
1596 SecServerCopyKeychainPlist(SecDbConnectionRef dbt,
1597                            SecurityClient *client,
1598                            keybag_handle_t src_keybag,
1599                            keybag_handle_t dest_keybag,
1600                            enum SecItemFilter filter,
1601                            CFErrorRef *error) {
1602     CFMutableDictionaryRef keychain;
1603     keychain = CFDictionaryCreateMutable(kCFAllocatorDefault, 0,
1604                                          &kCFTypeDictionaryKeyCallBacks, &kCFTypeDictionaryValueCallBacks);
1605     unsigned class_ix;
1606     bool inMultiUser = false;
1607     CFStringRef keybaguuid = NULL;
1608     Query q = { .q_keybag = src_keybag,
1609         .q_musrView = NULL
1610     };
1611
1612     if (!keychain) {
1613         if (error && !*error)
1614             SecError(errSecAllocate, error, CFSTR("Can't create keychain dictionary"));
1615         goto errOut;
1616     }
1617
1618     q.q_return_type =
1619         kSecReturnDataMask |
1620         kSecReturnAttributesMask |
1621         kSecReturnPersistentRefMask;
1622     q.q_limit = kSecMatchUnlimited;
1623     q.q_skip_acl_items = true;
1624
1625
1626 #if TARGET_OS_IPHONE
1627     if (client && client->inMultiUser) {
1628         q.q_musrView = SecMUSRCreateActiveUserUUID(client->uid);
1629         inMultiUser = true;
1630     } else
1631 #endif
1632     {
1633         q.q_musrView = SecMUSRGetSingleUserKeychainUUID();
1634         CFRetain(q.q_musrView);
1635
1636         keybaguuid = SecCreateKeybagUUID(dest_keybag);
1637         if (keybaguuid)
1638             CFDictionarySetValue(keychain, kSecBackupKeybagUUIDKey, keybaguuid);
1639     }
1640
1641     /* Get rid of this duplicate. */
1642     const SecDbClass *SecDbClasses[] = {
1643         genp_class(),
1644         inet_class(),
1645         cert_class(),
1646         keys_class()
1647     };
1648
1649     for (class_ix = 0; class_ix < array_size(SecDbClasses);
1650          ++class_ix) {
1651         q.q_class = SecDbClasses[class_ix];
1652         struct s3dl_export_row_ctx ctx = {
1653             .qc = { .q = &q, .dbt = dbt },
1654             .dest_keybag = dest_keybag, .filter = filter,
1655             .multiUser = inMultiUser,
1656         };
1657
1658         secnotice("item", "exporting class '%@'", q.q_class->name);
1659
1660         CFErrorRef localError = NULL;
1661         if (s3dl_query(s3dl_export_row, &ctx, &localError)) {
1662             if (CFArrayGetCount(ctx.qc.result)) {
1663                 SecSignpostBackupCount(SecSignpostImpulseBackupClassCount, q.q_class->name, CFArrayGetCount(ctx.qc.result), filter);
1664                 CFDictionaryAddValue(keychain, q.q_class->name, ctx.qc.result);
1665             }
1666
1667         } else {
1668             OSStatus status = (OSStatus)CFErrorGetCode(localError);
1669             if (status == errSecItemNotFound) {
1670                 CFRelease(localError);
1671             } else {
1672                 secerror("Export failed: %@", localError);
1673                 if (error) {
1674                     CFReleaseSafe(*error);
1675                     *error = localError;
1676                 } else {
1677                     CFRelease(localError);
1678                 }
1679                 CFReleaseNull(keychain);
1680                 CFReleaseNull(ctx.qc.result);
1681                 break;
1682             }
1683         }
1684         CFReleaseNull(ctx.qc.result);
1685     }
1686
1687 errOut:
1688     CFReleaseNull(q.q_musrView);
1689     CFReleaseNull(keybaguuid);
1690
1691     return keychain;
1692 }
1693
1694 struct SecServerImportClassState {
1695         SecDbConnectionRef dbt;
1696     CFErrorRef error;
1697     keybag_handle_t src_keybag;
1698     keybag_handle_t dest_keybag;
1699     SecurityClient *client;
1700     enum SecItemFilter filter;
1701 };
1702
1703 struct SecServerImportItemState {
1704     const SecDbClass *class;
1705         struct SecServerImportClassState *s;
1706 };
1707
1708 static void
1709 SecServerImportItem(const void *value, void *context)
1710 {
1711     struct SecServerImportItemState *state = (struct SecServerImportItemState *)context;
1712     bool inMultiUser = false;
1713 #if TARGET_OS_IPHONE
1714     if (state->s->client->inMultiUser)
1715         inMultiUser = true;
1716 #endif
1717
1718     if (state->s->error)
1719         return;
1720
1721     if (!isDictionary(value)) {
1722         SecError(errSecParam, &state->s->error, CFSTR("value %@ is not a dictionary"), value);
1723         return;
1724     }
1725
1726     CFDictionaryRef dict = (CFDictionaryRef)value;
1727
1728     secdebug("item", "Import Item : %@", dict);
1729
1730     /* We use the kSecSysBoundItemFilter to indicate that we don't
1731      * preserve rowid's during import.
1732      */
1733     if (state->s->filter == kSecBackupableItemFilter) {
1734         CFTypeRef pdmu;
1735
1736         /* We don't filter non sys_bound items during import since we know we
1737          * will never have any in this case.
1738          */
1739         if (SecItemIsSystemBound(dict, state->class, inMultiUser))
1740             return;
1741
1742         /*
1743          * Don't bother with u items when in edu mode since our current backup system
1744          * don't keep track of items that blongs to the device (u) but rather just
1745          * merge them into one blob.
1746          */
1747         if (inMultiUser && (pdmu = CFDictionaryGetValue(dict, kSecAttrAccessible))) {
1748             if (CFEqual(pdmu, kSecAttrAccessibleWhenUnlockedThisDeviceOnly)     ||
1749                 CFEqual(pdmu, kSecAttrAccessibleAfterFirstUnlockThisDeviceOnly) ||
1750                 CFEqual(pdmu, kSecAttrAccessibleWhenUnlockedThisDeviceOnly)     ||
1751                 CFEqual(pdmu, kSecAttrAccessibleWhenPasscodeSetThisDeviceOnly))
1752             {
1753                 secdebug("item", "Skipping KU item : %@", dict);
1754                 return;
1755             }
1756         }
1757
1758         /* Avoid importing token-based items.  Although newer backups should not have them,
1759          * older (iOS9, iOS10.0) produced backups with token-based items.
1760          */
1761         if (CFDictionaryContainsKey(dict, kSecAttrTokenID)) {
1762             secdebug("item", "Skipping token-based item : %@", dict);
1763             return;
1764         }
1765     }
1766
1767     SecDbItemRef item;
1768
1769     /* This is sligthly confusing:
1770      - During upgrade all items are exported with KEYBAG_NONE.
1771      - During restore from backup, existing sys_bound items are exported with KEYBAG_NONE, and are exported as dictionary of attributes.
1772      - Item in the actual backup are export with a real keybag, and are exported as encrypted v_Data and v_PersistentRef
1773      */
1774     if (state->s->src_keybag == KEYBAG_NONE) {
1775         item = SecDbItemCreateWithAttributes(kCFAllocatorDefault, state->class, dict, state->s->dest_keybag,  &state->s->error);
1776     } else {
1777         item = SecDbItemCreateWithBackupDictionary(kCFAllocatorDefault, state->class, dict, state->s->src_keybag, state->s->dest_keybag, &state->s->error);
1778     }
1779
1780     /*
1781      *
1782      */
1783
1784     if (item) {
1785         CFDataRef musr = NULL;
1786         CFDataRef musrBackup = CFDictionaryGetValue(dict, kSecAttrMultiUser);
1787         CFDataRef systemKeychainUUID = SecMUSRGetSystemKeychainUUID();
1788         bool systemKeychain = CFEqualSafe(musrBackup, systemKeychainUUID);
1789
1790 #if TARGET_OS_IPHONE
1791         if (state->s->client && state->s->client->inMultiUser) {
1792             if (systemKeychain) {
1793                 secwarning("system keychain not allowed in multi user mode for item: %@", item);
1794             } else {
1795                 musr = SecMUSRCreateActiveUserUUID(state->s->client->uid);
1796             }
1797         } else
1798 #endif
1799         {
1800             if (systemKeychain) {
1801                 musr = SecMUSRCopySystemKeychainUUID();
1802             } else {
1803                 musr = SecMUSRGetSingleUserKeychainUUID();
1804                 CFRetainSafe(musr);
1805             }
1806         }
1807         if (musr == NULL) {
1808             CFReleaseNull(item);
1809         } else {
1810             SecDbItemSetValueWithName(item, CFSTR("musr"), musr, &state->s->error);
1811             CFRelease(musr);
1812         }
1813     }
1814
1815     /*
1816      *
1817      */
1818
1819     if (item) {
1820         bool insertStatus;
1821
1822         if(state->s->filter != kSecSysBoundItemFilter) {
1823             SecDbItemExtractRowIdFromBackupDictionary(item, dict, &state->s->error);
1824         }
1825         SecDbItemInferSyncable(item, &state->s->error);
1826         insertStatus = SecDbItemInsert(item, state->s->dbt, &state->s->error);
1827         if (!insertStatus) {
1828             /*
1829              When running in EduMode, multiple users share the same
1830              keychain and unfortionaly the rowid is used a
1831              persistant reference and is part of the contraints (its
1832              UNIQUE), so lets clear the rowid and try to insert the
1833              entry again.
1834
1835              This even happens for normal operation because of
1836              SysBound entries, so in case of a failure, lets try
1837              again to insert the record.
1838              */
1839             SecDbItemClearRowId(item, NULL);
1840             SecDbItemInsert(item, state->s->dbt, &state->s->error);
1841         }
1842         }
1843
1844     /* Reset error if we had one, since we just skip the current item
1845      and continue importing what we can. */
1846     if (state->s->error) {
1847         secwarning("Failed to import an item (%@) of class '%@': %@ - ignoring error.",
1848                    item, state->class->name, state->s->error);
1849         CFReleaseNull(state->s->error);
1850     }
1851
1852     CFReleaseSafe(item);
1853 }
1854
1855 static void SecServerImportClass(const void *key, const void *value,
1856                                  void *context) {
1857     struct SecServerImportClassState *state =
1858     (struct SecServerImportClassState *)context;
1859     if (state->error)
1860         return;
1861     if (!isString(key)) {
1862         SecError(errSecParam, &state->error, CFSTR("class name %@ is not a string"), key);
1863         return;
1864     }
1865     /* ignore the Keybag UUID */
1866     if (CFEqual(key, kSecBackupKeybagUUIDKey))
1867         return;
1868     const SecDbClass *class = kc_class_with_name(key);
1869     if (!class) {
1870         secwarning("Ignoring unknown key class '%@'", key);
1871         return;
1872     }
1873     if (class == identity_class()) {
1874         SecError(errSecParam, &state->error, CFSTR("attempt to import an identity"));
1875         return;
1876     }
1877     struct SecServerImportItemState item_state = {
1878         .class = class, .s = state,
1879     };
1880     if (isArray(value)) {
1881         CFArrayRef items = (CFArrayRef)value;
1882         secwarning("Import %ld items of class %@ (filter %d)", (long)CFArrayGetCount(items), key, state->filter);
1883         SecSignpostBackupCount(SecSignpostImpulseRestoreClassCount, class->name, CFArrayGetCount(items), state->filter);
1884         CFArrayApplyFunction(items, CFRangeMake(0, CFArrayGetCount(items)),
1885                              SecServerImportItem, &item_state);
1886     } else if (isDictionary(value)) {
1887         CFDictionaryRef item = (CFDictionaryRef)value;
1888         secwarning("Import %ld items of class %@ (filter %d)", (long)1, key, state->filter);
1889         SecSignpostBackupCount(SecSignpostImpulseRestoreClassCount, class->name, 1, state->filter);
1890         SecServerImportItem(item, &item_state);
1891     } else {
1892         secwarning("Unknown value type for class %@ (filter %d)", key, state->filter);
1893     }
1894 }
1895
1896 bool SecServerImportKeychainInPlist(SecDbConnectionRef dbt, SecurityClient *client,
1897                                             keybag_handle_t src_keybag, keybag_handle_t dest_keybag,
1898                                             CFDictionaryRef keychain, enum SecItemFilter filter,
1899                                             bool removeKeychainContent, CFErrorRef *error) {
1900     CFStringRef keybaguuid = NULL;
1901     bool ok = true;
1902
1903     CFDictionaryRef sys_bound = NULL;
1904     if (filter == kSecBackupableItemFilter) {
1905         /* Grab a copy of all the items for which SecItemIsSystemBound()
1906          returns true. */
1907         require(sys_bound = SecServerCopyKeychainPlist(dbt, client, KEYBAG_DEVICE,
1908                                                        KEYBAG_NONE, kSecSysBoundItemFilter,
1909                                                        error), errOut);
1910     }
1911
1912     /*
1913      * Validate the uuid of the source keybag matches what we have in the backup
1914      */
1915     keybaguuid = SecCreateKeybagUUID(src_keybag);
1916     if (keybaguuid) {
1917         CFStringRef uuid = CFDictionaryGetValue(keychain, kSecBackupKeybagUUIDKey);
1918         if (isString(uuid)) {
1919             require_action(CFEqual(keybaguuid, uuid), errOut,
1920                            SecError(errSecDecode, error, CFSTR("Keybag UUID (%@) mismatch with backup (%@)"),
1921                                     keybaguuid, uuid));
1922         }
1923     }
1924
1925 #if TARGET_OS_IOS
1926     /*
1927      * Shared iPad is very special, it always delete's the user keychain, and never merge content
1928      */
1929     if (client->inMultiUser) {
1930         CFDataRef musrView = SecMUSRCreateActiveUserUUID(client->uid);
1931         require_action(musrView, errOut, ok = false);
1932         require_action(ok = SecServerDeleteAllForUser(dbt, musrView, true, error), errOut, CFReleaseNull(musrView));
1933         CFReleaseNull(musrView);
1934     } else
1935 #endif
1936     {
1937         /*
1938          * Delete everything in the keychain.
1939          * We don't want this if we're restoring backups because we probably already synced stuff over
1940          */
1941         if (removeKeychainContent) {
1942             require(ok = SecServerDeleteAll(dbt, error), errOut);
1943         } else {
1944             // Custom hack to support bluetooth's workflow for 11.3. Should be removed in a future release.
1945             __block CFErrorRef btError = NULL;
1946             bool deletedBT = kc_transaction(dbt, &btError, ^bool{
1947                 bool ok = SecDbExec(dbt, CFSTR("DELETE FROM genp WHERE sync = 0 AND NOT agrp IN ('com.apple.security.sos', 'com.apple.security.sos-usercredential', 'com.apple.security.ckks');"), &btError);
1948                 ok &= SecDbExec(dbt, CFSTR("DELETE FROM inet WHERE sync = 0 AND NOT agrp IN ('com.apple.security.sos', 'com.apple.security.sos-usercredential', 'com.apple.security.ckks');"), &btError);
1949                 ok &= SecDbExec(dbt, CFSTR("DELETE FROM cert WHERE sync = 0 AND NOT agrp IN ('com.apple.security.sos', 'com.apple.security.sos-usercredential', 'com.apple.security.ckks');"), &btError);
1950                 ok &= SecDbExec(dbt, CFSTR("DELETE FROM keys WHERE sync = 0 AND NOT agrp IN ('com.apple.security.sos', 'com.apple.security.sos-usercredential', 'com.apple.security.ckks');"), &btError);
1951                 return ok;
1952             });
1953             if (!deletedBT) {
1954                 secerror("Unable to delete nonsyncable items prior to keychain restore: %@", btError);
1955             } else {
1956                 secnotice("restore", "Successfully deleted nonsyncable items");
1957             }
1958             CFReleaseNull(btError);
1959         }
1960     }
1961
1962     struct SecServerImportClassState state = {
1963         .dbt = dbt,
1964         .src_keybag = src_keybag,
1965         .dest_keybag = dest_keybag,
1966         .client = client,
1967         .filter = filter,
1968     };
1969     /* Import the provided items, preserving rowids. */
1970     secwarning("Restoring backup items '%ld'", (long)CFDictionaryGetCount(keychain));
1971     CFDictionaryApplyFunction(keychain, SecServerImportClass, &state);
1972
1973     if (sys_bound) {
1974         state.src_keybag = KEYBAG_NONE;
1975         /* Import the items we preserved with random rowids. */
1976         state.filter = kSecSysBoundItemFilter;
1977         secwarning("Restoring sysbound items '%ld'", (long)CFDictionaryGetCount(sys_bound));
1978         CFDictionaryApplyFunction(sys_bound, SecServerImportClass, &state);
1979     }
1980     if (state.error) {
1981         if (error) {
1982             CFReleaseSafe(*error);
1983             *error = state.error;
1984         } else {
1985             CFRelease(state.error);
1986         }
1987         ok = false;
1988     }
1989
1990     // If CKKS had spun up, it's very likely that we just deleted its data.
1991     // Tell it to perform a local resync.
1992     SecCKKSPerformLocalResync();
1993
1994 errOut:
1995     CFReleaseSafe(sys_bound);
1996     CFReleaseSafe(keybaguuid);
1997
1998     return ok;
1999 }
2000
2001 CFStringRef
2002 SecServerBackupGetKeybagUUID(CFDictionaryRef keychain, CFErrorRef *error)
2003 {
2004     CFStringRef uuid = CFDictionaryGetValue(keychain, kSecBackupKeybagUUIDKey);
2005     if (!isString(uuid)) {
2006         SecError(errSecDecode, error, CFSTR("Missing or invalid %@ in backup dictionary"), kSecBackupKeybagUUIDKey);
2007         return NULL;
2008     }
2009     return uuid;
2010 }
2011
2012 #pragma mark - key rolling support
2013 #if USE_KEYSTORE
2014
2015 struct check_generation_ctx {
2016     struct s3dl_query_ctx query_ctx;
2017     uint32_t current_generation;
2018 };
2019
2020 static void check_generation(sqlite3_stmt *stmt, void *context) {
2021     struct check_generation_ctx *c = context;
2022     CFDataRef blob = NULL;
2023     size_t blobLen = 0;
2024     const uint8_t *cursor = NULL;
2025     uint32_t version;
2026     keyclass_t keyclass;
2027     uint32_t current_generation = c->current_generation;
2028
2029     require(blob = s3dl_copy_data_from_col(stmt, 1, &c->query_ctx.q->q_error), out);
2030     blobLen = CFDataGetLength(blob);
2031     cursor = CFDataGetBytePtr(blob);
2032
2033     /* Check for underflow, ensuring we have at least one full AES block left. */
2034     if (blobLen < sizeof(version) + sizeof(keyclass)) {
2035         SecError(errSecDecode, &c->query_ctx.q->q_error, CFSTR("check_generation: Check for underflow"));
2036         goto out;
2037     }
2038
2039     version = *((uint32_t *)cursor);
2040     cursor += sizeof(version);
2041
2042     (void) version; // TODO: do something with the version number.
2043
2044     keyclass = *((keyclass_t *)cursor);
2045
2046     // TODO: export get_key_gen macro
2047     if (((keyclass & ~key_class_last) == 0) != (current_generation == 0)) {
2048         c->query_ctx.found++;
2049     }
2050
2051     CFReleaseSafe(blob);
2052     return;
2053
2054 out:
2055     c->query_ctx.found++;
2056     CFReleaseSafe(blob);
2057 }
2058
2059 bool s3dl_dbt_keys_current(SecDbConnectionRef dbt, uint32_t current_generation, CFErrorRef *error) {
2060     CFErrorRef localError = NULL;
2061     struct check_generation_ctx ctx = { .query_ctx = { .dbt = dbt }, .current_generation = current_generation };
2062
2063     const SecDbClass *classes[] = {
2064         genp_class(),
2065         inet_class(),
2066         keys_class(),
2067         cert_class(),
2068     };
2069
2070     for (size_t class_ix = 0; class_ix < array_size(classes); ++class_ix) {
2071         Query *q = query_create(classes[class_ix], NULL, NULL, &localError);
2072         if (!q)
2073             return false;
2074
2075         ctx.query_ctx.q = q;
2076         q->q_limit = kSecMatchUnlimited;
2077
2078         bool ok = s3dl_query(check_generation, &ctx, &localError);
2079         query_destroy(q, NULL);
2080         CFReleaseNull(ctx.query_ctx.result);
2081
2082         if (!ok && localError && (CFErrorGetCode(localError) == errSecItemNotFound)) {
2083             CFReleaseNull(localError);
2084             continue;
2085         }
2086         secerror("Class %@ not up to date", classes[class_ix]->name);
2087         return false;
2088     }
2089     return true;
2090 }
2091
2092 bool s3dl_dbt_update_keys(SecDbConnectionRef dbt, SecurityClient *client, CFErrorRef *error) {
2093     return SecDbTransaction(dbt, kSecDbExclusiveTransactionType, error, ^(bool *commit) {
2094         __block bool ok = false;
2095         uint32_t keystore_generation_status;
2096
2097         /* can we migrate to new class keys right now? */
2098         if (!aks_generation(KEYBAG_DEVICE, generation_noop, &keystore_generation_status) &&
2099             (keystore_generation_status & generation_change_in_progress)) {
2100
2101             /* take a lock assertion */
2102             bool operated_while_unlocked = SecAKSDoWhileUserBagLocked(error, ^{
2103                 CFErrorRef localError = NULL;
2104                 CFDictionaryRef backup = SecServerCopyKeychainPlist(dbt, NULL,
2105                                                                     KEYBAG_DEVICE, KEYBAG_NONE, kSecNoItemFilter, &localError);
2106                 if (backup) {
2107                     if (localError) {
2108                         secerror("Ignoring export error: %@ during roll export", localError);
2109                         CFReleaseNull(localError);
2110                     }
2111                     // 'true' argument: we're replacing everything with newly wrapped entries so remove the old stuff
2112                     ok = SecServerImportKeychainInPlist(dbt, client, KEYBAG_NONE,
2113                                                         KEYBAG_DEVICE, backup, kSecNoItemFilter, true, &localError);
2114                     if (localError) {
2115                         secerror("Ignoring export error: %@ during roll export", localError);
2116                         CFReleaseNull(localError);
2117                     }
2118                     CFRelease(backup);
2119                 }
2120             });
2121             if (!operated_while_unlocked)
2122                 ok = false;
2123         } else {
2124             ok = SecError(errSecBadReq, error, CFSTR("No key roll in progress."));
2125         }
2126
2127         *commit = ok;
2128     });
2129 }
2130 #endif