OSX/sec/securityd/SecItemDb.c

   1 /*
   2  * Copyright (c) 2006-2014 Apple Inc. All Rights Reserved.
   3  *
   4  * @APPLE_LICENSE_HEADER_START@
   5  *
   6  * This file contains Original Code and/or Modifications of Original Code
   7  * as defined in and that are subject to the Apple Public Source License
   8  * Version 2.0 (the 'License'). You may not use this file except in
   9  * compliance with the License. Please obtain a copy of the License at
  10  * http://www.opensource.apple.com/apsl/ and read it before using this
  11  * file.
  12  *
  13  * The Original Code and all software distributed under the License are
  14  * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  15  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  16  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
  17  * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
  18  * Please see the License for the specific language governing rights and
  19  * limitations under the License.
  20  *
  21  * @APPLE_LICENSE_HEADER_END@
  22  */
  23
  24 /*
  25  *  SecItemDb.c - CoreFoundation-based constants and functions for
  26     access to Security items (certificates, keys, identities, and
  27     passwords.)
  28  */
  29
  30 #include <securityd/SecItemDb.h>
  31 #include <utilities/SecAKSWrappers.h>
  32
  33 #include <securityd/SecDbKeychainItem.h>
  34 #include <securityd/SecItemSchema.h>
  35 #include <securityd/SecItemServer.h>
  36 #include <Security/SecAccessControlPriv.h>
  37 #include <Security/SecBasePriv.h>
  38 #include <Security/SecItem.h>
  39 #include <Security/SecSignpost.h>
  40 #include <Security/SecItemPriv.h>
  41 #include <Security/SecItemInternal.h>
  42 #include <securityd/SOSCloudCircleServer.h>
  43 #include <utilities/array_size.h>
  44 #include <utilities/SecIOFormat.h>
  45 #include <utilities/SecCFCCWrappers.h>
  46 #include <SecAccessControlPriv.h>
  47 #include <uuid/uuid.h>
  48 #include "sec_action.h"
  49
  50 #include "keychain/ckks/CKKS.h"
  51
  52 #define kSecBackupKeybagUUIDKey CFSTR("keybag-uuid")
  53
  54 const SecDbAttr *SecDbAttrWithKey(const SecDbClass *c,
  55                                   CFTypeRef key,
  56                                   CFErrorRef *error) {
  57     /* Special case: identites can have all attributes of either cert
  58      or keys. */
  59     if (c == identity_class()) {
  60         const SecDbAttr *desc;
  61         if (!(desc = SecDbAttrWithKey(cert_class(), key, 0)))
  62             desc = SecDbAttrWithKey(keys_class(), key, error);
  63         return desc;
  64     }
  65
  66     if (isString(key)) {
  67         SecDbForEachAttr(c, a) {
  68             if (CFEqual(a->name, key))
  69                 return a;
  70         }
  71         if (CFEqual(kSecAttrNoLegacy, key)) {
  72             return NULL; /* results in no ops for this attribute */
  73         }
  74     }
  75
  76     SecError(errSecNoSuchAttr, error, CFSTR("attribute %@ not found in class %@"), key, c->name);
  77
  78     return NULL;
  79 }
  80
  81 bool kc_transaction(SecDbConnectionRef dbt, CFErrorRef *error, bool(^perform)(void)) {
  82     return kc_transaction_type(dbt, kSecDbExclusiveTransactionType, error, perform);
  83 }
  84
  85 bool kc_transaction_type(SecDbConnectionRef dbt, SecDbTransactionType type, CFErrorRef *error, bool(^perform)(void)) {
  86     __block bool ok = true;
  87     return ok && SecDbTransaction(dbt, type, error, ^(bool *commit) {
  88         ok = *commit = perform();
  89     });
  90 }
  91
  92 static CFStringRef SecDbGetKindSQL(SecDbAttrKind kind) {
  93     switch (kind) {
  94         case kSecDbBlobAttr:
  95         case kSecDbDataAttr:
  96         case kSecDbUUIDAttr:
  97         case kSecDbSHA1Attr:
  98         case kSecDbPrimaryKeyAttr:
  99         case kSecDbEncryptedDataAttr:
 100             return CFSTR("BLOB");
 101         case kSecDbAccessAttr:
 102         case kSecDbStringAttr:
 103             return CFSTR("TEXT");
 104         case kSecDbNumberAttr:
 105         case kSecDbSyncAttr:
 106         case kSecDbTombAttr:
 107             return CFSTR("INTEGER");
 108         case kSecDbDateAttr:
 109         case kSecDbCreationDateAttr:
 110         case kSecDbModificationDateAttr:
 111             return CFSTR("REAL");
 112         case kSecDbRowIdAttr:
 113             return CFSTR("INTEGER PRIMARY KEY AUTOINCREMENT");
 114         case kSecDbAccessControlAttr:
 115         case kSecDbUTombAttr:
 116             /* This attribute does not exist in the DB. */
 117             return NULL;
 118     }
 119 }
 120
 121 static void SecDbAppendUnique(CFMutableStringRef sql, CFStringRef value, bool *haveUnique) {
 122     assert(haveUnique);
 123     if (!*haveUnique)
 124         CFStringAppend(sql, CFSTR("UNIQUE("));
 125
 126     SecDbAppendElement(sql, value, haveUnique);
 127 }
 128
 129 static void SecDbAppendCreateTableWithClass(CFMutableStringRef sql, const SecDbClass *c) {
 130     CFStringAppendFormat(sql, 0, CFSTR("CREATE TABLE %@("), c->name);
 131     SecDbForEachAttrWithMask(c,desc,kSecDbInFlag) {
 132         CFStringAppendFormat(sql, 0, CFSTR("%@ %@"), desc->name, SecDbGetKindSQL(desc->kind));
 133         if (desc->flags & kSecDbNotNullFlag)
 134             CFStringAppend(sql, CFSTR(" NOT NULL"));
 135         if (desc->flags & kSecDbDefault0Flag)
 136             CFStringAppend(sql, CFSTR(" DEFAULT 0"));
 137         if (desc->flags & kSecDbDefaultEmptyFlag)
 138             CFStringAppend(sql, CFSTR(" DEFAULT ''"));
 139         CFStringAppend(sql, CFSTR(","));
 140     }
 141
 142     bool haveUnique = false;
 143     SecDbForEachAttrWithMask(c,desc,kSecDbPrimaryKeyFlag | kSecDbInFlag) {
 144         SecDbAppendUnique(sql, desc->name, &haveUnique);
 145     }
 146     if (haveUnique)
 147         CFStringAppend(sql, CFSTR(")"));
 148
 149     CFStringAppend(sql, CFSTR(");"));
 150
 151     // Create indices
 152     SecDbForEachAttrWithMask(c,desc, kSecDbIndexFlag | kSecDbInFlag) {
 153         if (desc->kind == kSecDbSyncAttr) {
 154             CFStringAppendFormat(sql, 0, CFSTR("CREATE INDEX %@%@0 ON %@(%@) WHERE %@=0;"), c->name, desc->name, c->name, desc->name, desc->name);
 155         } else {
 156             CFStringAppendFormat(sql, 0, CFSTR("CREATE INDEX %@%@ ON %@(%@);"), c->name, desc->name, c->name, desc->name);
 157         }
 158     }
 159 }
 160
 161 static void SecDbAppendDropTableWithClass(CFMutableStringRef sql, const SecDbClass *c) {
 162     CFStringAppendFormat(sql, 0, CFSTR("DROP TABLE %@;"), c->name);
 163 }
 164
 165 static CFDataRef SecPersistentRefCreateWithItem(SecDbItemRef item, CFErrorRef *error) {
 166     sqlite3_int64 row_id = SecDbItemGetRowId(item, error);
 167     if (row_id)
 168         return _SecItemCreatePersistentRef(SecDbItemGetClass(item)->name, row_id, item->attributes);
 169     return NULL;
 170 }
 171
 172 bool SecItemDbCreateSchema(SecDbConnectionRef dbt, const SecDbSchema *schema, CFArrayRef classIndexesForNewTables, bool includeVersion, CFErrorRef *error)
 173 {
 174     __block bool ok = true;
 175     CFMutableStringRef sql = CFStringCreateMutable(kCFAllocatorDefault, 0);
 176
 177     if (classIndexesForNewTables) {
 178         CFArrayForEach(classIndexesForNewTables, ^(const void* index) {
 179             const SecDbClass* class = schema->classes[(int)index];
 180             SecDbAppendCreateTableWithClass(sql, class);
 181         });
 182     }
 183     else {
 184         for (const SecDbClass * const *pclass = schema->classes; *pclass; ++pclass) {
 185             SecDbAppendCreateTableWithClass(sql, *pclass);
 186         }
 187     }
 188
 189     if (includeVersion) {
 190         CFStringAppendFormat(sql, NULL, CFSTR("INSERT INTO tversion(version,minor) VALUES(%d, %d);"),
 191                              schema->majorVersion, schema->minorVersion);
 192     }
 193     CFStringPerformWithCString(sql, ^(const char *sql_string) {
 194         ok = SecDbErrorWithDb(sqlite3_exec(SecDbHandle(dbt), sql_string, NULL, NULL, NULL),
 195                               SecDbHandle(dbt), error, CFSTR("sqlite3_exec: %s"), sql_string);
 196     });
 197     CFReleaseSafe(sql);
 198     return ok;
 199 }
 200
 201 bool SecItemDbDeleteSchema(SecDbConnectionRef dbt, const SecDbSchema *schema, CFErrorRef *error)
 202 {
 203     __block bool ok = true;
 204     CFMutableStringRef sql = CFStringCreateMutable(kCFAllocatorDefault, 0);
 205     for (const SecDbClass * const *pclass = schema->classes; *pclass; ++pclass) {
 206         SecDbAppendDropTableWithClass(sql, *pclass);
 207     }
 208     CFStringPerformWithCString(sql, ^(const char *sql_string) {
 209         ok = SecDbErrorWithDb(sqlite3_exec(SecDbHandle(dbt), sql_string, NULL, NULL, NULL),
 210                               SecDbHandle(dbt), error, CFSTR("sqlite3_exec: %s"), sql_string);
 211     });
 212     CFReleaseSafe(sql);
 213     return ok;
 214 }
 215
 216 CFTypeRef SecDbItemCopyResult(SecDbItemRef item, ReturnTypeMask return_type, CFErrorRef *error) {
 217     CFTypeRef a_result;
 218
 219         if (return_type == 0) {
 220                 /* Caller isn't interested in any results at all. */
 221                 a_result = kCFNull;
 222         } else if (return_type == kSecReturnDataMask) {
 223         a_result = SecDbItemGetCachedValueWithName(item, kSecValueData);
 224         if (a_result) {
 225             CFRetainSafe(a_result);
 226         } else {
 227             a_result = CFDataCreate(kCFAllocatorDefault, NULL, 0);
 228         }
 229         } else if (return_type == kSecReturnPersistentRefMask) {
 230                 a_result = SecPersistentRefCreateWithItem(item, error);
 231         } else {
 232         CFMutableDictionaryRef dict = CFDictionaryCreateMutableForCFTypes(CFGetAllocator(item));
 233                 /* We need to return more than one value. */
 234         if (return_type & kSecReturnRefMask) {
 235             CFDictionarySetValue(dict, kSecClass, SecDbItemGetClass(item)->name);
 236         }
 237         CFOptionFlags mask = (((return_type & kSecReturnDataMask || return_type & kSecReturnRefMask) ? kSecDbReturnDataFlag : 0) |
 238                               ((return_type & kSecReturnAttributesMask || return_type & kSecReturnRefMask) ? kSecDbReturnAttrFlag : 0));
 239         SecDbForEachAttr(SecDbItemGetClass(item), desc) {
 240             if ((desc->flags & mask) != 0) {
 241                 CFTypeRef value = SecDbItemGetValue(item, desc, error);
 242                 if (value && !CFEqual(kCFNull, value)) {
 243                     CFDictionarySetValue(dict, desc->name, value);
 244                 } else if (value == NULL) {
 245                     CFReleaseNull(dict);
 246                     break;
 247                 }
 248             }
 249         }
 250         CFDictionaryRemoveValue(dict, kSecAttrUUID);
 251
 252                 if (return_type & kSecReturnPersistentRefMask) {
 253             CFDataRef pref = SecPersistentRefCreateWithItem(item, error);
 254                         CFDictionarySetValue(dict, kSecValuePersistentRef, pref);
 255             CFReleaseSafe(pref);
 256                 }
 257
 258                 a_result = dict;
 259         }
 260
 261         return a_result;
 262 }
 263
 264 /* AUDIT[securityd](done):
 265  attributes (ok) is a caller provided dictionary, only its cf type has
 266  been checked.
 267  */
 268 bool
 269 s3dl_query_add(SecDbConnectionRef dbt, Query *q, CFTypeRef *result, CFErrorRef *error)
 270 {
 271     if (query_match_count(q) != 0)
 272         return errSecItemMatchUnsupported;
 273
 274     /* Add requires a class to be specified unless we are adding a ref. */
 275     if (q->q_use_item_list)
 276         return errSecUseItemListUnsupported;
 277
 278     /* Actual work here. */
 279     SecDbItemRef item = SecDbItemCreateWithAttributes(kCFAllocatorDefault, q->q_class, q->q_item, KEYBAG_DEVICE, error);
 280     if (!item)
 281         return false;
 282     if (SecDbItemIsTombstone(item))
 283         SecDbItemSetValue(item, &v7utomb, q->q_use_tomb ? q->q_use_tomb : kCFBooleanTrue, NULL);
 284
 285     bool ok = true;
 286     if (q->q_data)
 287         ok = SecDbItemSetValueWithName(item, CFSTR("v_Data"), q->q_data, error);
 288     if (q->q_row_id)
 289         ok = SecDbItemSetRowId(item, q->q_row_id, error);
 290     if (q->q_musrView)
 291         ok = SecDbItemSetValueWithName(item, CFSTR("musr"), q->q_musrView, error);
 292     SecDbItemSetCredHandle(item, q->q_use_cred_handle);
 293
 294 #if OCTAGON
 295     if(SecCKKSIsEnabled() && !SecCKKSTestDisableAutomaticUUID()) {
 296         s3dl_item_make_new_uuid(item, q->q_uuid_from_primary_key, error);
 297
 298         if(q->q_add_sync_callback) {
 299             CFTypeRef uuid = SecDbItemGetValue(item, &v10itemuuid, error);
 300             if(uuid) {
 301                 CKKSRegisterSyncStatusCallback(uuid, q->q_add_sync_callback);
 302             } else {
 303                 secerror("Couldn't fetch UUID from item; can't call callback");
 304             }
 305         }
 306     }
 307 #endif
 308
 309     if (ok)
 310         ok = SecDbItemInsert(item, dbt, error);
 311
 312     if (ok) {
 313         if (result && q->q_return_type) {
 314             *result = SecDbItemCopyResult(item, q->q_return_type, error);
 315         }
 316     }
 317     if (!ok && error && *error) {
 318         if (CFEqual(CFErrorGetDomain(*error), kSecDbErrorDomain) && CFErrorGetCode(*error) == SQLITE_CONSTRAINT) {
 319             CFReleaseNull(*error);
 320             SecError(errSecDuplicateItem, error, CFSTR("duplicate item %@"), item);
 321         } else if (CFEqual(CFErrorGetDomain(*error), kSecErrorDomain) && CFErrorGetCode(*error) == errSecDecode) { //handle situation when item have pdmn=akpu but passcode is not set
 322             CFTypeRef value = SecDbItemGetValue(item, SecDbClassAttrWithKind(item->class, kSecDbAccessAttr, error), error);
 323             if (value && CFEqual(value, kSecAttrAccessibleWhenPasscodeSetThisDeviceOnly)) {
 324                 CFReleaseNull(*error);
 325                 SecError(errSecAuthFailed, error, CFSTR("authentication failed"));
 326             }
 327         }
 328     }
 329
 330     if (ok) {
 331         q->q_changed = true;
 332         if (SecDbItemIsSyncable(item))
 333             q->q_sync_changed = true;
 334     }
 335
 336     secdebug("dbitem", "inserting item %@%s%@", item, ok ? "" : "failed: ", ok || error == NULL ? (CFErrorRef)CFSTR("") : *error);
 337
 338     CFRelease(item);
 339
 340         return ok;
 341 }
 342
 343 bool s3dl_item_make_new_uuid(SecDbItemRef item, bool uuid_from_primary_key, CFErrorRef* error) {
 344     if(!item) {
 345         return false;
 346     }
 347
 348     // Set the item UUID.
 349     CFUUIDRef uuid = NULL;
 350     // Were we asked to make the UUID static?
 351     if (uuid_from_primary_key) {
 352         // This UUID isn't used in any security mechanism, so we can
 353         // just use the first bits of the SHA256 hash.
 354         CFDataRef pkhash = SecDbKeychainItemCopySHA256PrimaryKey(item, error);
 355         if(CFDataGetLength(pkhash) >= 16) {
 356             UInt8 uuidBytes[16];
 357             CFRange range = CFRangeMake(0, 16);
 358             CFDataGetBytes(pkhash, range, uuidBytes);
 359
 360             uuid = CFUUIDCreateWithBytes(NULL,
 361                                          uuidBytes[ 0],
 362                                          uuidBytes[ 1],
 363                                          uuidBytes[ 2],
 364                                          uuidBytes[ 3],
 365                                          uuidBytes[ 4],
 366                                          uuidBytes[ 5],
 367                                          uuidBytes[ 6],
 368                                          uuidBytes[ 7],
 369                                          uuidBytes[ 8],
 370                                          uuidBytes[ 9],
 371                                          uuidBytes[10],
 372                                          uuidBytes[11],
 373                                          uuidBytes[12],
 374                                          uuidBytes[13],
 375                                          uuidBytes[14],
 376                                          uuidBytes[15]);
 377         }
 378         CFReleaseNull(pkhash);
 379     }
 380     if(uuid == NULL) {
 381         uuid = CFUUIDCreate(NULL);
 382     }
 383     SecDbItemSetValueWithName(item, kSecAttrUUID, uuid, error);
 384     CFReleaseNull(uuid);
 385     return true;
 386 }
 387
 388 typedef void (*s3dl_handle_row)(sqlite3_stmt *stmt, void *context);
 389
 390 static CFDataRef
 391 s3dl_copy_data_from_col(sqlite3_stmt *stmt, int col, CFErrorRef *error) {
 392     return CFDataCreateWithBytesNoCopy(0, sqlite3_column_blob(stmt, col),
 393                                        sqlite3_column_bytes(stmt, col),
 394                                        kCFAllocatorNull);
 395 }
 396
 397 static bool
 398 s3dl_item_from_col(sqlite3_stmt *stmt, Query *q, int col, CFArrayRef accessGroups,
 399                    CFMutableDictionaryRef *item, SecAccessControlRef *access_control, keyclass_t* keyclass, CFErrorRef *error) {
 400     CFDataRef edata = NULL;
 401     bool ok = false;
 402     require(edata = s3dl_copy_data_from_col(stmt, col, error), out);
 403     ok = s3dl_item_from_data(edata, q, accessGroups, item, access_control, keyclass, error);
 404
 405 out:
 406     CFReleaseSafe(edata);
 407     return ok;
 408 }
 409
 410 struct s3dl_query_ctx {
 411     Query *q;
 412     CFArrayRef accessGroups;
 413     SecDbConnectionRef dbt;
 414     CFTypeRef result;
 415     int found;
 416 };
 417
 418 /* Return whatever the caller requested based on the value of q->q_return_type.
 419  keys and values must be 3 larger than attr_count in size to accomadate the
 420  optional data, class and persistent ref results.  This is so we can use
 421  the CFDictionaryCreate() api here rather than appending to a
 422  mutable dictionary. */
 423 static CF_RETURNS_RETAINED CFTypeRef
 424 handle_result(Query *q,
 425               CFMutableDictionaryRef item,
 426               sqlite_int64 rowid)
 427 {
 428     CFTypeRef a_result;
 429     CFDataRef data;
 430     data = CFDictionaryGetValue(item, kSecValueData);
 431         if (q->q_return_type == 0) {
 432                 /* Caller isn't interested in any results at all. */
 433                 a_result = kCFNull;
 434         } else if (q->q_return_type == kSecReturnDataMask) {
 435         if (data) {
 436             a_result = data;
 437             CFRetain(a_result);
 438         } else {
 439             a_result = CFDataCreate(kCFAllocatorDefault, NULL, 0);
 440         }
 441         } else if (q->q_return_type == kSecReturnPersistentRefMask) {
 442                 a_result = _SecItemCreatePersistentRef(q->q_class->name, rowid, item);
 443         } else {
 444                 /* We need to return more than one value. */
 445         if (q->q_return_type & kSecReturnRefMask) {
 446             CFDictionarySetValue(item, kSecClass, q->q_class->name);
 447         } else if ((q->q_return_type & kSecReturnAttributesMask)) {
 448             if (!(q->q_return_type & kSecReturnDataMask)) {
 449                 CFDictionaryRemoveValue(item, kSecValueData);
 450             }
 451
 452             // Add any attributes which are supposed to be returned, are not present in the decrypted blob,
 453             // and have a way to generate themselves.
 454             SecDbItemRef itemRef = NULL;
 455             SecDbForEachAttrWithMask(q->q_class, attr, kSecDbReturnAttrFlag) {
 456                 if(!CFDictionaryGetValue(item, attr->name) && attr->copyValue) {
 457                     CFErrorRef cferror = NULL;
 458                     if(!itemRef) {
 459                         itemRef = SecDbItemCreateWithAttributes(NULL, q->q_class, item, KEYBAG_DEVICE, &cferror);
 460                     }
 461                     if(!cferror && itemRef) {
 462                         if (attr->kind != kSecDbSHA1Attr || (q->q_return_type & kSecReturnDataMask)) { // we'll skip returning the sha1 attribute unless the client has also asked us to return data, because without data our sha1 could be invalid
 463                             CFTypeRef attrValue = attr->copyValue(itemRef, attr, &cferror);
 464                             if (!cferror && attrValue) {
 465                                 CFDictionarySetValue(item, attr->name, attrValue);
 466                             }
 467                             CFReleaseNull(attrValue);
 468                         }
 469                     }
 470                     CFReleaseNull(cferror);
 471                 }
 472             }
 473             CFReleaseNull(itemRef);
 474
 475             CFDictionaryRemoveValue(item, kSecAttrUUID);
 476         } else {
 477             CFRetainSafe(data);
 478             CFDictionaryRemoveAllValues(item);
 479             if ((q->q_return_type & kSecReturnDataMask) && data) {
 480                 CFDictionarySetValue(item, kSecValueData, data);
 481             }
 482             CFReleaseSafe(data);
 483         }
 484                 if (q->q_return_type & kSecReturnPersistentRefMask) {
 485             CFDataRef pref = _SecItemCreatePersistentRef(q->q_class->name, rowid, item);
 486                         CFDictionarySetValue(item, kSecValuePersistentRef, pref);
 487             CFRelease(pref);
 488                 }
 489
 490                 a_result = item;
 491         CFRetain(item);
 492         }
 493
 494         return a_result;
 495 }
 496
 497 static void s3dl_merge_into_dict(const void *key, const void *value, void *context) {
 498     CFDictionarySetValue(context, key, value);
 499 }
 500
 501 static bool checkTokenObjectID(CFDataRef token_object_id, CFDataRef value_data) {
 502     bool equalOID = false;
 503     CFDictionaryRef itemValue = SecTokenItemValueCopy(value_data, NULL);
 504     require_quiet(itemValue, out);
 505     CFDataRef oID = CFDictionaryGetValue(itemValue, kSecTokenValueObjectIDKey);
 506     equalOID = CFEqualSafe(token_object_id, oID);
 507     CFRelease(itemValue);
 508 out:
 509     return equalOID;
 510 }
 511
 512 static void s3dl_query_row(sqlite3_stmt *stmt, void *context) {
 513     struct s3dl_query_ctx *c = context;
 514     Query *q = c->q;
 515     ReturnTypeMask saved_mask = q->q_return_type;
 516
 517     sqlite_int64 rowid = sqlite3_column_int64(stmt, 0);
 518     CFMutableDictionaryRef item = NULL;
 519     bool ok;
 520
 521 decode:
 522     ok = s3dl_item_from_col(stmt, q, 1, c->accessGroups, &item, NULL, NULL, &q->q_error);
 523     if (!ok) {
 524         OSStatus status = SecErrorGetOSStatus(q->q_error);
 525         // errSecDecode means the item is corrupted, stash it for delete.
 526         if (status == errSecDecode) {
 527             secwarning("ignoring corrupt %@,rowid=%" PRId64 " %@", q->q_class->name, rowid, q->q_error);
 528             {
 529                 CFDataRef edata = s3dl_copy_data_from_col(stmt, 1, NULL);
 530                 CFMutableStringRef edatastring =  CFStringCreateMutable(kCFAllocatorDefault, 0);
 531                 if(edatastring) {
 532                     CFStringAppendEncryptedData(edatastring, edata);
 533                     secnotice("item", "corrupted edata=%@", edatastring);
 534                 }
 535                 CFReleaseSafe(edata);
 536                 CFReleaseSafe(edatastring);
 537             }
 538             CFReleaseNull(q->q_error);
 539         } else if (status == errSecAuthNeeded) {
 540             secwarning("Authentication is needed for %@,rowid=%" PRId64 " (%" PRIdOSStatus "): %@", q->q_class->name, rowid, status, q->q_error);
 541         } else if (status == errSecInteractionNotAllowed) {
 542             static dispatch_once_t kclockedtoken;
 543             static sec_action_t kclockedaction;
 544             dispatch_once(&kclockedtoken, ^{
 545                 kclockedaction = sec_action_create("ratelimiterdisabledlogevent", 1);
 546                 sec_action_set_handler(kclockedaction, ^{
 547                     secerror("decode item failed, keychain is locked (%d)", (int)errSecInteractionNotAllowed);
 548                 });
 549             });
 550             sec_action_perform(kclockedaction);
 551         } else {
 552             secerror("decode %@,rowid=%" PRId64 " failed (%" PRIdOSStatus "): %@", q->q_class->name, rowid, status, q->q_error);
 553         }
 554         // q->q_error will be released appropriately by a call to query_error
 555         return;
 556     }
 557
 558     if (!item)
 559         goto out;
 560
 561     if (CFDictionaryContainsKey(item, kSecAttrTokenID) && (q->q_return_type & kSecReturnDataMask) == 0) {
 562         // For token-based items, to get really meaningful set of attributes we must provide also data field, so augment mask
 563         // and restart item decoding cycle.
 564         q->q_return_type |= kSecReturnDataMask;
 565         CFReleaseNull(item);
 566         goto decode;
 567     }
 568
 569     if (q->q_token_object_id != NULL && !checkTokenObjectID(q->q_token_object_id, CFDictionaryGetValue(item, kSecValueData)))
 570         goto out;
 571
 572     if (q->q_class == identity_class()) {
 573         // TODO: Use col 2 for key rowid and use both rowids in persistent ref.
 574
 575         CFMutableDictionaryRef key;
 576         /* TODO : if there is a errSecDecode error here, we should cleanup */
 577         if (!s3dl_item_from_col(stmt, q, 3, c->accessGroups, &key, NULL, NULL, &q->q_error) || !key)
 578             goto out;
 579
 580         CFDataRef certData = CFDictionaryGetValue(item, kSecValueData);
 581         if (certData) {
 582             CFDictionarySetValue(key, kSecAttrIdentityCertificateData, certData);
 583             CFDictionaryRemoveValue(item, kSecValueData);
 584         }
 585
 586         CFDataRef certTokenID = CFDictionaryGetValue(item, kSecAttrTokenID);
 587         if (certTokenID) {
 588             CFDictionarySetValue(key, kSecAttrIdentityCertificateTokenID, certTokenID);
 589             CFDictionaryRemoveValue(item, kSecAttrTokenID);
 590         }
 591         CFDictionaryApplyFunction(item, s3dl_merge_into_dict, key);
 592         CFRelease(item);
 593         item = key;
 594     }
 595
 596     if (!match_item(c->dbt, q, c->accessGroups, item))
 597         goto out;
 598
 599     CFTypeRef a_result = handle_result(q, item, rowid);
 600     if (a_result) {
 601         if (a_result == kCFNull) {
 602             /* Caller wasn't interested in a result, but we still
 603              count this row as found. */
 604             CFRelease(a_result);  // Help shut up clang
 605         } else if (q->q_limit == 1) {
 606             c->result = a_result;
 607         } else {
 608             CFArrayAppendValue((CFMutableArrayRef)c->result, a_result);
 609             CFRelease(a_result);
 610         }
 611         c->found++;
 612     }
 613
 614 out:
 615     q->q_return_type = saved_mask;
 616     CFReleaseSafe(item);
 617 }
 618
 619 static void
 620 SecDbAppendWhereROWID(CFMutableStringRef sql,
 621                       CFStringRef col, sqlite_int64 row_id,
 622                       bool *needWhere) {
 623     if (row_id > 0) {
 624         SecDbAppendWhereOrAnd(sql, needWhere);
 625         CFStringAppendFormat(sql, NULL, CFSTR("%@=%lld"), col, row_id);
 626     }
 627 }
 628
 629 static void
 630 SecDbAppendWhereAttrs(CFMutableStringRef sql, const Query *q, bool *needWhere) {
 631     CFIndex ix, attr_count = query_attr_count(q);
 632     for (ix = 0; ix < attr_count; ++ix) {
 633         SecDbAppendWhereOrAndEquals(sql, query_attr_at(q, ix).key, needWhere);
 634     }
 635 }
 636
 637 static void
 638 SecDbAppendWhereAccessGroups(CFMutableStringRef sql,
 639                              CFStringRef col,
 640                              CFArrayRef accessGroups,
 641                              bool *needWhere) {
 642     CFIndex ix, ag_count;
 643     if (!accessGroups || 0 == (ag_count = CFArrayGetCount(accessGroups))) {
 644         return;
 645     }
 646
 647     SecDbAppendWhereOrAnd(sql, needWhere);
 648     CFStringAppend(sql, col);
 649     CFStringAppend(sql, CFSTR(" IN (?"));
 650     for (ix = 1; ix < ag_count; ++ix) {
 651         CFStringAppend(sql, CFSTR(",?"));
 652     }
 653     CFStringAppend(sql, CFSTR(")"));
 654 }
 655
 656 static bool
 657 isQueryOverAllMUSRViews(CFTypeRef musrView)
 658 {
 659     return SecMUSRIsViewAllViews(musrView);
 660 }
 661
 662 static bool
 663 isQueryOverSingleUserView(CFTypeRef musrView)
 664 {
 665     return isNull(musrView);
 666 }
 667
 668 #if TARGET_OS_IPHONE
 669 static bool
 670 isQueryOverBothUserAndSystem(CFTypeRef musrView, uid_t *uid)
 671 {
 672     return SecMUSRGetBothUserAndSystemUUID(musrView, uid);
 673 }
 674 #endif
 675
 676 static void
 677 SecDbAppendWhereMusr(CFMutableStringRef sql,
 678                      const Query *q,
 679                      bool *needWhere)
 680 {
 681     SecDbAppendWhereOrAnd(sql, needWhere);
 682
 683 #if TARGET_OS_IPHONE
 684     if (isQueryOverBothUserAndSystem(q->q_musrView, NULL)) {
 685         CFStringAppend(sql, CFSTR("(musr = ? OR musr = ?)"));
 686     } else
 687 #endif
 688     if (isQueryOverAllMUSRViews(q->q_musrView)) {
 689             /* query over all items, regardless of view */
 690     } else if (isQueryOverSingleUserView(q->q_musrView)) {
 691         CFStringAppend(sql, CFSTR("musr = ?"));
 692     } else {
 693         CFStringAppend(sql, CFSTR("musr = ?"));
 694     }
 695 }
 696
 697 static void SecDbAppendWhereClause(CFMutableStringRef sql, const Query *q,
 698                                    CFArrayRef accessGroups) {
 699     bool needWhere = true;
 700     SecDbAppendWhereROWID(sql, CFSTR("ROWID"), q->q_row_id, &needWhere);
 701     SecDbAppendWhereAttrs(sql, q, &needWhere);
 702     SecDbAppendWhereMusr(sql, q, &needWhere);
 703     SecDbAppendWhereAccessGroups(sql, CFSTR("agrp"), accessGroups, &needWhere);
 704 }
 705
 706 static void SecDbAppendLimit(CFMutableStringRef sql, CFIndex limit) {
 707     if (limit != kSecMatchUnlimited)
 708         CFStringAppendFormat(sql, NULL, CFSTR(" LIMIT %" PRIdCFIndex), limit);
 709 }
 710
 711 static CFStringRef s3dl_create_select_sql(Query *q, CFArrayRef accessGroups) {
 712     CFMutableStringRef sql = CFStringCreateMutable(NULL, 0);
 713         if (q->q_class == identity_class()) {
 714         CFStringAppendFormat(sql, NULL, CFSTR("SELECT crowid, %@"
 715                                               ", rowid,data FROM "
 716                                               "(SELECT cert.rowid AS crowid, cert.labl AS labl,"
 717                                               " cert.issr AS issr, cert.slnr AS slnr, cert.skid AS skid,"
 718                                               " keys.*,cert.data AS %@"
 719                                               " FROM keys, cert"
 720                                               " WHERE keys.priv == 1 AND cert.pkhh == keys.klbl"),
 721                              kSecAttrIdentityCertificateData, kSecAttrIdentityCertificateData);
 722         SecDbAppendWhereAccessGroups(sql, CFSTR("cert.agrp"), accessGroups, 0);
 723         /* The next 3 SecDbAppendWhere calls are in the same order as in
 724          SecDbAppendWhereClause().  This makes sqlBindWhereClause() work,
 725          as long as we do an extra sqlBindAccessGroups first. */
 726         SecDbAppendWhereROWID(sql, CFSTR("crowid"), q->q_row_id, 0);
 727         CFStringAppend(sql, CFSTR(")"));
 728         bool needWhere = true;
 729         SecDbAppendWhereAttrs(sql, q, &needWhere);
 730         SecDbAppendWhereMusr(sql, q, &needWhere);
 731         SecDbAppendWhereAccessGroups(sql, CFSTR("agrp"), accessGroups, &needWhere);
 732         } else {
 733         CFStringAppend(sql, CFSTR("SELECT rowid, data FROM "));
 734                 CFStringAppend(sql, q->q_class->name);
 735         SecDbAppendWhereClause(sql, q, accessGroups);
 736     }
 737     //do not append limit for all queries which needs filtering
 738     if (q->q_match_issuer == NULL && q->q_match_policy == NULL && q->q_match_valid_on_date == NULL && q->q_match_trusted_only == NULL && q->q_token_object_id == NULL) {
 739         SecDbAppendLimit(sql, q->q_limit);
 740     }
 741
 742     return sql;
 743 }
 744
 745 static bool sqlBindMusr(sqlite3_stmt *stmt, const Query *q, int *pParam, CFErrorRef *error) {
 746     int param = *pParam;
 747     bool result = true;
 748 #if TARGET_OS_IPHONE
 749     uid_t uid;
 750
 751     if (isQueryOverBothUserAndSystem(q->q_musrView, &uid)) {
 752         /* network extensions are special and get to query both user and system views */
 753         CFDataRef systemUUID = SecMUSRGetSystemKeychainUUID();
 754         result = SecDbBindObject(stmt, param++, systemUUID, error);
 755         if (result) {
 756             CFDataRef activeUser = SecMUSRCreateActiveUserUUID(uid);
 757             result = SecDbBindObject(stmt, param++, activeUser, error);
 758             CFReleaseNull(activeUser);
 759         }
 760     } else
 761 #endif
 762     if (isQueryOverAllMUSRViews(q->q_musrView)) {
 763         /* query over all items, regardless of view */
 764     } else if (isQueryOverSingleUserView(q->q_musrView)) {
 765         CFDataRef singleUUID = SecMUSRGetSingleUserKeychainUUID();
 766         result = SecDbBindObject(stmt, param++, singleUUID, error);
 767     } else {
 768         result = SecDbBindObject(stmt, param++, q->q_musrView, error);
 769     }
 770
 771     *pParam = param;
 772     return result;
 773 }
 774
 775
 776 static bool sqlBindAccessGroups(sqlite3_stmt *stmt, CFArrayRef accessGroups,
 777                                 int *pParam, CFErrorRef *error) {
 778     bool result = true;
 779     int param = *pParam;
 780     CFIndex ix, count = accessGroups ? CFArrayGetCount(accessGroups) : 0;
 781     for (ix = 0; ix < count; ++ix) {
 782         result = SecDbBindObject(stmt, param++,
 783                                  CFArrayGetValueAtIndex(accessGroups, ix),
 784                                  error);
 785         if (!result)
 786             break;
 787     }
 788     *pParam = param;
 789     return result;
 790 }
 791
 792 static bool sqlBindWhereClause(sqlite3_stmt *stmt, const Query *q,
 793                                CFArrayRef accessGroups, int *pParam, CFErrorRef *error) {
 794     bool result = true;
 795     int param = *pParam;
 796     CFIndex ix, attr_count = query_attr_count(q);
 797     for (ix = 0; ix < attr_count; ++ix) {
 798         result = SecDbBindObject(stmt, param++, query_attr_at(q, ix).value, error);
 799         if (!result)
 800             break;
 801         }
 802
 803     if (result) {
 804         result = sqlBindMusr(stmt, q, &param, error);
 805     }
 806
 807     /* Bind the access group to the sql. */
 808     if (result) {
 809         result = sqlBindAccessGroups(stmt, accessGroups, &param, error);
 810     }
 811
 812     *pParam = param;
 813     return result;
 814 }
 815
 816 bool SecDbItemQuery(SecDbQueryRef query, CFArrayRef accessGroups, SecDbConnectionRef dbconn, CFErrorRef *error,
 817                     void (^handle_row)(SecDbItemRef item, bool *stop)) {
 818     __block bool ok = true;
 819     /* Sanity check the query. */
 820     if (query->q_ref)
 821         return SecError(errSecValueRefUnsupported, error, CFSTR("value ref not supported by queries"));
 822
 823     bool (^return_attr)(const SecDbAttr *attr) = ^bool (const SecDbAttr * attr) {
 824         // The attributes here must match field list hardcoded in s3dl_select_sql used below, which is
 825         // "rowid, data"
 826         return attr->kind == kSecDbRowIdAttr || attr->kind == kSecDbEncryptedDataAttr;
 827     };
 828
 829     CFStringRef sql = s3dl_create_select_sql(query, accessGroups);
 830     ok = sql;
 831     if (sql) {
 832         ok &= SecDbPrepare(dbconn, sql, error, ^(sqlite3_stmt *stmt) {
 833             /* Bind the values being searched for to the SELECT statement. */
 834             int param = 1;
 835             if (query->q_class == identity_class()) {
 836                 /* Bind the access groups to cert.agrp. */
 837                 ok &= sqlBindAccessGroups(stmt, accessGroups, &param, error);
 838             }
 839             if (ok)
 840                 ok &= sqlBindWhereClause(stmt, query, accessGroups, &param, error);
 841             if (ok) {
 842                 SecDbStep(dbconn, stmt, error, ^(bool *stop) {
 843                     SecDbItemRef itemFromStatement = SecDbItemCreateWithStatement(kCFAllocatorDefault, query->q_class, stmt, query->q_keybag, error, return_attr);
 844                     if (itemFromStatement) {
 845                         CFTransferRetained(itemFromStatement->credHandle, query->q_use_cred_handle);
 846                         if (match_item(dbconn, query, accessGroups, itemFromStatement->attributes))
 847                             handle_row(itemFromStatement, stop);
 848                         CFReleaseNull(itemFromStatement);
 849                     } else {
 850                         secerror("failed to create item from stmt: %@", error ? *error : (CFErrorRef)"no error");
 851                         if (error) {
 852                             CFReleaseNull(*error);
 853                         }
 854                         //*stop = true;
 855                         //ok = false;
 856                     }
 857                 });
 858             }
 859         });
 860         CFRelease(sql);
 861     }
 862
 863     return ok;
 864 }
 865
 866 static bool
 867 s3dl_query(s3dl_handle_row handle_row,
 868            void *context, CFErrorRef *error)
 869 {
 870     struct s3dl_query_ctx *c = context;
 871     SecDbConnectionRef dbt = c->dbt;
 872     Query *q = c->q;
 873     CFArrayRef accessGroups = c->accessGroups;
 874
 875     /* Sanity check the query. */
 876     if (q->q_ref)
 877         return SecError(errSecValueRefUnsupported, error, CFSTR("value ref not supported by queries"));
 878
 879         /* Actual work here. */
 880     if (q->q_limit == 1) {
 881         c->result = NULL;
 882     } else {
 883         c->result = CFArrayCreateMutable(NULL, 0, &kCFTypeArrayCallBacks);
 884     }
 885     CFStringRef sql = s3dl_create_select_sql(q, accessGroups);
 886     bool ok = SecDbWithSQL(dbt, sql, error, ^(sqlite3_stmt *stmt) {
 887         bool sql_ok = true;
 888         /* Bind the values being searched for to the SELECT statement. */
 889         int param = 1;
 890         if (q->q_class == identity_class()) {
 891             /* Bind the access groups to cert.agrp. */
 892             sql_ok = sqlBindAccessGroups(stmt, accessGroups, &param, error);
 893         }
 894         if (sql_ok)
 895             sql_ok = sqlBindWhereClause(stmt, q, accessGroups, &param, error);
 896         if (sql_ok) {
 897             SecDbForEach(dbt, stmt, error, ^bool (int row_index) {
 898                 handle_row(stmt, context);
 899
 900                 bool needs_auth = q->q_error && CFErrorGetCode(q->q_error) == errSecAuthNeeded;
 901                 if (q->q_skip_acl_items && needs_auth)
 902                     // Skip items needing authentication if we are told to do so.
 903                     CFReleaseNull(q->q_error);
 904
 905                 bool stop = q->q_limit != kSecMatchUnlimited && c->found >= q->q_limit;
 906                 stop = stop || (q->q_error && !needs_auth);
 907                 return !stop;
 908             });
 909         }
 910         return sql_ok;
 911     });
 912
 913     CFRelease(sql);
 914
 915     // First get the error from the query, since errSecDuplicateItem from an
 916     // update query should superceed the errSecItemNotFound below.
 917     if (!query_error(q, error))
 918         ok = false;
 919     if (ok && c->found == 0) {
 920         ok = SecError(errSecItemNotFound, error, CFSTR("no matching items found"));
 921         if (q->q_spindump_on_failure) {
 922             __security_stackshotreport(CFSTR("ItemNotFound"), __sec_exception_code_LostInMist);
 923         }
 924     }
 925
 926     return ok;
 927 }
 928
 929 bool
 930 s3dl_copy_matching(SecDbConnectionRef dbt, Query *q, CFTypeRef *result,
 931                    CFArrayRef accessGroups, CFErrorRef *error)
 932 {
 933     struct s3dl_query_ctx ctx = {
 934         .q = q, .accessGroups = accessGroups, .dbt = dbt,
 935     };
 936     if (q->q_row_id && query_attr_count(q))
 937         return SecError(errSecItemIllegalQuery, error,
 938                         CFSTR("attributes to query illegal; both row_id and other attributes can't be searched at the same time"));
 939     if (q->q_token_object_id && query_attr_count(q) != 1)
 940         return SecError(errSecItemIllegalQuery, error,
 941                         CFSTR("attributes to query illegal; both token persitent ref and other attributes can't be searched at the same time"));
 942
 943     // Only copy things that aren't tombstones unless the client explicitly asks otherwise.
 944     if (!CFDictionaryContainsKey(q->q_item, kSecAttrTombstone))
 945         query_add_attribute(kSecAttrTombstone, kCFBooleanFalse, q);
 946     bool ok = s3dl_query(s3dl_query_row, &ctx, error);
 947     if (ok && result)
 948         *result = ctx.result;
 949     else
 950         CFReleaseSafe(ctx.result);
 951
 952     return ok;
 953 }
 954
 955 typedef void (^s3dl_item_digest_callback)(CFDataRef persistantReference, CFDataRef encryptedData);
 956
 957 struct s3dl_digest_ctx {
 958     Query *q;
 959     SecDbConnectionRef dbt;
 960     s3dl_item_digest_callback item_callback;
 961 };
 962
 963 static void s3dl_query_row_digest(sqlite3_stmt *stmt, void *context) {
 964     struct s3dl_query_ctx *c = context;
 965     Query *q = c->q;
 966
 967     sqlite_int64 rowid = sqlite3_column_int64(stmt, 0);
 968     CFDataRef edata = s3dl_copy_data_from_col(stmt, 1, NULL);
 969     CFDataRef persistant_reference = _SecItemCreatePersistentRef(q->q_class->name, rowid, NULL);
 970     CFDataRef digest = NULL;
 971
 972     if (edata) {
 973         digest = CFDataCopySHA256Digest(edata, NULL);
 974     }
 975
 976     if (digest && persistant_reference) {
 977         CFDictionaryRef item = CFDictionaryCreateForCFTypes(NULL,
 978                                                             kSecValuePersistentRef, persistant_reference,
 979                                                             kSecValueData, digest,
 980                                                             NULL);
 981         if (item)
 982             CFArrayAppendValue((CFMutableArrayRef)c->result, item);
 983         CFReleaseNull(item);
 984         c->found++;
 985     } else {
 986         secinfo("item", "rowid %lu in %@ failed to create pref/digest", (unsigned long)rowid, q->q_class->name);
 987     }
 988     CFReleaseNull(digest);
 989     CFReleaseNull(edata);
 990     CFReleaseNull(persistant_reference);
 991 }
 992
 993
 994 bool
 995 s3dl_copy_digest(SecDbConnectionRef dbt, Query *q, CFArrayRef *result, CFArrayRef accessGroups, CFErrorRef *error)
 996 {
 997     struct s3dl_query_ctx ctx = {
 998         .q = q, .dbt = dbt, .accessGroups = accessGroups,
 999     };
1000     // Force to always return an array
1001     q->q_limit = kSecMatchUnlimited;
1002     // This interface only queries live data
1003     query_add_attribute(kSecAttrTombstone, kCFBooleanFalse, q);
1004     bool ok = s3dl_query(s3dl_query_row_digest, &ctx, error);
1005     if (ok && result)
1006         *result = (CFArrayRef)ctx.result;
1007     else
1008         CFReleaseSafe(ctx.result);
1009
1010     return ok;
1011 }
1012
1013 /* First remove key from q->q_pairs if it's present, then add the attribute again. */
1014 static void query_set_attribute_with_desc(const SecDbAttr *desc, const void *value, Query *q) {
1015     if (CFDictionaryContainsKey(q->q_item, desc->name)) {
1016         CFIndex ix;
1017         for (ix = 0; ix < q->q_attr_end; ++ix) {
1018             if (CFEqual(desc->name, q->q_pairs[ix].key)) {
1019                 CFReleaseSafe(q->q_pairs[ix].value);
1020                 --q->q_attr_end;
1021                 for (; ix < q->q_attr_end; ++ix) {
1022                     q->q_pairs[ix] = q->q_pairs[ix + 1];
1023                 }
1024                 CFDictionaryRemoveValue(q->q_item, desc->name);
1025                 break;
1026             }
1027         }
1028     }
1029     query_add_attribute_with_desc(desc, value, q);
1030 }
1031
1032 /* Update modification_date if needed. */
1033 static void query_pre_update(Query *q) {
1034     SecDbForEachAttr(q->q_class, desc) {
1035         if (desc->kind == kSecDbModificationDateAttr) {
1036             CFDateRef now = CFDateCreate(0, CFAbsoluteTimeGetCurrent());
1037             query_set_attribute_with_desc(desc, now, q);
1038             CFReleaseSafe(now);
1039         }
1040     }
1041 }
1042
1043 /* Make sure all attributes that are marked as not_null have a value.  If
1044  force_date is false, only set mdat and cdat if they aren't already set. */
1045 void query_pre_add(Query *q, bool force_date) {
1046     CFDateRef now = CFDateCreate(0, CFAbsoluteTimeGetCurrent());
1047     SecDbForEachAttrWithMask(q->q_class, desc, kSecDbInFlag) {
1048         if (desc->kind == kSecDbCreationDateAttr ||
1049             desc->kind == kSecDbModificationDateAttr) {
1050             if (force_date) {
1051                 query_set_attribute_with_desc(desc, now, q);
1052             } else if (!CFDictionaryContainsKey(q->q_item, desc->name)) {
1053                 query_add_attribute_with_desc(desc, now, q);
1054             }
1055         } else if ((desc->flags & kSecDbNotNullFlag) &&
1056                    !CFDictionaryContainsKey(q->q_item, desc->name)) {
1057             CFTypeRef value = NULL;
1058             if (desc->flags & kSecDbDefault0Flag) {
1059                 if (desc->kind == kSecDbDateAttr)
1060                     value = CFDateCreate(kCFAllocatorDefault, 0.0);
1061                 else {
1062                     SInt32 vzero = 0;
1063                     value = CFNumberCreate(0, kCFNumberSInt32Type, &vzero);
1064                 }
1065             } else if (desc->flags & kSecDbDefaultEmptyFlag) {
1066                 if (desc->kind == kSecDbDataAttr || desc->kind == kSecDbUUIDAttr)
1067                     value = CFDataCreate(kCFAllocatorDefault, NULL, 0);
1068                 else {
1069                     value = CFSTR("");
1070                     CFRetain(value);
1071                 }
1072             }
1073             if (value) {
1074                 /* Safe to use query_add_attribute here since the attr wasn't
1075                  set yet. */
1076                 query_add_attribute_with_desc(desc, value, q);
1077                 CFRelease(value);
1078             }
1079         }
1080     }
1081     CFReleaseSafe(now);
1082 }
1083
1084 // Return a tri state value false->never make a tombstone, true->always make a
1085 // tombstone, NULL->make a tombstone, but delete it if the tombstone itself is not currently being synced.
1086 static CFBooleanRef s3dl_should_make_tombstone(Query *q, bool item_is_syncable, SecDbItemRef item) {
1087     if (q->q_use_tomb)
1088         return q->q_use_tomb;
1089     else if (item_is_syncable && !SecDbItemIsTombstone(item))
1090         return NULL;
1091     else
1092         return kCFBooleanFalse;
1093 }
1094 /* AUDIT[securityd](done):
1095  attributesToUpdate (ok) is a caller provided dictionary,
1096  only its cf types have been checked.
1097  */
1098 bool
1099 s3dl_query_update(SecDbConnectionRef dbt, Query *q,
1100                   CFDictionaryRef attributesToUpdate, CFArrayRef accessGroups, CFErrorRef *error)
1101 {
1102     /* Sanity check the query. */
1103     if (query_match_count(q) != 0)
1104         return SecError(errSecItemMatchUnsupported, error, CFSTR("match not supported in attributes to update"));
1105     if (q->q_ref)
1106         return SecError(errSecValueRefUnsupported, error, CFSTR("value ref not supported in attributes to update"));
1107     if (q->q_row_id && query_attr_count(q))
1108         return SecError(errSecItemIllegalQuery, error, CFSTR("attributes to update illegal; both row_id and other attributes can't be updated at the same time"));
1109     if (q->q_token_object_id && query_attr_count(q) != 1)
1110         return SecError(errSecItemIllegalQuery, error, CFSTR("attributes to update illegal; both token persistent ref and other attributes can't be updated at the same time"));
1111
1112     __block bool result = true;
1113     Query *u = query_create(q->q_class, NULL, attributesToUpdate, error);
1114     if (u == NULL) return false;
1115     require_action_quiet(query_update_parse(u, attributesToUpdate, error), errOut, result = false);
1116     query_pre_update(u);
1117     result &= SecDbTransaction(dbt, kSecDbExclusiveTransactionType, error, ^(bool *commit) {
1118         // Make sure we only update real items, not tombstones, unless the client explicitly asks otherwise.
1119         if (!CFDictionaryContainsKey(q->q_item, kSecAttrTombstone))
1120             query_add_attribute(kSecAttrTombstone, kCFBooleanFalse, q);
1121         result &= SecDbItemQuery(q, accessGroups, dbt, error, ^(SecDbItemRef item, bool *stop) {
1122             // We always need to know the error here.
1123             CFErrorRef localError = NULL;
1124             if (q->q_token_object_id) {
1125                 const SecDbAttr *valueDataAttr = SecDbClassAttrWithKind(item->class, kSecDbDataAttr, NULL);
1126                 CFDataRef valueData = SecDbItemGetValue(item, valueDataAttr, NULL);
1127                 if (q->q_token_object_id != NULL && !checkTokenObjectID(q->q_token_object_id, valueData))
1128                     return;
1129             }
1130             // Cache the storedSHA1 digest so we use the one from the db not the recomputed one for notifications.
1131             const SecDbAttr *sha1attr = SecDbClassAttrWithKind(item->class, kSecDbSHA1Attr, NULL);
1132             CFDataRef storedSHA1 = CFRetainSafe(SecDbItemGetValue(item, sha1attr, NULL));
1133             SecDbItemRef new_item = SecDbItemCopyWithUpdates(item, u->q_item, &localError);
1134             SecDbItemSetValue(item, sha1attr, storedSHA1, NULL);
1135             CFReleaseSafe(storedSHA1);
1136             if (SecErrorGetOSStatus(localError) == errSecDecode) {
1137                 // We just ignore this, and treat as if item is not found.
1138                 secwarning("deleting corrupt %@,rowid=%" PRId64 " %@", q->q_class->name, SecDbItemGetRowId(item, NULL), localError);
1139                 CFReleaseNull(localError);
1140                 if (!SecDbItemDelete(item, dbt, false, &localError)) {
1141                     secerror("failed to delete corrupt %@,rowid=%" PRId64 " %@", q->q_class->name, SecDbItemGetRowId(item, NULL), localError);
1142                     CFReleaseNull(localError);
1143                 }
1144                 CFReleaseNull(new_item);
1145                 return;
1146             }
1147             if (new_item != NULL && u->q_access_control != NULL)
1148                 SecDbItemSetAccessControl(new_item, u->q_access_control, &localError);
1149             result = SecErrorPropagate(localError, error) && new_item;
1150             if (new_item) {
1151                 bool item_is_sync = SecDbItemIsSyncable(item);
1152                 result = SecDbItemUpdate(item, new_item, dbt, s3dl_should_make_tombstone(q, item_is_sync, item), q->q_uuid_from_primary_key, error);
1153                 if (result) {
1154                     q->q_changed = true;
1155                     if (item_is_sync || SecDbItemIsSyncable(new_item))
1156                         q->q_sync_changed = true;
1157                 }
1158                 CFRelease(new_item);
1159             }
1160         });
1161         if (!result)
1162             *commit = false;
1163     });
1164     if (result && !q->q_changed)
1165         result = SecError(errSecItemNotFound, error, CFSTR("No items updated"));
1166 errOut:
1167     if (!query_destroy(u, error))
1168         result = false;
1169     return result;
1170 }
1171
1172 static bool SecDbItemNeedAuth(SecDbItemRef item, CFErrorRef *error)
1173 {
1174     CFErrorRef localError = NULL;
1175     if (!SecDbItemEnsureDecrypted(item, true, &localError) && localError && CFErrorGetCode(localError) == errSecAuthNeeded) {
1176         if (error)
1177         *error = localError;
1178         return true;
1179     }
1180
1181     CFReleaseSafe(localError);
1182     return false;
1183 }
1184
1185 bool
1186 s3dl_query_delete(SecDbConnectionRef dbt, Query *q, CFArrayRef accessGroups, CFErrorRef *error)
1187 {
1188     __block bool ok = true;
1189     __block bool needAuth = false;
1190     // Only delete things that aren't tombstones, unless the client explicitly asks otherwise.
1191     if (!CFDictionaryContainsKey(q->q_item, kSecAttrTombstone))
1192         query_add_attribute(kSecAttrTombstone, kCFBooleanFalse, q);
1193     ok &= SecDbItemSelect(q, dbt, error, NULL, ^bool(const SecDbAttr *attr) {
1194         return false;
1195     },^bool(CFMutableStringRef sql, bool *needWhere) {
1196         SecDbAppendWhereClause(sql, q, accessGroups);
1197         return true;
1198     },^bool(sqlite3_stmt * stmt, int col) {
1199         return sqlBindWhereClause(stmt, q, accessGroups, &col, error);
1200     }, ^(SecDbItemRef item, bool *stop) {
1201         // Check if item for token persitence ref
1202         if (q->q_token_object_id) {
1203             const SecDbAttr *valueDataAttr = SecDbClassAttrWithKind(item->class, kSecDbDataAttr, NULL);
1204             CFDataRef valueData = SecDbItemGetValue(item, valueDataAttr, NULL);
1205             if (q->q_token_object_id != NULL && !checkTokenObjectID(q->q_token_object_id, valueData))
1206                 return;
1207         }
1208         // Check if item need to be authenticated by LocalAuthentication
1209         item->cryptoOp = kAKSKeyOpDelete;
1210         if (SecDbItemNeedAuth(item, error)) {
1211             needAuth = true;
1212             return;
1213         }
1214         // Cache the storedSHA1 digest so we use the one from the db not the recomputed one for notifications.
1215         const SecDbAttr *sha1attr = SecDbClassAttrWithKind(item->class, kSecDbSHA1Attr, NULL);
1216         CFDataRef storedSHA1 = CFRetainSafe(SecDbItemGetValue(item, sha1attr, NULL));
1217         bool item_is_sync = SecDbItemIsSyncable(item);
1218         SecDbItemSetValue(item, sha1attr, storedSHA1, NULL);
1219         CFReleaseSafe(storedSHA1);
1220         ok = SecDbItemDelete(item, dbt, s3dl_should_make_tombstone(q, item_is_sync, item), error);
1221         if (ok) {
1222             q->q_changed = true;
1223             if (item_is_sync)
1224                 q->q_sync_changed = true;
1225         }
1226     });
1227     if (ok && !q->q_changed && !needAuth) {
1228         ok = SecError(errSecItemNotFound, error, CFSTR("Delete failed to delete anything"));
1229     }
1230     return ok && !needAuth;
1231 }
1232
1233 static bool
1234 matchAnyString(CFStringRef needle, CFStringRef *haystack)
1235 {
1236     while (*haystack) {
1237         if (CFEqual(needle, *haystack))
1238             return true;
1239         haystack++;
1240     }
1241     return false;
1242 }
1243
1244 /* Return true iff the item in question should not be backed up, nor restored,
1245  but when restoring a backup the original version of the item should be
1246  added back to the keychain again after the restore completes. */
1247 bool SecItemIsSystemBound(CFDictionaryRef item, const SecDbClass *cls, bool multiUser) {
1248     CFNumberRef sysb =  CFDictionaryGetValue(item, kSecAttrSysBound);
1249     if (isNumber(sysb)) {
1250         int32_t num = 0;
1251         if (!CFNumberGetValue(sysb, kCFNumberSInt32Type, &num))
1252             return false;
1253         if (num == kSecSecAttrSysBoundNot) {
1254             return false;
1255         } else if (num == kSecSecAttrSysBoundPreserveDuringRestore) {
1256             return true;
1257         }
1258         return true;
1259     }
1260
1261     CFStringRef agrp = CFDictionaryGetValue(item, kSecAttrAccessGroup);
1262     if (!isString(agrp))
1263         return false;
1264
1265     if (CFEqualSafe(agrp, kSOSInternalAccessGroup)) {
1266         secdebug("backup", "found sysbound item: %@", item);
1267         return true;
1268     }
1269
1270     if (CFEqual(agrp, CFSTR("lockdown-identities"))) {
1271         secdebug("backup", "found sys_bound item: %@", item);
1272         return true;
1273     }
1274
1275     if (CFEqual(agrp, CFSTR("apple")) && cls == genp_class()) {
1276         CFStringRef service = CFDictionaryGetValue(item, kSecAttrService);
1277         CFStringRef account = CFDictionaryGetValue(item, kSecAttrAccount);
1278
1279         if (isString(service) && isString(account)) {
1280             static CFStringRef mcAccounts[] = {
1281                 CFSTR("Public"),
1282                 CFSTR("Private"),
1283                 NULL,
1284             };
1285
1286             if (CFEqual(service, CFSTR("com.apple.managedconfiguration"))
1287                 && matchAnyString(account, mcAccounts))
1288             {
1289                 secdebug("backup", "found sys_bound item: %@", item);
1290                 return true;
1291             }
1292         }
1293     }
1294
1295     if (multiUser && CFEqual(agrp, CFSTR("com.apple.apsd")) && cls == genp_class()) {
1296         static CFStringRef pushServices[] = {
1297             CFSTR("push.apple.com"),
1298             CFSTR("push.apple.com,PerAppToken.v0"),
1299             NULL
1300         };
1301         CFStringRef service = CFDictionaryGetValue(item, kSecAttrService);
1302
1303         if (isString(service) && matchAnyString(service, pushServices)) {
1304             secdebug("backup", "found sys_bound item: %@", item);
1305             return true;
1306         }
1307     }
1308
1309     if (multiUser && CFEqual(agrp, CFSTR("appleaccount")) && cls == genp_class()) {
1310         static CFStringRef accountServices[] = {
1311             CFSTR("com.apple.appleaccount.fmf.token"), /* temporary tokens while accout is being setup */
1312             CFSTR("com.apple.appleaccount.fmf.apptoken"),
1313             CFSTR("com.apple.appleaccount.fmip.siritoken"),
1314             CFSTR("com.apple.appleaccount.cloudkit.token"),
1315             NULL
1316         };
1317         CFStringRef service = CFDictionaryGetValue(item, kSecAttrService);
1318
1319         if (isString(service) && matchAnyString(service, accountServices)) {
1320             secdebug("backup", "found exact sys_bound item: %@", item);
1321             return true;
1322         }
1323     }
1324
1325     if (multiUser && CFEqual(agrp, CFSTR("apple")) && cls == genp_class()) {
1326         static CFStringRef accountServices[] = {
1327             /* accounts, remove with rdar://37595482 */
1328             CFSTR("com.apple.account.AppleAccount.token"),
1329             CFSTR("com.apple.account.AppleAccount.password"),
1330             CFSTR("com.apple.account.AppleAccount.rpassword"),
1331             CFSTR("com.apple.account.idms.token"),
1332             CFSTR("com.apple.account.idms.heartbeat-token"),
1333             CFSTR("com.apple.account.idms.continuation-key"),
1334             CFSTR("com.apple.account.CloudKit.token"),
1335             CFSTR("com.apple.account.IdentityServices.password"), /* accountsd for ids */
1336             CFSTR("com.apple.account.IdentityServices.rpassword"),
1337             CFSTR("com.apple.account.IdentityServices.token"),
1338             /* IDS stuff */
1339             CFSTR("BackupIDSAccountToken"),
1340             CFSTR("com.apple.ids"),
1341             CFSTR("ids"),
1342             CFSTR("IDS"),
1343             NULL
1344         };
1345         CFStringRef service = CFDictionaryGetValue(item, kSecAttrService);
1346
1347         if (isString(service) && matchAnyString(service, accountServices)) {
1348             secdebug("backup", "found exact sys_bound item: %@", item);
1349             return true;
1350         }
1351         if (isString(service) && CFStringHasPrefix(service, CFSTR("com.apple.gs."))) {
1352             secdebug("backup", "found exact sys_bound item: %@", item);
1353             return true;
1354         }
1355         if (isString(service) && CFEqual(service, CFSTR("com.apple.facetime"))) {
1356             CFStringRef account = CFDictionaryGetValue(item, kSecAttrAccount);
1357             if (isString(account) && CFEqual(account, CFSTR("registrationV1"))) {
1358                 secdebug("backup", "found exact sys_bound item: %@", item);
1359                 return true;
1360             }
1361         }
1362     }
1363
1364     /* accounts, remove with rdar://37595482 */
1365     if (multiUser && CFEqual(agrp, CFSTR("com.apple.ind")) && cls == genp_class()) {
1366         CFStringRef service = CFDictionaryGetValue(item, kSecAttrService);
1367         if (isString(service) && CFEqual(service, CFSTR("com.apple.ind.registration"))) {
1368             secdebug("backup", "found exact sys_bound item: %@", item);
1369             return true;
1370         }
1371     }
1372
1373     if (multiUser && CFEqual(agrp, CFSTR("ichat")) && cls == genp_class()) {
1374         static CFStringRef accountServices[] = {
1375             CFSTR("ids"),
1376             NULL
1377         };
1378         CFStringRef service = CFDictionaryGetValue(item, kSecAttrService);
1379
1380         if (isString(service) && matchAnyString(service, accountServices)) {
1381             secdebug("backup", "found exact sys_bound item: %@", item);
1382             return true;
1383         }
1384     }
1385
1386     if (multiUser && CFEqual(agrp, CFSTR("ichat")) && cls == keys_class()) {
1387         static CFStringRef exactMatchingLabel[] = {
1388             CFSTR("iMessage Encryption Key"),
1389             CFSTR("iMessage Signing Key"),
1390         };
1391         CFStringRef label = CFDictionaryGetValue(item, kSecAttrLabel);
1392         if (isString(label)) {
1393             if (matchAnyString(label, exactMatchingLabel)) {
1394                 secdebug("backup", "found exact sys_bound item: %@", item);
1395                 return true;
1396             }
1397         }
1398     }
1399
1400
1401     secdebug("backup", "found non sys_bound item: %@", item);
1402     return false;
1403 }
1404
1405 /* Delete all items from the current keychain.  If this is not an in
1406  place upgrade we don't delete items in the 'lockdown-identities'
1407  access group, this ensures that an import or restore of a backup
1408  will never overwrite an existing activation record. */
1409 static bool SecServerDeleteAll(SecDbConnectionRef dbt, CFErrorRef *error) {
1410     secwarning("SecServerDeleteAll");
1411
1412     return kc_transaction(dbt, error, ^{
1413
1414         bool ok = (SecDbExec(dbt, CFSTR("DELETE from genp;"), error) &&
1415                    SecDbExec(dbt, CFSTR("DELETE from inet;"), error) &&
1416                    SecDbExec(dbt, CFSTR("DELETE from cert;"), error) &&
1417                    SecDbExec(dbt, CFSTR("DELETE from keys;"), error));
1418         return ok;
1419     });
1420 }
1421
1422 #if TARGET_OS_IPHONE
1423
1424 static bool DeleteAllFromTableForMUSRView(SecDbConnectionRef dbt,
1425                                           CFStringRef sql,
1426                                           CFDataRef musr,
1427                                           bool keepU,
1428                                           CFErrorRef *error)
1429 {
1430     sqlite3_stmt *stmt = NULL;
1431     CFStringRef sql2 = NULL;
1432     bool ok = false;
1433
1434     if (keepU) {
1435         sql2 = CFStringCreateWithFormat(NULL, NULL, CFSTR("%@ AND pdmn NOT IN ('aku','akpu','cku','dku')"), sql);
1436     } else {
1437         sql2 = CFRetain(sql);
1438     }
1439     require(sql2, fail);
1440
1441     stmt = SecDbCopyStmt(dbt, sql2, NULL, error);
1442     require(stmt, fail);
1443
1444     ok = SecDbBindObject(stmt, 1, musr, error);
1445     require(ok, fail);
1446
1447     ok = SecDbStep(dbt, stmt, error, ^(bool *stop) { });
1448     require(ok, fail);
1449
1450 fail:
1451     if (stmt) {
1452         ok = SecDbFinalize(stmt, error);
1453     }
1454     if (!ok)
1455         secwarning("DeleteAllFromTableForMUSRView failed for %@ for musr: %@: %@", sql2, musr, error ? *error : NULL);
1456
1457     CFReleaseNull(sql2);
1458
1459     return ok;
1460 }
1461
1462 bool SecServerDeleteAllForUser(SecDbConnectionRef dbt, CFDataRef musrView, bool keepU, CFErrorRef *error) {
1463     secwarning("SecServerDeleteAllForUser for user: %@ keepU %s", musrView, keepU ? "yes" : "no");
1464
1465     return kc_transaction(dbt, error, ^{
1466         bool ok;
1467
1468         ok = (DeleteAllFromTableForMUSRView(dbt, CFSTR("DELETE FROM genp WHERE musr = ?"), musrView, keepU, error) &&
1469               DeleteAllFromTableForMUSRView(dbt, CFSTR("DELETE FROM inet WHERE musr = ?"), musrView, keepU, error) &&
1470               DeleteAllFromTableForMUSRView(dbt, CFSTR("DELETE FROM cert WHERE musr = ?"), musrView, keepU, error) &&
1471               DeleteAllFromTableForMUSRView(dbt, CFSTR("DELETE FROM keys WHERE musr = ?"), musrView, keepU, error));
1472
1473         return ok;
1474     });
1475 }
1476 #endif
1477
1478
1479 struct s3dl_export_row_ctx {
1480     struct s3dl_query_ctx qc;
1481     keybag_handle_t dest_keybag;
1482     enum SecItemFilter filter;
1483     bool multiUser;
1484 };
1485
1486 static void s3dl_export_row(sqlite3_stmt *stmt, void *context) {
1487     struct s3dl_export_row_ctx *c = context;
1488     Query *q = c->qc.q;
1489     SecAccessControlRef access_control = NULL;
1490     CFErrorRef localError = NULL;
1491
1492     /* Skip akpu items when backing up, those are intentionally lost across restores. The same applies to SEP-based keys */
1493     bool skip_akpu_or_token = c->filter == kSecBackupableItemFilter;
1494
1495     sqlite_int64 rowid = sqlite3_column_int64(stmt, 0);
1496     CFMutableDictionaryRef allAttributes = NULL;
1497     CFMutableDictionaryRef metadataAttributes = NULL;
1498     CFMutableDictionaryRef secretStuff = CFDictionaryCreateMutable(NULL, 0, &kCFTypeDictionaryKeyCallBacks, &kCFTypeDictionaryValueCallBacks);
1499     keyclass_t keyclass = 0;
1500     bool ok = s3dl_item_from_col(stmt, q, 1, c->qc.accessGroups, &allAttributes, &access_control, &keyclass, &localError);
1501
1502     if (ok) {
1503         metadataAttributes = CFDictionaryCreateMutableCopy(NULL, 0, allAttributes);
1504         SecDbForEachAttrWithMask(q->q_class, desc, kSecDbReturnDataFlag) {
1505             CFTypeRef value = CFDictionaryGetValue(metadataAttributes, desc->name);
1506             if (value) {
1507                 CFDictionarySetValue(secretStuff, desc->name, value);
1508                 CFDictionaryRemoveValue(metadataAttributes, desc->name);
1509             }
1510         }
1511     }
1512
1513     bool is_akpu = access_control ? CFEqualSafe(SecAccessControlGetProtection(access_control),
1514                                                 kSecAttrAccessibleWhenPasscodeSetThisDeviceOnly) : keyclass == key_class_akpu;
1515     bool is_token = (ok && allAttributes != NULL) ? CFDictionaryContainsKey(allAttributes, kSecAttrTokenID) : false;
1516
1517     if (ok && allAttributes && !(skip_akpu_or_token && (is_akpu || is_token))) {
1518         /* Only export sysbound items if do_sys_bound is true, only export non sysbound items otherwise. */
1519         bool do_sys_bound = c->filter == kSecSysBoundItemFilter;
1520         if (c->filter == kSecNoItemFilter ||
1521             SecItemIsSystemBound(allAttributes, q->q_class, c->multiUser) == do_sys_bound) {
1522             /* Re-encode the item. */
1523             secdebug("item", "export rowid %llu item: %@", rowid, allAttributes);
1524             /* The code below could be moved into handle_row. */
1525             CFDataRef pref = _SecItemCreatePersistentRef(q->q_class->name, rowid, allAttributes);
1526             if (pref) {
1527                 if (c->dest_keybag != KEYBAG_NONE) {
1528                     CFMutableDictionaryRef auth_attribs = CFDictionaryCreateMutable(NULL, 0, &kCFTypeDictionaryKeyCallBacks, &kCFTypeDictionaryValueCallBacks);
1529                     SecDbForEachAttrWithMask(q->q_class, desc, kSecDbInAuthenticatedDataFlag) {
1530                         CFTypeRef value = CFDictionaryGetValue(metadataAttributes, desc->name);
1531                         if(value) {
1532                             CFDictionaryAddValue(auth_attribs, desc->name, value);
1533                             CFDictionaryRemoveValue(metadataAttributes, desc->name);
1534                         }
1535                     }
1536
1537                     /* Encode and encrypt the item to the specified keybag. */
1538                     CFDataRef edata = NULL;
1539                     bool encrypted = ks_encrypt_data(c->dest_keybag, access_control, q->q_use_cred_handle, secretStuff, metadataAttributes, auth_attribs, &edata, false, &q->q_error);
1540                     CFDictionaryRemoveAllValues(allAttributes);
1541                     CFRelease(auth_attribs);
1542                     if (encrypted) {
1543                         CFDictionarySetValue(allAttributes, kSecValueData, edata);
1544                         CFReleaseSafe(edata);
1545                     } else {
1546                         seccritical("ks_encrypt_data %@,rowid=%" PRId64 ": failed: %@", q->q_class->name, rowid, q->q_error);
1547                         CFReleaseNull(q->q_error);
1548                     }
1549                 }
1550                 if (CFDictionaryGetCount(allAttributes)) {
1551                     CFDictionarySetValue(allAttributes, kSecValuePersistentRef, pref);
1552                     CFArrayAppendValue((CFMutableArrayRef)c->qc.result, allAttributes);
1553                     c->qc.found++;
1554                 }
1555                 CFReleaseSafe(pref);
1556             }
1557         }
1558     } else {
1559         OSStatus status = SecErrorGetOSStatus(localError);
1560
1561         if (status == errSecInteractionNotAllowed && is_akpu && skip_akpu_or_token) {
1562             // We expect akpu items to be inaccessible when the device is locked.
1563             CFReleaseNull(localError);
1564         } else {
1565             /* This happens a lot when trying to migrate keychain before first unlock, so only a notice */
1566             /* If the error is "corrupted item" then we just ignore it, otherwise we save it in the query */
1567             secinfo("item","Could not export item for rowid %llu: %@", rowid, localError);
1568
1569             if(status == errSecDecode) {
1570                 CFReleaseNull(localError);
1571             } else {
1572                 CFReleaseSafe(q->q_error);
1573                 q->q_error=localError;
1574             }
1575         }
1576     }
1577     CFReleaseNull(access_control);
1578     CFReleaseNull(allAttributes);
1579     CFReleaseNull(metadataAttributes);
1580     CFReleaseNull(secretStuff);
1581 }
1582
1583 static CFStringRef
1584 SecCreateKeybagUUID(keybag_handle_t keybag)
1585 {
1586 #if !TARGET_HAS_KEYSTORE
1587     return NULL;
1588 #else
1589     char uuidstr[37];
1590     uuid_t uuid;
1591     if (aks_get_bag_uuid(keybag, uuid) != KERN_SUCCESS)
1592         return NULL;
1593     uuid_unparse_lower(uuid, uuidstr);
1594     return CFStringCreateWithCString(NULL, uuidstr, kCFStringEncodingUTF8);
1595 #endif
1596 }
1597
1598
1599 CFDictionaryRef
1600 SecServerCopyKeychainPlist(SecDbConnectionRef dbt,
1601                            SecurityClient *client,
1602                            keybag_handle_t src_keybag,
1603                            keybag_handle_t dest_keybag,
1604                            enum SecItemFilter filter,
1605                            CFErrorRef *error) {
1606     CFMutableDictionaryRef keychain;
1607     keychain = CFDictionaryCreateMutable(kCFAllocatorDefault, 0,
1608                                          &kCFTypeDictionaryKeyCallBacks, &kCFTypeDictionaryValueCallBacks);
1609     unsigned class_ix;
1610     bool inMultiUser = false;
1611     CFStringRef keybaguuid = NULL;
1612     Query q = { .q_keybag = src_keybag,
1613         .q_musrView = NULL
1614     };
1615
1616     if (!keychain) {
1617         if (error && !*error)
1618             SecError(errSecAllocate, error, CFSTR("Can't create keychain dictionary"));
1619         goto errOut;
1620     }
1621
1622     q.q_return_type =
1623         kSecReturnDataMask |
1624         kSecReturnAttributesMask |
1625         kSecReturnPersistentRefMask;
1626     q.q_limit = kSecMatchUnlimited;
1627     q.q_skip_acl_items = true;
1628
1629
1630 #if TARGET_OS_IPHONE
1631     if (client && client->inMultiUser) {
1632         q.q_musrView = SecMUSRCreateActiveUserUUID(client->uid);
1633         inMultiUser = true;
1634     } else
1635 #endif
1636     {
1637         q.q_musrView = SecMUSRGetSingleUserKeychainUUID();
1638         CFRetain(q.q_musrView);
1639
1640         keybaguuid = SecCreateKeybagUUID(dest_keybag);
1641         if (keybaguuid)
1642             CFDictionarySetValue(keychain, kSecBackupKeybagUUIDKey, keybaguuid);
1643     }
1644
1645     /* Get rid of this duplicate. */
1646     const SecDbClass *SecDbClasses[] = {
1647         genp_class(),
1648         inet_class(),
1649         cert_class(),
1650         keys_class()
1651     };
1652
1653     for (class_ix = 0; class_ix < array_size(SecDbClasses);
1654          ++class_ix) {
1655         q.q_class = SecDbClasses[class_ix];
1656         struct s3dl_export_row_ctx ctx = {
1657             .qc = { .q = &q, .dbt = dbt },
1658             .dest_keybag = dest_keybag, .filter = filter,
1659             .multiUser = inMultiUser,
1660         };
1661
1662         secnotice("item", "exporting class '%@'", q.q_class->name);
1663
1664         CFErrorRef localError = NULL;
1665         if (s3dl_query(s3dl_export_row, &ctx, &localError)) {
1666             if (CFArrayGetCount(ctx.qc.result)) {
1667                 SecSignpostBackupCount(SecSignpostImpulseBackupClassCount, q.q_class->name, CFArrayGetCount(ctx.qc.result), filter);
1668                 CFDictionaryAddValue(keychain, q.q_class->name, ctx.qc.result);
1669             }
1670
1671         } else {
1672             OSStatus status = (OSStatus)CFErrorGetCode(localError);
1673             if (status == errSecItemNotFound) {
1674                 CFRelease(localError);
1675             } else {
1676                 secerror("Export failed: %@", localError);
1677                 if (error) {
1678                     CFReleaseSafe(*error);
1679                     *error = localError;
1680                 } else {
1681                     CFRelease(localError);
1682                 }
1683                 CFReleaseNull(keychain);
1684                 CFReleaseNull(ctx.qc.result);
1685                 break;
1686             }
1687         }
1688         CFReleaseNull(ctx.qc.result);
1689     }
1690
1691 errOut:
1692     CFReleaseNull(q.q_musrView);
1693     CFReleaseNull(keybaguuid);
1694
1695     return keychain;
1696 }
1697
1698 struct SecServerImportClassState {
1699         SecDbConnectionRef dbt;
1700     CFErrorRef error;
1701     keybag_handle_t src_keybag;
1702     keybag_handle_t dest_keybag;
1703     SecurityClient *client;
1704     enum SecItemFilter filter;
1705 };
1706
1707 struct SecServerImportItemState {
1708     const SecDbClass *class;
1709         struct SecServerImportClassState *s;
1710 };
1711
1712 static void
1713 SecServerImportItem(const void *value, void *context)
1714 {
1715     struct SecServerImportItemState *state = (struct SecServerImportItemState *)context;
1716     bool inMultiUser = false;
1717 #if TARGET_OS_IPHONE
1718     if (state->s->client->inMultiUser)
1719         inMultiUser = true;
1720 #endif
1721
1722     if (state->s->error)
1723         return;
1724
1725     if (!isDictionary(value)) {
1726         SecError(errSecParam, &state->s->error, CFSTR("value %@ is not a dictionary"), value);
1727         return;
1728     }
1729
1730     CFDictionaryRef dict = (CFDictionaryRef)value;
1731
1732     secdebug("item", "Import Item : %@", dict);
1733
1734     /* We use the kSecSysBoundItemFilter to indicate that we don't
1735      * preserve rowid's during import.
1736      */
1737     if (state->s->filter == kSecBackupableItemFilter) {
1738         CFTypeRef pdmu;
1739
1740         /* We don't filter non sys_bound items during import since we know we
1741          * will never have any in this case.
1742          */
1743         if (SecItemIsSystemBound(dict, state->class, inMultiUser))
1744             return;
1745
1746         /*
1747          * Don't bother with u items when in edu mode since our current backup system
1748          * don't keep track of items that blongs to the device (u) but rather just
1749          * merge them into one blob.
1750          */
1751         if (inMultiUser && (pdmu = CFDictionaryGetValue(dict, kSecAttrAccessible))) {
1752             if (CFEqual(pdmu, kSecAttrAccessibleWhenUnlockedThisDeviceOnly)     ||
1753                 CFEqual(pdmu, kSecAttrAccessibleAfterFirstUnlockThisDeviceOnly) ||
1754                 CFEqual(pdmu, kSecAttrAccessibleWhenUnlockedThisDeviceOnly)     ||
1755                 CFEqual(pdmu, kSecAttrAccessibleWhenPasscodeSetThisDeviceOnly))
1756             {
1757                 secdebug("item", "Skipping KU item : %@", dict);
1758                 return;
1759             }
1760         }
1761
1762         /* Avoid importing token-based items.  Although newer backups should not have them,
1763          * older (iOS9, iOS10.0) produced backups with token-based items.
1764          */
1765         if (CFDictionaryContainsKey(dict, kSecAttrTokenID)) {
1766             secdebug("item", "Skipping token-based item : %@", dict);
1767             return;
1768         }
1769     }
1770
1771     SecDbItemRef item;
1772
1773     /* This is sligthly confusing:
1774      - During upgrade all items are exported with KEYBAG_NONE.
1775      - During restore from backup, existing sys_bound items are exported with KEYBAG_NONE, and are exported as dictionary of attributes.
1776      - Item in the actual backup are export with a real keybag, and are exported as encrypted v_Data and v_PersistentRef
1777      */
1778     if (state->s->src_keybag == KEYBAG_NONE) {
1779         item = SecDbItemCreateWithAttributes(kCFAllocatorDefault, state->class, dict, state->s->dest_keybag,  &state->s->error);
1780     } else {
1781         item = SecDbItemCreateWithBackupDictionary(kCFAllocatorDefault, state->class, dict, state->s->src_keybag, state->s->dest_keybag, &state->s->error);
1782     }
1783
1784     /*
1785      *
1786      */
1787
1788     if (item) {
1789         CFDataRef musr = NULL;
1790         CFDataRef musrBackup = CFDictionaryGetValue(dict, kSecAttrMultiUser);
1791         CFDataRef systemKeychainUUID = SecMUSRGetSystemKeychainUUID();
1792         bool systemKeychain = CFEqualSafe(musrBackup, systemKeychainUUID);
1793
1794 #if TARGET_OS_IPHONE
1795         if (state->s->client && state->s->client->inMultiUser) {
1796             if (systemKeychain) {
1797                 secwarning("system keychain not allowed in multi user mode for item: %@", item);
1798             } else {
1799                 musr = SecMUSRCreateActiveUserUUID(state->s->client->uid);
1800             }
1801         } else
1802 #endif
1803         {
1804             if (systemKeychain) {
1805                 musr = SecMUSRCopySystemKeychainUUID();
1806             } else {
1807                 musr = SecMUSRGetSingleUserKeychainUUID();
1808                 CFRetainSafe(musr);
1809             }
1810         }
1811         if (musr == NULL) {
1812             CFReleaseNull(item);
1813         } else {
1814             SecDbItemSetValueWithName(item, CFSTR("musr"), musr, &state->s->error);
1815             CFRelease(musr);
1816         }
1817     }
1818
1819     /*
1820      *
1821      */
1822
1823     if (item) {
1824         bool insertStatus;
1825
1826         if(state->s->filter != kSecSysBoundItemFilter) {
1827             SecDbItemExtractRowIdFromBackupDictionary(item, dict, &state->s->error);
1828         }
1829         SecDbItemInferSyncable(item, &state->s->error);
1830         insertStatus = SecDbItemInsert(item, state->s->dbt, &state->s->error);
1831         if (!insertStatus) {
1832             /*
1833              When running in EduMode, multiple users share the same
1834              keychain and unfortionaly the rowid is used a
1835              persistant reference and is part of the contraints (its
1836              UNIQUE), so lets clear the rowid and try to insert the
1837              entry again.
1838
1839              This even happens for normal operation because of
1840              SysBound entries, so in case of a failure, lets try
1841              again to insert the record.
1842              */
1843             SecDbItemClearRowId(item, NULL);
1844             SecDbItemInsert(item, state->s->dbt, &state->s->error);
1845         }
1846         }
1847
1848     /* Reset error if we had one, since we just skip the current item
1849      and continue importing what we can. */
1850     if (state->s->error) {
1851         secwarning("Failed to import an item (%@) of class '%@': %@ - ignoring error.",
1852                    item, state->class->name, state->s->error);
1853         CFReleaseNull(state->s->error);
1854     }
1855
1856     CFReleaseSafe(item);
1857 }
1858
1859 static void SecServerImportClass(const void *key, const void *value,
1860                                  void *context) {
1861     struct SecServerImportClassState *state =
1862     (struct SecServerImportClassState *)context;
1863     if (state->error)
1864         return;
1865     if (!isString(key)) {
1866         SecError(errSecParam, &state->error, CFSTR("class name %@ is not a string"), key);
1867         return;
1868     }
1869     /* ignore the Keybag UUID */
1870     if (CFEqual(key, kSecBackupKeybagUUIDKey))
1871         return;
1872     const SecDbClass *class = kc_class_with_name(key);
1873     if (!class) {
1874         secwarning("Ignoring unknown key class '%@'", key);
1875         return;
1876     }
1877     if (class == identity_class()) {
1878         SecError(errSecParam, &state->error, CFSTR("attempt to import an identity"));
1879         return;
1880     }
1881     struct SecServerImportItemState item_state = {
1882         .class = class, .s = state,
1883     };
1884     if (isArray(value)) {
1885         CFArrayRef items = (CFArrayRef)value;
1886         secwarning("Import %ld items of class %@ (filter %d)", (long)CFArrayGetCount(items), key, state->filter);
1887         SecSignpostBackupCount(SecSignpostImpulseRestoreClassCount, class->name, CFArrayGetCount(items), state->filter);
1888         CFArrayApplyFunction(items, CFRangeMake(0, CFArrayGetCount(items)),
1889                              SecServerImportItem, &item_state);
1890     } else if (isDictionary(value)) {
1891         CFDictionaryRef item = (CFDictionaryRef)value;
1892         secwarning("Import %ld items of class %@ (filter %d)", (long)1, key, state->filter);
1893         SecSignpostBackupCount(SecSignpostImpulseRestoreClassCount, class->name, 1, state->filter);
1894         SecServerImportItem(item, &item_state);
1895     } else {
1896         secwarning("Unknown value type for class %@ (filter %d)", key, state->filter);
1897     }
1898 }
1899
1900 bool SecServerImportKeychainInPlist(SecDbConnectionRef dbt, SecurityClient *client,
1901                                             keybag_handle_t src_keybag, keybag_handle_t dest_keybag,
1902                                             CFDictionaryRef keychain, enum SecItemFilter filter,
1903                                             bool removeKeychainContent, CFErrorRef *error) {
1904     CFStringRef keybaguuid = NULL;
1905     bool ok = true;
1906
1907     CFDictionaryRef sys_bound = NULL;
1908     if (filter == kSecBackupableItemFilter) {
1909         /* Grab a copy of all the items for which SecItemIsSystemBound()
1910          returns true. */
1911         require(sys_bound = SecServerCopyKeychainPlist(dbt, client, KEYBAG_DEVICE,
1912                                                        KEYBAG_NONE, kSecSysBoundItemFilter,
1913                                                        error), errOut);
1914     }
1915
1916     /*
1917      * Validate the uuid of the source keybag matches what we have in the backup
1918      */
1919     keybaguuid = SecCreateKeybagUUID(src_keybag);
1920     if (keybaguuid) {
1921         CFStringRef uuid = CFDictionaryGetValue(keychain, kSecBackupKeybagUUIDKey);
1922         if (isString(uuid)) {
1923             require_action(CFEqual(keybaguuid, uuid), errOut,
1924                            SecError(errSecDecode, error, CFSTR("Keybag UUID (%@) mismatch with backup (%@)"),
1925                                     keybaguuid, uuid));
1926         }
1927     }
1928
1929 #if TARGET_OS_IOS
1930     /*
1931      * Shared iPad is very special, it always delete's the user keychain, and never merge content
1932      */
1933     if (client->inMultiUser) {
1934         CFDataRef musrView = SecMUSRCreateActiveUserUUID(client->uid);
1935         require_action(musrView, errOut, ok = false);
1936         require_action(ok = SecServerDeleteAllForUser(dbt, musrView, true, error), errOut, CFReleaseNull(musrView));
1937         CFReleaseNull(musrView);
1938     } else
1939 #endif
1940     {
1941         /*
1942          * Delete everything in the keychain.
1943          * We don't want this if we're restoring backups because we probably already synced stuff over
1944          */
1945         if (removeKeychainContent) {
1946             require(ok = SecServerDeleteAll(dbt, error), errOut);
1947         } else {
1948             // Custom hack to support bluetooth's workflow for 11.3. Should be removed in a future release.
1949             __block CFErrorRef btError = NULL;
1950             bool deletedBT = kc_transaction(dbt, &btError, ^bool{
1951                 bool ok = SecDbExec(dbt, CFSTR("DELETE FROM genp WHERE sync = 0 AND NOT agrp IN ('com.apple.security.sos', 'com.apple.security.sos-usercredential', 'com.apple.security.ckks');"), &btError);
1952                 ok &= SecDbExec(dbt, CFSTR("DELETE FROM inet WHERE sync = 0 AND NOT agrp IN ('com.apple.security.sos', 'com.apple.security.sos-usercredential', 'com.apple.security.ckks');"), &btError);
1953                 ok &= SecDbExec(dbt, CFSTR("DELETE FROM cert WHERE sync = 0 AND NOT agrp IN ('com.apple.security.sos', 'com.apple.security.sos-usercredential', 'com.apple.security.ckks');"), &btError);
1954                 ok &= SecDbExec(dbt, CFSTR("DELETE FROM keys WHERE sync = 0 AND NOT agrp IN ('com.apple.security.sos', 'com.apple.security.sos-usercredential', 'com.apple.security.ckks');"), &btError);
1955                 return ok;
1956             });
1957             if (!deletedBT) {
1958                 secerror("Unable to delete nonsyncable items prior to keychain restore: %@", btError);
1959             } else {
1960                 secnotice("restore", "Successfully deleted nonsyncable items");
1961             }
1962             CFReleaseNull(btError);
1963         }
1964     }
1965
1966     struct SecServerImportClassState state = {
1967         .dbt = dbt,
1968         .src_keybag = src_keybag,
1969         .dest_keybag = dest_keybag,
1970         .client = client,
1971         .filter = filter,
1972     };
1973     /* Import the provided items, preserving rowids. */
1974     secwarning("Restoring backup items '%ld'", (long)CFDictionaryGetCount(keychain));
1975     CFDictionaryApplyFunction(keychain, SecServerImportClass, &state);
1976
1977     if (sys_bound) {
1978         state.src_keybag = KEYBAG_NONE;
1979         /* Import the items we preserved with random rowids. */
1980         state.filter = kSecSysBoundItemFilter;
1981         secwarning("Restoring sysbound items '%ld'", (long)CFDictionaryGetCount(sys_bound));
1982         CFDictionaryApplyFunction(sys_bound, SecServerImportClass, &state);
1983     }
1984     if (state.error) {
1985         if (error) {
1986             CFReleaseSafe(*error);
1987             *error = state.error;
1988         } else {
1989             CFRelease(state.error);
1990         }
1991         ok = false;
1992     }
1993
1994     // If CKKS had spun up, it's very likely that we just deleted its data.
1995     // Tell it to perform a local resync.
1996     SecCKKSPerformLocalResync();
1997
1998 errOut:
1999     CFReleaseSafe(sys_bound);
2000     CFReleaseSafe(keybaguuid);
2001
2002     return ok;
2003 }
2004
2005 CFStringRef
2006 SecServerBackupGetKeybagUUID(CFDictionaryRef keychain, CFErrorRef *error)
2007 {
2008     CFStringRef uuid = CFDictionaryGetValue(keychain, kSecBackupKeybagUUIDKey);
2009     if (!isString(uuid)) {
2010         SecError(errSecDecode, error, CFSTR("Missing or invalid %@ in backup dictionary"), kSecBackupKeybagUUIDKey);
2011         return NULL;
2012     }
2013     return uuid;
2014 }
2015
2016 #pragma mark - key rolling support
2017 #if USE_KEYSTORE
2018
2019 struct check_generation_ctx {
2020     struct s3dl_query_ctx query_ctx;
2021     uint32_t current_generation;
2022 };
2023
2024 static void check_generation(sqlite3_stmt *stmt, void *context) {
2025     struct check_generation_ctx *c = context;
2026     CFDataRef blob = NULL;
2027     size_t blobLen = 0;
2028     const uint8_t *cursor = NULL;
2029     uint32_t version;
2030     keyclass_t keyclass;
2031     uint32_t current_generation = c->current_generation;
2032
2033     require(blob = s3dl_copy_data_from_col(stmt, 1, &c->query_ctx.q->q_error), out);
2034     blobLen = CFDataGetLength(blob);
2035     cursor = CFDataGetBytePtr(blob);
2036
2037     /* Check for underflow, ensuring we have at least one full AES block left. */
2038     if (blobLen < sizeof(version) + sizeof(keyclass)) {
2039         SecError(errSecDecode, &c->query_ctx.q->q_error, CFSTR("check_generation: Check for underflow"));
2040         goto out;
2041     }
2042
2043     version = *((uint32_t *)cursor);
2044     cursor += sizeof(version);
2045
2046     (void) version; // TODO: do something with the version number.
2047
2048     keyclass = *((keyclass_t *)cursor);
2049
2050     // TODO: export get_key_gen macro
2051     if (((keyclass & ~key_class_last) == 0) != (current_generation == 0)) {
2052         c->query_ctx.found++;
2053     }
2054
2055     CFReleaseSafe(blob);
2056     return;
2057
2058 out:
2059     c->query_ctx.found++;
2060     CFReleaseSafe(blob);
2061 }
2062
2063 bool s3dl_dbt_keys_current(SecDbConnectionRef dbt, uint32_t current_generation, CFErrorRef *error) {
2064     CFErrorRef localError = NULL;
2065     struct check_generation_ctx ctx = { .query_ctx = { .dbt = dbt }, .current_generation = current_generation };
2066
2067     const SecDbClass *classes[] = {
2068         genp_class(),
2069         inet_class(),
2070         keys_class(),
2071         cert_class(),
2072     };
2073
2074     for (size_t class_ix = 0; class_ix < array_size(classes); ++class_ix) {
2075         Query *q = query_create(classes[class_ix], NULL, NULL, &localError);
2076         if (!q)
2077             return false;
2078
2079         ctx.query_ctx.q = q;
2080         q->q_limit = kSecMatchUnlimited;
2081
2082         bool ok = s3dl_query(check_generation, &ctx, &localError);
2083         query_destroy(q, NULL);
2084         CFReleaseNull(ctx.query_ctx.result);
2085
2086         if (!ok && localError && (CFErrorGetCode(localError) == errSecItemNotFound)) {
2087             CFReleaseNull(localError);
2088             continue;
2089         }
2090         secerror("Class %@ not up to date", classes[class_ix]->name);
2091         return false;
2092     }
2093     return true;
2094 }
2095
2096 bool s3dl_dbt_update_keys(SecDbConnectionRef dbt, SecurityClient *client, CFErrorRef *error) {
2097     return SecDbTransaction(dbt, kSecDbExclusiveTransactionType, error, ^(bool *commit) {
2098         __block bool ok = false;
2099         uint32_t keystore_generation_status;
2100
2101         /* can we migrate to new class keys right now? */
2102         if (!aks_generation(KEYBAG_DEVICE, generation_noop, &keystore_generation_status) &&
2103             (keystore_generation_status & generation_change_in_progress)) {
2104
2105             /* take a lock assertion */
2106             bool operated_while_unlocked = SecAKSDoWhileUserBagLocked(error, ^{
2107                 CFErrorRef localError = NULL;
2108                 CFDictionaryRef backup = SecServerCopyKeychainPlist(dbt, NULL,
2109                                                                     KEYBAG_DEVICE, KEYBAG_NONE, kSecNoItemFilter, &localError);
2110                 if (backup) {
2111                     if (localError) {
2112                         secerror("Ignoring export error: %@ during roll export", localError);
2113                         CFReleaseNull(localError);
2114                     }
2115                     // 'true' argument: we're replacing everything with newly wrapped entries so remove the old stuff
2116                     ok = SecServerImportKeychainInPlist(dbt, client, KEYBAG_NONE,
2117                                                         KEYBAG_DEVICE, backup, kSecNoItemFilter, true, &localError);
2118                     if (localError) {
2119                         secerror("Ignoring export error: %@ during roll export", localError);
2120                         CFReleaseNull(localError);
2121                     }
2122                     CFRelease(backup);
2123                 }
2124             });
2125             if (!operated_while_unlocked)
2126                 ok = false;
2127         } else {
2128             ok = SecError(errSecBadReq, error, CFSTR("No key roll in progress."));
2129         }
2130
2131         *commit = ok;
2132     });
2133 }
2134 #endif