OSX/sec/securityd/SecItemDb.c

   1 /*
   2  * Copyright (c) 2006-2014 Apple Inc. All Rights Reserved.
   3  *
   4  * @APPLE_LICENSE_HEADER_START@
   5  *
   6  * This file contains Original Code and/or Modifications of Original Code
   7  * as defined in and that are subject to the Apple Public Source License
   8  * Version 2.0 (the 'License'). You may not use this file except in
   9  * compliance with the License. Please obtain a copy of the License at
  10  * http://www.opensource.apple.com/apsl/ and read it before using this
  11  * file.
  12  *
  13  * The Original Code and all software distributed under the License are
  14  * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  15  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  16  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
  17  * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
  18  * Please see the License for the specific language governing rights and
  19  * limitations under the License.
  20  *
  21  * @APPLE_LICENSE_HEADER_END@
  22  */
  23
  24 /*
  25  *  SecItemDb.c - CoreFoundation-based constants and functions for
  26     access to Security items (certificates, keys, identities, and
  27     passwords.)
  28  */
  29
  30 #include <securityd/SecItemDb.h>
  31 #include <utilities/SecAKSWrappers.h>
  32
  33 #include <securityd/SecDbKeychainItem.h>
  34 #include <securityd/SecItemSchema.h>
  35 #include <securityd/SecItemServer.h>
  36 #include <Security/SecAccessControlPriv.h>
  37 #include <Security/SecBasePriv.h>
  38 #include <Security/SecItem.h>
  39 #include <Security/SecItemPriv.h>
  40 #include <Security/SecItemInternal.h>
  41 #include <securityd/SOSCloudCircleServer.h>
  42 #include <utilities/array_size.h>
  43 #include <utilities/SecIOFormat.h>
  44 #include <utilities/SecCFCCWrappers.h>
  45 #include <SecAccessControlPriv.h>
  46 #include <uuid/uuid.h>
  47
  48 #include "keychain/ckks/CKKS.h"
  49
  50 #define kSecBackupKeybagUUIDKey CFSTR("keybag-uuid")
  51
  52 const SecDbAttr *SecDbAttrWithKey(const SecDbClass *c,
  53                                   CFTypeRef key,
  54                                   CFErrorRef *error) {
  55     /* Special case: identites can have all attributes of either cert
  56      or keys. */
  57     if (c == identity_class()) {
  58         const SecDbAttr *desc;
  59         if (!(desc = SecDbAttrWithKey(cert_class(), key, 0)))
  60             desc = SecDbAttrWithKey(keys_class(), key, error);
  61         return desc;
  62     }
  63
  64     if (isString(key)) {
  65         SecDbForEachAttr(c, a) {
  66             if (CFEqual(a->name, key))
  67                 return a;
  68         }
  69         if (CFEqual(kSecAttrNoLegacy, key)) {
  70             return NULL; /* results in no ops for this attribute */
  71         }
  72     }
  73
  74     SecError(errSecNoSuchAttr, error, CFSTR("attribute %@ not found in class %@"), key, c->name);
  75
  76     return NULL;
  77 }
  78
  79 bool kc_transaction(SecDbConnectionRef dbt, CFErrorRef *error, bool(^perform)()) {
  80     return kc_transaction_type(dbt, kSecDbExclusiveTransactionType, error, perform);
  81 }
  82
  83 bool kc_transaction_type(SecDbConnectionRef dbt, SecDbTransactionType type, CFErrorRef *error, bool(^perform)()) {
  84     __block bool ok = true;
  85     return ok && SecDbTransaction(dbt, type, error, ^(bool *commit) {
  86         ok = *commit = perform();
  87     });
  88 }
  89
  90 static CFStringRef SecDbGetKindSQL(SecDbAttrKind kind) {
  91     switch (kind) {
  92         case kSecDbBlobAttr:
  93         case kSecDbDataAttr:
  94         case kSecDbUUIDAttr:
  95         case kSecDbSHA1Attr:
  96         case kSecDbPrimaryKeyAttr:
  97         case kSecDbEncryptedDataAttr:
  98             return CFSTR("BLOB");
  99         case kSecDbAccessAttr:
 100         case kSecDbStringAttr:
 101             return CFSTR("TEXT");
 102         case kSecDbNumberAttr:
 103         case kSecDbSyncAttr:
 104         case kSecDbTombAttr:
 105             return CFSTR("INTEGER");
 106         case kSecDbDateAttr:
 107         case kSecDbCreationDateAttr:
 108         case kSecDbModificationDateAttr:
 109             return CFSTR("REAL");
 110         case kSecDbRowIdAttr:
 111             return CFSTR("INTEGER PRIMARY KEY AUTOINCREMENT");
 112         case kSecDbAccessControlAttr:
 113         case kSecDbUTombAttr:
 114             /* This attribute does not exist in the DB. */
 115             return NULL;
 116     }
 117 }
 118
 119 static void SecDbAppendUnique(CFMutableStringRef sql, CFStringRef value, bool *haveUnique) {
 120     assert(haveUnique);
 121     if (!*haveUnique)
 122         CFStringAppend(sql, CFSTR("UNIQUE("));
 123
 124     SecDbAppendElement(sql, value, haveUnique);
 125 }
 126
 127 static void SecDbAppendCreateTableWithClass(CFMutableStringRef sql, const SecDbClass *c) {
 128     CFStringAppendFormat(sql, 0, CFSTR("CREATE TABLE %@("), c->name);
 129     SecDbForEachAttrWithMask(c,desc,kSecDbInFlag) {
 130         CFStringAppendFormat(sql, 0, CFSTR("%@ %@"), desc->name, SecDbGetKindSQL(desc->kind));
 131         if (desc->flags & kSecDbNotNullFlag)
 132             CFStringAppend(sql, CFSTR(" NOT NULL"));
 133         if (desc->flags & kSecDbDefault0Flag)
 134             CFStringAppend(sql, CFSTR(" DEFAULT 0"));
 135         if (desc->flags & kSecDbDefaultEmptyFlag)
 136             CFStringAppend(sql, CFSTR(" DEFAULT ''"));
 137         CFStringAppend(sql, CFSTR(","));
 138     }
 139
 140     bool haveUnique = false;
 141     SecDbForEachAttrWithMask(c,desc,kSecDbPrimaryKeyFlag | kSecDbInFlag) {
 142         SecDbAppendUnique(sql, desc->name, &haveUnique);
 143     }
 144     if (haveUnique)
 145         CFStringAppend(sql, CFSTR(")"));
 146
 147     CFStringAppend(sql, CFSTR(");"));
 148
 149     // Create indicies
 150     SecDbForEachAttrWithMask(c,desc, kSecDbIndexFlag | kSecDbInFlag) {
 151         CFStringAppendFormat(sql, 0, CFSTR("CREATE INDEX %@%@ ON %@(%@);"), c->name, desc->name, c->name, desc->name);
 152     }
 153 }
 154
 155 static void SecDbAppendDropTableWithClass(CFMutableStringRef sql, const SecDbClass *c) {
 156     CFStringAppendFormat(sql, 0, CFSTR("DROP TABLE %@;"), c->name);
 157 }
 158
 159 static CFDataRef SecPersistentRefCreateWithItem(SecDbItemRef item, CFErrorRef *error) {
 160     sqlite3_int64 row_id = SecDbItemGetRowId(item, error);
 161     if (row_id)
 162         return _SecItemCreatePersistentRef(SecDbItemGetClass(item)->name, row_id, item->attributes);
 163     return NULL;
 164 }
 165
 166 bool SecItemDbCreateSchema(SecDbConnectionRef dbt, const SecDbSchema *schema, CFArrayRef classIndexesForNewTables, bool includeVersion, CFErrorRef *error)
 167 {
 168     __block bool ok = true;
 169     CFMutableStringRef sql = CFStringCreateMutable(kCFAllocatorDefault, 0);
 170
 171     if (classIndexesForNewTables) {
 172         CFArrayForEach(classIndexesForNewTables, ^(const void* index) {
 173             const SecDbClass* class = schema->classes[(int)index];
 174             SecDbAppendCreateTableWithClass(sql, class);
 175         });
 176     }
 177     else {
 178         for (const SecDbClass * const *pclass = schema->classes; *pclass; ++pclass) {
 179             SecDbAppendCreateTableWithClass(sql, *pclass);
 180         }
 181     }
 182
 183     if (includeVersion) {
 184         CFStringAppendFormat(sql, NULL, CFSTR("INSERT INTO tversion(version,minor) VALUES(%d, %d);"),
 185                              schema->majorVersion, schema->minorVersion);
 186     }
 187     CFStringPerformWithCString(sql, ^(const char *sql_string) {
 188         ok = SecDbErrorWithDb(sqlite3_exec(SecDbHandle(dbt), sql_string, NULL, NULL, NULL),
 189                               SecDbHandle(dbt), error, CFSTR("sqlite3_exec: %s"), sql_string);
 190     });
 191     CFReleaseSafe(sql);
 192     return ok;
 193 }
 194
 195 bool SecItemDbDeleteSchema(SecDbConnectionRef dbt, const SecDbSchema *schema, CFErrorRef *error)
 196 {
 197     __block bool ok = true;
 198     CFMutableStringRef sql = CFStringCreateMutable(kCFAllocatorDefault, 0);
 199     for (const SecDbClass * const *pclass = schema->classes; *pclass; ++pclass) {
 200         SecDbAppendDropTableWithClass(sql, *pclass);
 201     }
 202     CFStringPerformWithCString(sql, ^(const char *sql_string) {
 203         ok = SecDbErrorWithDb(sqlite3_exec(SecDbHandle(dbt), sql_string, NULL, NULL, NULL),
 204                               SecDbHandle(dbt), error, CFSTR("sqlite3_exec: %s"), sql_string);
 205     });
 206     CFReleaseSafe(sql);
 207     return ok;
 208 }
 209
 210 CFTypeRef SecDbItemCopyResult(SecDbItemRef item, ReturnTypeMask return_type, CFErrorRef *error) {
 211     CFTypeRef a_result;
 212
 213         if (return_type == 0) {
 214                 /* Caller isn't interested in any results at all. */
 215                 a_result = kCFNull;
 216         } else if (return_type == kSecReturnDataMask) {
 217         a_result = SecDbItemGetCachedValueWithName(item, kSecValueData);
 218         if (a_result) {
 219             CFRetainSafe(a_result);
 220         } else {
 221             a_result = CFDataCreate(kCFAllocatorDefault, NULL, 0);
 222         }
 223         } else if (return_type == kSecReturnPersistentRefMask) {
 224                 a_result = SecPersistentRefCreateWithItem(item, error);
 225         } else {
 226         CFMutableDictionaryRef dict = CFDictionaryCreateMutableForCFTypes(CFGetAllocator(item));
 227                 /* We need to return more than one value. */
 228         if (return_type & kSecReturnRefMask) {
 229             CFDictionarySetValue(dict, kSecClass, SecDbItemGetClass(item)->name);
 230         }
 231         CFOptionFlags mask = (((return_type & kSecReturnDataMask || return_type & kSecReturnRefMask) ? kSecDbReturnDataFlag : 0) |
 232                               ((return_type & kSecReturnAttributesMask || return_type & kSecReturnRefMask) ? kSecDbReturnAttrFlag : 0));
 233         SecDbForEachAttr(SecDbItemGetClass(item), desc) {
 234             if ((desc->flags & mask) != 0) {
 235                 CFTypeRef value = SecDbItemGetValue(item, desc, error);
 236                 if (value && !CFEqual(kCFNull, value)) {
 237                     CFDictionarySetValue(dict, desc->name, value);
 238                 } else if (value == NULL) {
 239                     CFReleaseNull(dict);
 240                     break;
 241                 }
 242             }
 243         }
 244         CFDictionaryRemoveValue(dict, kSecAttrUUID);
 245
 246                 if (return_type & kSecReturnPersistentRefMask) {
 247             CFDataRef pref = SecPersistentRefCreateWithItem(item, error);
 248                         CFDictionarySetValue(dict, kSecValuePersistentRef, pref);
 249             CFReleaseSafe(pref);
 250                 }
 251
 252                 a_result = dict;
 253         }
 254
 255         return a_result;
 256 }
 257
 258 /* AUDIT[securityd](done):
 259  attributes (ok) is a caller provided dictionary, only its cf type has
 260  been checked.
 261  */
 262 bool
 263 s3dl_query_add(SecDbConnectionRef dbt, Query *q, CFTypeRef *result, CFErrorRef *error)
 264 {
 265     if (query_match_count(q) != 0)
 266         return errSecItemMatchUnsupported;
 267
 268     /* Add requires a class to be specified unless we are adding a ref. */
 269     if (q->q_use_item_list)
 270         return errSecUseItemListUnsupported;
 271
 272     /* Actual work here. */
 273     SecDbItemRef item = SecDbItemCreateWithAttributes(kCFAllocatorDefault, q->q_class, q->q_item, KEYBAG_DEVICE, error);
 274     if (!item)
 275         return false;
 276     if (SecDbItemIsTombstone(item))
 277         SecDbItemSetValue(item, &v7utomb, q->q_use_tomb ? q->q_use_tomb : kCFBooleanTrue, NULL);
 278
 279     bool ok = true;
 280     if (q->q_data)
 281         ok = SecDbItemSetValueWithName(item, CFSTR("v_Data"), q->q_data, error);
 282     if (q->q_row_id)
 283         ok = SecDbItemSetRowId(item, q->q_row_id, error);
 284     if (q->q_musrView)
 285         ok = SecDbItemSetValueWithName(item, CFSTR("musr"), q->q_musrView, error);
 286     SecDbItemSetCredHandle(item, q->q_use_cred_handle);
 287
 288 #if OCTAGON
 289     if(SecCKKSIsEnabled() && !SecCKKSTestDisableAutomaticUUID()) {
 290         s3dl_item_make_new_uuid(item, q->q_uuid_from_primary_key, error);
 291
 292         if(q->q_add_sync_callback) {
 293             CFTypeRef uuid = SecDbItemGetValue(item, &v10itemuuid, error);
 294             if(uuid) {
 295                 CKKSRegisterSyncStatusCallback(uuid, q->q_add_sync_callback);
 296             } else {
 297                 secerror("Couldn't fetch UUID from item; can't call callback");
 298             }
 299         }
 300     }
 301 #endif
 302
 303     if (ok)
 304         ok = SecDbItemInsert(item, dbt, error);
 305
 306     if (ok) {
 307         if (result && q->q_return_type) {
 308             *result = SecDbItemCopyResult(item, q->q_return_type, error);
 309         }
 310     }
 311     if (!ok && error && *error) {
 312         if (CFEqual(CFErrorGetDomain(*error), kSecDbErrorDomain) && CFErrorGetCode(*error) == SQLITE_CONSTRAINT) {
 313             CFReleaseNull(*error);
 314             SecError(errSecDuplicateItem, error, CFSTR("duplicate item %@"), item);
 315         } else if (CFEqual(CFErrorGetDomain(*error), kSecErrorDomain) && CFErrorGetCode(*error) == errSecDecode) { //handle situation when item have pdmn=akpu but passcode is not set
 316             CFTypeRef value = SecDbItemGetValue(item, SecDbClassAttrWithKind(item->class, kSecDbAccessAttr, error), error);
 317             if (value && CFEqual(value, kSecAttrAccessibleWhenPasscodeSetThisDeviceOnly)) {
 318                 CFReleaseNull(*error);
 319                 SecError(errSecAuthFailed, error, CFSTR("authentication failed"));
 320             }
 321         }
 322     }
 323
 324     if (ok) {
 325         q->q_changed = true;
 326         if (SecDbItemIsSyncable(item))
 327             q->q_sync_changed = true;
 328     }
 329
 330     secdebug("dbitem", "inserting item %@%s%@", item, ok ? "" : "failed: ", ok || error == NULL ? (CFErrorRef)CFSTR("") : *error);
 331
 332     CFRelease(item);
 333
 334         return ok;
 335 }
 336
 337 bool s3dl_item_make_new_uuid(SecDbItemRef item, bool uuid_from_primary_key, CFErrorRef* error) {
 338     if(!item) {
 339         return false;
 340     }
 341
 342     // Set the item UUID.
 343     CFUUIDRef uuid = NULL;
 344     // Were we asked to make the UUID static?
 345     if (uuid_from_primary_key) {
 346         // This UUID isn't used in any security mechanism, so we can
 347         // just use the first bits of the SHA256 hash.
 348         CFDataRef pkhash = SecDbKeychainItemCopySHA256PrimaryKey(item, error);
 349         if(CFDataGetLength(pkhash) >= 16) {
 350             UInt8 uuidBytes[16];
 351             CFRange range = CFRangeMake(0, 16);
 352             CFDataGetBytes(pkhash, range, uuidBytes);
 353
 354             uuid = CFUUIDCreateWithBytes(NULL,
 355                                          uuidBytes[ 0],
 356                                          uuidBytes[ 1],
 357                                          uuidBytes[ 2],
 358                                          uuidBytes[ 3],
 359                                          uuidBytes[ 4],
 360                                          uuidBytes[ 5],
 361                                          uuidBytes[ 6],
 362                                          uuidBytes[ 7],
 363                                          uuidBytes[ 8],
 364                                          uuidBytes[ 9],
 365                                          uuidBytes[10],
 366                                          uuidBytes[11],
 367                                          uuidBytes[12],
 368                                          uuidBytes[13],
 369                                          uuidBytes[14],
 370                                          uuidBytes[15]);
 371         }
 372         CFReleaseNull(pkhash);
 373     }
 374     if(uuid == NULL) {
 375         uuid = CFUUIDCreate(NULL);
 376     }
 377     SecDbItemSetValueWithName(item, kSecAttrUUID, uuid, error);
 378     CFReleaseNull(uuid);
 379     return true;
 380 }
 381
 382 typedef void (*s3dl_handle_row)(sqlite3_stmt *stmt, void *context);
 383
 384 static CFDataRef
 385 s3dl_copy_data_from_col(sqlite3_stmt *stmt, int col, CFErrorRef *error) {
 386     return CFDataCreateWithBytesNoCopy(0, sqlite3_column_blob(stmt, col),
 387                                        sqlite3_column_bytes(stmt, col),
 388                                        kCFAllocatorNull);
 389 }
 390
 391 static bool
 392 s3dl_item_from_col(sqlite3_stmt *stmt, Query *q, int col, CFArrayRef accessGroups,
 393                    CFMutableDictionaryRef *item, SecAccessControlRef *access_control, CFErrorRef *error) {
 394     CFDataRef edata = NULL;
 395     bool ok = false;
 396     require(edata = s3dl_copy_data_from_col(stmt, col, error), out);
 397     ok = s3dl_item_from_data(edata, q, accessGroups, item, access_control, error);
 398
 399 out:
 400     CFReleaseSafe(edata);
 401     return ok;
 402 }
 403
 404 struct s3dl_query_ctx {
 405     Query *q;
 406     CFArrayRef accessGroups;
 407     SecDbConnectionRef dbt;
 408     CFTypeRef result;
 409     int found;
 410 };
 411
 412 /* Return whatever the caller requested based on the value of q->q_return_type.
 413  keys and values must be 3 larger than attr_count in size to accomadate the
 414  optional data, class and persistent ref results.  This is so we can use
 415  the CFDictionaryCreate() api here rather than appending to a
 416  mutable dictionary. */
 417 static CF_RETURNS_RETAINED CFTypeRef
 418 handle_result(Query *q,
 419               CFMutableDictionaryRef item,
 420               sqlite_int64 rowid)
 421 {
 422     CFTypeRef a_result;
 423     CFDataRef data;
 424     data = CFDictionaryGetValue(item, kSecValueData);
 425         if (q->q_return_type == 0) {
 426                 /* Caller isn't interested in any results at all. */
 427                 a_result = kCFNull;
 428         } else if (q->q_return_type == kSecReturnDataMask) {
 429         if (data) {
 430             a_result = data;
 431             CFRetain(a_result);
 432         } else {
 433             a_result = CFDataCreate(kCFAllocatorDefault, NULL, 0);
 434         }
 435         } else if (q->q_return_type == kSecReturnPersistentRefMask) {
 436                 a_result = _SecItemCreatePersistentRef(q->q_class->name, rowid, item);
 437         } else {
 438                 /* We need to return more than one value. */
 439         if (q->q_return_type & kSecReturnRefMask) {
 440             CFDictionarySetValue(item, kSecClass, q->q_class->name);
 441         } else if ((q->q_return_type & kSecReturnAttributesMask)) {
 442             if (!(q->q_return_type & kSecReturnDataMask)) {
 443                 CFDictionaryRemoveValue(item, kSecValueData);
 444             }
 445
 446             // Add any attributes which are supposed to be returned, are not present in the decrypted blob,
 447             // and have a way to generate themselves.
 448             SecDbItemRef itemRef = NULL;
 449             SecDbForEachAttrWithMask(q->q_class, attr, kSecDbReturnAttrFlag) {
 450                 if(!CFDictionaryGetValue(item, attr->name) && attr->copyValue) {
 451                     CFErrorRef cferror = NULL;
 452                     if(!itemRef) {
 453                         itemRef = SecDbItemCreateWithAttributes(NULL, q->q_class, item, KEYBAG_DEVICE, &cferror);
 454                     }
 455                     if(!cferror && itemRef) {
 456                         CFTypeRef attrValue = attr->copyValue(itemRef, attr, &cferror);
 457                         if(!cferror && attrValue) {
 458                             CFDictionarySetValue(item, attr->name, attrValue);
 459                         }
 460                         CFReleaseNull(attrValue);
 461                     }
 462                     CFReleaseNull(cferror);
 463                 }
 464             }
 465             CFReleaseNull(itemRef);
 466
 467             CFDictionaryRemoveValue(item, kSecAttrUUID);
 468         } else {
 469             CFRetainSafe(data);
 470             CFDictionaryRemoveAllValues(item);
 471             if ((q->q_return_type & kSecReturnDataMask) && data) {
 472                 CFDictionarySetValue(item, kSecValueData, data);
 473             }
 474             CFReleaseSafe(data);
 475         }
 476                 if (q->q_return_type & kSecReturnPersistentRefMask) {
 477             CFDataRef pref = _SecItemCreatePersistentRef(q->q_class->name, rowid, item);
 478                         CFDictionarySetValue(item, kSecValuePersistentRef, pref);
 479             CFRelease(pref);
 480                 }
 481
 482                 a_result = item;
 483         CFRetain(item);
 484         }
 485
 486         return a_result;
 487 }
 488
 489 static void s3dl_merge_into_dict(const void *key, const void *value, void *context) {
 490     CFDictionarySetValue(context, key, value);
 491 }
 492
 493 static bool checkTokenObjectID(CFDataRef token_object_id, CFDataRef value_data) {
 494     bool equalOID = false;
 495     CFDictionaryRef itemValue = SecTokenItemValueCopy(value_data, NULL);
 496     require_quiet(itemValue, out);
 497     CFDataRef oID = CFDictionaryGetValue(itemValue, kSecTokenValueObjectIDKey);
 498     equalOID = CFEqualSafe(token_object_id, oID);
 499     CFRelease(itemValue);
 500 out:
 501     return equalOID;
 502 }
 503
 504 static void s3dl_query_row(sqlite3_stmt *stmt, void *context) {
 505     struct s3dl_query_ctx *c = context;
 506     Query *q = c->q;
 507
 508     sqlite_int64 rowid = sqlite3_column_int64(stmt, 0);
 509     CFMutableDictionaryRef item = NULL;
 510     bool ok = s3dl_item_from_col(stmt, q, 1, c->accessGroups, &item, NULL, &q->q_error);
 511     if (!ok) {
 512         OSStatus status = SecErrorGetOSStatus(q->q_error);
 513         // errSecDecode means the item is corrupted, stash it for delete.
 514         if (status == errSecDecode) {
 515             secwarning("ignoring corrupt %@,rowid=%" PRId64 " %@", q->q_class->name, rowid, q->q_error);
 516             {
 517                 CFDataRef edata = s3dl_copy_data_from_col(stmt, 1, NULL);
 518                 CFMutableStringRef edatastring =  CFStringCreateMutable(kCFAllocatorDefault, 0);
 519                 if(edatastring) {
 520                     CFStringAppendEncryptedData(edatastring, edata);
 521                     secnotice("item", "corrupted edata=%@", edatastring);
 522                 }
 523                 CFReleaseSafe(edata);
 524                 CFReleaseSafe(edatastring);
 525             }
 526             CFReleaseNull(q->q_error);
 527         } else if (status == errSecAuthNeeded) {
 528             secwarning("Authentication is needed for %@,rowid=%" PRId64 " (%" PRIdOSStatus "): %@", q->q_class->name, rowid, status, q->q_error);
 529         } else {
 530             secerror("decode %@,rowid=%" PRId64 " failed (%" PRIdOSStatus "): %@", q->q_class->name, rowid, status, q->q_error);
 531         }
 532         // q->q_error will be released appropriately by a call to query_error
 533         return;
 534     }
 535
 536     if (!item)
 537         goto out;
 538
 539     if (q->q_token_object_id != NULL && !checkTokenObjectID(q->q_token_object_id, CFDictionaryGetValue(item, kSecValueData)))
 540         goto out;
 541
 542     if (q->q_class == identity_class()) {
 543         // TODO: Use col 2 for key rowid and use both rowids in persistent ref.
 544
 545         CFMutableDictionaryRef key;
 546         /* TODO : if there is a errSecDecode error here, we should cleanup */
 547         if (!s3dl_item_from_col(stmt, q, 3, c->accessGroups, &key, NULL, &q->q_error) || !key)
 548             goto out;
 549
 550         CFDataRef certData = CFDictionaryGetValue(item, kSecValueData);
 551         if (certData) {
 552             CFDictionarySetValue(key, kSecAttrIdentityCertificateData, certData);
 553             CFDictionaryRemoveValue(item, kSecValueData);
 554         }
 555
 556         CFDataRef certTokenID = CFDictionaryGetValue(item, kSecAttrTokenID);
 557         if (certTokenID) {
 558             CFDictionarySetValue(key, kSecAttrIdentityCertificateTokenID, certTokenID);
 559             CFDictionaryRemoveValue(item, kSecAttrTokenID);
 560         }
 561         CFDictionaryApplyFunction(item, s3dl_merge_into_dict, key);
 562         CFRelease(item);
 563         item = key;
 564     }
 565
 566     if (!match_item(c->dbt, q, c->accessGroups, item))
 567         goto out;
 568
 569     CFTypeRef a_result = handle_result(q, item, rowid);
 570     if (a_result) {
 571         if (a_result == kCFNull) {
 572             /* Caller wasn't interested in a result, but we still
 573              count this row as found. */
 574             CFRelease(a_result);  // Help shut up clang
 575         } else if (q->q_limit == 1) {
 576             c->result = a_result;
 577         } else {
 578             CFArrayAppendValue((CFMutableArrayRef)c->result, a_result);
 579             CFRelease(a_result);
 580         }
 581         c->found++;
 582     }
 583
 584 out:
 585     CFReleaseSafe(item);
 586 }
 587
 588 static void
 589 SecDbAppendWhereROWID(CFMutableStringRef sql,
 590                       CFStringRef col, sqlite_int64 row_id,
 591                       bool *needWhere) {
 592     if (row_id > 0) {
 593         SecDbAppendWhereOrAnd(sql, needWhere);
 594         CFStringAppendFormat(sql, NULL, CFSTR("%@=%lld"), col, row_id);
 595     }
 596 }
 597
 598 static void
 599 SecDbAppendWhereAttrs(CFMutableStringRef sql, const Query *q, bool *needWhere) {
 600     CFIndex ix, attr_count = query_attr_count(q);
 601     for (ix = 0; ix < attr_count; ++ix) {
 602         SecDbAppendWhereOrAndEquals(sql, query_attr_at(q, ix).key, needWhere);
 603     }
 604 }
 605
 606 static void
 607 SecDbAppendWhereAccessGroups(CFMutableStringRef sql,
 608                              CFStringRef col,
 609                              CFArrayRef accessGroups,
 610                              bool *needWhere) {
 611     CFIndex ix, ag_count;
 612     if (!accessGroups || 0 == (ag_count = CFArrayGetCount(accessGroups))) {
 613         return;
 614     }
 615
 616     SecDbAppendWhereOrAnd(sql, needWhere);
 617     CFStringAppend(sql, col);
 618     CFStringAppend(sql, CFSTR(" IN (?"));
 619     for (ix = 1; ix < ag_count; ++ix) {
 620         CFStringAppend(sql, CFSTR(",?"));
 621     }
 622     CFStringAppend(sql, CFSTR(")"));
 623 }
 624
 625 static bool
 626 isQueryOverAllMUSRViews(CFTypeRef musrView)
 627 {
 628     return SecMUSRIsViewAllViews(musrView);
 629 }
 630
 631 static bool
 632 isQueryOverSingleUserView(CFTypeRef musrView)
 633 {
 634     return isNull(musrView);
 635 }
 636
 637 #if TARGET_OS_IPHONE
 638 static bool
 639 isQueryOverBothUserAndSystem(CFTypeRef musrView, uid_t *uid)
 640 {
 641     return SecMUSRGetBothUserAndSystemUUID(musrView, uid);
 642 }
 643 #endif
 644
 645 static void
 646 SecDbAppendWhereMusr(CFMutableStringRef sql,
 647                      const Query *q,
 648                      bool *needWhere)
 649 {
 650     SecDbAppendWhereOrAnd(sql, needWhere);
 651
 652 #if TARGET_OS_IPHONE
 653     if (isQueryOverBothUserAndSystem(q->q_musrView, NULL)) {
 654         CFStringAppend(sql, CFSTR("(musr = ? OR musr = ?)"));
 655     } else
 656 #endif
 657     if (isQueryOverAllMUSRViews(q->q_musrView)) {
 658             /* query over all items, regardless of view */
 659     } else if (isQueryOverSingleUserView(q->q_musrView)) {
 660         CFStringAppend(sql, CFSTR("musr = ?"));
 661     } else {
 662         CFStringAppend(sql, CFSTR("musr = ?"));
 663     }
 664 }
 665
 666 static void SecDbAppendWhereClause(CFMutableStringRef sql, const Query *q,
 667                                    CFArrayRef accessGroups) {
 668     bool needWhere = true;
 669     SecDbAppendWhereROWID(sql, CFSTR("ROWID"), q->q_row_id, &needWhere);
 670     SecDbAppendWhereAttrs(sql, q, &needWhere);
 671     SecDbAppendWhereMusr(sql, q, &needWhere);
 672     SecDbAppendWhereAccessGroups(sql, CFSTR("agrp"), accessGroups, &needWhere);
 673 }
 674
 675 static void SecDbAppendLimit(CFMutableStringRef sql, CFIndex limit) {
 676     if (limit != kSecMatchUnlimited)
 677         CFStringAppendFormat(sql, NULL, CFSTR(" LIMIT %" PRIdCFIndex), limit);
 678 }
 679
 680 static CFStringRef s3dl_create_select_sql(Query *q, CFArrayRef accessGroups) {
 681     CFMutableStringRef sql = CFStringCreateMutable(NULL, 0);
 682         if (q->q_class == identity_class()) {
 683         CFStringAppendFormat(sql, NULL, CFSTR("SELECT crowid, %@"
 684                                               ", rowid,data FROM "
 685                                               "(SELECT cert.rowid AS crowid, cert.labl AS labl,"
 686                                               " cert.issr AS issr, cert.slnr AS slnr, cert.skid AS skid,"
 687                                               " keys.*,cert.data AS %@"
 688                                               " FROM keys, cert"
 689                                               " WHERE keys.priv == 1 AND cert.pkhh == keys.klbl"),
 690                              kSecAttrIdentityCertificateData, kSecAttrIdentityCertificateData);
 691         SecDbAppendWhereAccessGroups(sql, CFSTR("cert.agrp"), accessGroups, 0);
 692         /* The next 3 SecDbAppendWhere calls are in the same order as in
 693          SecDbAppendWhereClause().  This makes sqlBindWhereClause() work,
 694          as long as we do an extra sqlBindAccessGroups first. */
 695         SecDbAppendWhereROWID(sql, CFSTR("crowid"), q->q_row_id, 0);
 696         CFStringAppend(sql, CFSTR(")"));
 697         bool needWhere = true;
 698         SecDbAppendWhereAttrs(sql, q, &needWhere);
 699         SecDbAppendWhereMusr(sql, q, &needWhere);
 700         SecDbAppendWhereAccessGroups(sql, CFSTR("agrp"), accessGroups, &needWhere);
 701         } else {
 702         CFStringAppend(sql, CFSTR("SELECT rowid, data FROM "));
 703                 CFStringAppend(sql, q->q_class->name);
 704         SecDbAppendWhereClause(sql, q, accessGroups);
 705     }
 706     //do not append limit for all queries which needs filtering
 707     if (q->q_match_issuer == NULL && q->q_match_policy == NULL && q->q_match_valid_on_date == NULL && q->q_match_trusted_only == NULL && q->q_token_object_id == NULL) {
 708         SecDbAppendLimit(sql, q->q_limit);
 709     }
 710
 711     return sql;
 712 }
 713
 714 static bool sqlBindMusr(sqlite3_stmt *stmt, const Query *q, int *pParam, CFErrorRef *error) {
 715     int param = *pParam;
 716     bool result = true;
 717 #if TARGET_OS_IPHONE
 718     uid_t uid;
 719
 720     if (isQueryOverBothUserAndSystem(q->q_musrView, &uid)) {
 721         /* network extensions are special and get to query both user and system views */
 722         CFDataRef systemUUID = SecMUSRGetSystemKeychainUUID();
 723         result = SecDbBindObject(stmt, param++, systemUUID, error);
 724         if (result) {
 725             CFDataRef activeUser = SecMUSRCreateActiveUserUUID(uid);
 726             result = SecDbBindObject(stmt, param++, activeUser, error);
 727             CFReleaseNull(activeUser);
 728         }
 729     } else
 730 #endif
 731     if (isQueryOverAllMUSRViews(q->q_musrView)) {
 732         /* query over all items, regardless of view */
 733     } else if (isQueryOverSingleUserView(q->q_musrView)) {
 734         CFDataRef singleUUID = SecMUSRGetSingleUserKeychainUUID();
 735         result = SecDbBindObject(stmt, param++, singleUUID, error);
 736     } else {
 737         result = SecDbBindObject(stmt, param++, q->q_musrView, error);
 738     }
 739
 740     *pParam = param;
 741     return result;
 742 }
 743
 744
 745 static bool sqlBindAccessGroups(sqlite3_stmt *stmt, CFArrayRef accessGroups,
 746                                 int *pParam, CFErrorRef *error) {
 747     bool result = true;
 748     int param = *pParam;
 749     CFIndex ix, count = accessGroups ? CFArrayGetCount(accessGroups) : 0;
 750     for (ix = 0; ix < count; ++ix) {
 751         result = SecDbBindObject(stmt, param++,
 752                                  CFArrayGetValueAtIndex(accessGroups, ix),
 753                                  error);
 754         if (!result)
 755             break;
 756     }
 757     *pParam = param;
 758     return result;
 759 }
 760
 761 static bool sqlBindWhereClause(sqlite3_stmt *stmt, const Query *q,
 762                                CFArrayRef accessGroups, int *pParam, CFErrorRef *error) {
 763     bool result = true;
 764     int param = *pParam;
 765     CFIndex ix, attr_count = query_attr_count(q);
 766     for (ix = 0; ix < attr_count; ++ix) {
 767         result = SecDbBindObject(stmt, param++, query_attr_at(q, ix).value, error);
 768         if (!result)
 769             break;
 770         }
 771
 772     if (result) {
 773         result = sqlBindMusr(stmt, q, &param, error);
 774     }
 775
 776     /* Bind the access group to the sql. */
 777     if (result) {
 778         result = sqlBindAccessGroups(stmt, accessGroups, &param, error);
 779     }
 780
 781     *pParam = param;
 782     return result;
 783 }
 784
 785 bool SecDbItemQuery(SecDbQueryRef query, CFArrayRef accessGroups, SecDbConnectionRef dbconn, CFErrorRef *error,
 786                     void (^handle_row)(SecDbItemRef item, bool *stop)) {
 787     __block bool ok = true;
 788     /* Sanity check the query. */
 789     if (query->q_ref)
 790         return SecError(errSecValueRefUnsupported, error, CFSTR("value ref not supported by queries"));
 791
 792     bool (^return_attr)(const SecDbAttr *attr) = ^bool (const SecDbAttr * attr) {
 793         // The attributes here must match field list hardcoded in s3dl_select_sql used below, which is
 794         // "rowid, data"
 795         return attr->kind == kSecDbRowIdAttr || attr->kind == kSecDbEncryptedDataAttr;
 796     };
 797
 798     CFStringRef sql = s3dl_create_select_sql(query, accessGroups);
 799     ok = sql;
 800     if (sql) {
 801         ok &= SecDbPrepare(dbconn, sql, error, ^(sqlite3_stmt *stmt) {
 802             /* Bind the values being searched for to the SELECT statement. */
 803             int param = 1;
 804             if (query->q_class == identity_class()) {
 805                 /* Bind the access groups to cert.agrp. */
 806                 ok &= sqlBindAccessGroups(stmt, accessGroups, &param, error);
 807             }
 808             if (ok)
 809                 ok &= sqlBindWhereClause(stmt, query, accessGroups, &param, error);
 810             if (ok) {
 811                 SecDbStep(dbconn, stmt, error, ^(bool *stop) {
 812                     SecDbItemRef itemFromStatement = SecDbItemCreateWithStatement(kCFAllocatorDefault, query->q_class, stmt, query->q_keybag, error, return_attr);
 813                     if (itemFromStatement) {
 814                         CFTransferRetained(itemFromStatement->credHandle, query->q_use_cred_handle);
 815                         if (match_item(dbconn, query, accessGroups, itemFromStatement->attributes))
 816                             handle_row(itemFromStatement, stop);
 817                         CFReleaseNull(itemFromStatement);
 818                     } else {
 819                         secerror("failed to create item from stmt: %@", error ? *error : (CFErrorRef)"no error");
 820                         if (error) {
 821                             CFReleaseNull(*error);
 822                         }
 823                         //*stop = true;
 824                         //ok = false;
 825                     }
 826                 });
 827             }
 828         });
 829         CFRelease(sql);
 830     }
 831
 832     return ok;
 833 }
 834
 835 static bool
 836 s3dl_query(s3dl_handle_row handle_row,
 837            void *context, CFErrorRef *error)
 838 {
 839     struct s3dl_query_ctx *c = context;
 840     SecDbConnectionRef dbt = c->dbt;
 841     Query *q = c->q;
 842     CFArrayRef accessGroups = c->accessGroups;
 843
 844     /* Sanity check the query. */
 845     if (q->q_ref)
 846         return SecError(errSecValueRefUnsupported, error, CFSTR("value ref not supported by queries"));
 847
 848         /* Actual work here. */
 849     if (q->q_limit == 1) {
 850         c->result = NULL;
 851     } else {
 852         c->result = CFArrayCreateMutable(NULL, 0, &kCFTypeArrayCallBacks);
 853     }
 854     CFStringRef sql = s3dl_create_select_sql(q, accessGroups);
 855     bool ok = SecDbWithSQL(dbt, sql, error, ^(sqlite3_stmt *stmt) {
 856         bool sql_ok = true;
 857         /* Bind the values being searched for to the SELECT statement. */
 858         int param = 1;
 859         if (q->q_class == identity_class()) {
 860             /* Bind the access groups to cert.agrp. */
 861             sql_ok = sqlBindAccessGroups(stmt, accessGroups, &param, error);
 862         }
 863         if (sql_ok)
 864             sql_ok = sqlBindWhereClause(stmt, q, accessGroups, &param, error);
 865         if (sql_ok) {
 866             SecDbForEach(dbt, stmt, error, ^bool (int row_index) {
 867                 handle_row(stmt, context);
 868
 869                 bool needs_auth = q->q_error && CFErrorGetCode(q->q_error) == errSecAuthNeeded;
 870                 if (q->q_skip_acl_items && needs_auth)
 871                     // Skip items needing authentication if we are told to do so.
 872                     CFReleaseNull(q->q_error);
 873
 874                 bool stop = q->q_limit != kSecMatchUnlimited && c->found >= q->q_limit;
 875                 stop = stop || (q->q_error && !needs_auth);
 876                 return !stop;
 877             });
 878         }
 879         return sql_ok;
 880     });
 881
 882     CFRelease(sql);
 883
 884     // First get the error from the query, since errSecDuplicateItem from an
 885     // update query should superceed the errSecItemNotFound below.
 886     if (!query_error(q, error))
 887         ok = false;
 888     if (ok && c->found == 0) {
 889         ok = SecError(errSecItemNotFound, error, CFSTR("no matching items found"));
 890         if (q->q_spindump_on_failure) {
 891             __security_stackshotreport(CFSTR("ItemNotFound"), __sec_exception_code_LostInMist);
 892         }
 893     }
 894
 895     return ok;
 896 }
 897
 898 bool
 899 s3dl_copy_matching(SecDbConnectionRef dbt, Query *q, CFTypeRef *result,
 900                    CFArrayRef accessGroups, CFErrorRef *error)
 901 {
 902     struct s3dl_query_ctx ctx = {
 903         .q = q, .accessGroups = accessGroups, .dbt = dbt,
 904     };
 905     if (q->q_row_id && query_attr_count(q))
 906         return SecError(errSecItemIllegalQuery, error,
 907                         CFSTR("attributes to query illegal; both row_id and other attributes can't be searched at the same time"));
 908     if (q->q_token_object_id && query_attr_count(q) != 1)
 909         return SecError(errSecItemIllegalQuery, error,
 910                         CFSTR("attributes to query illegal; both token persitent ref and other attributes can't be searched at the same time"));
 911
 912     // Only copy things that aren't tombstones unless the client explicitly asks otherwise.
 913     if (!CFDictionaryContainsKey(q->q_item, kSecAttrTombstone))
 914         query_add_attribute(kSecAttrTombstone, kCFBooleanFalse, q);
 915     bool ok = s3dl_query(s3dl_query_row, &ctx, error);
 916     if (ok && result)
 917         *result = ctx.result;
 918     else
 919         CFReleaseSafe(ctx.result);
 920
 921     return ok;
 922 }
 923
 924 typedef void (^s3dl_item_digest_callback)(CFDataRef persistantReference, CFDataRef encryptedData);
 925
 926 struct s3dl_digest_ctx {
 927     Query *q;
 928     SecDbConnectionRef dbt;
 929     s3dl_item_digest_callback item_callback;
 930 };
 931
 932 static void s3dl_query_row_digest(sqlite3_stmt *stmt, void *context) {
 933     struct s3dl_query_ctx *c = context;
 934     Query *q = c->q;
 935
 936     sqlite_int64 rowid = sqlite3_column_int64(stmt, 0);
 937     CFDataRef edata = s3dl_copy_data_from_col(stmt, 1, NULL);
 938     CFDataRef persistant_reference = _SecItemCreatePersistentRef(q->q_class->name, rowid, NULL);
 939     CFDataRef digest = NULL;
 940
 941     if (edata) {
 942         digest = CFDataCopySHA256Digest(edata, NULL);
 943     }
 944
 945     if (digest && persistant_reference) {
 946         CFDictionaryRef item = CFDictionaryCreateForCFTypes(NULL,
 947                                                             kSecValuePersistentRef, persistant_reference,
 948                                                             kSecValueData, digest,
 949                                                             NULL);
 950         if (item)
 951             CFArrayAppendValue((CFMutableArrayRef)c->result, item);
 952         CFReleaseNull(item);
 953         c->found++;
 954     } else {
 955         secinfo("item", "rowid %lu in %@ failed to create pref/digest", (unsigned long)rowid, q->q_class->name);
 956     }
 957     CFReleaseNull(digest);
 958     CFReleaseNull(edata);
 959     CFReleaseNull(persistant_reference);
 960 }
 961
 962
 963 bool
 964 s3dl_copy_digest(SecDbConnectionRef dbt, Query *q, CFArrayRef *result, CFArrayRef accessGroups, CFErrorRef *error)
 965 {
 966     struct s3dl_query_ctx ctx = {
 967         .q = q, .dbt = dbt, .accessGroups = accessGroups,
 968     };
 969     // Force to always return an array
 970     q->q_limit = kSecMatchUnlimited;
 971     // This interface only queries live data
 972     query_add_attribute(kSecAttrTombstone, kCFBooleanFalse, q);
 973     bool ok = s3dl_query(s3dl_query_row_digest, &ctx, error);
 974     if (ok && result)
 975         *result = (CFArrayRef)ctx.result;
 976     else
 977         CFReleaseSafe(ctx.result);
 978
 979     return ok;
 980 }
 981
 982 /* First remove key from q->q_pairs if it's present, then add the attribute again. */
 983 static void query_set_attribute_with_desc(const SecDbAttr *desc, const void *value, Query *q) {
 984     if (CFDictionaryContainsKey(q->q_item, desc->name)) {
 985         CFIndex ix;
 986         for (ix = 0; ix < q->q_attr_end; ++ix) {
 987             if (CFEqual(desc->name, q->q_pairs[ix].key)) {
 988                 CFReleaseSafe(q->q_pairs[ix].value);
 989                 --q->q_attr_end;
 990                 for (; ix < q->q_attr_end; ++ix) {
 991                     q->q_pairs[ix] = q->q_pairs[ix + 1];
 992                 }
 993                 CFDictionaryRemoveValue(q->q_item, desc->name);
 994                 break;
 995             }
 996         }
 997     }
 998     query_add_attribute_with_desc(desc, value, q);
 999 }
1000
1001 /* Update modification_date if needed. */
1002 static void query_pre_update(Query *q) {
1003     SecDbForEachAttr(q->q_class, desc) {
1004         if (desc->kind == kSecDbModificationDateAttr) {
1005             CFDateRef now = CFDateCreate(0, CFAbsoluteTimeGetCurrent());
1006             query_set_attribute_with_desc(desc, now, q);
1007             CFReleaseSafe(now);
1008         }
1009     }
1010 }
1011
1012 /* Make sure all attributes that are marked as not_null have a value.  If
1013  force_date is false, only set mdat and cdat if they aren't already set. */
1014 void query_pre_add(Query *q, bool force_date) {
1015     CFDateRef now = CFDateCreate(0, CFAbsoluteTimeGetCurrent());
1016     SecDbForEachAttrWithMask(q->q_class, desc, kSecDbInFlag) {
1017         if (desc->kind == kSecDbCreationDateAttr ||
1018             desc->kind == kSecDbModificationDateAttr) {
1019             if (force_date) {
1020                 query_set_attribute_with_desc(desc, now, q);
1021             } else if (!CFDictionaryContainsKey(q->q_item, desc->name)) {
1022                 query_add_attribute_with_desc(desc, now, q);
1023             }
1024         } else if ((desc->flags & kSecDbNotNullFlag) &&
1025                    !CFDictionaryContainsKey(q->q_item, desc->name)) {
1026             CFTypeRef value = NULL;
1027             if (desc->flags & kSecDbDefault0Flag) {
1028                 if (desc->kind == kSecDbDateAttr)
1029                     value = CFDateCreate(kCFAllocatorDefault, 0.0);
1030                 else {
1031                     SInt32 vzero = 0;
1032                     value = CFNumberCreate(0, kCFNumberSInt32Type, &vzero);
1033                 }
1034             } else if (desc->flags & kSecDbDefaultEmptyFlag) {
1035                 if (desc->kind == kSecDbDataAttr || desc->kind == kSecDbUUIDAttr)
1036                     value = CFDataCreate(kCFAllocatorDefault, NULL, 0);
1037                 else {
1038                     value = CFSTR("");
1039                     CFRetain(value);
1040                 }
1041             }
1042             if (value) {
1043                 /* Safe to use query_add_attribute here since the attr wasn't
1044                  set yet. */
1045                 query_add_attribute_with_desc(desc, value, q);
1046                 CFRelease(value);
1047             }
1048         }
1049     }
1050     CFReleaseSafe(now);
1051 }
1052
1053 // Return a tri state value false->never make a tombstone, true->always make a
1054 // tombstone, NULL->make a tombstone, but delete it if the tombstone itself is not currently being synced.
1055 static CFBooleanRef s3dl_should_make_tombstone(Query *q, bool item_is_syncable, SecDbItemRef item) {
1056     if (q->q_use_tomb)
1057         return q->q_use_tomb;
1058     else if (item_is_syncable && !SecDbItemIsTombstone(item))
1059         return NULL;
1060     else
1061         return kCFBooleanFalse;
1062 }
1063 /* AUDIT[securityd](done):
1064  attributesToUpdate (ok) is a caller provided dictionary,
1065  only its cf types have been checked.
1066  */
1067 bool
1068 s3dl_query_update(SecDbConnectionRef dbt, Query *q,
1069                   CFDictionaryRef attributesToUpdate, CFArrayRef accessGroups, CFErrorRef *error)
1070 {
1071     /* Sanity check the query. */
1072     if (query_match_count(q) != 0)
1073         return SecError(errSecItemMatchUnsupported, error, CFSTR("match not supported in attributes to update"));
1074     if (q->q_ref)
1075         return SecError(errSecValueRefUnsupported, error, CFSTR("value ref not supported in attributes to update"));
1076     if (q->q_row_id && query_attr_count(q))
1077         return SecError(errSecItemIllegalQuery, error, CFSTR("attributes to update illegal; both row_id and other attributes can't be updated at the same time"));
1078     if (q->q_token_object_id && query_attr_count(q) != 1)
1079         return SecError(errSecItemIllegalQuery, error, CFSTR("attributes to update illegal; both token persistent ref and other attributes can't be updated at the same time"));
1080
1081     __block bool result = true;
1082     Query *u = query_create(q->q_class, NULL, attributesToUpdate, error);
1083     if (u == NULL) return false;
1084     require_action_quiet(query_update_parse(u, attributesToUpdate, error), errOut, result = false);
1085     query_pre_update(u);
1086     result &= SecDbTransaction(dbt, kSecDbExclusiveTransactionType, error, ^(bool *commit) {
1087         // Make sure we only update real items, not tombstones, unless the client explicitly asks otherwise.
1088         if (!CFDictionaryContainsKey(q->q_item, kSecAttrTombstone))
1089             query_add_attribute(kSecAttrTombstone, kCFBooleanFalse, q);
1090         result &= SecDbItemQuery(q, accessGroups, dbt, error, ^(SecDbItemRef item, bool *stop) {
1091             // We always need to know the error here.
1092             CFErrorRef localError = NULL;
1093             if (q->q_token_object_id) {
1094                 const SecDbAttr *valueDataAttr = SecDbClassAttrWithKind(item->class, kSecDbDataAttr, NULL);
1095                 CFDataRef valueData = SecDbItemGetValue(item, valueDataAttr, NULL);
1096                 if (q->q_token_object_id != NULL && !checkTokenObjectID(q->q_token_object_id, valueData))
1097                     return;
1098             }
1099             // Cache the storedSHA1 digest so we use the one from the db not the recomputed one for notifications.
1100             const SecDbAttr *sha1attr = SecDbClassAttrWithKind(item->class, kSecDbSHA1Attr, NULL);
1101             CFDataRef storedSHA1 = CFRetainSafe(SecDbItemGetValue(item, sha1attr, NULL));
1102             SecDbItemRef new_item = SecDbItemCopyWithUpdates(item, u->q_item, &localError);
1103             SecDbItemSetValue(item, sha1attr, storedSHA1, NULL);
1104             CFReleaseSafe(storedSHA1);
1105             if (SecErrorGetOSStatus(localError) == errSecDecode) {
1106                 // We just ignore this, and treat as if item is not found.
1107                 secwarning("deleting corrupt %@,rowid=%" PRId64 " %@", q->q_class->name, SecDbItemGetRowId(item, NULL), localError);
1108                 CFReleaseNull(localError);
1109                 if (!SecDbItemDelete(item, dbt, false, &localError)) {
1110                     secerror("failed to delete corrupt %@,rowid=%" PRId64 " %@", q->q_class->name, SecDbItemGetRowId(item, NULL), localError);
1111                     CFReleaseNull(localError);
1112                 }
1113                 return;
1114             }
1115             if (new_item != NULL && u->q_access_control != NULL)
1116                 SecDbItemSetAccessControl(new_item, u->q_access_control, &localError);
1117             result = SecErrorPropagate(localError, error) && new_item;
1118             if (new_item) {
1119                 bool item_is_sync = SecDbItemIsSyncable(item);
1120                 result = SecDbItemUpdate(item, new_item, dbt, s3dl_should_make_tombstone(q, item_is_sync, item), q->q_uuid_from_primary_key, error);
1121                 if (result) {
1122                     q->q_changed = true;
1123                     if (item_is_sync || SecDbItemIsSyncable(new_item))
1124                         q->q_sync_changed = true;
1125                 }
1126                 CFRelease(new_item);
1127             }
1128         });
1129         if (!result)
1130             *commit = false;
1131     });
1132     if (result && !q->q_changed)
1133         result = SecError(errSecItemNotFound, error, CFSTR("No items updated"));
1134 errOut:
1135     if (!query_destroy(u, error))
1136         result = false;
1137     return result;
1138 }
1139
1140 static bool SecDbItemNeedAuth(SecDbItemRef item, CFErrorRef *error)
1141 {
1142     CFErrorRef localError = NULL;
1143     if (!SecDbItemEnsureDecrypted(item, &localError) && localError && CFErrorGetCode(localError) == errSecAuthNeeded) {
1144         if (error)
1145         *error = localError;
1146         return true;
1147     }
1148
1149     CFReleaseSafe(localError);
1150     return false;
1151 }
1152
1153 bool
1154 s3dl_query_delete(SecDbConnectionRef dbt, Query *q, CFArrayRef accessGroups, CFErrorRef *error)
1155 {
1156     __block bool ok = true;
1157     __block bool needAuth = false;
1158     // Only delete things that aren't tombstones, unless the client explicitly asks otherwise.
1159     if (!CFDictionaryContainsKey(q->q_item, kSecAttrTombstone))
1160         query_add_attribute(kSecAttrTombstone, kCFBooleanFalse, q);
1161     ok &= SecDbItemSelect(q, dbt, error, NULL, ^bool(const SecDbAttr *attr) {
1162         return false;
1163     },^bool(CFMutableStringRef sql, bool *needWhere) {
1164         SecDbAppendWhereClause(sql, q, accessGroups);
1165         return true;
1166     },^bool(sqlite3_stmt * stmt, int col) {
1167         return sqlBindWhereClause(stmt, q, accessGroups, &col, error);
1168     }, ^(SecDbItemRef item, bool *stop) {
1169         // Check if item for token persitence ref
1170         if (q->q_token_object_id) {
1171             const SecDbAttr *valueDataAttr = SecDbClassAttrWithKind(item->class, kSecDbDataAttr, NULL);
1172             CFDataRef valueData = SecDbItemGetValue(item, valueDataAttr, NULL);
1173             if (q->q_token_object_id != NULL && !checkTokenObjectID(q->q_token_object_id, valueData))
1174                 return;
1175         }
1176         // Check if item need to be authenticated by LocalAuthentication
1177         item->cryptoOp = kAKSKeyOpDelete;
1178         if (SecDbItemNeedAuth(item, error)) {
1179             needAuth = true;
1180             return;
1181         }
1182         // Cache the storedSHA1 digest so we use the one from the db not the recomputed one for notifications.
1183         const SecDbAttr *sha1attr = SecDbClassAttrWithKind(item->class, kSecDbSHA1Attr, NULL);
1184         CFDataRef storedSHA1 = CFRetainSafe(SecDbItemGetValue(item, sha1attr, NULL));
1185         bool item_is_sync = SecDbItemIsSyncable(item);
1186         SecDbItemSetValue(item, sha1attr, storedSHA1, NULL);
1187         CFReleaseSafe(storedSHA1);
1188         ok = SecDbItemDelete(item, dbt, s3dl_should_make_tombstone(q, item_is_sync, item), error);
1189         if (ok) {
1190             q->q_changed = true;
1191             if (item_is_sync)
1192                 q->q_sync_changed = true;
1193         }
1194     });
1195     if (ok && !q->q_changed && !needAuth) {
1196         ok = SecError(errSecItemNotFound, error, CFSTR("Delete failed to delete anything"));
1197     }
1198     return ok && !needAuth;
1199 }
1200
1201 static bool
1202 matchAnyString(CFStringRef needle, CFStringRef *haystack)
1203 {
1204     while (*haystack) {
1205         if (CFEqual(needle, *haystack))
1206             return true;
1207         haystack++;
1208     }
1209     return false;
1210 }
1211
1212 /* Return true iff the item in question should not be backed up, nor restored,
1213  but when restoring a backup the original version of the item should be
1214  added back to the keychain again after the restore completes. */
1215 static bool SecItemIsSystemBound(CFDictionaryRef item, const SecDbClass *cls, bool multiUser) {
1216     CFNumberRef sysb =  CFDictionaryGetValue(item, kSecAttrSysBound);
1217     if (isNumber(sysb)) {
1218         int32_t num = 0;
1219         if (!CFNumberGetValue(sysb, kCFNumberSInt32Type, &num))
1220             return false;
1221         if (num == kSecSecAttrSysBoundNot) {
1222             return false;
1223         } else if (num == kSecSecAttrSysBoundPreserveDuringRestore) {
1224             return true;
1225         }
1226         return true;
1227     }
1228
1229     CFStringRef agrp = CFDictionaryGetValue(item, kSecAttrAccessGroup);
1230     if (!isString(agrp))
1231         return false;
1232
1233     if (CFEqualSafe(agrp, kSOSInternalAccessGroup)) {
1234         secdebug("backup", "found sysbound item: %@", item);
1235         return true;
1236     }
1237
1238     if (CFEqual(agrp, CFSTR("lockdown-identities"))) {
1239         secdebug("backup", "found sys_bound item: %@", item);
1240         return true;
1241     }
1242
1243     if (CFEqual(agrp, CFSTR("apple")) && cls == genp_class()) {
1244         CFStringRef service = CFDictionaryGetValue(item, kSecAttrService);
1245         CFStringRef account = CFDictionaryGetValue(item, kSecAttrAccount);
1246
1247         if (isString(service) && isString(account)) {
1248             static CFStringRef mcAccounts[] = {
1249                 CFSTR("Public"),
1250                 CFSTR("Private"),
1251                 NULL,
1252             };
1253
1254             if (CFEqual(service, CFSTR("com.apple.managedconfiguration"))
1255                 && matchAnyString(account, mcAccounts))
1256             {
1257                 secdebug("backup", "found sys_bound item: %@", item);
1258                 return true;
1259             }
1260         }
1261     }
1262
1263     if (multiUser && CFEqual(agrp, CFSTR("com.apple.apsd")) && cls == genp_class()) {
1264         static CFStringRef pushServices[] = {
1265             CFSTR("push.apple.com"),
1266             CFSTR("push.apple.com,PerAppToken.v0"),
1267             NULL
1268         };
1269         CFStringRef service = CFDictionaryGetValue(item, kSecAttrService);
1270
1271         if (isString(service) && matchAnyString(service, pushServices)) {
1272             secdebug("backup", "found sys_bound item: %@", item);
1273             return true;
1274         }
1275     }
1276
1277     if (multiUser && CFEqual(agrp, CFSTR("appleaccount")) && cls == genp_class()) {
1278         static CFStringRef accountServices[] = {
1279             CFSTR("com.apple.appleaccount.fmf.token"), /* temporary tokens while accout is being setup */
1280             CFSTR("com.apple.appleaccount.fmf.apptoken"),
1281             CFSTR("com.apple.appleaccount.fmip.siritoken"),
1282             CFSTR("com.apple.appleaccount.cloudkit.token"),
1283             NULL
1284         };
1285         CFStringRef service = CFDictionaryGetValue(item, kSecAttrService);
1286
1287         if (isString(service) && matchAnyString(service, accountServices)) {
1288             secdebug("backup", "found exact sys_bound item: %@", item);
1289             return true;
1290         }
1291     }
1292
1293     if (multiUser && CFEqual(agrp, CFSTR("apple")) && cls == genp_class()) {
1294         static CFStringRef accountServices[] = {
1295             CFSTR("com.apple.account.AppleAccount.token"),
1296             CFSTR("com.apple.account.AppleAccount.password"),
1297             CFSTR("com.apple.account.AppleAccount.rpassword"),
1298             CFSTR("com.apple.account.idms.token"),
1299             CFSTR("com.apple.account.idms.heartbeat-token"),
1300             CFSTR("com.apple.account.idms.continuation-key"),
1301             CFSTR("com.apple.account.CloudKit.token"),
1302             CFSTR("com.apple.account.IdentityServices.password"), /* accountsd for ids */
1303             CFSTR("com.apple.account.IdentityServices.rpassword"),
1304             CFSTR("com.apple.account.IdentityServices.token"),
1305             CFSTR("BackupIDSAccountToken"),
1306             CFSTR("com.apple.ids"),
1307             CFSTR("ids"),
1308             CFSTR("IDS"),
1309             NULL
1310         };
1311         CFStringRef service = CFDictionaryGetValue(item, kSecAttrService);
1312
1313         if (isString(service) && matchAnyString(service, accountServices)) {
1314             secdebug("backup", "found exact sys_bound item: %@", item);
1315             return true;
1316         }
1317         if (isString(service) && CFEqual(service, CFSTR("com.apple.facetime"))) {
1318             CFStringRef account = CFDictionaryGetValue(item, kSecAttrAccount);
1319             if (isString(account) && CFEqual(account, CFSTR("registrationV1"))) {
1320                 secdebug("backup", "found exact sys_bound item: %@", item);
1321                 return true;
1322             }
1323         }
1324     }
1325
1326     if (multiUser && CFEqual(agrp, CFSTR("ichat")) && cls == genp_class()) {
1327         static CFStringRef accountServices[] = {
1328             CFSTR("ids"),
1329             NULL
1330         };
1331         CFStringRef service = CFDictionaryGetValue(item, kSecAttrService);
1332
1333         if (isString(service) && matchAnyString(service, accountServices)) {
1334             secdebug("backup", "found exact sys_bound item: %@", item);
1335             return true;
1336         }
1337     }
1338
1339     if (multiUser && CFEqual(agrp, CFSTR("ichat")) && cls == keys_class()) {
1340         static CFStringRef exactMatchingLabel[] = {
1341             CFSTR("iMessage Encryption Key"),
1342             CFSTR("iMessage Signing Key"),
1343         };
1344         CFStringRef label = CFDictionaryGetValue(item, kSecAttrLabel);
1345         if (isString(label)) {
1346             if (matchAnyString(label, exactMatchingLabel)) {
1347                 secdebug("backup", "found exact sys_bound item: %@", item);
1348                 return true;
1349             }
1350         }
1351     }
1352
1353
1354     secdebug("backup", "found non sys_bound item: %@", item);
1355     return false;
1356 }
1357
1358 /* Delete all items from the current keychain.  If this is not an in
1359  place upgrade we don't delete items in the 'lockdown-identities'
1360  access group, this ensures that an import or restore of a backup
1361  will never overwrite an existing activation record. */
1362 static bool SecServerDeleteAll(SecDbConnectionRef dbt, CFErrorRef *error) {
1363     secwarning("SecServerDeleteAll");
1364
1365     return kc_transaction(dbt, error, ^{
1366
1367         bool ok = (SecDbExec(dbt, CFSTR("DELETE from genp;"), error) &&
1368                    SecDbExec(dbt, CFSTR("DELETE from inet;"), error) &&
1369                    SecDbExec(dbt, CFSTR("DELETE from cert;"), error) &&
1370                    SecDbExec(dbt, CFSTR("DELETE from keys;"), error));
1371         return ok;
1372     });
1373 }
1374
1375 #if TARGET_OS_IPHONE
1376
1377 static bool DeleteAllFromTableForMUSRView(SecDbConnectionRef dbt,
1378                                           CFStringRef sql,
1379                                           CFDataRef musr,
1380                                           bool keepU,
1381                                           CFErrorRef *error)
1382 {
1383     sqlite3_stmt *stmt = NULL;
1384     CFStringRef sql2 = NULL;
1385     bool ok = false;
1386
1387     if (keepU) {
1388         sql2 = CFStringCreateWithFormat(NULL, NULL, CFSTR("%@ AND pdmn NOT IN ('aku','akpu','cku','dku')"), sql);
1389     } else {
1390         sql2 = CFRetain(sql);
1391     }
1392     require(sql2, fail);
1393
1394     stmt = SecDbCopyStmt(dbt, sql2, NULL, error);
1395     require(stmt, fail);
1396
1397     ok = SecDbBindObject(stmt, 1, musr, error);
1398     require(ok, fail);
1399
1400     ok = SecDbStep(dbt, stmt, error, ^(bool *stop) { });
1401     require(ok, fail);
1402
1403 fail:
1404     if (stmt) {
1405         ok = SecDbFinalize(stmt, error);
1406     }
1407     if (!ok)
1408         secwarning("DeleteAllFromTableForMUSRView failed for %@ for musr: %@: %@", sql2, musr, error ? *error : NULL);
1409
1410     CFReleaseNull(sql2);
1411
1412     return ok;
1413 }
1414
1415 bool SecServerDeleteAllForUser(SecDbConnectionRef dbt, CFDataRef musrView, bool keepU, CFErrorRef *error) {
1416     secwarning("SecServerDeleteAllForUser for user: %@ keepU %s", musrView, keepU ? "yes" : "no");
1417
1418     return kc_transaction(dbt, error, ^{
1419         bool ok;
1420
1421         ok = (DeleteAllFromTableForMUSRView(dbt, CFSTR("DELETE FROM genp WHERE musr = ?"), musrView, keepU, error) &&
1422               DeleteAllFromTableForMUSRView(dbt, CFSTR("DELETE FROM inet WHERE musr = ?"), musrView, keepU, error) &&
1423               DeleteAllFromTableForMUSRView(dbt, CFSTR("DELETE FROM cert WHERE musr = ?"), musrView, keepU, error) &&
1424               DeleteAllFromTableForMUSRView(dbt, CFSTR("DELETE FROM keys WHERE musr = ?"), musrView, keepU, error));
1425
1426         return ok;
1427     });
1428 }
1429 #endif
1430
1431
1432 struct s3dl_export_row_ctx {
1433     struct s3dl_query_ctx qc;
1434     keybag_handle_t dest_keybag;
1435     enum SecItemFilter filter;
1436     bool multiUser;
1437 };
1438
1439 static void s3dl_export_row(sqlite3_stmt *stmt, void *context) {
1440     struct s3dl_export_row_ctx *c = context;
1441     Query *q = c->qc.q;
1442     SecAccessControlRef access_control = NULL;
1443     CFErrorRef localError = NULL;
1444
1445     /* Skip akpu items when backing up, those are intentionally lost across restores. The same applies to SEP-based keys */
1446     bool skip_akpu_or_token = c->filter == kSecBackupableItemFilter;
1447
1448     sqlite_int64 rowid = sqlite3_column_int64(stmt, 0);
1449     CFMutableDictionaryRef item = NULL;
1450     bool ok = s3dl_item_from_col(stmt, q, 1, c->qc.accessGroups, &item, &access_control, &localError);
1451
1452     bool is_akpu = access_control ? CFEqualSafe(SecAccessControlGetProtection(access_control),
1453                                                 kSecAttrAccessibleWhenPasscodeSetThisDeviceOnly) : false;
1454     bool is_token = (ok && item != NULL) ? CFDictionaryContainsKey(item, kSecAttrTokenID) : false;
1455
1456     if (ok && item && !(skip_akpu_or_token && (is_akpu || is_token))) {
1457         /* Only export sysbound items if do_sys_bound is true, only export non sysbound items otherwise. */
1458         bool do_sys_bound = c->filter == kSecSysBoundItemFilter;
1459         if (c->filter == kSecNoItemFilter ||
1460             SecItemIsSystemBound(item, q->q_class, c->multiUser) == do_sys_bound) {
1461             /* Re-encode the item. */
1462             secdebug("item", "export rowid %llu item: %@", rowid, item);
1463             /* The code below could be moved into handle_row. */
1464             CFDataRef pref = _SecItemCreatePersistentRef(q->q_class->name, rowid, item);
1465             if (pref) {
1466                 if (c->dest_keybag != KEYBAG_NONE) {
1467                     CFMutableDictionaryRef auth_attribs = CFDictionaryCreateMutable(NULL, 0, &kCFTypeDictionaryKeyCallBacks, &kCFTypeDictionaryValueCallBacks);
1468                     SecDbForEachAttrWithMask(q->q_class, desc, kSecDbInAuthenticatedDataFlag) {
1469                         CFTypeRef value = CFDictionaryGetValue(item, desc->name);
1470                         if(value) {
1471                             CFDictionaryAddValue(auth_attribs, desc->name, value);
1472                             CFDictionaryRemoveValue(item, desc->name);
1473                         }
1474                     }
1475
1476                     /* Encode and encrypt the item to the specified keybag. */
1477                     CFDataRef edata = NULL;
1478                     bool encrypted = ks_encrypt_data(c->dest_keybag, access_control, q->q_use_cred_handle, item, auth_attribs, &edata, false, &q->q_error);
1479                     CFDictionaryRemoveAllValues(item);
1480                     CFRelease(auth_attribs);
1481                     if (encrypted) {
1482                         CFDictionarySetValue(item, kSecValueData, edata);
1483                         CFReleaseSafe(edata);
1484                     } else {
1485                         seccritical("ks_encrypt_data %@,rowid=%" PRId64 ": failed: %@", q->q_class->name, rowid, q->q_error);
1486                         CFReleaseNull(q->q_error);
1487                     }
1488                 }
1489                 if (CFDictionaryGetCount(item)) {
1490                     CFDictionarySetValue(item, kSecValuePersistentRef, pref);
1491                     CFArrayAppendValue((CFMutableArrayRef)c->qc.result, item);
1492                     c->qc.found++;
1493                 }
1494                 CFReleaseSafe(pref);
1495             }
1496         }
1497         CFRelease(item);
1498     } else {
1499         OSStatus status = SecErrorGetOSStatus(localError);
1500
1501         if (status == errSecInteractionNotAllowed && is_akpu && skip_akpu_or_token) {
1502             // We expect akpu items to be inaccessible when the device is locked.
1503             CFReleaseNull(localError);
1504         } else {
1505             /* This happens a lot when trying to migrate keychain before first unlock, so only a notice */
1506             /* If the error is "corrupted item" then we just ignore it, otherwise we save it in the query */
1507             secinfo("item","Could not export item for rowid %llu: %@", rowid, localError);
1508
1509             if(status == errSecDecode) {
1510                 CFReleaseNull(localError);
1511             } else {
1512                 CFReleaseSafe(q->q_error);
1513                 q->q_error=localError;
1514             }
1515         }
1516     }
1517     CFReleaseSafe(access_control);
1518 }
1519
1520 static CFStringRef
1521 SecCreateKeybagUUID(keybag_handle_t keybag)
1522 {
1523 #if !TARGET_HAS_KEYSTORE
1524     return NULL;
1525 #else
1526     char uuidstr[37];
1527     uuid_t uuid;
1528     if (aks_get_bag_uuid(keybag, uuid) != KERN_SUCCESS)
1529         return NULL;
1530     uuid_unparse_lower(uuid, uuidstr);
1531     return CFStringCreateWithCString(NULL, uuidstr, kCFStringEncodingUTF8);
1532 #endif
1533 }
1534
1535
1536 CFDictionaryRef
1537 SecServerCopyKeychainPlist(SecDbConnectionRef dbt,
1538                            SecurityClient *client,
1539                            keybag_handle_t src_keybag,
1540                            keybag_handle_t dest_keybag,
1541                            enum SecItemFilter filter,
1542                            CFErrorRef *error) {
1543     CFMutableDictionaryRef keychain;
1544     keychain = CFDictionaryCreateMutable(kCFAllocatorDefault, 0,
1545                                          &kCFTypeDictionaryKeyCallBacks, &kCFTypeDictionaryValueCallBacks);
1546     unsigned class_ix;
1547     bool inMultiUser = false;
1548     CFStringRef keybaguuid = NULL;
1549     Query q = { .q_keybag = src_keybag,
1550         .q_musrView = NULL
1551     };
1552
1553     if (!keychain) {
1554         if (error && !*error)
1555             SecError(errSecAllocate, error, CFSTR("Can't create keychain dictionary"));
1556         goto errOut;
1557     }
1558
1559     q.q_return_type =
1560         kSecReturnDataMask |
1561         kSecReturnAttributesMask |
1562         kSecReturnPersistentRefMask;
1563     q.q_limit = kSecMatchUnlimited;
1564     q.q_skip_acl_items = true;
1565
1566
1567 #if TARGET_OS_IPHONE
1568     if (client && client->inMultiUser) {
1569         q.q_musrView = SecMUSRCreateActiveUserUUID(client->uid);
1570         inMultiUser = true;
1571     } else
1572 #endif
1573     {
1574         q.q_musrView = SecMUSRGetSingleUserKeychainUUID();
1575         CFRetain(q.q_musrView);
1576
1577         keybaguuid = SecCreateKeybagUUID(dest_keybag);
1578         if (keybaguuid)
1579             CFDictionarySetValue(keychain, kSecBackupKeybagUUIDKey, keybaguuid);
1580     }
1581
1582     /* Get rid of this duplicate. */
1583     const SecDbClass *SecDbClasses[] = {
1584         genp_class(),
1585         inet_class(),
1586         cert_class(),
1587         keys_class()
1588     };
1589
1590     for (class_ix = 0; class_ix < array_size(SecDbClasses);
1591          ++class_ix) {
1592         q.q_class = SecDbClasses[class_ix];
1593         struct s3dl_export_row_ctx ctx = {
1594             .qc = { .q = &q, .dbt = dbt },
1595             .dest_keybag = dest_keybag, .filter = filter,
1596             .multiUser = inMultiUser,
1597         };
1598
1599         secnotice("item", "exporting class '%@'", q.q_class->name);
1600
1601         CFErrorRef localError = NULL;
1602         if (s3dl_query(s3dl_export_row, &ctx, &localError)) {
1603             if (CFArrayGetCount(ctx.qc.result))
1604                 CFDictionaryAddValue(keychain, q.q_class->name, ctx.qc.result);
1605
1606         } else {
1607             OSStatus status = (OSStatus)CFErrorGetCode(localError);
1608             if (status == errSecItemNotFound) {
1609                 CFRelease(localError);
1610             } else {
1611                 secerror("Export failed: %@", localError);
1612                 if (error) {
1613                     CFReleaseSafe(*error);
1614                     *error = localError;
1615                 } else {
1616                     CFRelease(localError);
1617                 }
1618                 CFReleaseNull(keychain);
1619                 CFReleaseNull(ctx.qc.result);
1620                 break;
1621             }
1622         }
1623         CFReleaseNull(ctx.qc.result);
1624     }
1625
1626 errOut:
1627     CFReleaseNull(q.q_musrView);
1628     CFReleaseNull(keybaguuid);
1629
1630     return keychain;
1631 }
1632
1633 struct SecServerImportClassState {
1634         SecDbConnectionRef dbt;
1635     CFErrorRef error;
1636     keybag_handle_t src_keybag;
1637     keybag_handle_t dest_keybag;
1638     SecurityClient *client;
1639     enum SecItemFilter filter;
1640 };
1641
1642 struct SecServerImportItemState {
1643     const SecDbClass *class;
1644         struct SecServerImportClassState *s;
1645 };
1646
1647 static void
1648 SecServerImportItem(const void *value, void *context)
1649 {
1650     struct SecServerImportItemState *state = (struct SecServerImportItemState *)context;
1651     bool inMultiUser = false;
1652 #if TARGET_OS_IPHONE
1653     if (state->s->client->inMultiUser)
1654         inMultiUser = true;
1655 #endif
1656
1657     if (state->s->error)
1658         return;
1659
1660     if (!isDictionary(value)) {
1661         SecError(errSecParam, &state->s->error, CFSTR("value %@ is not a dictionary"), value);
1662         return;
1663     }
1664
1665     CFDictionaryRef dict = (CFDictionaryRef)value;
1666
1667     secdebug("item", "Import Item : %@", dict);
1668
1669     /* We use the kSecSysBoundItemFilter to indicate that we don't
1670      * preserve rowid's during import.
1671      */
1672     if (state->s->filter == kSecBackupableItemFilter) {
1673         CFTypeRef pdmu;
1674
1675         /* We don't filter non sys_bound items during import since we know we
1676          * will never have any in this case.
1677          */
1678         if (SecItemIsSystemBound(dict, state->class, inMultiUser))
1679             return;
1680
1681         /*
1682          * Don't bother with u items when in edu mode since our current backup system
1683          * don't keep track of items that blongs to the device (u) but rather just
1684          * merge them into one blob.
1685          */
1686         if (inMultiUser && (pdmu = CFDictionaryGetValue(dict, kSecAttrAccessible))) {
1687             if (CFEqual(pdmu, kSecAttrAccessibleWhenUnlockedThisDeviceOnly)     ||
1688                 CFEqual(pdmu, kSecAttrAccessibleAfterFirstUnlockThisDeviceOnly) ||
1689                 CFEqual(pdmu, kSecAttrAccessibleWhenUnlockedThisDeviceOnly)     ||
1690                 CFEqual(pdmu, kSecAttrAccessibleWhenPasscodeSetThisDeviceOnly))
1691             {
1692                 secdebug("item", "Skipping KU item : %@", dict);
1693                 return;
1694             }
1695         }
1696
1697         /* Avoid importing token-based items.  Although newer backups should not have them,
1698          * older (iOS9, iOS10.0) produced backups with token-based items.
1699          */
1700         if (CFDictionaryContainsKey(dict, kSecAttrTokenID)) {
1701             secdebug("item", "Skipping token-based item : %@", dict);
1702             return;
1703         }
1704     }
1705
1706     SecDbItemRef item;
1707
1708     /* This is sligthly confusing:
1709      - During upgrade all items are exported with KEYBAG_NONE.
1710      - During restore from backup, existing sys_bound items are exported with KEYBAG_NONE, and are exported as dictionary of attributes.
1711      - Item in the actual backup are export with a real keybag, and are exported as encrypted v_Data and v_PersistentRef
1712      */
1713     if (state->s->src_keybag == KEYBAG_NONE) {
1714         item = SecDbItemCreateWithAttributes(kCFAllocatorDefault, state->class, dict, state->s->dest_keybag,  &state->s->error);
1715     } else {
1716         item = SecDbItemCreateWithBackupDictionary(kCFAllocatorDefault, state->class, dict, state->s->src_keybag, state->s->dest_keybag, &state->s->error);
1717     }
1718
1719     /*
1720      *
1721      */
1722
1723     if (item) {
1724         CFDataRef musr = NULL;
1725         CFDataRef musrBackup = CFDictionaryGetValue(dict, kSecAttrMultiUser);
1726         CFDataRef systemKeychainUUID = SecMUSRGetSystemKeychainUUID();
1727         bool systemKeychain = CFEqualSafe(musrBackup, systemKeychainUUID);
1728
1729 #if TARGET_OS_IPHONE
1730         if (state->s->client && state->s->client->inMultiUser) {
1731             if (systemKeychain) {
1732                 secwarning("system keychain not allowed in multi user mode for item: %@", item);
1733             } else {
1734                 musr = SecMUSRCreateActiveUserUUID(state->s->client->uid);
1735             }
1736         } else
1737 #endif
1738         {
1739             if (systemKeychain) {
1740                 musr = SecMUSRCopySystemKeychainUUID();
1741             } else {
1742                 musr = SecMUSRGetSingleUserKeychainUUID();
1743                 CFRetainSafe(musr);
1744             }
1745         }
1746         if (musr == NULL) {
1747             CFReleaseNull(item);
1748         } else {
1749             SecDbItemSetValueWithName(item, CFSTR("musr"), musr, &state->s->error);
1750             CFRelease(musr);
1751         }
1752     }
1753
1754     /*
1755      *
1756      */
1757
1758     if (item) {
1759         bool insertStatus;
1760
1761         if(state->s->filter != kSecSysBoundItemFilter) {
1762             SecDbItemExtractRowIdFromBackupDictionary(item, dict, &state->s->error);
1763         }
1764         SecDbItemInferSyncable(item, &state->s->error);
1765         insertStatus = SecDbItemInsert(item, state->s->dbt, &state->s->error);
1766         if (!insertStatus) {
1767             /*
1768              When running in EduMode, multiple users share the same
1769              keychain and unfortionaly the rowid is used a
1770              persistant reference and is part of the contraints (its
1771              UNIQUE), so lets clear the rowid and try to insert the
1772              entry again.
1773
1774              This even happens for normal operation because of
1775              SysBound entries, so in case of a failure, lets try
1776              again to insert the record.
1777              */
1778             SecDbItemClearRowId(item, NULL);
1779             SecDbItemInsert(item, state->s->dbt, &state->s->error);
1780         }
1781         }
1782
1783     /* Reset error if we had one, since we just skip the current item
1784      and continue importing what we can. */
1785     if (state->s->error) {
1786         secwarning("Failed to import an item (%@) of class '%@': %@ - ignoring error.",
1787                    item, state->class->name, state->s->error);
1788         CFReleaseNull(state->s->error);
1789     }
1790
1791     CFReleaseSafe(item);
1792 }
1793
1794 static void SecServerImportClass(const void *key, const void *value,
1795                                  void *context) {
1796     struct SecServerImportClassState *state =
1797     (struct SecServerImportClassState *)context;
1798     if (state->error)
1799         return;
1800     if (!isString(key)) {
1801         SecError(errSecParam, &state->error, CFSTR("class name %@ is not a string"), key);
1802         return;
1803     }
1804     /* ignore the Keybag UUID */
1805     if (CFEqual(key, kSecBackupKeybagUUIDKey))
1806         return;
1807     const SecDbClass *class = kc_class_with_name(key);
1808     if (!class) {
1809         secwarning("Ignoring unknown key class '%@'", key);
1810         return;
1811     }
1812     if (class == identity_class()) {
1813         SecError(errSecParam, &state->error, CFSTR("attempt to import an identity"));
1814         return;
1815     }
1816     struct SecServerImportItemState item_state = {
1817         .class = class, .s = state,
1818     };
1819     if (isArray(value)) {
1820         CFArrayRef items = (CFArrayRef)value;
1821         secwarning("Import %ld items of class %@ (filter %d)", (long)CFArrayGetCount(items), key, state->filter);
1822         CFArrayApplyFunction(items, CFRangeMake(0, CFArrayGetCount(items)),
1823                              SecServerImportItem, &item_state);
1824     } else if (isDictionary(value)) {
1825         CFDictionaryRef item = (CFDictionaryRef)value;
1826         secwarning("Import %ld items of class %@ (filter %d)", (long)CFDictionaryGetCount(item), key, state->filter);
1827         SecServerImportItem(item, &item_state);
1828     } else {
1829         secwarning("Unknown value type for class %@ (filter %d)", key, state->filter);
1830     }
1831 }
1832
1833 bool SecServerImportKeychainInPlist(SecDbConnectionRef dbt, SecurityClient *client,
1834                                            keybag_handle_t src_keybag, keybag_handle_t dest_keybag,
1835                                            CFDictionaryRef keychain, enum SecItemFilter filter, CFErrorRef *error) {
1836     CFStringRef keybaguuid = NULL;
1837     bool ok = true;
1838
1839     CFDictionaryRef sys_bound = NULL;
1840     if (filter == kSecBackupableItemFilter) {
1841         /* Grab a copy of all the items for which SecItemIsSystemBound()
1842          returns true. */
1843         require(sys_bound = SecServerCopyKeychainPlist(dbt, client, KEYBAG_DEVICE,
1844                                                        KEYBAG_NONE, kSecSysBoundItemFilter,
1845                                                        error), errOut);
1846     }
1847
1848     /*
1849      * Validate the uuid of the source keybag matches what we have in the backup
1850      */
1851     keybaguuid = SecCreateKeybagUUID(src_keybag);
1852     if (keybaguuid) {
1853         CFStringRef uuid = CFDictionaryGetValue(keychain, kSecBackupKeybagUUIDKey);
1854         if (isString(uuid)) {
1855             require_action(CFEqual(keybaguuid, uuid), errOut,
1856                            SecError(errSecDecode, error, CFSTR("Keybag UUID (%@) mismatch with backup (%@)"),
1857                                     keybaguuid, uuid));
1858         }
1859     }
1860
1861     /* Delete everything in the keychain. */
1862 #if TARGET_OS_IPHONE
1863     if (client->inMultiUser) {
1864         CFDataRef musrView = SecMUSRCreateActiveUserUUID(client->uid);
1865         require_action(musrView, errOut, ok = false);
1866         require_action(ok = SecServerDeleteAllForUser(dbt, musrView, true, error), errOut, CFReleaseNull(musrView));
1867         CFReleaseNull(musrView);
1868     } else
1869 #endif
1870     {
1871         require(ok = SecServerDeleteAll(dbt, error), errOut);
1872     }
1873
1874     struct SecServerImportClassState state = {
1875         .dbt = dbt,
1876         .src_keybag = src_keybag,
1877         .dest_keybag = dest_keybag,
1878         .client = client,
1879         .filter = filter,
1880     };
1881     /* Import the provided items, preserving rowids. */
1882     secwarning("Restoring backup items '%ld'", (long)CFDictionaryGetCount(keychain));
1883     CFDictionaryApplyFunction(keychain, SecServerImportClass, &state);
1884
1885     if (sys_bound) {
1886         state.src_keybag = KEYBAG_NONE;
1887         /* Import the items we preserved with random rowids. */
1888         state.filter = kSecSysBoundItemFilter;
1889         secwarning("Restoring sysbound items '%ld'", (long)CFDictionaryGetCount(sys_bound));
1890         CFDictionaryApplyFunction(sys_bound, SecServerImportClass, &state);
1891     }
1892     if (state.error) {
1893         if (error) {
1894             CFReleaseSafe(*error);
1895             *error = state.error;
1896         } else {
1897             CFRelease(state.error);
1898         }
1899         ok = false;
1900     }
1901
1902 errOut:
1903     CFReleaseSafe(sys_bound);
1904     CFReleaseSafe(keybaguuid);
1905
1906     return ok;
1907 }
1908
1909 CFStringRef
1910 SecServerBackupGetKeybagUUID(CFDictionaryRef keychain, CFErrorRef *error)
1911 {
1912     CFStringRef uuid = CFDictionaryGetValue(keychain, kSecBackupKeybagUUIDKey);
1913     if (!isString(uuid)) {
1914         SecError(errSecDecode, error, CFSTR("Missing or invalid %@ in backup dictionary"), kSecBackupKeybagUUIDKey);
1915         return NULL;
1916     }
1917     return uuid;
1918 }
1919
1920 #pragma mark - key rolling support
1921 #if USE_KEYSTORE
1922
1923 struct check_generation_ctx {
1924     struct s3dl_query_ctx query_ctx;
1925     uint32_t current_generation;
1926 };
1927
1928 static void check_generation(sqlite3_stmt *stmt, void *context) {
1929     struct check_generation_ctx *c = context;
1930     CFDataRef blob = NULL;
1931     size_t blobLen = 0;
1932     const uint8_t *cursor = NULL;
1933     uint32_t version;
1934     keyclass_t keyclass;
1935     uint32_t current_generation = c->current_generation;
1936
1937     require(blob = s3dl_copy_data_from_col(stmt, 1, &c->query_ctx.q->q_error), out);
1938     blobLen = CFDataGetLength(blob);
1939     cursor = CFDataGetBytePtr(blob);
1940
1941     /* Check for underflow, ensuring we have at least one full AES block left. */
1942     if (blobLen < sizeof(version) + sizeof(keyclass)) {
1943         SecError(errSecDecode, &c->query_ctx.q->q_error, CFSTR("check_generation: Check for underflow"));
1944         goto out;
1945     }
1946
1947     version = *((uint32_t *)cursor);
1948     cursor += sizeof(version);
1949
1950     (void) version; // TODO: do something with the version number.
1951
1952     keyclass = *((keyclass_t *)cursor);
1953
1954     // TODO: export get_key_gen macro
1955     if (((keyclass & ~key_class_last) == 0) != (current_generation == 0)) {
1956         c->query_ctx.found++;
1957     }
1958
1959     CFReleaseSafe(blob);
1960     return;
1961
1962 out:
1963     c->query_ctx.found++;
1964     CFReleaseSafe(blob);
1965 }
1966
1967 bool s3dl_dbt_keys_current(SecDbConnectionRef dbt, uint32_t current_generation, CFErrorRef *error) {
1968     CFErrorRef localError = NULL;
1969     struct check_generation_ctx ctx = { .query_ctx = { .dbt = dbt }, .current_generation = current_generation };
1970
1971     const SecDbClass *classes[] = {
1972         genp_class(),
1973         inet_class(),
1974         keys_class(),
1975         cert_class(),
1976     };
1977
1978     for (size_t class_ix = 0; class_ix < array_size(classes); ++class_ix) {
1979         Query *q = query_create(classes[class_ix], NULL, NULL, &localError);
1980         if (!q)
1981             return false;
1982
1983         ctx.query_ctx.q = q;
1984         q->q_limit = kSecMatchUnlimited;
1985
1986         bool ok = s3dl_query(check_generation, &ctx, &localError);
1987         query_destroy(q, NULL);
1988         CFReleaseNull(ctx.query_ctx.result);
1989
1990         if (!ok && localError && (CFErrorGetCode(localError) == errSecItemNotFound)) {
1991             CFReleaseNull(localError);
1992             continue;
1993         }
1994         secerror("Class %@ not up to date", classes[class_ix]->name);
1995         return false;
1996     }
1997     return true;
1998 }
1999
2000 bool s3dl_dbt_update_keys(SecDbConnectionRef dbt, SecurityClient *client, CFErrorRef *error) {
2001     return SecDbTransaction(dbt, kSecDbExclusiveTransactionType, error, ^(bool *commit) {
2002         __block bool ok = false;
2003         uint32_t keystore_generation_status;
2004
2005         /* can we migrate to new class keys right now? */
2006         if (!aks_generation(KEYBAG_DEVICE, generation_noop, &keystore_generation_status) &&
2007             (keystore_generation_status & generation_change_in_progress)) {
2008
2009             /* take a lock assertion */
2010             bool operated_while_unlocked = SecAKSDoWhileUserBagLocked(error, ^{
2011                 CFErrorRef localError = NULL;
2012                 CFDictionaryRef backup = SecServerCopyKeychainPlist(dbt, NULL,
2013                                                                     KEYBAG_DEVICE, KEYBAG_NONE, kSecNoItemFilter, &localError);
2014                 if (backup) {
2015                     if (localError) {
2016                         secerror("Ignoring export error: %@ during roll export", localError);
2017                         CFReleaseNull(localError);
2018                     }
2019                     ok = SecServerImportKeychainInPlist(dbt, client, KEYBAG_NONE,
2020                                                         KEYBAG_DEVICE, backup, kSecNoItemFilter, &localError);
2021                     if (localError) {
2022                         secerror("Ignoring export error: %@ during roll export", localError);
2023                         CFReleaseNull(localError);
2024                     }
2025                     CFRelease(backup);
2026                 }
2027             });
2028             if (!operated_while_unlocked)
2029                 ok = false;
2030         } else {
2031             ok = SecError(errSecBadReq, error, CFSTR("No key roll in progress."));
2032         }
2033
2034         *commit = ok;
2035     });
2036 }
2037 #endif