OSX/sec/securityd/SecDbItem.c

   1 /*
   2  * Copyright (c) 2012-2014 Apple Inc. All Rights Reserved.
   3  *
   4  * @APPLE_LICENSE_HEADER_START@
   5  *
   6  * This file contains Original Code and/or Modifications of Original Code
   7  * as defined in and that are subject to the Apple Public Source License
   8  * Version 2.0 (the 'License'). You may not use this file except in
   9  * compliance with the License. Please obtain a copy of the License at
  10  * http://www.opensource.apple.com/apsl/ and read it before using this
  11  * file.
  12  *
  13  * The Original Code and all software distributed under the License are
  14  * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  15  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  16  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
  17  * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
  18  * Please see the License for the specific language governing rights and
  19  * limitations under the License.
  20  *
  21  * @APPLE_LICENSE_HEADER_END@
  22  */
  23
  24 /*
  25  * SecDbItem.c - CoreFoundation-based constants and functions representing
  26  * database items (certificates, keys, identities, and passwords.)
  27  */
  28
  29 #if TARGET_DARWINOS
  30 #undef OCTAGON
  31 #undef SECUREOBJECTSYNC
  32 #undef SHAREDWEBCREDENTIALS
  33 #endif
  34
  35 #include <securityd/SecDbItem.h>
  36 #include <securityd/SecDbKeychainItem.h>
  37 #include <securityd/SecItemDb.h>
  38 #include <utilities/SecCFWrappers.h>
  39 #include <utilities/SecCFCCWrappers.h>
  40 #include <utilities/der_date.h>
  41 #include <utilities/der_plist.h>
  42 #include <utilities/debugging.h>
  43
  44 #include <Security/SecBasePriv.h>
  45 #include <Security/SecInternal.h>
  46 #include <corecrypto/ccsha1.h>
  47 #include <Security/SecItem.h>
  48 #include <Security/SecItemPriv.h>
  49 #include <Security/SecAccessControl.h>
  50 #include <Security/SecAccessControlPriv.h>
  51 #include <securityd/SecItemSchema.h>
  52
  53 #include <keychain/ckks/CKKS.h>
  54
  55 // MARK: type converters
  56
  57 CFStringRef copyString(CFTypeRef obj) {
  58     CFTypeID tid = CFGetTypeID(obj);
  59     if (tid == CFStringGetTypeID()) {
  60         return CFStringCreateCopy(0, obj);
  61     }else if (tid == CFDataGetTypeID()) {
  62         return CFStringCreateFromExternalRepresentation(0, obj, kCFStringEncodingUTF8);
  63     } else if (tid == CFUUIDGetTypeID()) {
  64         return CFUUIDCreateString(NULL, obj);
  65     } else {
  66         return NULL;
  67     }
  68 }
  69
  70 CFDataRef copyData(CFTypeRef obj) {
  71     CFTypeID tid = CFGetTypeID(obj);
  72     if (tid == CFDataGetTypeID()) {
  73         return CFDataCreateCopy(0, obj);
  74     } else if (tid == CFStringGetTypeID()) {
  75         return CFStringCreateExternalRepresentation(0, obj, kCFStringEncodingUTF8, 0);
  76     } else if (tid == CFNumberGetTypeID()) {
  77         SInt32 value;
  78         CFNumberGetValue(obj, kCFNumberSInt32Type, &value);
  79         return CFDataCreate(0, (const UInt8 *)&value, sizeof(value));
  80     } else {
  81         return NULL;
  82     }
  83 }
  84
  85 CFTypeRef copyUUID(CFTypeRef obj) {
  86     CFTypeID tid = CFGetTypeID(obj);
  87     if (tid == CFDataGetTypeID()) {
  88         CFIndex length = CFDataGetLength(obj);
  89         if (length != 0 && length != 16)
  90             return NULL;
  91         return CFDataCreateCopy(NULL, obj);
  92     } else if (tid == CFNullGetTypeID()) {
  93         return CFDataCreate(NULL, NULL, 0);
  94     } else if (tid == CFUUIDGetTypeID()) {
  95         CFUUIDBytes uuidbytes = CFUUIDGetUUIDBytes(obj);
  96         CFDataRef uuiddata = CFDataCreate(NULL, (void*) &uuidbytes, sizeof(uuidbytes));
  97         return uuiddata;
  98     } else {
  99         return NULL;
 100     }
 101 }
 102
 103
 104 CFTypeRef copyBlob(CFTypeRef obj) {
 105     CFTypeID tid = CFGetTypeID(obj);
 106     if (tid == CFDataGetTypeID()) {
 107         return CFDataCreateCopy(0, obj);
 108     } else if (tid == CFStringGetTypeID()) {
 109         return CFStringCreateCopy(0, obj);
 110     } else if (tid == CFNumberGetTypeID()) {
 111         CFRetain(obj);
 112         return obj;
 113     } else {
 114         return NULL;
 115     }
 116 }
 117
 118 CFDataRef copySHA1(CFTypeRef obj) {
 119     CFTypeID tid = CFGetTypeID(obj);
 120     if (tid == CFDataGetTypeID() && CFDataGetLength(obj) == CCSHA1_OUTPUT_SIZE) {
 121         return CFDataCreateCopy(CFGetAllocator(obj), obj);
 122     } else {
 123         return NULL;
 124     }
 125 }
 126
 127 CFTypeRef copyNumber(CFTypeRef obj) {
 128     CFTypeID tid = CFGetTypeID(obj);
 129     if (tid == CFNumberGetTypeID()) {
 130         CFRetain(obj);
 131         return obj;
 132     } else if (tid == CFBooleanGetTypeID()) {
 133         SInt32 value = CFBooleanGetValue(obj);
 134         return CFNumberCreate(0, kCFNumberSInt32Type, &value);
 135     } else if (tid == CFStringGetTypeID()) {
 136         SInt32 value = CFStringGetIntValue(obj);
 137         CFStringRef t = CFStringCreateWithFormat(0, 0, CFSTR("%ld"), (long) value);
 138         /* If a string converted to an int isn't equal to the int printed as
 139            a string, return a CFStringRef instead. */
 140         if (!CFEqual(t, obj)) {
 141             CFRelease(t);
 142             return CFStringCreateCopy(0, obj);
 143         }
 144         CFRelease(t);
 145         return CFNumberCreate(0, kCFNumberSInt32Type, &value);
 146     } else
 147         return NULL;
 148 }
 149
 150 CFDateRef copyDate(CFTypeRef obj) {
 151     CFTypeID tid = CFGetTypeID(obj);
 152     if (tid == CFDateGetTypeID()) {
 153         CFRetain(obj);
 154         return obj;
 155     } else
 156         return NULL;
 157 }
 158
 159 // MARK: SecDbColumn accessors, to retrieve values as CF types in SecDbStep.
 160
 161 static CFDataRef SecDbColumnCopyData(CFAllocatorRef allocator, sqlite3_stmt *stmt, int col, CFErrorRef *error) {
 162     return CFDataCreate(allocator, sqlite3_column_blob(stmt, col),
 163                         sqlite3_column_bytes(stmt, col));
 164     //return CFDataCreateWithBytesNoCopy(0, sqlite3_column_blob(stmt, col),
 165     //                                   sqlite3_column_bytes(stmt, col),
 166     //                                   kCFAllocatorNull);
 167 }
 168
 169 static CFDateRef SecDbColumnCopyDate(CFAllocatorRef allocator, sqlite3_stmt *stmt, int col, CFErrorRef *error) {
 170     return CFDateCreate(allocator, sqlite3_column_double(stmt, col));
 171 }
 172
 173 static CFNumberRef SecDbColumnCopyDouble(CFAllocatorRef allocator, sqlite3_stmt *stmt, int col, CFErrorRef *error) {
 174     double number = sqlite3_column_double(stmt, col);
 175     return CFNumberCreate(allocator, kCFNumberDoubleType, &number);
 176 }
 177
 178 static CFNumberRef SecDbColumnCopyNumber64(CFAllocatorRef allocator, sqlite3_stmt *stmt, int col, CFErrorRef *error) {
 179     sqlite_int64 number = sqlite3_column_int64(stmt, col);
 180     return CFNumberCreate(allocator, kCFNumberSInt64Type, &number);
 181 }
 182
 183 static CFNumberRef SecDbColumnCopyNumber(CFAllocatorRef allocator, sqlite3_stmt *stmt, int col, CFErrorRef *error) {
 184     sqlite_int64 number = sqlite3_column_int64(stmt, col);
 185     if (INT32_MIN <= number && number <= INT32_MAX) {
 186         int32_t num32 = (int32_t)number;
 187         return CFNumberCreate(allocator, kCFNumberSInt32Type, &num32);
 188     } else {
 189         return CFNumberCreate(allocator, kCFNumberSInt64Type, &number);
 190     }
 191 }
 192
 193 static CFTypeRef SecDbColumnCopyString(CFAllocatorRef allocator, sqlite3_stmt *stmt, int col, CFErrorRef *error,
 194                                        CFOptionFlags flags) {
 195     const unsigned char *text = sqlite3_column_text(stmt, col);
 196     if (!text || 0 == strlen((const char *)text)) {
 197         if (flags & kSecDbDefaultEmptyFlag) {
 198             return CFSTR("");
 199         } else if (flags & kSecDbDefault0Flag) {
 200             return CFSTR("0");
 201         } else {
 202             return kCFNull;
 203         }
 204     }
 205     return CFStringCreateWithBytes(allocator, text, strlen((const char *)text), kCFStringEncodingUTF8, false);
 206 }
 207
 208 // MARK: SecDbClass helpers
 209
 210 const SecDbAttr *SecDbClassAttrWithKind(const SecDbClass *class, SecDbAttrKind kind, CFErrorRef *error) {
 211     const SecDbAttr *result = NULL;
 212     SecDbForEachAttr(class, desc) {
 213         if (desc->kind == kind)
 214             result = desc;
 215     }
 216
 217     if (!result)
 218         SecError(errSecInternal, error, CFSTR("Can't find attribute of kind %d in class %@"), kind, class->name);
 219
 220     return result;
 221 }
 222
 223 // MARK: SecDbAttr helpers
 224
 225 static bool SecDbIsTombstoneDbSelectAttr(const SecDbAttr *attr) {
 226     return attr->flags & kSecDbPrimaryKeyFlag || attr->kind == kSecDbTombAttr;
 227 }
 228
 229 #if 0
 230 static bool SecDbIsTombstoneDbInsertAttr(const SecDbAttr *attr) {
 231     return SecDbIsTombstoneDbSelectAttr(attr) || attr->kind == kSecDbAccessAttr || attr->kind == kSecDbCreationDateAttr || attr->kind == kSecDbModificationDateAttr;
 232 }
 233 #endif
 234
 235 static bool SecDbIsTombstoneDbUpdateAttr(const SecDbAttr *attr) {
 236     // We add AuthenticatedData to include UUIDs, which can't be primary keys
 237     return SecDbIsTombstoneDbSelectAttr(attr) || attr->kind == kSecDbAccessAttr || attr->kind == kSecDbCreationDateAttr || attr->kind == kSecDbRowIdAttr || (attr->flags & kSecDbInAuthenticatedDataFlag);
 238 }
 239
 240 CFTypeRef SecDbAttrCopyDefaultValue(const SecDbAttr *attr, CFErrorRef *error) {
 241     CFTypeRef value = NULL;
 242     switch (attr->kind) {
 243         case kSecDbAccessAttr:
 244         case kSecDbStringAttr:
 245         case kSecDbAccessControlAttr:
 246             value = CFSTR("");
 247             break;
 248         case kSecDbBlobAttr:
 249         case kSecDbDataAttr:
 250             value = CFDataCreate(kCFAllocatorDefault, NULL, 0);
 251             break;
 252         case kSecDbUUIDAttr:
 253             value = CFDataCreate(kCFAllocatorDefault, NULL, 0);
 254             break;
 255         case kSecDbNumberAttr:
 256         case kSecDbSyncAttr:
 257         case kSecDbTombAttr:
 258         {
 259             int32_t zero = 0;
 260             value = CFNumberCreate(kCFAllocatorDefault, kCFNumberSInt32Type, &zero);
 261             break;
 262         }
 263         case kSecDbDateAttr:
 264             value = CFDateCreate(kCFAllocatorDefault, 0.0);
 265             break;
 266         case kSecDbCreationDateAttr:
 267         case kSecDbModificationDateAttr:
 268             value = CFDateCreate(kCFAllocatorDefault, CFAbsoluteTimeGetCurrent());
 269             break;
 270         default:
 271             SecError(errSecInternal, error, CFSTR("attr %@ has no default value"), attr->name);
 272             value = NULL;
 273     }
 274
 275     return value;
 276 }
 277
 278 static CFTypeRef SecDbAttrCopyValueForDb(const SecDbAttr *attr, CFTypeRef value, CFErrorRef *error) {
 279     CFDataRef data = NULL;
 280     CFTypeRef result = NULL;
 281
 282     if (value == NULL)
 283         value = kCFNull;
 284
 285     if (CFEqual(value, kCFNull) && attr->flags & kSecDbPrimaryKeyFlag) {
 286         // SQLITE3 doesn't like NULL for primary key attributes, pretend kSecDbDefaultEmptyFlag was specified
 287         require_quiet(result = SecDbAttrCopyDefaultValue(attr, error), out);
 288     } else {
 289         result = CFRetain(value);
 290     }
 291
 292     if (attr->flags & kSecDbSHA1ValueInFlag && !CFEqual(result, kCFNull)) {
 293         require_action_quiet(data = copyData(result), out,
 294                              SecError(errSecInternal, error, CFSTR("failed to get attribute %@ data"), attr->name);
 295                              CFReleaseNull(result));
 296         CFAssignRetained(result, CFDataCopySHA1Digest(data, error));
 297     }
 298
 299 out:
 300     CFReleaseSafe(data);
 301     return result;
 302 }
 303
 304 static CFStringRef SecDbAttrGetHashName(const SecDbAttr *attr) {
 305     if ((attr->flags & kSecDbSHA1ValueInFlag) == 0) {
 306         return attr->name;
 307     }
 308
 309     static dispatch_once_t once;
 310     static CFMutableDictionaryRef hash_store;
 311     static dispatch_queue_t queue;
 312     dispatch_once(&once, ^{
 313         queue = dispatch_queue_create("secd-hash-name", NULL);
 314         hash_store = CFDictionaryCreateMutableForCFTypes(NULL);
 315     });
 316
 317     __block CFStringRef name;
 318     dispatch_sync(queue, ^{
 319         name = CFDictionaryGetValue(hash_store, attr->name);
 320         if (name == NULL) {
 321             name = CFStringCreateWithFormat(NULL, NULL, CFSTR("#%@"), attr->name);
 322             CFDictionarySetValue(hash_store, attr->name, name);
 323             CFRelease(name);
 324         }
 325     });
 326     return name;
 327 }
 328
 329 // MARK: SecDbItem
 330
 331 CFTypeRef SecDbItemGetCachedValueWithName(SecDbItemRef item, CFStringRef name) {
 332     return CFDictionaryGetValue(item->attributes, name);
 333 }
 334
 335 static CFTypeRef SecDbItemGetCachedValue(SecDbItemRef item, const SecDbAttr *desc) {
 336     return CFDictionaryGetValue(item->attributes, desc->name);
 337 }
 338
 339 CFMutableDictionaryRef SecDbItemCopyPListWithMask(SecDbItemRef item, CFOptionFlags mask, CFErrorRef *error) {
 340     CFMutableDictionaryRef dict = CFDictionaryCreateMutableForCFTypes(kCFAllocatorDefault);
 341     SecDbForEachAttrWithMask(item->class, desc, mask) {
 342         CFTypeRef value = SecDbItemGetValue(item, desc, error);
 343         if (value) {
 344             if (!CFEqual(kCFNull, value)) {
 345                 CFDictionarySetValue(dict, desc->name, value);
 346             } else if (desc->flags & kSecDbNotNullFlag) {
 347                 SecError(errSecDecode, error, CFSTR("attribute %@ has NULL value"), desc->name);
 348                 secerror("%@", error ? *error : (CFErrorRef)CFSTR("error == NULL"));
 349                 CFReleaseNull(dict);
 350                 break;
 351             }
 352         } else {
 353             CFReleaseNull(dict);
 354             break;
 355         }
 356     }
 357     return dict;
 358 }
 359
 360 void SecDbItemSetCredHandle(SecDbItemRef item, CFTypeRef cred_handle) {
 361     CFRetainAssign(item->credHandle, cred_handle);
 362 }
 363
 364 void SecDbItemSetCallerAccessGroups(SecDbItemRef item, CFArrayRef caller_access_groups) {
 365     CFRetainAssign(item->callerAccessGroups, caller_access_groups);
 366 }
 367
 368 CFDataRef SecDbItemCopyEncryptedDataToBackup(SecDbItemRef item, uint64_t handle, CFErrorRef *error) {
 369     CFDataRef edata = NULL;
 370     keybag_handle_t keybag = (keybag_handle_t)handle;
 371     CFMutableDictionaryRef attributes = SecDbItemCopyPListWithMask(item, kSecDbInCryptoDataFlag, error);
 372     CFMutableDictionaryRef auth_attributes = SecDbItemCopyPListWithMask(item, kSecDbInAuthenticatedDataFlag, error);
 373     if (attributes || auth_attributes) {
 374         SecAccessControlRef access_control = SecDbItemCopyAccessControl(item, error);
 375         if (access_control) {
 376             if (ks_encrypt_data_legacy(keybag, access_control, item->credHandle, attributes, auth_attributes, &edata, false, error)) {
 377                 item->_edataState = kSecDbItemEncrypting;
 378             } else {
 379                 seccritical("ks_encrypt_data (db): failed: %@", error ? *error : (CFErrorRef)CFSTR(""));
 380             }
 381             CFRelease(access_control);
 382         }
 383         CFReleaseNull(attributes);
 384         CFReleaseNull(auth_attributes);
 385     }
 386
 387     return edata;
 388 }
 389
 390 bool SecDbItemEnsureDecrypted(SecDbItemRef item, bool decryptSecretData, CFErrorRef *error) {
 391
 392     // If we haven't yet decrypted the item, make sure we do so now
 393     bool result = true;
 394     if (item->_edataState == kSecDbItemEncrypted || (decryptSecretData && item->_edataState == kSecDbItemSecretEncrypted)) {
 395         const SecDbAttr *attr = SecDbClassAttrWithKind(item->class, kSecDbEncryptedDataAttr, error);
 396         if (attr) {
 397             CFDataRef edata = SecDbItemGetCachedValue(item, attr);
 398             if (!edata)
 399                 return SecError(errSecInternal, error, CFSTR("state= encrypted but edata is NULL"));
 400             // Decrypt calls set value a bunch of times which clears our edata and changes our state.
 401             item->_edataState = kSecDbItemDecrypting;
 402             result = SecDbItemDecrypt(item, decryptSecretData, edata, error);
 403             if (result)
 404                 item->_edataState = decryptSecretData ? kSecDbItemClean : kSecDbItemSecretEncrypted;
 405             else
 406                 item->_edataState = kSecDbItemEncrypted;
 407         }
 408     }
 409     return result;
 410 }
 411
 412 // Only called if cached value is not found.
 413 static CFTypeRef SecDbItemCopyValue(SecDbItemRef item, const SecDbAttr *attr, CFErrorRef *error) {
 414     if (attr->copyValue) {
 415         return attr->copyValue(item, attr, error);
 416     }
 417
 418     CFTypeRef value = NULL;
 419     switch (attr->kind) {
 420             // These have an explicit copyValue; here to shut up compiler
 421         case kSecDbSHA1Attr:
 422         case kSecDbEncryptedDataAttr:
 423         case kSecDbPrimaryKeyAttr:
 424             value = NULL;
 425             break;
 426         case kSecDbAccessAttr:
 427         case kSecDbStringAttr:
 428         case kSecDbBlobAttr:
 429         case kSecDbAccessControlAttr:
 430             if (attr->flags & kSecDbNotNullFlag) {
 431                 if (attr->flags & kSecDbDefault0Flag) {
 432                     value = CFSTR("0");
 433                     break;
 434                 } else if (attr->kind != kSecDbBlobAttr && attr->flags & kSecDbDefaultEmptyFlag) {
 435                     // blob drops through to data everything else is empty string
 436                     value = CFSTR("");
 437                     break;
 438                 }
 439             }
 440             //DROPTHROUGH
 441         case kSecDbDataAttr:
 442             if (attr->flags & kSecDbNotNullFlag && attr->flags & kSecDbDefaultEmptyFlag) {
 443                 value = CFDataCreate(CFGetAllocator(item), NULL, 0);
 444             } else {
 445                 value = kCFNull;
 446             }
 447             break;
 448         case kSecDbUUIDAttr:
 449             value = CFDataCreate(CFGetAllocator(item), NULL, 0);
 450             break;
 451         case kSecDbNumberAttr:
 452         case kSecDbSyncAttr:
 453         case kSecDbTombAttr:
 454             if (attr->flags & kSecDbNotNullFlag) {
 455                 int32_t zero = 0;
 456                 value = CFNumberCreate(CFGetAllocator(item), kCFNumberSInt32Type, &zero);
 457             } else {
 458                 value = kCFNull;
 459             }
 460             break;
 461         case kSecDbDateAttr:
 462             if (attr->flags & kSecDbNotNullFlag && attr->flags & kSecDbDefault0Flag) {
 463                 value = CFDateCreate(kCFAllocatorDefault, 0.0);
 464             } else {
 465                 value = kCFNull;
 466             }
 467             break;
 468         case kSecDbRowIdAttr:
 469             if (attr->flags & kSecDbNotNullFlag) {
 470                 // No can do, error?
 471             }
 472             value = kCFNull;
 473             break;
 474         case kSecDbCreationDateAttr:
 475         case kSecDbModificationDateAttr:
 476             value = CFDateCreate(CFGetAllocator(item), CFAbsoluteTimeGetCurrent());
 477             break;
 478         case kSecDbUTombAttr:
 479             value = kCFNull;
 480             break;
 481     }
 482
 483     return value;
 484 }
 485
 486 // SecDbItemGetValue will return kCFNull if there is no value for an attribute and this was not
 487 // an error.  It will return NULL and optionally set *error if there was an error computing an
 488 // attribute, or if a required attribute was missing a value and had no known way to compute
 489 // it's value.
 490 CFTypeRef SecDbItemGetValue(SecDbItemRef item, const SecDbAttr *desc, CFErrorRef *error) {
 491     // Propagate chained errors
 492     if (!desc)
 493         return NULL;
 494
 495     if (desc->flags & kSecDbInCryptoDataFlag || desc->flags & kSecDbInAuthenticatedDataFlag || desc->flags & kSecDbReturnDataFlag) {
 496         if (!SecDbItemEnsureDecrypted(item, desc->flags & kSecDbReturnDataFlag, error))
 497             return NULL;
 498     }
 499
 500     CFTypeRef value = SecDbItemGetCachedValue(item, desc);
 501     if (!value) {
 502         value = SecDbItemCopyValue(item, desc, error);
 503         if (value) {
 504             if (CFEqual(kCFNull, value)) {
 505                 CFRelease(value); // This is redundant but it shuts clang's static analyzer up.
 506                 value = kCFNull;
 507             } else {
 508                 SecDbItemSetValue(item, desc, value, error);
 509                 CFRelease(value);
 510                 value = SecDbItemGetCachedValue(item, desc);
 511             }
 512         }
 513     }
 514     return value;
 515 }
 516
 517 CFTypeRef SecDbItemGetValueKind(SecDbItemRef item, SecDbAttrKind descKind, CFErrorRef *error) {
 518     CFTypeRef result = NULL;
 519
 520     const SecDbClass * itemClass = SecDbItemGetClass(item);
 521     const SecDbAttr * desc = SecDbClassAttrWithKind(itemClass, descKind, error);
 522
 523     if (desc) {
 524         result = SecDbItemGetValue(item, desc, error);
 525     }
 526
 527     return result;
 528 }
 529
 530
 531 // Similar as SecDbItemGetValue, but if attr represents attribute stored into DB field as hash, returns
 532 // hashed value for the attribute.
 533 static CFTypeRef SecDbItemCopyValueForDb(SecDbItemRef item, const SecDbAttr *desc, CFErrorRef *error) {
 534     CFTypeRef value = NULL;
 535     CFStringRef hash_name = NULL;
 536     hash_name = SecDbAttrGetHashName(desc);
 537     if ((desc->flags & kSecDbSHA1ValueInFlag) && (desc->flags & kSecDbInFlag)) {
 538         value = CFRetainSafe(CFDictionaryGetValue(item->attributes, hash_name));
 539     }
 540
 541     if (value == NULL) {
 542         require_quiet(value = SecDbItemGetValue(item, desc, error), out);
 543         require_action_quiet(value = SecDbAttrCopyValueForDb(desc, value, error), out, CFReleaseNull(value));
 544         if ((desc->flags & kSecDbSHA1ValueInFlag) != 0) {
 545             CFDictionarySetValue(item->attributes, hash_name, value);
 546         }
 547     }
 548
 549 out:
 550     return value;
 551 }
 552
 553 static bool SecDbItemGetBoolValue(SecDbItemRef item, const SecDbAttr *desc, bool *bvalue, CFErrorRef *error) {
 554     CFTypeRef value = SecDbItemGetValue(item, desc, error);
 555     if (!value)
 556         return false;
 557     char cvalue;
 558     *bvalue = (isNumber(value) && CFNumberGetValue(value, kCFNumberCharType, &cvalue) && cvalue == 1);
 559     return true;
 560 }
 561
 562 static CFStringRef SecDbItemCopyFormatDescription(CFTypeRef cf, CFDictionaryRef formatOptions) {
 563     CFStringRef desc;
 564     if (isDictionary(formatOptions) && CFDictionaryContainsKey(formatOptions, kSecDebugFormatOption)) {
 565         SecDbItemRef item = (SecDbItemRef)cf;
 566         CFMutableStringRef mdesc = CFStringCreateMutable(CFGetAllocator(cf), 0);
 567         CFStringAppendFormat(mdesc, NULL, CFSTR("<%@"), item->class->name);
 568         SecDbForEachAttr(item->class, attr) {
 569                 CFTypeRef value = SecDbItemGetValue(item, attr, NULL);
 570                 if (value) {
 571                     CFStringAppend(mdesc, CFSTR(","));
 572                     CFStringAppend(mdesc, attr->name);
 573                     CFStringAppend(mdesc, CFSTR("="));
 574                     if (CFEqual(CFSTR("data"), attr->name)) {
 575                         CFStringAppendEncryptedData(mdesc, value);
 576                     } else if (CFEqual(CFSTR("v_Data"), attr->name)) {
 577                         CFStringAppend(mdesc, CFSTR("<?>"));
 578                     } else if (isData(value)) {
 579                         CFStringAppendHexData(mdesc, value);
 580                     } else {
 581                         CFStringAppendFormat(mdesc, 0, CFSTR("%@"), value);
 582                     }
 583                 }
 584         }
 585         CFStringAppend(mdesc, CFSTR(">"));
 586         desc = mdesc;
 587     } else {
 588         SecDbItemRef item = (SecDbItemRef)cf;
 589         const UInt8 zero4[4] = {};
 590         const UInt8 *pk = &zero4[0], *sha1 = &zero4[0];
 591         char sync = 0;
 592         char tomb = 0;
 593         SInt64 rowid = 0;
 594         CFStringRef access = NULL;
 595         uint8_t mdatbuf[32] = {};
 596         uint8_t *mdat = &mdatbuf[0];
 597         CFMutableStringRef attrs = CFStringCreateMutable(kCFAllocatorDefault, 0);
 598         CFStringRef agrp = NULL;
 599         CFBooleanRef utomb = NULL;
 600
 601         SecDbForEachAttr(item->class, attr) {
 602             CFTypeRef value;
 603             switch (attr->kind) {
 604                 case kSecDbBlobAttr:
 605                 case kSecDbDataAttr:
 606                 case kSecDbStringAttr:
 607                 case kSecDbNumberAttr:
 608                 case kSecDbDateAttr:
 609                 case kSecDbEncryptedDataAttr:
 610                     if (attr->flags & (kSecDbReturnAttrFlag | kSecDbReturnDataFlag) && (value = SecDbItemGetValue(item, attr, NULL)) && !CFEqual(value, kCFNull)) {
 611                         if (isString(value) && CFEqual(attr->name, kSecAttrAccessGroup)) {
 612                             agrp = value;
 613                         } else {
 614                             // We don't log these, just record that we saw the attribute.
 615                             CFStringAppend(attrs, CFSTR(","));
 616                             CFStringAppend(attrs, attr->name);
 617                         }
 618                     }
 619                     break;
 620                 case kSecDbUUIDAttr:
 621                     if ((value = SecDbItemGetValue(item, attr, NULL))) {
 622                         if (CFEqual(attr->name, kSecAttrMultiUser)) {
 623                             if (isData(value)) {
 624                                 CFStringAppend(attrs, CFSTR(","));
 625                                 if (CFDataGetLength(value)) {
 626                                     CFStringAppendHexData(attrs, value);
 627                                 } else {
 628                                     CFStringAppend(attrs, attr->name);
 629                                 }
 630                             }
 631                         }
 632                     }
 633                     break;
 634                 case kSecDbCreationDateAttr:
 635                     // We don't care about this and every object has one.
 636                     break;
 637                 case kSecDbModificationDateAttr:
 638                     value = SecDbItemGetValue(item, attr, NULL);
 639                     if (isDate(value))
 640                         mdat = der_encode_generalizedtime_body(CFDateGetAbsoluteTime(value), NULL, mdat, &mdatbuf[31]);
 641                     break;
 642                 case kSecDbSHA1Attr:
 643                     value = SecDbItemGetValue(item, attr, NULL);
 644                     if (isData(value) && CFDataGetLength(value) >= (CFIndex)sizeof(zero4))
 645                         sha1 = CFDataGetBytePtr(value);
 646                     break;
 647                 case kSecDbRowIdAttr:
 648                     value = SecDbItemGetValue(item, attr, NULL);
 649                     if (isNumber(value))
 650                         CFNumberGetValue(value, kCFNumberSInt64Type, &rowid);
 651                     break;
 652                 case kSecDbPrimaryKeyAttr:
 653                     value = SecDbItemGetValue(item, attr, NULL);
 654                     if (isData(value))
 655                         pk = CFDataGetBytePtr(value);
 656                     break;
 657                 case kSecDbSyncAttr:
 658                     value = SecDbItemGetValue(item, attr, NULL);
 659                     if (isNumber(value))
 660                         CFNumberGetValue(value, kCFNumberCharType, &sync);
 661                     break;
 662                 case kSecDbTombAttr:
 663                     value = SecDbItemGetValue(item, attr, NULL);
 664                     if (isNumber(value))
 665                         CFNumberGetValue(value, kCFNumberCharType, &tomb);
 666                     break;
 667                 case kSecDbAccessAttr:
 668                     value = SecDbItemGetValue(item, attr, NULL);
 669                     if (isString(value))
 670                         access = value;
 671                     break;
 672                 case kSecDbUTombAttr:
 673                     value = SecDbItemGetValue(item, attr, NULL);
 674                     if (isBoolean(value))
 675                         utomb = value;
 676                 case kSecDbAccessControlAttr:
 677                     /* TODO: Add formatting of ACLs. */
 678                     break;
 679             }
 680         }
 681
 682         desc = CFStringCreateWithFormat(CFGetAllocator(cf), NULL,
 683             CFSTR(
 684                   "%s,"
 685                   "%@,"
 686                   "%02X%02X%02X%02X,"
 687                   "%s,"
 688                   "%@,"
 689                   "%@,"
 690                   "%"PRId64
 691                   "%@,"
 692                   "%s,"
 693                   "%s"
 694                   "%02X%02X%02X%02X"),
 695             tomb ? "T" : "O",
 696             item->class->name,
 697             pk[0], pk[1], pk[2], pk[3],
 698             sync ? "S" : "L",
 699             access,
 700             agrp,
 701             rowid,
 702             attrs,
 703             mdat,
 704             utomb ? (CFEqual(utomb, kCFBooleanFalse) ? "F," : "T,") : "",
 705             sha1[0], sha1[1], sha1[2], sha1[3]);
 706         CFReleaseSafe(attrs);
 707     }
 708
 709     return desc;
 710 }
 711
 712 static void SecDbItemDestroy(CFTypeRef cf) {
 713     SecDbItemRef item = (SecDbItemRef)cf;
 714     CFReleaseSafe(item->attributes);
 715     CFReleaseSafe(item->credHandle);
 716     CFReleaseSafe(item->callerAccessGroups);
 717     CFReleaseSafe(item->cryptoOp);
 718 }
 719
 720 static CFHashCode SecDbItemHash(CFTypeRef cf) {
 721     SecDbItemRef item = (SecDbItemRef)cf;
 722     CFDataRef digest = SecDbItemGetSHA1(item, NULL);
 723     CFHashCode code;
 724     const UInt8 *p = CFDataGetBytePtr(digest);
 725     // Read first 8 bytes of digest in order
 726     code = p[0] + ((p[1] + ((p[2] + ((p[3] + ((p[4] + ((p[5] + ((p[6] + (p[7] << 8)) << 8)) << 8)) << 8)) << 8)) << 8)) << 8);
 727     return code;
 728 }
 729
 730 static Boolean SecDbItemCompare(CFTypeRef cf1, CFTypeRef cf2) {
 731     SecDbItemRef item1 = (SecDbItemRef)cf1;
 732     SecDbItemRef item2 = (SecDbItemRef)cf2;
 733     CFDataRef digest1 = NULL;
 734     CFDataRef digest2 = NULL;
 735     if (item1)
 736         digest1 = SecDbItemGetSHA1(item1, NULL);
 737     if (item2)
 738         digest2 = SecDbItemGetSHA1(item2, NULL);
 739     Boolean equal = CFEqual(digest1, digest2);
 740     return equal;
 741 }
 742
 743 CFGiblisWithHashFor(SecDbItem)
 744
 745 static SecDbItemRef SecDbItemCreate(CFAllocatorRef allocator, const SecDbClass *class, keybag_handle_t keybag) {
 746     SecDbItemRef item = CFTypeAllocate(SecDbItem, struct SecDbItem, allocator);
 747     item->class = class;
 748     item->attributes = CFDictionaryCreateMutableForCFTypes(allocator);
 749     item->keybag = keybag;
 750     item->_edataState = kSecDbItemDirty;
 751     item->cryptoOp = kAKSKeyOpDecrypt;
 752
 753     return item;
 754 }
 755
 756 const SecDbClass *SecDbItemGetClass(SecDbItemRef item) {
 757     return item->class;
 758 }
 759
 760 keybag_handle_t SecDbItemGetKeybag(SecDbItemRef item) {
 761     return item->keybag;
 762 }
 763
 764 bool SecDbItemSetKeybag(SecDbItemRef item, keybag_handle_t keybag, CFErrorRef *error) {
 765     if (!SecDbItemEnsureDecrypted(item, true, error))
 766         return false;
 767     if (item->keybag != keybag) {
 768         item->keybag = keybag;
 769         if (item->_edataState == kSecDbItemClean) {
 770             SecDbItemSetValue(item, SecDbClassAttrWithKind(item->class, kSecDbEncryptedDataAttr, NULL), kCFNull, NULL);
 771         }
 772     }
 773
 774     return true;
 775 }
 776
 777 bool SecDbItemSetValue(SecDbItemRef item, const SecDbAttr *desc, CFTypeRef value, CFErrorRef *error) {
 778     // Propagate chained errors.
 779     if (!desc)
 780         return false;
 781
 782     if (!value)
 783         value = kCFNull;
 784
 785     if (desc->setValue)
 786         return desc->setValue(item, desc, value, error);
 787
 788     if (desc->flags & kSecDbInCryptoDataFlag || desc->flags & kSecDbInAuthenticatedDataFlag) {
 789         if (!SecDbItemEnsureDecrypted(item, true, error)) {
 790             return false;
 791         }
 792     }
 793
 794     bool changed = false;
 795     CFTypeRef attr = NULL;
 796     switch (desc->kind) {
 797         case kSecDbPrimaryKeyAttr:
 798         case kSecDbDataAttr:
 799             attr = copyData(value);
 800             break;
 801         case kSecDbEncryptedDataAttr:
 802             attr = copyData(value);
 803             if (attr) {
 804                 if (item->_edataState == kSecDbItemEncrypting)
 805                     item->_edataState = kSecDbItemClean;
 806                 else
 807                     item->_edataState = kSecDbItemEncrypted;
 808             } else if (!value || CFEqual(kCFNull, value)) {
 809                 item->_edataState = kSecDbItemDirty;
 810             }
 811             break;
 812         case kSecDbBlobAttr:
 813         case kSecDbAccessControlAttr:
 814             attr = copyBlob(value);
 815             break;
 816         case kSecDbDateAttr:
 817         case kSecDbCreationDateAttr:
 818         case kSecDbModificationDateAttr:
 819             attr = copyDate(value);
 820             break;
 821         case kSecDbNumberAttr:
 822         case kSecDbSyncAttr:
 823         case kSecDbTombAttr:
 824         case kSecDbRowIdAttr:
 825             attr = copyNumber(value);
 826             break;
 827         case kSecDbAccessAttr:
 828         case kSecDbStringAttr:
 829             attr = copyString(value);
 830             break;
 831         case kSecDbSHA1Attr:
 832             attr = copySHA1(value);
 833             break;
 834         case kSecDbUTombAttr:
 835             attr = CFRetainSafe(asBoolean(value, NULL));
 836             break;
 837         case kSecDbUUIDAttr:
 838             attr = copyUUID(value);
 839             break;
 840     }
 841
 842     if (attr) {
 843         CFTypeRef ovalue = CFDictionaryGetValue(item->attributes, desc->name);
 844         changed = (!ovalue || !CFEqual(ovalue, attr));
 845         CFDictionarySetValue(item->attributes, desc->name, attr);
 846         CFRelease(attr);
 847     } else {
 848         if (value && !CFEqual(kCFNull, value)) {
 849             SecError(errSecItemInvalidValue, error, CFSTR("attribute %@: value: %@ failed to convert"), desc->name, value);
 850             return false;
 851         }
 852         CFTypeRef ovalue = CFDictionaryGetValue(item->attributes, desc->name);
 853         changed = (ovalue && !CFEqual(ovalue, kCFNull));
 854         CFDictionaryRemoveValue(item->attributes, desc->name);
 855     }
 856
 857     if (changed) {
 858         if (desc->flags & kSecDbInHashFlag)
 859             SecDbItemSetValue(item, SecDbClassAttrWithKind(item->class, kSecDbSHA1Attr, NULL), kCFNull, NULL);
 860         if (desc->flags & kSecDbPrimaryKeyFlag)
 861             SecDbItemSetValue(item, SecDbClassAttrWithKind(item->class, kSecDbPrimaryKeyAttr, NULL), kCFNull, NULL);
 862         if ((desc->flags & kSecDbInCryptoDataFlag || desc->flags & kSecDbInAuthenticatedDataFlag) && (item->_edataState == kSecDbItemClean || (item->_edataState == kSecDbItemSecretEncrypted && (desc->flags & kSecDbReturnDataFlag) == 0)))
 863             SecDbItemSetValue(item, SecDbClassAttrWithKind(item->class, kSecDbEncryptedDataAttr, NULL), kCFNull, NULL);
 864         if (desc->flags & kSecDbSHA1ValueInFlag)
 865             CFDictionaryRemoveValue(item->attributes, SecDbAttrGetHashName(desc));
 866     }
 867
 868     return true;
 869 }
 870
 871 bool SecDbItemSetValues(SecDbItemRef item, CFDictionaryRef values, CFErrorRef *error) {
 872     SecDbForEachAttr(item->class, attr) {
 873         CFTypeRef value = CFDictionaryGetValue(values, attr->name);
 874         if (value && !SecDbItemSetValue(item, attr, value, error))
 875             return false;
 876     }
 877     return true;
 878 }
 879
 880 bool SecDbItemSetValueWithName(SecDbItemRef item, CFStringRef name, CFTypeRef value, CFErrorRef *error) {
 881     SecDbForEachAttr(item->class, attr) {
 882         if (CFEqual(attr->name, name)) {
 883             return SecDbItemSetValue(item, attr, value, error);
 884         }
 885     }
 886     return false;
 887 }
 888
 889 bool SecDbItemSetAccessControl(SecDbItemRef item, SecAccessControlRef access_control, CFErrorRef *error) {
 890     bool ok = true;
 891     if (item->_edataState == kSecDbItemClean)
 892         ok = SecDbItemSetValue(item, SecDbClassAttrWithKind(item->class, kSecDbEncryptedDataAttr, error), kCFNull, error);
 893     if (ok && access_control) { //added check for access_control because ks_decrypt_data can leave NULL in access_control in case of error
 894         item->_edataState = kSecDbItemDirty;
 895         CFDataRef data = SecAccessControlCopyData(access_control);
 896         ok = SecDbItemSetValue(item, SecDbClassAttrWithKind(item->class, kSecDbAccessControlAttr, error), data, error);
 897         CFRelease(data);
 898     }
 899     return ok;
 900 }
 901
 902 SecDbItemRef SecDbItemCreateWithAttributes(CFAllocatorRef allocator, const SecDbClass *class, CFDictionaryRef attributes, keybag_handle_t keybag, CFErrorRef *error) {
 903     SecDbItemRef item = SecDbItemCreate(kCFAllocatorDefault, class, keybag);
 904     if (item && !SecDbItemSetValues(item, attributes, error))
 905         CFReleaseNull(item);
 906     return item;
 907 }
 908
 909 static CFTypeRef
 910 SecDbColumnCopyValueWithAttr(CFAllocatorRef allocator, sqlite3_stmt *stmt, const SecDbAttr *attr, int col, CFErrorRef *error) {
 911     CFTypeRef value = NULL;
 912     switch (attr->kind) {
 913         case kSecDbDateAttr:
 914         case kSecDbCreationDateAttr:
 915         case kSecDbModificationDateAttr:
 916             value = SecDbColumnCopyDate(allocator, stmt, col, error);
 917             break;
 918         case kSecDbBlobAttr:
 919         case kSecDbNumberAttr:
 920             switch (sqlite3_column_type(stmt, col)) {
 921                 case SQLITE_INTEGER:
 922                     value = SecDbColumnCopyNumber(allocator, stmt, col, error);
 923                     break;
 924                 case SQLITE_FLOAT:
 925                     value = SecDbColumnCopyDouble(allocator, stmt, col, error);
 926                     break;
 927                 case SQLITE_TEXT:
 928                     value = SecDbColumnCopyString(allocator, stmt, col, error,
 929                                                   attr->flags);
 930                     break;
 931                 case SQLITE_BLOB:
 932                     value = SecDbColumnCopyData(allocator, stmt, col, error);
 933                     break;
 934                 case SQLITE_NULL:
 935                     value = kCFNull;
 936                     break;
 937             }
 938             break;
 939         case kSecDbAccessAttr:
 940         case kSecDbStringAttr:
 941             value = SecDbColumnCopyString(allocator, stmt, col, error,
 942                                           attr->flags);
 943             break;
 944         case kSecDbDataAttr:
 945         case kSecDbUUIDAttr:
 946         case kSecDbSHA1Attr:
 947         case kSecDbPrimaryKeyAttr:
 948         case kSecDbEncryptedDataAttr:
 949             value = SecDbColumnCopyData(allocator, stmt, col, error);
 950             break;
 951         case kSecDbSyncAttr:
 952         case kSecDbTombAttr:
 953             value = SecDbColumnCopyNumber(allocator, stmt, col, error);
 954             break;
 955         case kSecDbRowIdAttr:
 956             value = SecDbColumnCopyNumber64(allocator, stmt, col, error);
 957             break;
 958         case kSecDbAccessControlAttr:
 959         case kSecDbUTombAttr:
 960             /* This attributes does not have any database column associated, exists only inside encrypted blob as metadata. */
 961             break;
 962     }
 963     return value;
 964 }
 965
 966 SecDbItemRef SecDbItemCreateWithStatement(CFAllocatorRef allocator, const SecDbClass *class, sqlite3_stmt *stmt, keybag_handle_t keybag, CFErrorRef *error, bool (^return_attr)(const SecDbAttr *attr)) {
 967     SecDbItemRef item = SecDbItemCreate(allocator, class, keybag);
 968     int col = 0;
 969     SecDbForEachAttr(class, attr) {
 970         if (return_attr(attr)) {
 971             CFTypeRef value = SecDbColumnCopyValueWithAttr(allocator, stmt, attr, col++, error);
 972             require_action_quiet(value, errOut, CFReleaseNull(item));
 973
 974             CFDictionarySetValue(item->attributes, SecDbAttrGetHashName(attr), value);
 975             CFRelease(value);
 976         }
 977
 978         const SecDbAttr *data_attr = SecDbClassAttrWithKind(class, kSecDbEncryptedDataAttr, NULL);
 979         if (data_attr != NULL && CFDictionaryGetValue(item->attributes, data_attr->name) != NULL) {
 980             item->_edataState = kSecDbItemEncrypted;
 981         }
 982     }
 983
 984 errOut:
 985     return item;
 986 }
 987
 988 SecDbItemRef SecDbItemCreateWithEncryptedData(CFAllocatorRef allocator, const SecDbClass *class,
 989                                               CFDataRef edata, keybag_handle_t keybag, CFErrorRef *error) {
 990     SecDbItemRef item = SecDbItemCreate(allocator, class, keybag);
 991     const SecDbAttr *edata_attr = SecDbClassAttrWithKind(class, kSecDbEncryptedDataAttr, error);
 992     if (edata_attr) {
 993         if (!SecDbItemSetValue(item, edata_attr, edata, error))
 994             CFReleaseNull(item);
 995     }
 996     return item;
 997 }
 998
 999 // TODO: Hack -- Replace with real filtering
1000
1001 // Return true iff an item for which SecDbItemIsSyncable() already returns true should be part of the v2 view.
1002 bool SecDbItemInV2(SecDbItemRef item) {
1003     const SecDbClass *iclass = SecDbItemGetClass(item);
1004     return  (SecDbItemGetCachedValueWithName(item, kSecAttrSyncViewHint) == NULL &&
1005              (iclass == genp_class() || iclass == inet_class() || iclass == keys_class() || iclass == cert_class()));
1006 }
1007
1008 // Return true iff an item for which SecDbItemIsSyncable() and SecDbItemInV2() already return true should be part of the v0 view.
1009 bool SecDbItemInV2AlsoInV0(SecDbItemRef item) {
1010     return  (SecDbItemGetCachedValueWithName(item, kSecAttrTokenID) == NULL && SecDbItemGetClass(item) != cert_class());
1011 }
1012
1013 SecDbItemRef SecDbItemCopyWithUpdates(SecDbItemRef item, CFDictionaryRef updates, CFErrorRef *error) {
1014     SecDbItemRef new_item = SecDbItemCreate(CFGetAllocator(item), item->class, item->keybag);
1015     SecDbItemSetCredHandle(new_item, item->credHandle);
1016     SecDbForEachAttr(item->class, attr) {
1017         // Copy each attribute, except the mod date attribute (it will be reset to now when needed),
1018         // from the updates dict unless it's not there in which case we copy the attribute from the passed in item.
1019         if (attr->kind != kSecDbModificationDateAttr && attr->kind != kSecDbEncryptedDataAttr && attr->kind != kSecDbSHA1Attr && attr->kind != kSecDbPrimaryKeyAttr) {
1020             CFTypeRef value = NULL;
1021             if (CFDictionaryGetValueIfPresent(updates, attr->name, &value)) {
1022                 if (!value)
1023                     SecError(errSecParam, error, CFSTR("NULL value in dictionary"));
1024             } else {
1025                 value = SecDbItemGetValue(item, attr, error);
1026             }
1027             if (!value || !SecDbItemSetValue(new_item, attr, value, error)) {
1028                 CFReleaseNull(new_item);
1029                 break;
1030             }
1031         }
1032     }
1033     return new_item;
1034 }
1035
1036 // Ensure that the date value of attr of new_item is greater than that of old_item.
1037 static bool SecDbItemMakeAttrYounger(SecDbItemRef new_item, SecDbItemRef old_item, const SecDbAttr *attr, CFErrorRef *error) {
1038     CFDateRef old_date = SecDbItemGetValue(old_item, attr, error);
1039     if (!old_date)
1040         return false;
1041     CFDateRef new_date = SecDbItemGetValue(new_item, attr, error);
1042     if (!new_date)
1043         return false;
1044     bool ok = true;
1045     if (CFDateCompare(new_date, old_date, NULL) != kCFCompareGreaterThan) {
1046         CFDateRef adjusted_date = CFDateCreate(kCFAllocatorDefault, CFDateGetAbsoluteTime(old_date) + 0.001);
1047         if (adjusted_date) {
1048             ok = SecDbItemSetValue(new_item, attr, adjusted_date, error);
1049             CFRelease(adjusted_date);
1050         }
1051     }
1052     return ok;
1053 }
1054
1055 // Ensure that the mod date of new_item is greater than that of old_item.
1056 static bool SecDbItemMakeYounger(SecDbItemRef new_item, SecDbItemRef old_item, CFErrorRef *error) {
1057     const SecDbAttr *attr = SecDbClassAttrWithKind(new_item->class, kSecDbModificationDateAttr, error);
1058     return attr && SecDbItemMakeAttrYounger(new_item, old_item, attr, error);
1059 }
1060
1061 static SecDbItemRef SecDbItemCopyTombstone(SecDbItemRef item, CFBooleanRef makeTombStone, CFErrorRef *error) {
1062     SecDbItemRef new_item = SecDbItemCreate(CFGetAllocator(item), item->class, item->keybag);
1063     SecDbForEachAttr(item->class, attr) {
1064         if (attr->kind == kSecDbTombAttr) {
1065             // Set the tomb attr to true to indicate a tombstone.
1066             if (!SecDbItemSetValue(new_item, attr, kCFBooleanTrue, error)) {
1067                 CFReleaseNull(new_item);
1068                 break;
1069             }
1070         } else if (SecDbIsTombstoneDbUpdateAttr(attr)) {
1071             // Copy all primary key attributes and creation timestamps from the original item.
1072             CFTypeRef value = SecDbItemGetValue(item, attr, error);
1073             if (!value || (!CFEqual(kCFNull, value) && !SecDbItemSetValue(new_item, attr, value, error))) {
1074                 CFReleaseNull(new_item);
1075                 break;
1076             }
1077         } else if (attr->kind == kSecDbModificationDateAttr) {
1078             if (!SecDbItemMakeAttrYounger(new_item, item, attr, error)) {
1079                 CFReleaseNull(new_item);
1080                 break;
1081             }
1082         } else if (makeTombStone && attr->kind == kSecDbUTombAttr) {
1083             if (makeTombStone)
1084                 SecDbItemSetValue(new_item, attr, makeTombStone, error);
1085         }
1086     }
1087
1088     return new_item;
1089 }
1090
1091 bool SecDbItemIsEngineInternalState(SecDbItemRef itemObject) {
1092     // Only used for controlling logging
1093     // Use agrp=com.apple.security.sos, since it is not encrypted
1094     if (!itemObject) {
1095         return false;
1096     }
1097     const SecDbAttr *agrp = SecDbAttrWithKey(SecDbItemGetClass(itemObject), kSecAttrAccessGroup, NULL);
1098     CFTypeRef cfval = SecDbItemGetValue(itemObject, agrp, NULL);
1099     return cfval && CFStringCompareSafe(cfval, kSOSInternalAccessGroup, NULL) == kCFCompareEqualTo;
1100 }
1101
1102
1103 // MARK: -
1104 // MARK: SQL Construction helpers -- These should become private in the future
1105
1106 void SecDbAppendElement(CFMutableStringRef sql, CFStringRef value, bool *needComma) {
1107     assert(needComma);
1108     if (*needComma) {
1109         CFStringAppend(sql, CFSTR(","));
1110     } else {
1111         *needComma = true;
1112     }
1113     CFStringAppend(sql, value);
1114 }
1115
1116 static void SecDbAppendElementEquals(CFMutableStringRef sql, CFStringRef value, bool *needComma) {
1117     SecDbAppendElement(sql, value, needComma);
1118     CFStringAppend(sql, CFSTR("=?"));
1119 }
1120
1121 /* Append AND is needWhere is NULL or *needWhere is false.  Append WHERE
1122  otherwise.  Upon return *needWhere will be false.  */
1123 void
1124 SecDbAppendWhereOrAnd(CFMutableStringRef sql, bool *needWhere) {
1125     if (!needWhere || !*needWhere) {
1126         CFStringAppend(sql, CFSTR(" AND "));
1127     } else {
1128         CFStringAppend(sql, CFSTR(" WHERE "));
1129         *needWhere = false;
1130     }
1131 }
1132
1133 void
1134 SecDbAppendWhereOrAndEquals(CFMutableStringRef sql, CFStringRef col, bool *needWhere) {
1135     SecDbAppendWhereOrAnd(sql, needWhere);
1136     CFStringAppend(sql, col);
1137     CFStringAppend(sql, CFSTR("=?"));
1138 }
1139
1140 void
1141 SecDbAppendWhereOrAndNotEquals(CFMutableStringRef sql, CFStringRef col, bool *needWhere) {
1142     SecDbAppendWhereOrAnd(sql, needWhere);
1143     CFStringAppend(sql, col);
1144     CFStringAppend(sql, CFSTR("!=?"));
1145 }
1146
1147 static void SecDbAppendCountArgsAndCloseParen(CFMutableStringRef sql, CFIndex count) {
1148     bool needComma = false;
1149     while (count-- > 0)
1150         SecDbAppendElement(sql, CFSTR("?"), &needComma);
1151     CFStringAppend(sql, CFSTR(")"));
1152 }
1153
1154 void
1155 SecDbAppendWhereOrAndIn(CFMutableStringRef sql, CFStringRef col, bool *needWhere, CFIndex count) {
1156     if (count == 1)
1157         return SecDbAppendWhereOrAndEquals(sql, col, needWhere);
1158     SecDbAppendWhereOrAnd(sql, needWhere);
1159     CFStringAppend(sql, col);
1160     CFStringAppend(sql, CFSTR(" IN ("));
1161     SecDbAppendCountArgsAndCloseParen(sql, count);
1162 }
1163
1164 void
1165 SecDbAppendWhereOrAndNotIn(CFMutableStringRef sql, CFStringRef col, bool *needWhere, CFIndex count) {
1166     if (count == 1)
1167         return SecDbAppendWhereOrAndNotEquals(sql, col, needWhere);
1168     SecDbAppendWhereOrAnd(sql, needWhere);
1169     CFStringAppend(sql, col);
1170     CFStringAppend(sql, CFSTR(" NOT IN ("));
1171     SecDbAppendCountArgsAndCloseParen(sql, count);
1172 }
1173
1174 static CFStringRef SecDbItemCopyInsertSQL(SecDbItemRef item, bool(^use_attr)(const SecDbAttr *attr)) {
1175     CFMutableStringRef sql = CFStringCreateMutable(CFGetAllocator(item), 0);
1176     CFStringAppend(sql, CFSTR("INSERT INTO "));
1177     CFStringAppend(sql, item->class->name);
1178     CFStringAppend(sql, CFSTR("("));
1179     bool needComma = false;
1180     CFIndex used_attr = 0;
1181     SecDbForEachAttr(item->class, attr) {
1182         if (use_attr(attr)) {
1183             ++used_attr;
1184             SecDbAppendElement(sql, attr->name, &needComma);
1185         }
1186     }
1187     CFStringAppend(sql, CFSTR(")VALUES(?"));
1188     while (used_attr-- > 1) {
1189         CFStringAppend(sql, CFSTR(",?"));
1190     }
1191     CFStringAppend(sql, CFSTR(")"));
1192     return sql;
1193
1194 }
1195
1196 static bool SecDbItemInsertBind(SecDbItemRef item, sqlite3_stmt *stmt, CFErrorRef *error, bool(^use_attr)(const SecDbAttr *attr)) {
1197     bool ok = true;
1198     int param = 0;
1199     SecDbForEachAttr(item->class, attr) {
1200         if (use_attr(attr)) {
1201             CFTypeRef value = SecDbItemCopyValueForDb(item, attr, error);
1202             ok = value && SecDbBindObject(stmt, ++param, value, error);
1203             CFReleaseSafe(value);
1204             if (!ok)
1205                 break;
1206         }
1207     }
1208     return ok;
1209 }
1210
1211 sqlite3_int64 SecDbItemGetRowId(SecDbItemRef item, CFErrorRef *error) {
1212     sqlite3_int64 row_id = 0;
1213     const SecDbAttr *attr = SecDbClassAttrWithKind(item->class, kSecDbRowIdAttr, error);
1214     if (attr) {
1215         CFNumberRef number = SecDbItemGetValue(item, attr, error);
1216         if (!isNumber(number)|| !CFNumberGetValue(number, kCFNumberSInt64Type, &row_id))
1217             SecDbError(SQLITE_ERROR, error, CFSTR("rowid %@ is not a 64 bit number"), number);
1218     }
1219
1220     return row_id;
1221 }
1222
1223 static CFNumberRef SecDbItemCreateRowId(SecDbItemRef item, sqlite3_int64 rowid, CFErrorRef *error) {
1224     return CFNumberCreate(CFGetAllocator(item), kCFNumberSInt64Type, &rowid);
1225 }
1226
1227 bool SecDbItemSetRowId(SecDbItemRef item, sqlite3_int64 rowid, CFErrorRef *error) {
1228     bool ok = true;
1229     const SecDbAttr *attr = SecDbClassAttrWithKind(item->class, kSecDbRowIdAttr, error);
1230     if (attr) {
1231         CFNumberRef value = SecDbItemCreateRowId(item, rowid, error);
1232         if (!value)
1233             return false;
1234
1235         ok = SecDbItemSetValue(item, attr, value, error);
1236         CFRelease(value);
1237     }
1238     return ok;
1239 }
1240
1241 bool SecDbItemClearRowId(SecDbItemRef item, CFErrorRef *error) {
1242     bool ok = true;
1243     const SecDbAttr *attr = SecDbClassAttrWithKind(item->class, kSecDbRowIdAttr, error);
1244     if (attr) {
1245         CFDictionaryRemoveValue(item->attributes, attr->name);
1246         //ok = SecDbItemSetValue(item, attr, kCFNull, error);
1247     }
1248     return ok;
1249 }
1250
1251 static bool SecDbItemSetLastInsertRowId(SecDbItemRef item, SecDbConnectionRef dbconn, CFErrorRef *error) {
1252     sqlite3_int64 rowid = sqlite3_last_insert_rowid(SecDbHandle(dbconn));
1253     return SecDbItemSetRowId(item, rowid, error);
1254 }
1255
1256 bool SecDbItemIsSyncableOrCorrupted(SecDbItemRef item) {
1257     bool is_syncable_or_corrupted = false;
1258     CFErrorRef localError = NULL;
1259     if (!SecDbItemGetBoolValue(item, SecDbClassAttrWithKind(item->class, kSecDbSyncAttr, &localError),
1260                                &is_syncable_or_corrupted, &localError)) {
1261         is_syncable_or_corrupted = SecErrorGetOSStatus(localError) == errSecDecode;
1262     }
1263     CFReleaseSafe(localError);
1264     return is_syncable_or_corrupted;
1265 }
1266
1267 bool SecDbItemIsSyncable(SecDbItemRef item) {
1268     bool is_syncable;
1269     if (SecDbItemGetBoolValue(item, SecDbClassAttrWithKind(item->class, kSecDbSyncAttr, NULL), &is_syncable, NULL))
1270         return is_syncable;
1271     return false;
1272 }
1273
1274 bool SecDbItemSetSyncable(SecDbItemRef item, bool sync, CFErrorRef *error)
1275 {
1276     return SecDbItemSetValue(item, SecDbClassAttrWithKind(item->class, kSecDbSyncAttr, error), sync ? kCFBooleanTrue : kCFBooleanFalse, error);
1277 }
1278
1279 bool SecDbItemIsTombstone(SecDbItemRef item) {
1280     bool is_tomb;
1281     if (SecDbItemGetBoolValue(item, SecDbClassAttrWithKind(item->class, kSecDbTombAttr, NULL), &is_tomb, NULL))
1282         return is_tomb;
1283     return false;
1284 }
1285
1286 CFDataRef SecDbItemGetPrimaryKey(SecDbItemRef item, CFErrorRef *error) {
1287     return SecDbItemGetValue(item, SecDbClassAttrWithKind(item->class, kSecDbPrimaryKeyAttr, error), error);
1288 }
1289
1290 CFDataRef SecDbItemGetSHA1(SecDbItemRef item, CFErrorRef *error) {
1291     return SecDbItemGetValue(item, SecDbClassAttrWithKind(item->class, kSecDbSHA1Attr, error), error);
1292 }
1293
1294 static SecDbQueryRef SecDbQueryCreateWithItemPrimaryKey(SecDbItemRef item, CFErrorRef *error) {
1295     CFMutableDictionaryRef dict = SecDbItemCopyPListWithMask(item, kSecDbPrimaryKeyFlag, error);
1296     if (!dict)
1297         return NULL;
1298
1299     SecDbQueryRef query = query_create(item->class, NULL, NULL, error);
1300     if (query) {
1301         CFReleaseSafe(query->q_item);
1302         query->q_item = dict;
1303     }
1304     else
1305         CFRelease(dict);
1306
1307     return query;
1308 }
1309
1310 static bool SecDbItemIsCorrupt(SecDbItemRef item, bool *is_corrupt, CFErrorRef *error) {
1311     CFErrorRef localError = NULL;
1312     // Cache the storedSHA1 digest so we use the one from the db not the recomputed one for notifications.
1313     const struct SecDbAttr *sha1attr = SecDbClassAttrWithKind(item->class, kSecDbSHA1Attr, &localError);
1314     CFDataRef storedSHA1 = CFRetainSafe(SecDbItemGetValue(item, sha1attr, &localError));
1315     bool akpu = false;
1316
1317     if (localError || !SecDbItemEnsureDecrypted(item, true, &localError)) {
1318         if (SecErrorGetOSStatus(localError) == errSecDecode) {
1319             // We failed to decrypt the item
1320             const SecDbAttr *desc = SecDbClassAttrWithKind(item->class, kSecDbAccessControlAttr, &localError);
1321             SecAccessControlRef accc = NULL;
1322             CFDataRef acccData = NULL;
1323
1324             acccData = (CFDataRef)SecDbItemGetValue(item, desc, &localError);
1325             if (isData(acccData)) {
1326                 accc = SecAccessControlCreateFromData(CFGetAllocator(item), acccData, &localError);
1327             }
1328
1329             if (accc && CFEqualSafe(SecAccessControlGetProtection(accc), kSecAttrAccessibleWhenPasscodeSetThisDeviceOnly)) {
1330                 akpu = true;
1331                 secwarning("cannot decrypt item %@, item is irrecoverably lost with older passcode (error %@)", item, localError);
1332             } else {
1333                 secerror("error %@ reading item %@ (corrupted)", localError, item);
1334                 __security_simulatecrash(CFSTR("Corrupted item found in keychain"), __sec_exception_code_CorruptItem);
1335             }
1336             CFReleaseNull(localError);
1337             *is_corrupt = true;
1338         }
1339     }
1340
1341     // Recompute sha1 hash attribute and compare with the cached one.
1342     CFDataRef computedSHA1 = SecDbItemCopyValue(item, sha1attr, &localError);
1343     if (storedSHA1 && computedSHA1 && !CFEqual(storedSHA1, computedSHA1)) {
1344         CFStringRef storedHex = CFDataCopyHexString(storedSHA1), computedHex = CFDataCopyHexString(computedSHA1);
1345         secerror("error %@ %@ != %@ item %@ (corrupted)", sha1attr->name, storedHex, computedHex, item);
1346         __security_simulatecrash(CFSTR("Corrupted item (sha1 mismatch) found in keychain"), __sec_exception_code_CorruptItem);
1347         CFReleaseSafe(storedHex);
1348         CFReleaseSafe(computedHex);
1349         *is_corrupt = true;
1350     }
1351
1352     // Sanity check that all attributes that must not be NULL actually aren't
1353     if (!localError) SecDbForEachAttr(item->class, attr) {
1354         if (attr->flags & (kSecDbInCryptoDataFlag | kSecDbInAuthenticatedDataFlag)) {
1355             CFTypeRef value = SecDbItemGetValue(item, attr, &localError);
1356             if (value) {
1357                 if (CFEqual(kCFNull, value) && attr->flags & kSecDbNotNullFlag) {
1358                     secerror("error attribute %@ has NULL value in item %@ (corrupted)", attr->name, item);
1359                     __security_simulatecrash(CFSTR("Corrupted item (attr NULL) found in keychain"), __sec_exception_code_CorruptItem);
1360                     *is_corrupt = true;
1361                     break;
1362                 }
1363             } else {
1364                 if (SecErrorGetOSStatus(localError) == errSecDecode) {
1365                     // We failed to decrypt the item
1366                     if (akpu) {
1367                         secwarning("attribute %@: %@ item %@ (item lost with older passcode)", attr->name, localError, item);
1368                     } else {
1369                         secerror("error attribute %@: %@ item %@ (corrupted)", attr->name, localError, item);
1370                         __security_simulatecrash(CFSTR("Corrupted item found in keychain"), __sec_exception_code_CorruptItem);
1371                     }
1372                     *is_corrupt = true;
1373                     CFReleaseNull(localError);
1374                 }
1375                 break;
1376             }
1377         }
1378     }
1379
1380     CFReleaseSafe(computedSHA1);
1381     CFReleaseSafe(storedSHA1);
1382     return SecErrorPropagate(localError, error);
1383 }
1384
1385 static void SecDbItemRecordUpdate(SecDbConnectionRef dbconn, SecDbItemRef deleted, SecDbItemRef inserted) {
1386     SecDbRecordChange(dbconn, deleted, inserted);
1387 }
1388
1389 static bool SecDbItemDoInsert(SecDbItemRef item, SecDbConnectionRef dbconn, CFErrorRef *error) {
1390     bool (^use_attr)(const SecDbAttr *attr) = ^bool(const SecDbAttr *attr) {
1391         return (attr->flags & kSecDbInFlag);
1392     };
1393
1394     if (!SecDbItemEnsureDecrypted(item, true, error)) {
1395         return false;
1396     }
1397
1398     CFStringRef sql = SecDbItemCopyInsertSQL(item, use_attr);
1399     __block bool ok = sql;
1400     if (sql) {
1401         ok &= SecDbPrepare(dbconn, sql, error, ^(sqlite3_stmt *stmt) {
1402             ok = (SecDbItemInsertBind(item, stmt, error, use_attr) &&
1403                   SecDbStep(dbconn, stmt, error, NULL) &&
1404                   SecDbItemSetLastInsertRowId(item, dbconn, error));
1405         });
1406         CFRelease(sql);
1407     }
1408     if (ok) {
1409         secnotice("item", "inserted %@", item);
1410         SecDbItemRecordUpdate(dbconn, NULL, item);
1411     } else {
1412         if (SecDbItemIsEngineInternalState(item)) {
1413             secdebug ("item", "insert failed for item %@ with %@", item, error ? *error : NULL);
1414         } else {
1415             secnotice("item", "insert failed for item %@ with %@", item, error ? *error : NULL);
1416         }
1417     }
1418
1419     return ok;
1420 }
1421
1422 bool SecErrorIsSqliteDuplicateItemError(CFErrorRef error) {
1423     return error && CFErrorGetCode(error) == SQLITE_CONSTRAINT && CFEqual(kSecDbErrorDomain, CFErrorGetDomain(error));
1424 }
1425
1426 bool SecDbItemInsertOrReplace(SecDbItemRef item, SecDbConnectionRef dbconn, CFErrorRef *error, void(^duplicate)(SecDbItemRef item, SecDbItemRef *replace)) {
1427     __block CFErrorRef localError = NULL;
1428     __block bool ok = SecDbItemDoInsert(item, dbconn, &localError);
1429     if (!ok && localError && CFErrorGetCode(localError) == SQLITE_CONSTRAINT && CFEqual(kSecDbErrorDomain, CFErrorGetDomain(localError))) {
1430         SecDbQueryRef query = SecDbQueryCreateWithItemPrimaryKey(item, error);
1431         if (query) {
1432             CFRetainAssign(query->q_use_cred_handle, item->credHandle);
1433             SecDbItemSelect(query, dbconn, error, NULL, ^bool(const SecDbAttr *attr) {
1434                 return attr->flags & kSecDbPrimaryKeyFlag;
1435             }, NULL, NULL, ^(SecDbItemRef old_item, bool *stop) {
1436                 bool is_corrupt = false;
1437                 ok = SecDbItemIsCorrupt(old_item, &is_corrupt, error);
1438                 SecDbItemRef replace = NULL;
1439                 if (is_corrupt) {
1440                     // If old_item is corrupted pretend it's not there and just replace it.
1441                     replace = item;
1442                     CFRetain(replace);
1443                     if(error)
1444                         CFReleaseNull(*error);  //item is corrupted and will be replaced, so drop the error
1445                 } else if (ok && duplicate) {
1446                     duplicate(old_item, &replace);
1447                 }
1448                 if (replace) {
1449                     const SecDbAttr *rowid_attr = SecDbClassAttrWithKind(old_item->class, kSecDbRowIdAttr, error);
1450                     CFNumberRef oldrowid = SecDbItemGetCachedValue(old_item, rowid_attr);
1451                     if (oldrowid) {
1452                         ok = SecDbItemSetValue(replace, rowid_attr, oldrowid, &localError);
1453                         if (ok && !is_corrupt) {
1454                             ok = SecDbItemMakeYounger(replace, old_item, error);
1455                         }
1456                         ok = ok && SecDbItemDoUpdate(old_item, replace, dbconn, &localError, ^bool (const SecDbAttr *attr) {
1457                             return attr->kind == kSecDbRowIdAttr;
1458                         });
1459                     } else {
1460                         ok = SecError(errSecInternal, &localError, CFSTR("no rowid for %@"), old_item);
1461                     }
1462                     CFRelease(replace);
1463                     if (ok)
1464                         CFReleaseNull(localError); // Clear the error, since we replaced the item.
1465                 }
1466             });
1467             SecDbItemSetCredHandle(item, query->q_use_cred_handle);
1468             ok &= query_destroy(query, error);
1469         }
1470     }
1471
1472     return ok & SecErrorPropagate(localError, error); // Don't use && here!
1473 }
1474
1475 bool SecDbItemInsert(SecDbItemRef item, SecDbConnectionRef dbconn, CFErrorRef *error) {
1476     return SecDbItemInsertOrReplace(item, dbconn, error, ^(SecDbItemRef old_item, SecDbItemRef *replace) {
1477         if (SecDbItemIsTombstone(old_item)) {
1478             CFRetain(item);
1479             *replace = item;
1480         }
1481     });
1482 }
1483
1484 static CFStringRef SecDbItemCopyUpdateSQL(SecDbItemRef old_item, SecDbItemRef new_item, bool(^use_attr_in_where)(const SecDbAttr *attr)) {
1485     CFMutableStringRef sql = CFStringCreateMutable(CFGetAllocator(new_item), 0);
1486     CFStringAppend(sql, CFSTR("UPDATE "));
1487     CFStringAppend(sql, new_item->class->name);
1488     CFStringAppend(sql, CFSTR(" SET "));
1489     bool needComma = false;
1490     CFIndex used_attr = 0;
1491     SecDbForEachAttrWithMask(new_item->class, attr, kSecDbInFlag) {
1492         ++used_attr;
1493         SecDbAppendElementEquals(sql, attr->name, &needComma);
1494     }
1495
1496     bool needWhere = true;
1497     SecDbForEachAttr(old_item->class, attr) {
1498         if (use_attr_in_where(attr)) {
1499             SecDbAppendWhereOrAndEquals(sql, attr->name, &needWhere);
1500         }
1501     }
1502
1503     return sql;
1504 }
1505
1506 static bool SecDbItemUpdateBind(SecDbItemRef old_item, SecDbItemRef new_item, sqlite3_stmt *stmt, CFErrorRef *error, bool(^use_attr_in_where)(const SecDbAttr *attr)) {
1507     bool ok = true;
1508     int param = 0;
1509     SecDbForEachAttrWithMask(new_item->class, attr, kSecDbInFlag) {
1510         CFTypeRef value = SecDbItemCopyValueForDb(new_item, attr, error);
1511         ok &= value && SecDbBindObject(stmt, ++param, value, error);
1512         CFReleaseSafe(value);
1513         if (!ok)
1514             break;
1515     }
1516     SecDbForEachAttr(old_item->class, attr) {
1517         if (use_attr_in_where(attr)) {
1518             CFTypeRef value = SecDbItemCopyValueForDb(old_item, attr, error);
1519             ok &= value && SecDbBindObject(stmt, ++param, value, error);
1520             CFReleaseSafe(value);
1521             if (!ok)
1522                 break;
1523         }
1524     }
1525     return ok;
1526 }
1527
1528 // Primary keys are the same -- do an update
1529 bool SecDbItemDoUpdate(SecDbItemRef old_item, SecDbItemRef new_item, SecDbConnectionRef dbconn, CFErrorRef *error, bool (^use_attr_in_where)(const SecDbAttr *attr)) {
1530     CFStringRef sql = SecDbItemCopyUpdateSQL(old_item, new_item, use_attr_in_where);
1531     __block bool ok = sql;
1532     if (sql) {
1533         ok &= SecDbPrepare(dbconn, sql, error, ^(sqlite3_stmt *stmt) {
1534             ok = SecDbItemUpdateBind(old_item, new_item, stmt, error, use_attr_in_where) && SecDbStep(dbconn, stmt, error, NULL);
1535         });
1536         CFRelease(sql);
1537     }
1538     if (ok) {
1539         if (SecDbItemIsEngineInternalState(old_item)) {
1540             secdebug ("item", "replaced %@ in %@", old_item, dbconn);
1541             secdebug ("item", "    with %@ in %@", new_item, dbconn);
1542         } else {
1543             secnotice("item", "replaced %@ in %@", old_item, dbconn);
1544             secnotice("item", "    with %@ in %@", new_item, dbconn);
1545         }
1546         SecDbItemRecordUpdate(dbconn, old_item, new_item);
1547     }
1548     return ok;
1549 }
1550
1551 static CFStringRef SecDbItemCopyDeleteSQL(SecDbItemRef item, bool(^use_attr_in_where)(const SecDbAttr *attr)) {
1552     CFMutableStringRef sql = CFStringCreateMutable(CFGetAllocator(item), 0);
1553     CFStringAppend(sql, CFSTR("DELETE FROM "));
1554     CFStringAppend(sql, item->class->name);
1555     bool needWhere = true;
1556     SecDbForEachAttr(item->class, attr) {
1557         if (use_attr_in_where(attr)) {
1558             SecDbAppendWhereOrAndEquals(sql, attr->name, &needWhere);
1559         }
1560     }
1561
1562     return sql;
1563 }
1564
1565 static bool SecDbItemDeleteBind(SecDbItemRef item, sqlite3_stmt *stmt, CFErrorRef *error, bool(^use_attr_in_where)(const SecDbAttr *attr)) {
1566     bool ok = true;
1567     int param = 0;
1568     SecDbForEachAttr(item->class, attr) {
1569         if (use_attr_in_where(attr)) {
1570             CFTypeRef value = SecDbItemCopyValueForDb(item, attr, error);
1571             ok &= value && SecDbBindObject(stmt, ++param, value, error);
1572             CFReleaseSafe(value);
1573             if (!ok)
1574                 break;
1575         }
1576     }
1577     return ok;
1578 }
1579
1580 static bool SecDbItemDoDeleteOnly(SecDbItemRef item, SecDbConnectionRef dbconn, CFErrorRef *error, bool (^use_attr_in_where)(const SecDbAttr *attr)) {
1581     CFStringRef sql = SecDbItemCopyDeleteSQL(item, use_attr_in_where);
1582     __block bool ok = sql;
1583     if (sql) {
1584         ok &= SecDbPrepare(dbconn, sql, error, ^(sqlite3_stmt *stmt) {
1585             ok = SecDbItemDeleteBind(item, stmt, error, use_attr_in_where) && SecDbStep(dbconn, stmt, error, NULL);
1586         });
1587         CFRelease(sql);
1588     }
1589     return ok;
1590 }
1591
1592 bool SecDbItemDoDeleteSilently(SecDbItemRef item, SecDbConnectionRef dbconn, CFErrorRef *error) {
1593     return SecDbItemDoDeleteOnly(item, dbconn, error, ^bool(const SecDbAttr *attr) {
1594         return attr->kind == kSecDbRowIdAttr;
1595     });
1596 }
1597
1598 static bool SecDbItemDoDelete(SecDbItemRef item, SecDbConnectionRef dbconn, CFErrorRef *error, bool (^use_attr_in_where)(const SecDbAttr *attr)) {
1599     bool ok = SecDbItemDoDeleteOnly(item, dbconn, error, use_attr_in_where);
1600     if (ok) {
1601         secnotice("item", "deleted %@ from %@", item, dbconn);
1602         SecDbItemRecordUpdate(dbconn, item, NULL);
1603     }
1604     return ok;
1605 }
1606
1607 #if 0
1608 static bool SecDbItemDeleteTombstone(SecDbItemRef item, SecDbConnectionRef dbconn, CFErrorRef *error) {
1609     bool ok = true;
1610     // TODO: Treat non decryptable items like tombstones here too and delete them
1611     SecDbItemRef tombstone = SecDbItemCopyTombstone(item, error);
1612     ok = tombstone;
1613     if (tombstone) {
1614         ok = SecDbItemClearRowId(tombstone, error);
1615         if (ok) {
1616             ok = SecDbItemDoDelete(tombstone, dbconn, error, ^bool (const SecDbAttr *attr) {
1617                 return SecDbIsTombstoneDbSelectAttr(attr);
1618             });
1619         }
1620         CFRelease(tombstone);
1621     }
1622     return ok;
1623 }
1624 #endif
1625
1626 static bool
1627 isCKKSEnabled(void)
1628 {
1629 #if OCTAGON
1630     return SecCKKSIsEnabled();
1631 #else
1632     return false;
1633 #endif
1634 }
1635
1636 // Replace old_item with new_item.  If primary keys are the same this does an update otherwise it does a delete + add
1637 bool SecDbItemUpdate(SecDbItemRef old_item, SecDbItemRef new_item, SecDbConnectionRef dbconn, CFBooleanRef makeTombstone, bool uuid_from_primary_key, CFErrorRef *error) {
1638     __block bool ok = true;
1639     __block CFErrorRef localError = NULL;
1640
1641     CFDataRef old_pk = SecDbItemGetPrimaryKey(old_item, error);
1642     CFDataRef new_pk = SecDbItemGetPrimaryKey(new_item, error);
1643
1644     ok = old_pk && new_pk;
1645
1646     bool pk_equal = ok && CFEqual(old_pk, new_pk);
1647     if (pk_equal) {
1648         ok = SecDbItemMakeYounger(new_item, old_item, error);
1649     } else if(!CFEqualSafe(makeTombstone, kCFBooleanFalse) && isCKKSEnabled()) {
1650         // The primary keys aren't equal, and we're going to make a tombstone.
1651         // Help CKKS out: the tombstone should have the existing item's UUID, and the newly updated item should have a new UUID.
1652
1653         s3dl_item_make_new_uuid(new_item, uuid_from_primary_key, error);
1654     }
1655     ok = ok && SecDbItemDoUpdate(old_item, new_item, dbconn, &localError, ^bool(const SecDbAttr *attr) {
1656         return attr->kind == kSecDbRowIdAttr;
1657     });
1658
1659     if (localError) {
1660         if(CFErrorGetCode(localError) == SQLITE_CONSTRAINT && CFEqual(kSecDbErrorDomain, CFErrorGetDomain(localError))) {
1661             /* Update failed because we changed the PrimaryKey and there was a dup.
1662                Find the dup and see if it is a tombstone or corrupted item. */
1663             SecDbQueryRef query = SecDbQueryCreateWithItemPrimaryKey(new_item, error);
1664             ok = query;
1665             if (query) {
1666                 ok &= SecDbItemSelect(query, dbconn, error, NULL, ^bool(const SecDbAttr *attr) {
1667                     return attr->flags & kSecDbPrimaryKeyFlag;
1668                 }, NULL, NULL, ^(SecDbItemRef duplicate_item, bool *stop) {
1669                     bool is_corrupt = false;
1670                     bool is_tomb = false;
1671                     ok = SecDbItemIsCorrupt(duplicate_item, &is_corrupt, error);
1672                     if (ok && !is_corrupt) {
1673                         if ((is_tomb = SecDbItemIsTombstone(duplicate_item)))
1674                             ok = SecDbItemMakeYounger(new_item, duplicate_item, error);
1675                     }
1676                     if (ok && (is_corrupt || is_tomb)) {
1677                         ok = SecDbItemDoDelete(old_item, dbconn, error, ^bool (const SecDbAttr *attr) {
1678                             return attr->kind == kSecDbRowIdAttr;
1679                         });
1680                         ok = ok && SecDbItemDoUpdate(duplicate_item, new_item, dbconn, error, ^bool (const SecDbAttr *attr) {
1681                             return attr->kind == kSecDbRowIdAttr;
1682                         });
1683                         CFReleaseNull(localError);
1684                     }
1685                 });
1686                 ok &= query_destroy(query, error);
1687             }
1688         }
1689
1690         if (localError) {
1691             ok = false;
1692             if (error && *error == NULL) {
1693                 *error = localError;
1694                 localError = NULL;
1695             }
1696             CFReleaseSafe(localError);
1697         }
1698     }
1699
1700     if (ok && !pk_equal && !CFEqualSafe(makeTombstone, kCFBooleanFalse)) {
1701         /* The primary key of new_item is different than that of old_item, we
1702            have been asked to make a tombstone so leave one for the old_item. */
1703         SecDbItemRef tombstone = SecDbItemCopyTombstone(old_item, makeTombstone, error);
1704         ok = tombstone;
1705         if (tombstone) {
1706             ok = (SecDbItemClearRowId(tombstone, error) &&
1707                   SecDbItemDoInsert(tombstone, dbconn, error));
1708             CFRelease(tombstone);
1709         }
1710     }
1711
1712     return ok;
1713 }
1714
1715 // Replace the object with a tombstone
1716 bool SecDbItemDelete(SecDbItemRef item, SecDbConnectionRef dbconn, CFBooleanRef makeTombstone, CFErrorRef *error) {
1717     bool ok = false;
1718     if (!CFEqualSafe(makeTombstone, kCFBooleanFalse)) {
1719         SecDbItemRef tombstone = SecDbItemCopyTombstone(item, makeTombstone, error);
1720         if (tombstone) {
1721             ok = SecDbItemDoUpdate(item, tombstone, dbconn, error, ^bool(const SecDbAttr *attr) {
1722                 return attr->kind == kSecDbRowIdAttr;
1723             });
1724             CFRelease(tombstone);
1725         }
1726     } else {
1727         ok = SecDbItemDoDelete(item, dbconn, error, ^bool(const SecDbAttr *attr) {
1728             return attr->kind == kSecDbRowIdAttr;
1729         });
1730     }
1731     return ok;
1732 }
1733
1734 CFStringRef SecDbItemCopySelectSQL(SecDbQueryRef query,
1735                                    bool (^return_attr)(const SecDbAttr *attr),
1736                                    bool (^use_attr_in_where)(const SecDbAttr *attr),
1737                                    bool (^add_where_sql)(CFMutableStringRef sql, bool *needWhere)) {
1738     CFMutableStringRef sql = CFStringCreateMutable(kCFAllocatorDefault, 0);
1739     CFStringAppend(sql, CFSTR("SELECT "));
1740     // What are we selecting?
1741     bool needComma = false;
1742     SecDbForEachAttr(query->q_class, attr) {
1743         if (return_attr(attr))
1744             SecDbAppendElement(sql, attr->name, &needComma);
1745     }
1746
1747     // From which table?
1748     CFStringAppend(sql, CFSTR(" FROM "));
1749     CFStringAppend(sql, query->q_class->name);
1750
1751     // And which elements do we want to select
1752     bool needWhere = true;
1753     SecDbForEachAttr(query->q_class, attr) {
1754         if (use_attr_in_where(attr)) {
1755             CFTypeRef value = CFDictionaryGetValue(query->q_item, attr->name);
1756             if (isArray(value)) {
1757                 CFArrayRef array = (CFArrayRef)value;
1758                 CFIndex length = CFArrayGetCount(array);
1759                 if (length > 0) {
1760                     CFTypeRef head = CFArrayGetValueAtIndex(array, 0);
1761                     if (CFEqualSafe(head, kCFNull)) {
1762                         SecDbAppendWhereOrAndNotIn(sql, attr->name, &needWhere, length - 1);
1763                     } else {
1764                         SecDbAppendWhereOrAndIn(sql, attr->name, &needWhere, length);
1765                     }
1766                 }
1767             } else {
1768                 SecDbAppendWhereOrAndEquals(sql, attr->name, &needWhere);
1769             }
1770         }
1771     }
1772     // Append SQL for access groups and limits.
1773     if (add_where_sql)
1774         add_where_sql(sql, &needWhere);
1775
1776     return sql;
1777 }
1778
1779 static bool SecDbItemSelectBindValue(SecDbQueryRef query, sqlite3_stmt *stmt, int param, const SecDbAttr *attr, CFTypeRef inValue, CFErrorRef *error) {
1780     bool ok = true;
1781     CFTypeRef value = NULL;
1782     if (attr->kind == kSecDbRowIdAttr) {
1783         // TODO: Ignores inValue and uses rowid directly instead HACK should go
1784         value = CFNumberCreate(NULL, kCFNumberSInt64Type, &query->q_row_id);
1785     } else {
1786         value = SecDbAttrCopyValueForDb(attr, inValue, error);
1787     }
1788     ok = ok && value != NULL && SecDbBindObject(stmt, ++param, value, error);
1789     CFReleaseSafe(value);
1790     return ok;
1791 }
1792
1793 bool SecDbItemSelectBind(SecDbQueryRef query, sqlite3_stmt *stmt, CFErrorRef *error,
1794                          bool (^use_attr_in_where)(const SecDbAttr *attr),
1795                          bool (^bind_added_where)(sqlite3_stmt *stmt, int col)) {
1796     __block bool ok = true;
1797     __block int param = 0;
1798     SecDbForEachAttr(query->q_class, attr) {
1799         if (use_attr_in_where(attr)) {
1800             CFTypeRef value = CFDictionaryGetValue(query->q_item, attr->name);
1801             if (isArray(value)) {
1802                 CFArrayRef array = (CFArrayRef)value;
1803                 CFRange range = {.location = 0, .length = CFArrayGetCount(array) };
1804                 if (range.length > 0) {
1805                     CFTypeRef head = CFArrayGetValueAtIndex(array, 0);
1806                     if (CFEqualSafe(head, kCFNull)) {
1807                         range.length--;
1808                         range.location++;
1809                     }
1810                 }
1811                 CFArrayApplyFunction(array, range, apply_block_1, (void (^)(const void *value)) ^(const void *arrayValue) {
1812                     ok = SecDbItemSelectBindValue(query, stmt, param++, attr, arrayValue, error);
1813                 });
1814             } else {
1815                 ok = SecDbItemSelectBindValue(query, stmt, param++, attr, value, error);
1816             }
1817
1818             if (!ok)
1819                 break;
1820         }
1821     }
1822     // TODO: Bind arguments for access groups and limits.
1823     if (bind_added_where)
1824         bind_added_where(stmt, ++param);
1825
1826     return ok;
1827 }
1828
1829 bool SecDbItemSelect(SecDbQueryRef query, SecDbConnectionRef dbconn, CFErrorRef *error,
1830                      bool (^return_attr)(const SecDbAttr *attr),
1831                      bool (^use_attr_in_where)(const SecDbAttr *attr),
1832                      bool (^add_where_sql)(CFMutableStringRef sql, bool *needWhere),
1833                      bool (^bind_added_where)(sqlite3_stmt *stmt, int col),
1834                      void (^handle_row)(SecDbItemRef item, bool *stop)) {
1835     __block bool ok = true;
1836     if (return_attr == NULL) {
1837         return_attr = ^bool (const SecDbAttr * attr) {
1838             return attr->kind == kSecDbRowIdAttr || attr->kind == kSecDbEncryptedDataAttr || attr->kind == kSecDbSHA1Attr;
1839         };
1840     }
1841     if (use_attr_in_where == NULL) {
1842         use_attr_in_where = ^bool (const SecDbAttr* attr) { return false; };
1843     }
1844
1845     CFStringRef sql = SecDbItemCopySelectSQL(query, return_attr, use_attr_in_where, add_where_sql);
1846     if (sql) {
1847         ok &= SecDbPrepare(dbconn, sql, error, ^(sqlite3_stmt *stmt) {
1848             ok = (SecDbItemSelectBind(query, stmt, error, use_attr_in_where, bind_added_where) &&
1849                   SecDbStep(dbconn, stmt, error, ^(bool *stop) {
1850                 SecDbItemRef item = SecDbItemCreateWithStatement(kCFAllocatorDefault, query->q_class, stmt, query->q_keybag, error, return_attr);
1851                 if (item) {
1852                     CFRetainAssign(item->credHandle, query->q_use_cred_handle);
1853                     handle_row(item, stop);
1854                     CFRelease(item);
1855                 } else {
1856                     //*stop = true;
1857                     //ok = false;
1858                 }
1859             }));
1860         });
1861         CFRelease(sql);
1862     } else {
1863         ok = false;
1864     }
1865     return ok;
1866 }
1867