OSX/sec/securityd/SecDbItem.c

   1 /*
   2  * Copyright (c) 2012-2014 Apple Inc. All Rights Reserved.
   3  *
   4  * @APPLE_LICENSE_HEADER_START@
   5  *
   6  * This file contains Original Code and/or Modifications of Original Code
   7  * as defined in and that are subject to the Apple Public Source License
   8  * Version 2.0 (the 'License'). You may not use this file except in
   9  * compliance with the License. Please obtain a copy of the License at
  10  * http://www.opensource.apple.com/apsl/ and read it before using this
  11  * file.
  12  *
  13  * The Original Code and all software distributed under the License are
  14  * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  15  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  16  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
  17  * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
  18  * Please see the License for the specific language governing rights and
  19  * limitations under the License.
  20  *
  21  * @APPLE_LICENSE_HEADER_END@
  22  */
  23
  24 /*
  25  * SecDbItem.c - CoreFoundation-based constants and functions representing
  26  * database items (certificates, keys, identities, and passwords.)
  27  */
  28
  29 #include <securityd/SecDbItem.h>
  30 #include <securityd/SecDbKeychainItem.h>
  31 #include <securityd/SecItemDb.h>
  32 #include <utilities/SecCFWrappers.h>
  33 #include <utilities/SecCFCCWrappers.h>
  34 #include <utilities/der_date.h>
  35 #include <utilities/der_plist.h>
  36 #include <utilities/debugging.h>
  37
  38 #include <Security/SecBasePriv.h>
  39 #include <Security/SecInternal.h>
  40 #include <corecrypto/ccsha1.h>
  41 #include <Security/SecItem.h>
  42 #include <Security/SecItemPriv.h>
  43 #include <Security/SecAccessControl.h>
  44 #include <Security/SecAccessControlPriv.h>
  45 #include <securityd/SecItemSchema.h>
  46
  47 // MARK: type converters
  48
  49 CFStringRef copyString(CFTypeRef obj) {
  50     CFTypeID tid = CFGetTypeID(obj);
  51     if (tid == CFStringGetTypeID())
  52         return CFStringCreateCopy(0, obj);
  53     else if (tid == CFDataGetTypeID())
  54         return CFStringCreateFromExternalRepresentation(0, obj, kCFStringEncodingUTF8);
  55     else
  56         return NULL;
  57 }
  58
  59 CFDataRef copyData(CFTypeRef obj) {
  60     CFTypeID tid = CFGetTypeID(obj);
  61     if (tid == CFDataGetTypeID()) {
  62         return CFDataCreateCopy(0, obj);
  63     } else if (tid == CFStringGetTypeID()) {
  64         return CFStringCreateExternalRepresentation(0, obj, kCFStringEncodingUTF8, 0);
  65     } else if (tid == CFNumberGetTypeID()) {
  66         SInt32 value;
  67         CFNumberGetValue(obj, kCFNumberSInt32Type, &value);
  68         return CFDataCreate(0, (const UInt8 *)&value, sizeof(value));
  69     } else {
  70         return NULL;
  71     }
  72 }
  73
  74 CFTypeRef copyUUID(CFTypeRef obj) {
  75     CFTypeID tid = CFGetTypeID(obj);
  76     if (tid == CFDataGetTypeID()) {
  77         CFIndex length = CFDataGetLength(obj);
  78         if (length != 0 && length != 16)
  79             return NULL;
  80         return CFDataCreateCopy(NULL, obj);
  81     } if (tid == CFNullGetTypeID()) {
  82         return CFDataCreate(NULL, NULL, 0);
  83     } else {
  84         return NULL;
  85     }
  86 }
  87
  88
  89 CFTypeRef copyBlob(CFTypeRef obj) {
  90     CFTypeID tid = CFGetTypeID(obj);
  91     if (tid == CFDataGetTypeID()) {
  92         return CFDataCreateCopy(0, obj);
  93     } else if (tid == CFStringGetTypeID()) {
  94         return CFStringCreateCopy(0, obj);
  95     } else if (tid == CFNumberGetTypeID()) {
  96         CFRetain(obj);
  97         return obj;
  98     } else {
  99         return NULL;
 100     }
 101 }
 102
 103 CFDataRef copySHA1(CFTypeRef obj) {
 104     CFTypeID tid = CFGetTypeID(obj);
 105     if (tid == CFDataGetTypeID() && CFDataGetLength(obj) == CCSHA1_OUTPUT_SIZE) {
 106         return CFDataCreateCopy(CFGetAllocator(obj), obj);
 107     } else {
 108         return NULL;
 109     }
 110 }
 111
 112 CFTypeRef copyNumber(CFTypeRef obj) {
 113     CFTypeID tid = CFGetTypeID(obj);
 114     if (tid == CFNumberGetTypeID()) {
 115         CFRetain(obj);
 116         return obj;
 117     } else if (tid == CFBooleanGetTypeID()) {
 118         SInt32 value = CFBooleanGetValue(obj);
 119         return CFNumberCreate(0, kCFNumberSInt32Type, &value);
 120     } else if (tid == CFStringGetTypeID()) {
 121         SInt32 value = CFStringGetIntValue(obj);
 122         CFStringRef t = CFStringCreateWithFormat(0, 0, CFSTR("%ld"), (long) value);
 123         /* If a string converted to an int isn't equal to the int printed as
 124            a string, return a CFStringRef instead. */
 125         if (!CFEqual(t, obj)) {
 126             CFRelease(t);
 127             return CFStringCreateCopy(0, obj);
 128         }
 129         CFRelease(t);
 130         return CFNumberCreate(0, kCFNumberSInt32Type, &value);
 131     } else
 132         return NULL;
 133 }
 134
 135 CFDateRef copyDate(CFTypeRef obj) {
 136     CFTypeID tid = CFGetTypeID(obj);
 137     if (tid == CFDateGetTypeID()) {
 138         CFRetain(obj);
 139         return obj;
 140     } else
 141         return NULL;
 142 }
 143
 144 // MARK: SecDbColumn accessors, to retrieve values as CF types in SecDbStep.
 145
 146 static CFDataRef SecDbColumnCopyData(CFAllocatorRef allocator, sqlite3_stmt *stmt, int col, CFErrorRef *error) {
 147     return CFDataCreate(allocator, sqlite3_column_blob(stmt, col),
 148                         sqlite3_column_bytes(stmt, col));
 149     //return CFDataCreateWithBytesNoCopy(0, sqlite3_column_blob(stmt, col),
 150     //                                   sqlite3_column_bytes(stmt, col),
 151     //                                   kCFAllocatorNull);
 152 }
 153
 154 static CFDateRef SecDbColumnCopyDate(CFAllocatorRef allocator, sqlite3_stmt *stmt, int col, CFErrorRef *error) {
 155     return CFDateCreate(allocator, sqlite3_column_double(stmt, col));
 156 }
 157
 158 static CFNumberRef SecDbColumnCopyDouble(CFAllocatorRef allocator, sqlite3_stmt *stmt, int col, CFErrorRef *error) {
 159     double number = sqlite3_column_double(stmt, col);
 160     return CFNumberCreate(allocator, kCFNumberDoubleType, &number);
 161 }
 162
 163 static CFNumberRef SecDbColumnCopyNumber64(CFAllocatorRef allocator, sqlite3_stmt *stmt, int col, CFErrorRef *error) {
 164     sqlite_int64 number = sqlite3_column_int64(stmt, col);
 165     return CFNumberCreate(allocator, kCFNumberSInt64Type, &number);
 166 }
 167
 168 static CFNumberRef SecDbColumnCopyNumber(CFAllocatorRef allocator, sqlite3_stmt *stmt, int col, CFErrorRef *error) {
 169     sqlite_int64 number = sqlite3_column_int64(stmt, col);
 170     if (INT32_MIN <= number && number <= INT32_MAX) {
 171         int32_t num32 = (int32_t)number;
 172         return CFNumberCreate(allocator, kCFNumberSInt32Type, &num32);
 173     } else {
 174         return CFNumberCreate(allocator, kCFNumberSInt64Type, &number);
 175     }
 176 }
 177
 178 static CFTypeRef SecDbColumnCopyString(CFAllocatorRef allocator, sqlite3_stmt *stmt, int col, CFErrorRef *error,
 179                                        CFOptionFlags flags) {
 180     const unsigned char *text = sqlite3_column_text(stmt, col);
 181     if (!text || 0 == strlen((const char *)text)) {
 182         if (flags & kSecDbDefaultEmptyFlag) {
 183             return CFSTR("");
 184         } else if (flags & kSecDbDefault0Flag) {
 185             return CFSTR("0");
 186         } else {
 187             return kCFNull;
 188         }
 189     }
 190     return CFStringCreateWithBytes(allocator, text, strlen((const char *)text), kCFStringEncodingUTF8, false);
 191 }
 192
 193 // MARK: SecDbClass helpers
 194
 195 const SecDbAttr *SecDbClassAttrWithKind(const SecDbClass *class, SecDbAttrKind kind, CFErrorRef *error) {
 196     const SecDbAttr *result = NULL;
 197     SecDbForEachAttr(class, desc) {
 198         if (desc->kind == kind)
 199             result = desc;
 200     }
 201
 202     if (!result)
 203         SecError(errSecInternal, error, CFSTR("Can't find attribute of kind %d in class %@"), kind, class->name);
 204
 205     return result;
 206 }
 207
 208 // MARK: SecDbAttr helpers
 209
 210 static bool SecDbIsTombstoneDbSelectAttr(const SecDbAttr *attr) {
 211     return attr->flags & kSecDbPrimaryKeyFlag || attr->kind == kSecDbTombAttr;
 212 }
 213
 214 #if 0
 215 static bool SecDbIsTombstoneDbInsertAttr(const SecDbAttr *attr) {
 216     return SecDbIsTombstoneDbSelectAttr(attr) || attr->kind == kSecDbAccessAttr || attr->kind == kSecDbCreationDateAttr || attr->kind == kSecDbModificationDateAttr;
 217 }
 218 #endif
 219
 220 static bool SecDbIsTombstoneDbUpdateAttr(const SecDbAttr *attr) {
 221     return SecDbIsTombstoneDbSelectAttr(attr) || attr->kind == kSecDbAccessAttr || attr->kind == kSecDbCreationDateAttr || attr->kind == kSecDbRowIdAttr;
 222 }
 223
 224 CFTypeRef SecDbAttrCopyDefaultValue(const SecDbAttr *attr, CFErrorRef *error) {
 225     CFTypeRef value = NULL;
 226     switch (attr->kind) {
 227         case kSecDbAccessAttr:
 228         case kSecDbStringAttr:
 229         case kSecDbAccessControlAttr:
 230             value = CFSTR("");
 231             break;
 232         case kSecDbBlobAttr:
 233         case kSecDbDataAttr:
 234             value = CFDataCreate(kCFAllocatorDefault, NULL, 0);
 235             break;
 236         case kSecDbUUIDAttr:
 237             value = CFDataCreate(kCFAllocatorDefault, NULL, 0);
 238             break;
 239         case kSecDbNumberAttr:
 240         case kSecDbSyncAttr:
 241         case kSecDbTombAttr:
 242         {
 243             int32_t zero = 0;
 244             value = CFNumberCreate(kCFAllocatorDefault, kCFNumberSInt32Type, &zero);
 245             break;
 246         }
 247         case kSecDbDateAttr:
 248             value = CFDateCreate(kCFAllocatorDefault, 0.0);
 249             break;
 250         case kSecDbCreationDateAttr:
 251         case kSecDbModificationDateAttr:
 252             value = CFDateCreate(kCFAllocatorDefault, CFAbsoluteTimeGetCurrent());
 253             break;
 254         default:
 255             SecError(errSecInternal, error, CFSTR("attr %@ has no default value"), attr->name);
 256             value = NULL;
 257     }
 258
 259     return value;
 260 }
 261
 262 static CFTypeRef SecDbAttrCopyValueForDb(const SecDbAttr *attr, CFTypeRef value, CFErrorRef *error) {
 263     CFDataRef data = NULL;
 264     CFTypeRef result = NULL;
 265
 266     if (value == NULL)
 267         value = kCFNull;
 268
 269     if (CFEqual(value, kCFNull) && attr->flags & kSecDbPrimaryKeyFlag) {
 270         // SQLITE3 doesn't like NULL for primary key attributes, pretend kSecDbDefaultEmptyFlag was specified
 271         require_quiet(result = SecDbAttrCopyDefaultValue(attr, error), out);
 272     } else {
 273         result = CFRetain(value);
 274     }
 275
 276     if (attr->flags & kSecDbSHA1ValueInFlag && !CFEqual(result, kCFNull)) {
 277         require_action_quiet(data = copyData(result), out,
 278                              SecError(errSecInternal, error, CFSTR("failed to get attribute %@ data"), attr->name);
 279                              CFReleaseNull(result));
 280         CFAssignRetained(result, CFDataCopySHA1Digest(data, error));
 281     }
 282
 283 out:
 284     CFReleaseSafe(data);
 285     return result;
 286 }
 287
 288 static CFStringRef SecDbAttrGetHashName(const SecDbAttr *attr) {
 289     if ((attr->flags & kSecDbSHA1ValueInFlag) == 0) {
 290         return attr->name;
 291     }
 292
 293     static dispatch_once_t once;
 294     static CFMutableDictionaryRef hash_store;
 295     static dispatch_queue_t queue;
 296     dispatch_once(&once, ^{
 297         queue = dispatch_queue_create("secd-hash-name", NULL);
 298         hash_store = CFDictionaryCreateMutableForCFTypes(NULL);
 299     });
 300
 301     __block CFStringRef name;
 302     dispatch_sync(queue, ^{
 303         name = CFDictionaryGetValue(hash_store, attr->name);
 304         if (name == NULL) {
 305             name = CFStringCreateWithFormat(NULL, NULL, CFSTR("#%@"), attr->name);
 306             CFDictionarySetValue(hash_store, attr->name, name);
 307             CFRelease(name);
 308         }
 309     });
 310     return name;
 311 }
 312
 313 // MARK: SecDbItem
 314
 315 CFTypeRef SecDbItemGetCachedValueWithName(SecDbItemRef item, CFStringRef name) {
 316     return CFDictionaryGetValue(item->attributes, name);
 317 }
 318
 319 static CFTypeRef SecDbItemGetCachedValue(SecDbItemRef item, const SecDbAttr *desc) {
 320     return CFDictionaryGetValue(item->attributes, desc->name);
 321 }
 322
 323 CFMutableDictionaryRef SecDbItemCopyPListWithMask(SecDbItemRef item, CFOptionFlags mask, CFErrorRef *error) {
 324     CFMutableDictionaryRef dict = CFDictionaryCreateMutableForCFTypes(kCFAllocatorDefault);
 325     SecDbForEachAttrWithMask(item->class, desc, mask) {
 326         CFTypeRef value = SecDbItemGetValue(item, desc, error);
 327         if (value) {
 328             if (!CFEqual(kCFNull, value)) {
 329                 CFDictionarySetValue(dict, desc->name, value);
 330             } else if (desc->flags & kSecDbNotNullFlag) {
 331                 SecError(errSecDecode, error, CFSTR("attribute %@ has NULL value"), desc->name);
 332                 secerror("%@", error ? *error : (CFErrorRef)CFSTR("error == NULL"));
 333                 CFReleaseNull(dict);
 334                 break;
 335             }
 336         } else {
 337             CFReleaseNull(dict);
 338             break;
 339         }
 340     }
 341     return dict;
 342 }
 343
 344 void SecDbItemSetCredHandle(SecDbItemRef item, CFTypeRef cred_handle) {
 345     CFRetainAssign(item->credHandle, cred_handle);
 346 }
 347
 348 void SecDbItemSetCallerAccessGroups(SecDbItemRef item, CFArrayRef caller_access_groups) {
 349     CFRetainAssign(item->callerAccessGroups, caller_access_groups);
 350 }
 351
 352 CFDataRef SecDbItemCopyEncryptedDataToBackup(SecDbItemRef item, uint64_t handle, CFErrorRef *error) {
 353     CFDataRef edata = NULL;
 354     keybag_handle_t keybag = (keybag_handle_t)handle;
 355     CFMutableDictionaryRef attributes = SecDbItemCopyPListWithMask(item, kSecDbInCryptoDataFlag, error);
 356     CFMutableDictionaryRef auth_attributes = SecDbItemCopyPListWithMask(item, kSecDbInAuthenticatedDataFlag, error);
 357     if (attributes || auth_attributes) {
 358         SecAccessControlRef access_control = SecDbItemCopyAccessControl(item, error);
 359         if (access_control) {
 360             if (ks_encrypt_data(keybag, access_control, item->credHandle, attributes, auth_attributes, &edata, false, error)) {
 361                 item->_edataState = kSecDbItemEncrypting;
 362             } else {
 363                 seccritical("ks_encrypt_data (db): failed: %@", error ? *error : (CFErrorRef)CFSTR(""));
 364             }
 365             CFRelease(access_control);
 366         }
 367         CFReleaseSafe(attributes);
 368         CFReleaseSafe(auth_attributes);
 369     }
 370     return edata;
 371 }
 372
 373 bool SecDbItemEnsureDecrypted(SecDbItemRef item, CFErrorRef *error) {
 374
 375     // If we haven't yet decrypted the item, make sure we do so now
 376     bool result = true;
 377     if (item->_edataState == kSecDbItemEncrypted) {
 378         const SecDbAttr *attr = SecDbClassAttrWithKind(item->class, kSecDbEncryptedDataAttr, error);
 379         if (attr) {
 380             CFDataRef edata = SecDbItemGetCachedValue(item, attr);
 381             if (!edata)
 382                 return SecError(errSecInternal, error, CFSTR("state= encrypted but edata is NULL"));
 383             // Decrypt calls set value a bunch of times which clears our edata and changes our state.
 384             item->_edataState = kSecDbItemDecrypting;
 385             result = SecDbItemDecrypt(item, edata, error);
 386             if (result)
 387                 item->_edataState = kSecDbItemClean;
 388             else
 389                 item->_edataState = kSecDbItemEncrypted;
 390         }
 391     }
 392     return result;
 393 }
 394
 395 // Only called if cached value is not found.
 396 static CFTypeRef SecDbItemCopyValue(SecDbItemRef item, const SecDbAttr *attr, CFErrorRef *error) {
 397     if (attr->copyValue)
 398         return attr->copyValue(item, attr, error);
 399
 400     CFTypeRef value = NULL;
 401     switch (attr->kind) {
 402             // These have an explicit copyValue; here to shut up compiler
 403         case kSecDbSHA1Attr:
 404         case kSecDbEncryptedDataAttr:
 405         case kSecDbPrimaryKeyAttr:
 406             value = NULL;
 407             break;
 408         case kSecDbAccessAttr:
 409         case kSecDbStringAttr:
 410         case kSecDbBlobAttr:
 411         case kSecDbAccessControlAttr:
 412             if (attr->flags & kSecDbNotNullFlag) {
 413                 if (attr->flags & kSecDbDefault0Flag) {
 414                     value = CFSTR("0");
 415                     break;
 416                 } else if (attr->kind != kSecDbBlobAttr && attr->flags & kSecDbDefaultEmptyFlag) {
 417                     // blob drops through to data everything else is empty string
 418                     value = CFSTR("");
 419                     break;
 420                 }
 421             }
 422             //DROPTHROUGH
 423         case kSecDbDataAttr:
 424             if (attr->flags & kSecDbNotNullFlag && attr->flags & kSecDbDefaultEmptyFlag) {
 425                 value = CFDataCreate(CFGetAllocator(item), NULL, 0);
 426             } else {
 427                 value = kCFNull;
 428             }
 429             break;
 430         case kSecDbUUIDAttr:
 431             value = CFDataCreate(CFGetAllocator(item), NULL, 0);
 432             break;
 433         case kSecDbNumberAttr:
 434         case kSecDbSyncAttr:
 435         case kSecDbTombAttr:
 436             if (attr->flags & kSecDbNotNullFlag) {
 437                 int32_t zero = 0;
 438                 value = CFNumberCreate(CFGetAllocator(item), kCFNumberSInt32Type, &zero);
 439             } else {
 440                 value = kCFNull;
 441             }
 442             break;
 443         case kSecDbDateAttr:
 444             if (attr->flags & kSecDbNotNullFlag && attr->flags & kSecDbDefault0Flag) {
 445                 value = CFDateCreate(kCFAllocatorDefault, 0.0);
 446             } else {
 447                 value = kCFNull;
 448             }
 449             break;
 450         case kSecDbRowIdAttr:
 451             if (attr->flags & kSecDbNotNullFlag) {
 452                 // No can do, error?
 453             }
 454             value = kCFNull;
 455             break;
 456         case kSecDbCreationDateAttr:
 457         case kSecDbModificationDateAttr:
 458             value = CFDateCreate(CFGetAllocator(item), CFAbsoluteTimeGetCurrent());
 459             break;
 460         case kSecDbUTombAttr:
 461             value = kCFNull;
 462             break;
 463     }
 464
 465     return value;
 466 }
 467
 468 // SecDbItemGetValue will return kCFNull if there is no value for an attribute and this was not
 469 // an error.  It will return NULL and optionally set *error if there was an error computing an
 470 // attribute, or if a required attribute was missing a value and had no known way to compute
 471 // it's value.
 472 CFTypeRef SecDbItemGetValue(SecDbItemRef item, const SecDbAttr *desc, CFErrorRef *error) {
 473     // Propagate chained errors
 474     if (!desc)
 475         return NULL;
 476
 477     if (desc->flags & kSecDbInCryptoDataFlag || desc->flags & kSecDbInAuthenticatedDataFlag) {
 478         if (!SecDbItemEnsureDecrypted(item, error))
 479             return NULL;
 480     }
 481
 482     CFTypeRef value = SecDbItemGetCachedValue(item, desc);
 483     if (!value) {
 484         value = SecDbItemCopyValue(item, desc, error);
 485         if (value) {
 486             if (CFEqual(kCFNull, value)) {
 487                 CFRelease(value); // This is redundant but it shuts clang's static analyzer up.
 488                 value = kCFNull;
 489             } else {
 490                 SecDbItemSetValue(item, desc, value, error);
 491                 CFRelease(value);
 492                 value = SecDbItemGetCachedValue(item, desc);
 493             }
 494         }
 495     }
 496     return value;
 497 }
 498
 499 CFTypeRef SecDbItemGetValueKind(SecDbItemRef item, SecDbAttrKind descKind, CFErrorRef *error) {
 500     CFTypeRef result = NULL;
 501
 502     const SecDbClass * itemClass = SecDbItemGetClass(item);
 503     const SecDbAttr * desc = SecDbClassAttrWithKind(itemClass, descKind, error);
 504
 505     if (desc) {
 506         result = SecDbItemGetValue(item, desc, error);
 507     }
 508
 509     return result;
 510 }
 511
 512
 513 // Similar as SecDbItemGetValue, but if attr represents attribute stored into DB field as hash, returns
 514 // hashed value for the attribute.
 515 static CFTypeRef SecDbItemCopyValueForDb(SecDbItemRef item, const SecDbAttr *desc, CFErrorRef *error) {
 516     CFTypeRef value = NULL;
 517     CFStringRef hash_name = NULL;
 518     hash_name = SecDbAttrGetHashName(desc);
 519     if ((desc->flags & kSecDbSHA1ValueInFlag) && (desc->flags & kSecDbInFlag)) {
 520         value = CFRetainSafe(CFDictionaryGetValue(item->attributes, hash_name));
 521     }
 522
 523     if (value == NULL) {
 524         require_quiet(value = SecDbItemGetValue(item, desc, error), out);
 525         require_action_quiet(value = SecDbAttrCopyValueForDb(desc, value, error), out, CFReleaseNull(value));
 526         if ((desc->flags & kSecDbSHA1ValueInFlag) != 0) {
 527             CFDictionarySetValue(item->attributes, hash_name, value);
 528         }
 529     }
 530
 531 out:
 532     return value;
 533 }
 534
 535 static bool SecDbItemGetBoolValue(SecDbItemRef item, const SecDbAttr *desc, bool *bvalue, CFErrorRef *error) {
 536     CFTypeRef value = SecDbItemGetValue(item, desc, error);
 537     if (!value)
 538         return false;
 539     char cvalue;
 540     *bvalue = (isNumber(value) && CFNumberGetValue(value, kCFNumberCharType, &cvalue) && cvalue == 1);
 541     return true;
 542 }
 543
 544 static CFStringRef SecDbItemCopyFormatDescription(CFTypeRef cf, CFDictionaryRef formatOptions) {
 545     CFStringRef desc;
 546     if (isDictionary(formatOptions) && CFDictionaryContainsKey(formatOptions, kSecDebugFormatOption)) {
 547         SecDbItemRef item = (SecDbItemRef)cf;
 548         CFMutableStringRef mdesc = CFStringCreateMutable(CFGetAllocator(cf), 0);
 549         CFStringAppendFormat(mdesc, NULL, CFSTR("<%@"), item->class->name);
 550         SecDbForEachAttr(item->class, attr) {
 551                 CFTypeRef value = SecDbItemGetValue(item, attr, NULL);
 552                 if (value) {
 553                     CFStringAppend(mdesc, CFSTR(","));
 554                     CFStringAppend(mdesc, attr->name);
 555                     CFStringAppend(mdesc, CFSTR("="));
 556                     if (CFEqual(CFSTR("data"), attr->name)) {
 557                         CFStringAppendEncryptedData(mdesc, value);
 558                     } else if (CFEqual(CFSTR("v_Data"), attr->name)) {
 559                         CFStringAppend(mdesc, CFSTR("<?>"));
 560                     } else if (isData(value)) {
 561                         CFStringAppendHexData(mdesc, value);
 562                     } else {
 563                         CFStringAppendFormat(mdesc, 0, CFSTR("%@"), value);
 564                     }
 565                 }
 566         }
 567         CFStringAppend(mdesc, CFSTR(">"));
 568         desc = mdesc;
 569     } else {
 570         SecDbItemRef item = (SecDbItemRef)cf;
 571         const UInt8 zero4[4] = {};
 572         const UInt8 *pk = &zero4[0], *sha1 = &zero4[0];
 573         char sync = 0;
 574         char tomb = 0;
 575         SInt64 rowid = 0;
 576         CFStringRef access = NULL;
 577         uint8_t mdatbuf[32] = {};
 578         uint8_t *mdat = &mdatbuf[0];
 579         CFMutableStringRef attrs = CFStringCreateMutable(kCFAllocatorDefault, 0);
 580         CFStringRef agrp = NULL;
 581         CFBooleanRef utomb = NULL;
 582
 583         SecDbForEachAttr(item->class, attr) {
 584             CFTypeRef value;
 585             switch (attr->kind) {
 586                 case kSecDbBlobAttr:
 587                 case kSecDbDataAttr:
 588                 case kSecDbStringAttr:
 589                 case kSecDbNumberAttr:
 590                 case kSecDbDateAttr:
 591                 case kSecDbEncryptedDataAttr:
 592                     if (attr->flags & (kSecDbReturnAttrFlag | kSecDbReturnDataFlag) && (value = SecDbItemGetValue(item, attr, NULL)) && !CFEqual(value, kCFNull)) {
 593                         if (isString(value) && CFEqual(attr->name, kSecAttrAccessGroup)) {
 594                             agrp = value;
 595                         } else {
 596                             // We don't log these, just record that we saw the attribute.
 597                             CFStringAppend(attrs, CFSTR(","));
 598                             CFStringAppend(attrs, attr->name);
 599                         }
 600                     }
 601                     break;
 602                 case kSecDbUUIDAttr:
 603                     if ((value = SecDbItemGetValue(item, attr, NULL))) {
 604                         if (CFEqual(attr->name, kSecAttrMultiUser)) {
 605                             if (isData(value)) {
 606                                 CFStringAppend(attrs, CFSTR(","));
 607                                 if (CFDataGetLength(value)) {
 608                                     CFStringAppendHexData(attrs, value);
 609                                 } else {
 610                                     CFStringAppend(attrs, attr->name);
 611                                 }
 612                             }
 613                         }
 614                     }
 615                     break;
 616                 case kSecDbCreationDateAttr:
 617                     // We don't care about this and every object has one.
 618                     break;
 619                 case kSecDbModificationDateAttr:
 620                     value = SecDbItemGetValue(item, attr, NULL);
 621                     if (isDate(value))
 622                         mdat = der_encode_generalizedtime_body(CFDateGetAbsoluteTime(value), NULL, mdat, &mdatbuf[31]);
 623                     break;
 624                 case kSecDbSHA1Attr:
 625                     value = SecDbItemGetValue(item, attr, NULL);
 626                     if (isData(value) && CFDataGetLength(value) >= (CFIndex)sizeof(zero4))
 627                         sha1 = CFDataGetBytePtr(value);
 628                     break;
 629                 case kSecDbRowIdAttr:
 630                     value = SecDbItemGetValue(item, attr, NULL);
 631                     if (isNumber(value))
 632                         CFNumberGetValue(value, kCFNumberSInt64Type, &rowid);
 633                     break;
 634                 case kSecDbPrimaryKeyAttr:
 635                     value = SecDbItemGetValue(item, attr, NULL);
 636                     if (isData(value))
 637                         pk = CFDataGetBytePtr(value);
 638                     break;
 639                 case kSecDbSyncAttr:
 640                     value = SecDbItemGetValue(item, attr, NULL);
 641                     if (isNumber(value))
 642                         CFNumberGetValue(value, kCFNumberCharType, &sync);
 643                     break;
 644                 case kSecDbTombAttr:
 645                     value = SecDbItemGetValue(item, attr, NULL);
 646                     if (isNumber(value))
 647                         CFNumberGetValue(value, kCFNumberCharType, &tomb);
 648                     break;
 649                 case kSecDbAccessAttr:
 650                     value = SecDbItemGetValue(item, attr, NULL);
 651                     if (isString(value))
 652                         access = value;
 653                     break;
 654                 case kSecDbUTombAttr:
 655                     value = SecDbItemGetValue(item, attr, NULL);
 656                     if (isBoolean(value))
 657                         utomb = value;
 658                 case kSecDbAccessControlAttr:
 659                     /* TODO: Add formatting of ACLs. */
 660                     break;
 661             }
 662         }
 663
 664         desc = CFStringCreateWithFormat(CFGetAllocator(cf), NULL,
 665             CFSTR(
 666                   "%s,"
 667                   "%@,"
 668                   "%02X%02X%02X%02X,"
 669                   "%s,"
 670                   "%@,"
 671                   "%@,"
 672                   "%"PRId64
 673                   "%@,"
 674                   "%s,"
 675                   "%s"
 676                   "%02X%02X%02X%02X"),
 677             tomb ? "T" : "O",
 678             item->class->name,
 679             pk[0], pk[1], pk[2], pk[3],
 680             sync ? "S" : "L",
 681             access,
 682             agrp,
 683             rowid,
 684             attrs,
 685             mdat,
 686             utomb ? (CFEqual(utomb, kCFBooleanFalse) ? "F," : "T,") : "",
 687             sha1[0], sha1[1], sha1[2], sha1[3]);
 688         CFReleaseSafe(attrs);
 689     }
 690
 691     return desc;
 692 }
 693
 694 static void SecDbItemDestroy(CFTypeRef cf) {
 695     SecDbItemRef item = (SecDbItemRef)cf;
 696     CFReleaseSafe(item->attributes);
 697     CFReleaseSafe(item->credHandle);
 698     CFReleaseSafe(item->callerAccessGroups);
 699     CFReleaseSafe(item->cryptoOp);
 700 }
 701
 702 static CFHashCode SecDbItemHash(CFTypeRef cf) {
 703     SecDbItemRef item = (SecDbItemRef)cf;
 704     CFDataRef digest = SecDbItemGetSHA1(item, NULL);
 705     CFHashCode code;
 706     const UInt8 *p = CFDataGetBytePtr(digest);
 707     // Read first 8 bytes of digest in order
 708     code = p[0] + ((p[1] + ((p[2] + ((p[3] + ((p[4] + ((p[5] + ((p[6] + (p[7] << 8)) << 8)) << 8)) << 8)) << 8)) << 8)) << 8);
 709     return code;
 710 }
 711
 712 static Boolean SecDbItemCompare(CFTypeRef cf1, CFTypeRef cf2) {
 713     SecDbItemRef item1 = (SecDbItemRef)cf1;
 714     SecDbItemRef item2 = (SecDbItemRef)cf2;
 715     CFDataRef digest1 = NULL;
 716     CFDataRef digest2 = NULL;
 717     if (item1)
 718         digest1 = SecDbItemGetSHA1(item1, NULL);
 719     if (item2)
 720         digest2 = SecDbItemGetSHA1(item2, NULL);
 721     Boolean equal = CFEqual(digest1, digest2);
 722     return equal;
 723 }
 724
 725 CFGiblisWithHashFor(SecDbItem)
 726
 727 static SecDbItemRef SecDbItemCreate(CFAllocatorRef allocator, const SecDbClass *class, keybag_handle_t keybag) {
 728     SecDbItemRef item = CFTypeAllocate(SecDbItem, struct SecDbItem, allocator);
 729     item->class = class;
 730     item->attributes = CFDictionaryCreateMutableForCFTypes(allocator);
 731     item->keybag = keybag;
 732     item->_edataState = kSecDbItemDirty;
 733     item->cryptoOp = kAKSKeyOpDecrypt;
 734     return item;
 735 }
 736
 737 const SecDbClass *SecDbItemGetClass(SecDbItemRef item) {
 738     return item->class;
 739 }
 740
 741 keybag_handle_t SecDbItemGetKeybag(SecDbItemRef item) {
 742     return item->keybag;
 743 }
 744
 745 bool SecDbItemSetKeybag(SecDbItemRef item, keybag_handle_t keybag, CFErrorRef *error) {
 746     if (!SecDbItemEnsureDecrypted(item, error))
 747         return false;
 748     if (item->keybag != keybag) {
 749         item->keybag = keybag;
 750         if (item->_edataState == kSecDbItemClean) {
 751             SecDbItemSetValue(item, SecDbClassAttrWithKind(item->class, kSecDbEncryptedDataAttr, NULL), kCFNull, NULL);
 752         }
 753     }
 754
 755     return true;
 756 }
 757
 758 bool SecDbItemSetValue(SecDbItemRef item, const SecDbAttr *desc, CFTypeRef value, CFErrorRef *error) {
 759     // Propagate chained errors.
 760     if (!desc)
 761         return false;
 762
 763     if (!value)
 764         value = kCFNull;
 765
 766     if (desc->setValue)
 767         return desc->setValue(item, desc, value, error);
 768
 769     if (desc->flags & kSecDbInCryptoDataFlag || desc->flags & kSecDbInAuthenticatedDataFlag)
 770         if (!SecDbItemEnsureDecrypted(item, error))
 771             return false;
 772
 773     bool changed = false;
 774     CFTypeRef attr = NULL;
 775     switch (desc->kind) {
 776         case kSecDbPrimaryKeyAttr:
 777         case kSecDbDataAttr:
 778             attr = copyData(value);
 779             break;
 780         case kSecDbEncryptedDataAttr:
 781             attr = copyData(value);
 782             if (attr) {
 783                 if (item->_edataState == kSecDbItemEncrypting)
 784                     item->_edataState = kSecDbItemClean;
 785                 else
 786                     item->_edataState = kSecDbItemEncrypted;
 787             } else if (!value || CFEqual(kCFNull, value)) {
 788                 item->_edataState = kSecDbItemDirty;
 789             }
 790             break;
 791         case kSecDbBlobAttr:
 792         case kSecDbAccessControlAttr:
 793             attr = copyBlob(value);
 794             break;
 795         case kSecDbDateAttr:
 796         case kSecDbCreationDateAttr:
 797         case kSecDbModificationDateAttr:
 798             attr = copyDate(value);
 799             break;
 800         case kSecDbNumberAttr:
 801         case kSecDbSyncAttr:
 802         case kSecDbTombAttr:
 803         case kSecDbRowIdAttr:
 804             attr = copyNumber(value);
 805             break;
 806         case kSecDbAccessAttr:
 807         case kSecDbStringAttr:
 808             attr = copyString(value);
 809             break;
 810         case kSecDbSHA1Attr:
 811             attr = copySHA1(value);
 812             break;
 813         case kSecDbUTombAttr:
 814             attr = CFRetainSafe(asBoolean(value, NULL));
 815             break;
 816         case kSecDbUUIDAttr:
 817             attr = copyUUID(value);
 818             break;
 819     }
 820
 821     if (attr) {
 822         CFTypeRef ovalue = CFDictionaryGetValue(item->attributes, desc->name);
 823         changed = (!ovalue || !CFEqual(ovalue, attr));
 824         CFDictionarySetValue(item->attributes, desc->name, attr);
 825         CFRelease(attr);
 826     } else {
 827         if (value && !CFEqual(kCFNull, value)) {
 828             SecError(errSecItemInvalidValue, error, CFSTR("attribute %@: value: %@ failed to convert"), desc->name, value);
 829             return false;
 830         }
 831         CFTypeRef ovalue = CFDictionaryGetValue(item->attributes, desc->name);
 832         changed = (ovalue && !CFEqual(ovalue, kCFNull));
 833         CFDictionaryRemoveValue(item->attributes, desc->name);
 834     }
 835
 836     if (changed) {
 837         if (desc->flags & kSecDbInHashFlag)
 838             SecDbItemSetValue(item, SecDbClassAttrWithKind(item->class, kSecDbSHA1Attr, NULL), kCFNull, NULL);
 839         if (desc->flags & kSecDbPrimaryKeyFlag)
 840             SecDbItemSetValue(item, SecDbClassAttrWithKind(item->class, kSecDbPrimaryKeyAttr, NULL), kCFNull, NULL);
 841         if ((desc->flags & kSecDbInCryptoDataFlag || desc->flags & kSecDbInAuthenticatedDataFlag) && item->_edataState == kSecDbItemClean)
 842             SecDbItemSetValue(item, SecDbClassAttrWithKind(item->class, kSecDbEncryptedDataAttr, NULL), kCFNull, NULL);
 843         if (desc->flags & kSecDbSHA1ValueInFlag)
 844             CFDictionaryRemoveValue(item->attributes, SecDbAttrGetHashName(desc));
 845     }
 846
 847     return true;
 848 }
 849
 850 bool SecDbItemSetValues(SecDbItemRef item, CFDictionaryRef values, CFErrorRef *error) {
 851     SecDbForEachAttr(item->class, attr) {
 852         CFTypeRef value = CFDictionaryGetValue(values, attr->name);
 853         if (value && !SecDbItemSetValue(item, attr, value, error))
 854             return false;
 855     }
 856     return true;
 857 }
 858
 859 bool SecDbItemSetValueWithName(SecDbItemRef item, CFStringRef name, CFTypeRef value, CFErrorRef *error) {
 860     SecDbForEachAttr(item->class, attr) {
 861         if (CFEqual(attr->name, name)) {
 862             return SecDbItemSetValue(item, attr, value, error);
 863         }
 864     }
 865     return false;
 866 }
 867
 868 bool SecDbItemSetAccessControl(SecDbItemRef item, SecAccessControlRef access_control, CFErrorRef *error) {
 869     bool ok = true;
 870     if (item->_edataState == kSecDbItemClean)
 871         ok = SecDbItemSetValue(item, SecDbClassAttrWithKind(item->class, kSecDbEncryptedDataAttr, error), kCFNull, error);
 872     if (ok && access_control) { //added check for access_control because ks_decrypt_data can leave NULL in access_control in case of error
 873         item->_edataState = kSecDbItemDirty;
 874         CFDataRef data = SecAccessControlCopyData(access_control);
 875         ok = SecDbItemSetValue(item, SecDbClassAttrWithKind(item->class, kSecDbAccessControlAttr, error), data, error);
 876         CFRelease(data);
 877     }
 878     return ok;
 879 }
 880
 881 SecDbItemRef SecDbItemCreateWithAttributes(CFAllocatorRef allocator, const SecDbClass *class, CFDictionaryRef attributes, keybag_handle_t keybag, CFErrorRef *error) {
 882     SecDbItemRef item = SecDbItemCreate(kCFAllocatorDefault, class, keybag);
 883     if (item && !SecDbItemSetValues(item, attributes, error))
 884         CFReleaseNull(item);
 885     return item;
 886 }
 887
 888 static CFTypeRef
 889 SecDbColumnCopyValueWithAttr(CFAllocatorRef allocator, sqlite3_stmt *stmt, const SecDbAttr *attr, int col, CFErrorRef *error) {
 890     CFTypeRef value = NULL;
 891     switch (attr->kind) {
 892         case kSecDbDateAttr:
 893         case kSecDbCreationDateAttr:
 894         case kSecDbModificationDateAttr:
 895             value = SecDbColumnCopyDate(allocator, stmt, col, error);
 896             break;
 897         case kSecDbBlobAttr:
 898         case kSecDbNumberAttr:
 899             switch (sqlite3_column_type(stmt, col)) {
 900                 case SQLITE_INTEGER:
 901                     value = SecDbColumnCopyNumber(allocator, stmt, col, error);
 902                     break;
 903                 case SQLITE_FLOAT:
 904                     value = SecDbColumnCopyDouble(allocator, stmt, col, error);
 905                     break;
 906                 case SQLITE_TEXT:
 907                     value = SecDbColumnCopyString(allocator, stmt, col, error,
 908                                                   attr->flags);
 909                     break;
 910                 case SQLITE_BLOB:
 911                     value = SecDbColumnCopyData(allocator, stmt, col, error);
 912                     break;
 913                 case SQLITE_NULL:
 914                     value = kCFNull;
 915                     break;
 916             }
 917             break;
 918         case kSecDbAccessAttr:
 919         case kSecDbStringAttr:
 920             value = SecDbColumnCopyString(allocator, stmt, col, error,
 921                                           attr->flags);
 922             break;
 923         case kSecDbDataAttr:
 924         case kSecDbUUIDAttr:
 925         case kSecDbSHA1Attr:
 926         case kSecDbPrimaryKeyAttr:
 927         case kSecDbEncryptedDataAttr:
 928             value = SecDbColumnCopyData(allocator, stmt, col, error);
 929             break;
 930         case kSecDbSyncAttr:
 931         case kSecDbTombAttr:
 932             value = SecDbColumnCopyNumber(allocator, stmt, col, error);
 933             break;
 934         case kSecDbRowIdAttr:
 935             value = SecDbColumnCopyNumber64(allocator, stmt, col, error);
 936             break;
 937         case kSecDbAccessControlAttr:
 938         case kSecDbUTombAttr:
 939             /* This attributes does not have any database column associated, exists only inside encrypted blob as metadata. */
 940             break;
 941     }
 942     return value;
 943 }
 944
 945 SecDbItemRef SecDbItemCreateWithStatement(CFAllocatorRef allocator, const SecDbClass *class, sqlite3_stmt *stmt, keybag_handle_t keybag, CFErrorRef *error, bool (^return_attr)(const SecDbAttr *attr)) {
 946     SecDbItemRef item = SecDbItemCreate(allocator, class, keybag);
 947     int col = 0;
 948     SecDbForEachAttr(class, attr) {
 949         if (return_attr(attr)) {
 950             CFTypeRef value = SecDbColumnCopyValueWithAttr(allocator, stmt, attr, col++, error);
 951             require_action_quiet(value, errOut, CFReleaseNull(item));
 952
 953             CFDictionarySetValue(item->attributes, SecDbAttrGetHashName(attr), value);
 954             CFRelease(value);
 955         }
 956
 957         const SecDbAttr *data_attr = SecDbClassAttrWithKind(class, kSecDbEncryptedDataAttr, NULL);
 958         if (data_attr != NULL && CFDictionaryGetValue(item->attributes, data_attr->name) != NULL) {
 959             item->_edataState = kSecDbItemEncrypted;
 960         }
 961     }
 962
 963 errOut:
 964     return item;
 965 }
 966
 967 SecDbItemRef SecDbItemCreateWithEncryptedData(CFAllocatorRef allocator, const SecDbClass *class,
 968                                               CFDataRef edata, keybag_handle_t keybag, CFErrorRef *error) {
 969     SecDbItemRef item = SecDbItemCreate(allocator, class, keybag);
 970     const SecDbAttr *edata_attr = SecDbClassAttrWithKind(class, kSecDbEncryptedDataAttr, error);
 971     if (edata_attr) {
 972         if (!SecDbItemSetValue(item, edata_attr, edata, error))
 973             CFReleaseNull(item);
 974     }
 975     return item;
 976 }
 977
 978 // TODO: Hack -- Replace with real filtering
 979
 980 // Return true iff an item for which SecDbItemIsSyncable() already returns true should be part of the v2 view.
 981 bool SecDbItemInV2(SecDbItemRef item) {
 982     const SecDbClass *iclass = SecDbItemGetClass(item);
 983     return  (SecDbItemGetCachedValueWithName(item, kSecAttrSyncViewHint) == NULL &&
 984              (iclass == &genp_class || iclass == &inet_class || iclass == &keys_class || iclass == &cert_class));
 985 }
 986
 987 // Return true iff an item for which SecDbItemIsSyncable() and SecDbItemInV2() already return true should be part of the v0 view.
 988 bool SecDbItemInV2AlsoInV0(SecDbItemRef item) {
 989     return  (SecDbItemGetCachedValueWithName(item, kSecAttrTokenID) == NULL && SecDbItemGetClass(item) != &cert_class);
 990 }
 991
 992 SecDbItemRef SecDbItemCopyWithUpdates(SecDbItemRef item, CFDictionaryRef updates, CFErrorRef *error) {
 993     SecDbItemRef new_item = SecDbItemCreate(CFGetAllocator(item), item->class, item->keybag);
 994     SecDbItemSetCredHandle(new_item, item->credHandle);
 995     SecDbForEachAttr(item->class, attr) {
 996         // Copy each attribute, except the mod date attribute (it will be reset to now when needed),
 997         // from the updates dict unless it's not there in which case we copy the attribute from the passed in item.
 998         if (attr->kind != kSecDbModificationDateAttr && attr->kind != kSecDbEncryptedDataAttr && attr->kind != kSecDbSHA1Attr && attr->kind != kSecDbPrimaryKeyAttr) {
 999             CFTypeRef value = NULL;
1000             if (CFDictionaryGetValueIfPresent(updates, attr->name, &value)) {
1001                 if (!value)
1002                     SecError(errSecParam, error, CFSTR("NULL value in dictionary"));
1003             } else {
1004                 value = SecDbItemGetValue(item, attr, error);
1005             }
1006             if (!value || !SecDbItemSetValue(new_item, attr, value, error)) {
1007                 CFReleaseNull(new_item);
1008                 break;
1009             }
1010         }
1011     }
1012     return new_item;
1013 }
1014
1015 // Ensure that the date value of attr of new_item is greater than that of old_item.
1016 static bool SecDbItemMakeAttrYounger(SecDbItemRef new_item, SecDbItemRef old_item, const SecDbAttr *attr, CFErrorRef *error) {
1017     CFDateRef old_date = SecDbItemGetValue(old_item, attr, error);
1018     if (!old_date)
1019         return false;
1020     CFDateRef new_date = SecDbItemGetValue(new_item, attr, error);
1021     if (!new_date)
1022         return false;
1023     bool ok = true;
1024     if (CFDateCompare(new_date, old_date, NULL) != kCFCompareGreaterThan) {
1025         CFDateRef adjusted_date = CFDateCreate(kCFAllocatorDefault, CFDateGetAbsoluteTime(old_date) + 0.001);
1026         if (adjusted_date) {
1027             ok = SecDbItemSetValue(new_item, attr, adjusted_date, error);
1028             CFRelease(adjusted_date);
1029         }
1030     }
1031     return ok;
1032 }
1033
1034 // Ensure that the mod date of new_item is greater than that of old_item.
1035 static bool SecDbItemMakeYounger(SecDbItemRef new_item, SecDbItemRef old_item, CFErrorRef *error) {
1036     const SecDbAttr *attr = SecDbClassAttrWithKind(new_item->class, kSecDbModificationDateAttr, error);
1037     return attr && SecDbItemMakeAttrYounger(new_item, old_item, attr, error);
1038 }
1039
1040 static SecDbItemRef SecDbItemCopyTombstone(SecDbItemRef item, CFBooleanRef makeTombStone, CFErrorRef *error) {
1041     SecDbItemRef new_item = SecDbItemCreate(CFGetAllocator(item), item->class, item->keybag);
1042     SecDbForEachAttr(item->class, attr) {
1043         if (attr->kind == kSecDbTombAttr) {
1044             // Set the tomb attr to true to indicate a tombstone.
1045             if (!SecDbItemSetValue(new_item, attr, kCFBooleanTrue, error)) {
1046                 CFReleaseNull(new_item);
1047                 break;
1048             }
1049         } else if (SecDbIsTombstoneDbUpdateAttr(attr)) {
1050             // Copy all primary key attributes and creation timestamps from the original item.
1051             CFTypeRef value = SecDbItemGetValue(item, attr, error);
1052             if (!value || (!CFEqual(kCFNull, value) && !SecDbItemSetValue(new_item, attr, value, error))) {
1053                 CFReleaseNull(new_item);
1054                 break;
1055             }
1056         } else if (attr->kind == kSecDbModificationDateAttr) {
1057             if (!SecDbItemMakeAttrYounger(new_item, item, attr, error)) {
1058                 CFReleaseNull(new_item);
1059                 break;
1060             }
1061         } else if (makeTombStone && attr->kind == kSecDbUTombAttr) {
1062             if (makeTombStone)
1063                 SecDbItemSetValue(new_item, attr, makeTombStone, error);
1064         }
1065     }
1066
1067     return new_item;
1068 }
1069
1070 // MARK: -
1071 // MARK: SQL Construction helpers -- These should become private in the future
1072
1073 void SecDbAppendElement(CFMutableStringRef sql, CFStringRef value, bool *needComma) {
1074     assert(needComma);
1075     if (*needComma) {
1076         CFStringAppend(sql, CFSTR(","));
1077     } else {
1078         *needComma = true;
1079     }
1080     CFStringAppend(sql, value);
1081 }
1082
1083 static void SecDbAppendElementEquals(CFMutableStringRef sql, CFStringRef value, bool *needComma) {
1084     SecDbAppendElement(sql, value, needComma);
1085     CFStringAppend(sql, CFSTR("=?"));
1086 }
1087
1088 /* Append AND is needWhere is NULL or *needWhere is false.  Append WHERE
1089  otherwise.  Upon return *needWhere will be false.  */
1090 void
1091 SecDbAppendWhereOrAnd(CFMutableStringRef sql, bool *needWhere) {
1092     if (!needWhere || !*needWhere) {
1093         CFStringAppend(sql, CFSTR(" AND "));
1094     } else {
1095         CFStringAppend(sql, CFSTR(" WHERE "));
1096         *needWhere = false;
1097     }
1098 }
1099
1100 void
1101 SecDbAppendWhereOrAndEquals(CFMutableStringRef sql, CFStringRef col, bool *needWhere) {
1102     SecDbAppendWhereOrAnd(sql, needWhere);
1103     CFStringAppend(sql, col);
1104     CFStringAppend(sql, CFSTR("=?"));
1105 }
1106
1107 void
1108 SecDbAppendWhereOrAndNotEquals(CFMutableStringRef sql, CFStringRef col, bool *needWhere) {
1109     SecDbAppendWhereOrAnd(sql, needWhere);
1110     CFStringAppend(sql, col);
1111     CFStringAppend(sql, CFSTR("!=?"));
1112 }
1113
1114 static void SecDbAppendCountArgsAndCloseParen(CFMutableStringRef sql, CFIndex count) {
1115     bool needComma = false;
1116     while (count-- > 0)
1117         SecDbAppendElement(sql, CFSTR("?"), &needComma);
1118     CFStringAppend(sql, CFSTR(")"));
1119 }
1120
1121 void
1122 SecDbAppendWhereOrAndIn(CFMutableStringRef sql, CFStringRef col, bool *needWhere, CFIndex count) {
1123     if (count == 1)
1124         return SecDbAppendWhereOrAndEquals(sql, col, needWhere);
1125     SecDbAppendWhereOrAnd(sql, needWhere);
1126     CFStringAppend(sql, col);
1127     CFStringAppend(sql, CFSTR(" IN ("));
1128     SecDbAppendCountArgsAndCloseParen(sql, count);
1129 }
1130
1131 void
1132 SecDbAppendWhereOrAndNotIn(CFMutableStringRef sql, CFStringRef col, bool *needWhere, CFIndex count) {
1133     if (count == 1)
1134         return SecDbAppendWhereOrAndNotEquals(sql, col, needWhere);
1135     SecDbAppendWhereOrAnd(sql, needWhere);
1136     CFStringAppend(sql, col);
1137     CFStringAppend(sql, CFSTR(" NOT IN ("));
1138     SecDbAppendCountArgsAndCloseParen(sql, count);
1139 }
1140
1141 static CFStringRef SecDbItemCopyInsertSQL(SecDbItemRef item, bool(^use_attr)(const SecDbAttr *attr)) {
1142     CFMutableStringRef sql = CFStringCreateMutable(CFGetAllocator(item), 0);
1143     CFStringAppend(sql, CFSTR("INSERT INTO "));
1144     CFStringAppend(sql, item->class->name);
1145     CFStringAppend(sql, CFSTR("("));
1146     bool needComma = false;
1147     CFIndex used_attr = 0;
1148     SecDbForEachAttr(item->class, attr) {
1149         if (use_attr(attr)) {
1150             ++used_attr;
1151             SecDbAppendElement(sql, attr->name, &needComma);
1152         }
1153     }
1154     CFStringAppend(sql, CFSTR(")VALUES(?"));
1155     while (used_attr-- > 1) {
1156         CFStringAppend(sql, CFSTR(",?"));
1157     }
1158     CFStringAppend(sql, CFSTR(")"));
1159     return sql;
1160
1161 }
1162
1163 static bool SecDbItemInsertBind(SecDbItemRef item, sqlite3_stmt *stmt, CFErrorRef *error, bool(^use_attr)(const SecDbAttr *attr)) {
1164     bool ok = true;
1165     int param = 0;
1166     SecDbForEachAttr(item->class, attr) {
1167         if (use_attr(attr)) {
1168             CFTypeRef value = SecDbItemCopyValueForDb(item, attr, error);
1169             ok = value && SecDbBindObject(stmt, ++param, value, error);
1170             CFReleaseSafe(value);
1171             if (!ok)
1172                 break;
1173         }
1174     }
1175     return ok;
1176 }
1177
1178 sqlite3_int64 SecDbItemGetRowId(SecDbItemRef item, CFErrorRef *error) {
1179     sqlite3_int64 row_id = 0;
1180     const SecDbAttr *attr = SecDbClassAttrWithKind(item->class, kSecDbRowIdAttr, error);
1181     if (attr) {
1182         CFNumberRef number = SecDbItemGetValue(item, attr, error);
1183         if (!isNumber(number)|| !CFNumberGetValue(number, kCFNumberSInt64Type, &row_id))
1184             SecDbError(SQLITE_ERROR, error, CFSTR("rowid %@ is not a 64 bit number"), number);
1185     }
1186
1187     return row_id;
1188 }
1189
1190 static CFNumberRef SecDbItemCreateRowId(SecDbItemRef item, sqlite3_int64 rowid, CFErrorRef *error) {
1191     return CFNumberCreate(CFGetAllocator(item), kCFNumberSInt64Type, &rowid);
1192 }
1193
1194 bool SecDbItemSetRowId(SecDbItemRef item, sqlite3_int64 rowid, CFErrorRef *error) {
1195     bool ok = true;
1196     const SecDbAttr *attr = SecDbClassAttrWithKind(item->class, kSecDbRowIdAttr, error);
1197     if (attr) {
1198         CFNumberRef value = SecDbItemCreateRowId(item, rowid, error);
1199         if (!value)
1200             return false;
1201
1202         ok = SecDbItemSetValue(item, attr, value, error);
1203         CFRelease(value);
1204     }
1205     return ok;
1206 }
1207
1208 bool SecDbItemClearRowId(SecDbItemRef item, CFErrorRef *error) {
1209     bool ok = true;
1210     const SecDbAttr *attr = SecDbClassAttrWithKind(item->class, kSecDbRowIdAttr, error);
1211     if (attr) {
1212         CFDictionaryRemoveValue(item->attributes, attr->name);
1213         //ok = SecDbItemSetValue(item, attr, kCFNull, error);
1214     }
1215     return ok;
1216 }
1217
1218 static bool SecDbItemSetLastInsertRowId(SecDbItemRef item, SecDbConnectionRef dbconn, CFErrorRef *error) {
1219     sqlite3_int64 rowid = sqlite3_last_insert_rowid(SecDbHandle(dbconn));
1220     return SecDbItemSetRowId(item, rowid, error);
1221 }
1222
1223 bool SecDbItemIsSyncableOrCorrupted(SecDbItemRef item) {
1224     bool is_syncable_or_corrupted = false;
1225     CFErrorRef localError = NULL;
1226     if (!SecDbItemGetBoolValue(item, SecDbClassAttrWithKind(item->class, kSecDbSyncAttr, &localError),
1227                                &is_syncable_or_corrupted, &localError)) {
1228         is_syncable_or_corrupted = SecErrorGetOSStatus(localError) == errSecDecode;
1229     }
1230     CFReleaseSafe(localError);
1231     return is_syncable_or_corrupted;
1232 }
1233
1234 bool SecDbItemIsSyncable(SecDbItemRef item) {
1235     bool is_syncable;
1236     if (SecDbItemGetBoolValue(item, SecDbClassAttrWithKind(item->class, kSecDbSyncAttr, NULL), &is_syncable, NULL))
1237         return is_syncable;
1238     return false;
1239 }
1240
1241 bool SecDbItemSetSyncable(SecDbItemRef item, bool sync, CFErrorRef *error)
1242 {
1243     return SecDbItemSetValue(item, SecDbClassAttrWithKind(item->class, kSecDbSyncAttr, error), sync ? kCFBooleanTrue : kCFBooleanFalse, error);
1244 }
1245
1246 bool SecDbItemIsTombstone(SecDbItemRef item) {
1247     bool is_tomb;
1248     if (SecDbItemGetBoolValue(item, SecDbClassAttrWithKind(item->class, kSecDbTombAttr, NULL), &is_tomb, NULL))
1249         return is_tomb;
1250     return false;
1251 }
1252
1253 CFDataRef SecDbItemGetPrimaryKey(SecDbItemRef item, CFErrorRef *error) {
1254     return SecDbItemGetValue(item, SecDbClassAttrWithKind(item->class, kSecDbPrimaryKeyAttr, error), error);
1255 }
1256
1257 CFDataRef SecDbItemGetSHA1(SecDbItemRef item, CFErrorRef *error) {
1258     return SecDbItemGetValue(item, SecDbClassAttrWithKind(item->class, kSecDbSHA1Attr, error), error);
1259 }
1260
1261 static SecDbQueryRef SecDbQueryCreateWithItemPrimaryKey(SecDbItemRef item, CFErrorRef *error) {
1262     CFMutableDictionaryRef dict = SecDbItemCopyPListWithMask(item, kSecDbPrimaryKeyFlag, error);
1263     if (!dict)
1264         return NULL;
1265
1266     SecDbQueryRef query = query_create(item->class, NULL, NULL, error);
1267     if (query) {
1268         CFReleaseSafe(query->q_item);
1269         query->q_item = dict;
1270     }
1271     else
1272         CFRelease(dict);
1273
1274     return query;
1275 }
1276
1277 static bool SecDbItemIsCorrupt(SecDbItemRef item, bool *is_corrupt, CFErrorRef *error) {
1278     CFErrorRef localError = NULL;
1279     // Cache the storedSHA1 digest so we use the one from the db not the recomputed one for notifications.
1280     const struct SecDbAttr *sha1attr = SecDbClassAttrWithKind(item->class, kSecDbSHA1Attr, &localError);
1281     CFDataRef storedSHA1 = CFRetainSafe(SecDbItemGetValue(item, sha1attr, &localError));
1282     bool akpu = false;
1283
1284     if (localError || !SecDbItemEnsureDecrypted(item, &localError)) {
1285         if (SecErrorGetOSStatus(localError) == errSecDecode) {
1286             // We failed to decrypt the item
1287             const SecDbAttr *desc = SecDbClassAttrWithKind(item->class, kSecDbAccessControlAttr, &localError);
1288             SecAccessControlRef accc = NULL;
1289             CFDataRef acccData = NULL;
1290
1291             acccData = (CFDataRef)SecDbItemGetValue(item, desc, &localError);
1292             if (isData(acccData)) {
1293                 accc = SecAccessControlCreateFromData(CFGetAllocator(item), acccData, &localError);
1294             }
1295
1296             if (accc && CFEqualSafe(SecAccessControlGetProtection(accc), kSecAttrAccessibleWhenPasscodeSetThisDeviceOnly)) {
1297                 akpu = true;
1298                 secwarning("cannot decrypt item %@, item is irrecoverably lost with older passcode (error %@)", item, localError);
1299             } else {
1300                 secerror("error %@ reading item %@ (corrupted)", localError, item);
1301                 __security_simulatecrash(CFSTR("Corrupted item found in keychain"), __sec_exception_code_CorruptItem);
1302             }
1303             CFReleaseNull(localError);
1304             *is_corrupt = true;
1305         }
1306     }
1307
1308     // Recompute sha1 hash attribute and compare with the cached one.
1309     CFDataRef computedSHA1 = SecDbItemCopyValue(item, sha1attr, &localError);
1310     if (storedSHA1 && computedSHA1 && !CFEqual(storedSHA1, computedSHA1)) {
1311         CFStringRef storedHex = CFDataCopyHexString(storedSHA1), computedHex = CFDataCopyHexString(computedSHA1);
1312         secerror("error %@ %@ != %@ item %@ (corrupted)", sha1attr->name, storedHex, computedHex, item);
1313         __security_simulatecrash(CFSTR("Corrupted item (sha1 mismatch) found in keychain"), __sec_exception_code_CorruptItem);
1314         CFReleaseSafe(storedHex);
1315         CFReleaseSafe(computedHex);
1316         *is_corrupt = true;
1317     }
1318
1319     // Sanity check that all attributes that must not be NULL actually aren't
1320     if (!localError) SecDbForEachAttr(item->class, attr) {
1321         if (attr->flags & (kSecDbInCryptoDataFlag | kSecDbInAuthenticatedDataFlag)) {
1322             CFTypeRef value = SecDbItemGetValue(item, attr, &localError);
1323             if (value) {
1324                 if (CFEqual(kCFNull, value) && attr->flags & kSecDbNotNullFlag) {
1325                     secerror("error attribute %@ has NULL value in item %@ (corrupted)", attr->name, item);
1326                     __security_simulatecrash(CFSTR("Corrupted item (attr NULL) found in keychain"), __sec_exception_code_CorruptItem);
1327                     *is_corrupt = true;
1328                     break;
1329                 }
1330             } else {
1331                 if (SecErrorGetOSStatus(localError) == errSecDecode) {
1332                     // We failed to decrypt the item
1333                     if (akpu) {
1334                         secwarning("attribute %@: %@ item %@ (item lost with older passcode)", attr->name, localError, item);
1335                     } else {
1336                         secerror("error attribute %@: %@ item %@ (corrupted)", attr->name, localError, item);
1337                         __security_simulatecrash(CFSTR("Corrupted item found in keychain"), __sec_exception_code_CorruptItem);
1338                     }
1339                     *is_corrupt = true;
1340                     CFReleaseNull(localError);
1341                 }
1342                 break;
1343             }
1344         }
1345     }
1346
1347     CFReleaseSafe(computedSHA1);
1348     CFReleaseSafe(storedSHA1);
1349     return SecErrorPropagate(localError, error);
1350 }
1351
1352 static void SecDbItemRecordUpdate(SecDbConnectionRef dbconn, SecDbItemRef deleted, SecDbItemRef inserted) {
1353     SecDbRecordChange(dbconn, deleted, inserted);
1354 }
1355
1356 static bool SecDbItemDoInsert(SecDbItemRef item, SecDbConnectionRef dbconn, CFErrorRef *error) {
1357     bool (^use_attr)(const SecDbAttr *attr) = ^bool(const SecDbAttr *attr) {
1358         return (attr->flags & kSecDbInFlag);
1359     };
1360     CFStringRef sql = SecDbItemCopyInsertSQL(item, use_attr);
1361     __block bool ok = sql;
1362     if (sql) {
1363         ok &= SecDbPrepare(dbconn, sql, error, ^(sqlite3_stmt *stmt) {
1364             ok = (SecDbItemInsertBind(item, stmt, error, use_attr) &&
1365                   SecDbStep(dbconn, stmt, error, NULL) &&
1366                   SecDbItemSetLastInsertRowId(item, dbconn, error));
1367         });
1368         CFRelease(sql);
1369     }
1370     if (ok) {
1371         secnotice("item", "inserted %@", item);
1372         SecDbItemRecordUpdate(dbconn, NULL, item);
1373     } else {
1374         secnotice("item", "insert failed for item %@ with %@", item, error ? *error : NULL);
1375     }
1376
1377     return ok;
1378 }
1379
1380 bool SecDbItemInsertOrReplace(SecDbItemRef item, SecDbConnectionRef dbconn, CFErrorRef *error, void(^duplicate)(SecDbItemRef item, SecDbItemRef *replace)) {
1381     __block CFErrorRef localError = NULL;
1382     __block bool ok = SecDbItemDoInsert(item, dbconn, &localError);
1383     if (!ok && localError && CFErrorGetCode(localError) == SQLITE_CONSTRAINT && CFEqual(kSecDbErrorDomain, CFErrorGetDomain(localError))) {
1384         SecDbQueryRef query = SecDbQueryCreateWithItemPrimaryKey(item, error);
1385         if (query) {
1386             CFRetainAssign(query->q_use_cred_handle, item->credHandle);
1387             SecDbItemSelect(query, dbconn, error, NULL, ^bool(const SecDbAttr *attr) {
1388                 return attr->flags & kSecDbPrimaryKeyFlag;
1389             }, NULL, NULL, ^(SecDbItemRef old_item, bool *stop) {
1390                 bool is_corrupt = false;
1391                 ok = SecDbItemIsCorrupt(old_item, &is_corrupt, error);
1392                 SecDbItemRef replace = NULL;
1393                 if (is_corrupt) {
1394                     // If old_item is corrupted pretend it's not there and just replace it.
1395                     replace = item;
1396                     CFRetain(replace);
1397                     if(error)
1398                         CFReleaseNull(*error);  //item is corrupted and will be replaced, so drop the error
1399                 } else if (ok && duplicate) {
1400                     duplicate(old_item, &replace);
1401                 }
1402                 if (replace) {
1403                     const SecDbAttr *rowid_attr = SecDbClassAttrWithKind(old_item->class, kSecDbRowIdAttr, error);
1404                     CFNumberRef oldrowid = SecDbItemGetCachedValue(old_item, rowid_attr);
1405                     if (oldrowid) {
1406                         ok = SecDbItemSetValue(replace, rowid_attr, oldrowid, &localError);
1407                         if (ok && !is_corrupt) {
1408                             ok = SecDbItemMakeYounger(replace, old_item, error);
1409                         }
1410                         ok = ok && SecDbItemDoUpdate(old_item, replace, dbconn, &localError, ^bool (const SecDbAttr *attr) {
1411                             return attr->kind == kSecDbRowIdAttr;
1412                         });
1413                     } else {
1414                         ok = SecError(errSecInternal, &localError, CFSTR("no rowid for %@"), old_item);
1415                     }
1416                     CFRelease(replace);
1417                     if (ok)
1418                         CFReleaseNull(localError); // Clear the error, since we replaced the item.
1419                 }
1420             });
1421             SecDbItemSetCredHandle(item, query->q_use_cred_handle);
1422             ok &= query_destroy(query, error);
1423         }
1424     }
1425
1426     return ok & SecErrorPropagate(localError, error); // Don't use && here!
1427 }
1428
1429 bool SecDbItemInsert(SecDbItemRef item, SecDbConnectionRef dbconn, CFErrorRef *error) {
1430     return SecDbItemInsertOrReplace(item, dbconn, error, ^(SecDbItemRef old_item, SecDbItemRef *replace) {
1431         if (SecDbItemIsTombstone(old_item)) {
1432             CFRetain(item);
1433             *replace = item;
1434         }
1435     });
1436 }
1437
1438 static CFStringRef SecDbItemCopyUpdateSQL(SecDbItemRef old_item, SecDbItemRef new_item, bool(^use_attr_in_where)(const SecDbAttr *attr)) {
1439     CFMutableStringRef sql = CFStringCreateMutable(CFGetAllocator(new_item), 0);
1440     CFStringAppend(sql, CFSTR("UPDATE "));
1441     CFStringAppend(sql, new_item->class->name);
1442     CFStringAppend(sql, CFSTR(" SET "));
1443     bool needComma = false;
1444     CFIndex used_attr = 0;
1445     SecDbForEachAttrWithMask(new_item->class, attr, kSecDbInFlag) {
1446         ++used_attr;
1447         SecDbAppendElementEquals(sql, attr->name, &needComma);
1448     }
1449
1450     bool needWhere = true;
1451     SecDbForEachAttr(old_item->class, attr) {
1452         if (use_attr_in_where(attr)) {
1453             SecDbAppendWhereOrAndEquals(sql, attr->name, &needWhere);
1454         }
1455     }
1456
1457     return sql;
1458 }
1459
1460 static bool SecDbItemUpdateBind(SecDbItemRef old_item, SecDbItemRef new_item, sqlite3_stmt *stmt, CFErrorRef *error, bool(^use_attr_in_where)(const SecDbAttr *attr)) {
1461     bool ok = true;
1462     int param = 0;
1463     SecDbForEachAttrWithMask(new_item->class, attr, kSecDbInFlag) {
1464         CFTypeRef value = SecDbItemCopyValueForDb(new_item, attr, error);
1465         ok &= value && SecDbBindObject(stmt, ++param, value, error);
1466         CFReleaseSafe(value);
1467         if (!ok)
1468             break;
1469     }
1470     SecDbForEachAttr(old_item->class, attr) {
1471         if (use_attr_in_where(attr)) {
1472             CFTypeRef value = SecDbItemCopyValueForDb(old_item, attr, error);
1473             ok &= value && SecDbBindObject(stmt, ++param, value, error);
1474             CFReleaseSafe(value);
1475             if (!ok)
1476                 break;
1477         }
1478     }
1479     return ok;
1480 }
1481
1482 // Primary keys are the same -- do an update
1483 bool SecDbItemDoUpdate(SecDbItemRef old_item, SecDbItemRef new_item, SecDbConnectionRef dbconn, CFErrorRef *error, bool (^use_attr_in_where)(const SecDbAttr *attr)) {
1484     CFStringRef sql = SecDbItemCopyUpdateSQL(old_item, new_item, use_attr_in_where);
1485     __block bool ok = sql;
1486     if (sql) {
1487         ok &= SecDbPrepare(dbconn, sql, error, ^(sqlite3_stmt *stmt) {
1488             ok = SecDbItemUpdateBind(old_item, new_item, stmt, error, use_attr_in_where) && SecDbStep(dbconn, stmt, error, NULL);
1489         });
1490         CFRelease(sql);
1491     }
1492     if (ok) {
1493         secnotice("item", "replaced %@ with %@ in %@", old_item, new_item, dbconn);
1494         SecDbItemRecordUpdate(dbconn, old_item, new_item);
1495     }
1496     return ok;
1497 }
1498
1499 static CFStringRef SecDbItemCopyDeleteSQL(SecDbItemRef item, bool(^use_attr_in_where)(const SecDbAttr *attr)) {
1500     CFMutableStringRef sql = CFStringCreateMutable(CFGetAllocator(item), 0);
1501     CFStringAppend(sql, CFSTR("DELETE FROM "));
1502     CFStringAppend(sql, item->class->name);
1503     bool needWhere = true;
1504     SecDbForEachAttr(item->class, attr) {
1505         if (use_attr_in_where(attr)) {
1506             SecDbAppendWhereOrAndEquals(sql, attr->name, &needWhere);
1507         }
1508     }
1509
1510     return sql;
1511 }
1512
1513 static bool SecDbItemDeleteBind(SecDbItemRef item, sqlite3_stmt *stmt, CFErrorRef *error, bool(^use_attr_in_where)(const SecDbAttr *attr)) {
1514     bool ok = true;
1515     int param = 0;
1516     SecDbForEachAttr(item->class, attr) {
1517         if (use_attr_in_where(attr)) {
1518             CFTypeRef value = SecDbItemCopyValueForDb(item, attr, error);
1519             ok &= value && SecDbBindObject(stmt, ++param, value, error);
1520             CFReleaseSafe(value);
1521             if (!ok)
1522                 break;
1523         }
1524     }
1525     return ok;
1526 }
1527
1528 static bool SecDbItemDoDeleteOnly(SecDbItemRef item, SecDbConnectionRef dbconn, CFErrorRef *error, bool (^use_attr_in_where)(const SecDbAttr *attr)) {
1529     CFStringRef sql = SecDbItemCopyDeleteSQL(item, use_attr_in_where);
1530     __block bool ok = sql;
1531     if (sql) {
1532         ok &= SecDbPrepare(dbconn, sql, error, ^(sqlite3_stmt *stmt) {
1533             ok = SecDbItemDeleteBind(item, stmt, error, use_attr_in_where) && SecDbStep(dbconn, stmt, error, NULL);
1534         });
1535         CFRelease(sql);
1536     }
1537     return ok;
1538 }
1539
1540 bool SecDbItemDoDeleteSilently(SecDbItemRef item, SecDbConnectionRef dbconn, CFErrorRef *error) {
1541     return SecDbItemDoDeleteOnly(item, dbconn, error, ^bool(const SecDbAttr *attr) {
1542         return attr->kind == kSecDbRowIdAttr;
1543     });
1544 }
1545
1546 static bool SecDbItemDoDelete(SecDbItemRef item, SecDbConnectionRef dbconn, CFErrorRef *error, bool (^use_attr_in_where)(const SecDbAttr *attr)) {
1547     bool ok = SecDbItemDoDeleteOnly(item, dbconn, error, use_attr_in_where);
1548     if (ok) {
1549         secnotice("item", "deleted %@ from %@", item, dbconn);
1550         SecDbItemRecordUpdate(dbconn, item, NULL);
1551     }
1552     return ok;
1553 }
1554
1555 #if 0
1556 static bool SecDbItemDeleteTombstone(SecDbItemRef item, SecDbConnectionRef dbconn, CFErrorRef *error) {
1557     bool ok = true;
1558     // TODO: Treat non decryptable items like tombstones here too and delete them
1559     SecDbItemRef tombstone = SecDbItemCopyTombstone(item, error);
1560     ok = tombstone;
1561     if (tombstone) {
1562         ok = SecDbItemClearRowId(tombstone, error);
1563         if (ok) {
1564             ok = SecDbItemDoDelete(tombstone, dbconn, error, ^bool (const SecDbAttr *attr) {
1565                 return SecDbIsTombstoneDbSelectAttr(attr);
1566             });
1567         }
1568         CFRelease(tombstone);
1569     }
1570     return ok;
1571 }
1572 #endif
1573
1574 // Replace old_item with new_item.  If primary keys are the same this does an update otherwise it does a delete + add
1575 bool SecDbItemUpdate(SecDbItemRef old_item, SecDbItemRef new_item, SecDbConnectionRef dbconn, CFBooleanRef makeTombstone, CFErrorRef *error) {
1576     __block bool ok = true;
1577     __block CFErrorRef localError = NULL;
1578
1579     CFDataRef old_pk = SecDbItemGetPrimaryKey(old_item, error);
1580     CFDataRef new_pk = SecDbItemGetPrimaryKey(new_item, error);
1581
1582     ok = old_pk && new_pk;
1583
1584     bool pk_equal = ok && CFEqual(old_pk, new_pk);
1585     if (pk_equal) {
1586         ok = SecDbItemMakeYounger(new_item, old_item, error);
1587     }
1588     ok = ok && SecDbItemDoUpdate(old_item, new_item, dbconn, &localError, ^bool(const SecDbAttr *attr) {
1589         return attr->kind == kSecDbRowIdAttr;
1590     });
1591
1592     if (localError) {
1593         if(CFErrorGetCode(localError) == SQLITE_CONSTRAINT && CFEqual(kSecDbErrorDomain, CFErrorGetDomain(localError))) {
1594             /* Update failed because we changed the PrimaryKey and there was a dup.
1595                Find the dup and see if it is a tombstone or corrupted item. */
1596             SecDbQueryRef query = SecDbQueryCreateWithItemPrimaryKey(new_item, error);
1597             ok = query;
1598             if (query) {
1599                 ok &= SecDbItemSelect(query, dbconn, error, NULL, ^bool(const SecDbAttr *attr) {
1600                     return attr->flags & kSecDbPrimaryKeyFlag;
1601                 }, NULL, NULL, ^(SecDbItemRef duplicate_item, bool *stop) {
1602                     bool is_corrupt = false;
1603                     bool is_tomb = false;
1604                     ok = SecDbItemIsCorrupt(duplicate_item, &is_corrupt, error);
1605                     if (ok && !is_corrupt) {
1606                         if ((is_tomb = SecDbItemIsTombstone(duplicate_item)))
1607                             ok = SecDbItemMakeYounger(new_item, duplicate_item, error);
1608                     }
1609                     if (ok && (is_corrupt || is_tomb)) {
1610                         ok = SecDbItemDoDelete(old_item, dbconn, error, ^bool (const SecDbAttr *attr) {
1611                             return attr->kind == kSecDbRowIdAttr;
1612                         });
1613                         ok = ok && SecDbItemDoUpdate(duplicate_item, new_item, dbconn, error, ^bool (const SecDbAttr *attr) {
1614                             return attr->kind == kSecDbRowIdAttr;
1615                         });
1616                         CFReleaseNull(localError);
1617                     }
1618                 });
1619                 ok &= query_destroy(query, error);
1620             }
1621         }
1622
1623         if (localError) {
1624             ok = false;
1625             if (error && *error == NULL) {
1626                 *error = localError;
1627                 localError = NULL;
1628             }
1629             CFReleaseSafe(localError);
1630         }
1631     }
1632
1633     if (ok && !pk_equal && !CFEqualSafe(makeTombstone, kCFBooleanFalse)) {
1634         /* The primary key of new_item is different than that of old_item, we
1635            have been asked to make a tombstone so leave one for the old_item. */
1636         SecDbItemRef tombstone = SecDbItemCopyTombstone(old_item, makeTombstone, error);
1637         ok = tombstone;
1638         if (tombstone) {
1639             ok = (SecDbItemClearRowId(tombstone, error) &&
1640                   SecDbItemDoInsert(tombstone, dbconn, error));
1641             CFRelease(tombstone);
1642         }
1643     }
1644
1645     return ok;
1646 }
1647
1648 // Replace the object with a tombstone
1649 bool SecDbItemDelete(SecDbItemRef item, SecDbConnectionRef dbconn, CFBooleanRef makeTombstone, CFErrorRef *error) {
1650     bool ok = false;
1651     if (!CFEqualSafe(makeTombstone, kCFBooleanFalse)) {
1652         SecDbItemRef tombstone = SecDbItemCopyTombstone(item, makeTombstone, error);
1653         if (tombstone) {
1654             ok = SecDbItemDoUpdate(item, tombstone, dbconn, error, ^bool(const SecDbAttr *attr) {
1655                 return attr->kind == kSecDbRowIdAttr;
1656             });
1657             CFRelease(tombstone);
1658         }
1659     } else {
1660         ok = SecDbItemDoDelete(item, dbconn, error, ^bool(const SecDbAttr *attr) {
1661             return attr->kind == kSecDbRowIdAttr;
1662         });
1663     }
1664     return ok;
1665 }
1666
1667 CFStringRef SecDbItemCopySelectSQL(SecDbQueryRef query,
1668                                    bool (^return_attr)(const SecDbAttr *attr),
1669                                    bool (^use_attr_in_where)(const SecDbAttr *attr),
1670                                    bool (^add_where_sql)(CFMutableStringRef sql, bool *needWhere)) {
1671     CFMutableStringRef sql = CFStringCreateMutable(kCFAllocatorDefault, 0);
1672     CFStringAppend(sql, CFSTR("SELECT "));
1673     // What are we selecting?
1674     bool needComma = false;
1675     SecDbForEachAttr(query->q_class, attr) {
1676         if (return_attr(attr))
1677             SecDbAppendElement(sql, attr->name, &needComma);
1678     }
1679
1680     // From which table?
1681     CFStringAppend(sql, CFSTR(" FROM "));
1682     CFStringAppend(sql, query->q_class->name);
1683
1684     // And which elements do we want to select
1685     bool needWhere = true;
1686     SecDbForEachAttr(query->q_class, attr) {
1687         if (use_attr_in_where(attr)) {
1688             CFTypeRef value = CFDictionaryGetValue(query->q_item, attr->name);
1689             if (isArray(value)) {
1690                 CFArrayRef array = (CFArrayRef)value;
1691                 CFIndex length = CFArrayGetCount(array);
1692                 if (length > 0) {
1693                     CFTypeRef head = CFArrayGetValueAtIndex(array, 0);
1694                     if (CFEqualSafe(head, kCFNull)) {
1695                         SecDbAppendWhereOrAndNotIn(sql, attr->name, &needWhere, length - 1);
1696                     } else {
1697                         SecDbAppendWhereOrAndIn(sql, attr->name, &needWhere, length);
1698                     }
1699                 }
1700             } else {
1701                 SecDbAppendWhereOrAndEquals(sql, attr->name, &needWhere);
1702             }
1703         }
1704     }
1705     // Append SQL for access groups and limits.
1706     if (add_where_sql)
1707         add_where_sql(sql, &needWhere);
1708
1709     return sql;
1710 }
1711
1712 static bool SecDbItemSelectBindValue(SecDbQueryRef query, sqlite3_stmt *stmt, int param, const SecDbAttr *attr, CFTypeRef inValue, CFErrorRef *error) {
1713     bool ok = true;
1714     CFTypeRef value = NULL;
1715     if (attr->kind == kSecDbRowIdAttr) {
1716         // TODO: Ignores inValue and uses rowid directly instead HACK should go
1717         value = CFNumberCreate(NULL, kCFNumberSInt64Type, &query->q_row_id);
1718     } else {
1719         value = SecDbAttrCopyValueForDb(attr, inValue, error);
1720     }
1721     ok = ok && value != NULL && SecDbBindObject(stmt, ++param, value, error);
1722     CFReleaseSafe(value);
1723     return ok;
1724 }
1725
1726 bool SecDbItemSelectBind(SecDbQueryRef query, sqlite3_stmt *stmt, CFErrorRef *error,
1727                          bool (^use_attr_in_where)(const SecDbAttr *attr),
1728                          bool (^bind_added_where)(sqlite3_stmt *stmt, int col)) {
1729     __block bool ok = true;
1730     __block int param = 0;
1731     SecDbForEachAttr(query->q_class, attr) {
1732         if (use_attr_in_where(attr)) {
1733             CFTypeRef value = CFDictionaryGetValue(query->q_item, attr->name);
1734             if (isArray(value)) {
1735                 CFArrayRef array = (CFArrayRef)value;
1736                 CFRange range = {.location = 0, .length = CFArrayGetCount(array) };
1737                 if (range.length > 0) {
1738                     CFTypeRef head = CFArrayGetValueAtIndex(array, 0);
1739                     if (CFEqualSafe(head, kCFNull)) {
1740                         range.length--;
1741                         range.location++;
1742                     }
1743                 }
1744                 CFArrayApplyFunction(array, range, apply_block_1, (void (^)(const void *value)) ^(const void *value) {
1745                     ok = SecDbItemSelectBindValue(query, stmt, param++, attr, value, error);
1746                 });
1747             } else {
1748                 ok = SecDbItemSelectBindValue(query, stmt, param++, attr, value, error);
1749             }
1750
1751             if (!ok)
1752                 break;
1753         }
1754     }
1755     // TODO: Bind arguments for access groups and limits.
1756     if (bind_added_where)
1757         bind_added_where(stmt, ++param);
1758
1759     return ok;
1760 }
1761
1762 bool SecDbItemSelect(SecDbQueryRef query, SecDbConnectionRef dbconn, CFErrorRef *error,
1763                      bool (^return_attr)(const SecDbAttr *attr),
1764                      bool (^use_attr_in_where)(const SecDbAttr *attr),
1765                      bool (^add_where_sql)(CFMutableStringRef sql, bool *needWhere),
1766                      bool (^bind_added_where)(sqlite3_stmt *stmt, int col),
1767                      void (^handle_row)(SecDbItemRef item, bool *stop)) {
1768     __block bool ok = true;
1769     if (return_attr == NULL) {
1770         return_attr = ^bool (const SecDbAttr * attr) {
1771             return attr->kind == kSecDbRowIdAttr || attr->kind == kSecDbEncryptedDataAttr || attr->kind == kSecDbSHA1Attr;
1772         };
1773     }
1774
1775     CFStringRef sql = SecDbItemCopySelectSQL(query, return_attr, use_attr_in_where, add_where_sql);
1776     if (sql) {
1777         ok &= SecDbPrepare(dbconn, sql, error, ^(sqlite3_stmt *stmt) {
1778             ok = (SecDbItemSelectBind(query, stmt, error, use_attr_in_where, bind_added_where) &&
1779                   SecDbStep(dbconn, stmt, error, ^(bool *stop) {
1780                 SecDbItemRef item = SecDbItemCreateWithStatement(kCFAllocatorDefault, query->q_class, stmt, query->q_keybag, error, return_attr);
1781                 if (item) {
1782                     CFRetainAssign(item->credHandle, query->q_use_cred_handle);
1783                     handle_row(item, stop);
1784                     CFRelease(item);
1785                 } else {
1786                     //*stop = true;
1787                     //ok = false;
1788                 }
1789             }));
1790         });
1791         CFRelease(sql);
1792     } else {
1793         ok = false;
1794     }
1795     return ok;
1796 }
1797