OSX/utilities/src/SecDb.c

   1 /*
   2  * Copyright (c) 2012-2014 Apple Inc. All Rights Reserved.
   3  *
   4  * @APPLE_LICENSE_HEADER_START@
   5  *
   6  * This file contains Original Code and/or Modifications of Original Code
   7  * as defined in and that are subject to the Apple Public Source License
   8  * Version 2.0 (the 'License'). You may not use this file except in
   9  * compliance with the License. Please obtain a copy of the License at
  10  * http://www.opensource.apple.com/apsl/ and read it before using this
  11  * file.
  12  *
  13  * The Original Code and all software distributed under the License are
  14  * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  15  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  16  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
  17  * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
  18  * Please see the License for the specific language governing rights and
  19  * limitations under the License.
  20  *
  21  * @APPLE_LICENSE_HEADER_END@
  22  */
  23
  24
  25 #include "SecDb.h"
  26 #include "debugging.h"
  27
  28 #include <sqlite3.h>
  29 #include <sqlite3_private.h>
  30 #include <CoreFoundation/CoreFoundation.h>
  31 #include <libgen.h>
  32 #include <sys/stat.h>
  33 #include <AssertMacros.h>
  34 #include "SecCFWrappers.h"
  35 #include "SecCFError.h"
  36 #include "SecIOFormat.h"
  37 #include <stdio.h>
  38 #include "Security/SecBase.h"
  39
  40
  41 //
  42 // Architecturally inverted files
  43 // These are in SecureObjectSync but utilities depends on them
  44 // <rdar://problem/20802079> Fix layer violation (SOSDigestVector, SOSManifest, SecDB.c)
  45 //
  46 #include <Security/SecureObjectSync/SOSDigestVector.h>
  47 #include <Security/SecureObjectSync/SOSManifest.h>
  48
  49 #define LOGE(ARG,...) secerror(ARG, ## __VA_ARGS__)
  50 #define LOGV(ARG,...) secdebug("secdb", ARG, ## __VA_ARGS__)
  51 #define LOGD(ARG,...) secdebug("secdb", ARG, ## __VA_ARGS__)
  52
  53 #define HAVE_UNLOCK_NOTIFY  0
  54 #define USE_BUSY_HANDLER  1
  55
  56 struct __OpaqueSecDbStatement {
  57     CFRuntimeBase _base;
  58
  59     SecDbConnectionRef dbconn;
  60     sqlite3_stmt *stmt;
  61 };
  62
  63 struct __OpaqueSecDbConnection {
  64     CFRuntimeBase _base;
  65
  66     //CFMutableDictionaryRef statements;
  67
  68     SecDbRef db;     // NONRETAINED, since db or block retains us
  69     bool readOnly;
  70     bool inTransaction;
  71     SecDbTransactionSource source;
  72     bool isCorrupted;
  73     sqlite3 *handle;
  74     // Pending deletions and additions for the current transaction
  75     // Entires are either:
  76     // 1) a CFArrayRef of 1 element representing a deletion,
  77     // 2) a CFArrayRef of 2 elements representing the element 0 having been replaced with element 1
  78     // 3) a CFTypeRef that is not a CFArrayRef, representing an add of the element in question.
  79     CFMutableArrayRef changes;
  80 };
  81
  82 struct __OpaqueSecDb {
  83     CFRuntimeBase _base;
  84
  85     CFStringRef db_path;
  86     dispatch_queue_t queue;
  87     CFMutableArrayRef connections;
  88     dispatch_semaphore_t write_semaphore;
  89     dispatch_semaphore_t read_semaphore;
  90     bool didFirstOpen;
  91     bool (^opened)(SecDbConnectionRef dbconn, bool didCreate, bool *callMeAgainForNextConnection, CFErrorRef *error);
  92     bool callOpenedHandlerForNextConnection;
  93     dispatch_queue_t notifyQueue;
  94     SecDBNotifyBlock notifyPhase;
  95 };
  96
  97 // MARK: Error domains and error helper functions
  98
  99 CFStringRef kSecDbErrorDomain = CFSTR("com.apple.utilities.sqlite3");
 100
 101 bool SecDbError(int sql_code, CFErrorRef *error, CFStringRef format, ...) {
 102     if (sql_code == SQLITE_OK) return true;
 103     if (error) {
 104         va_list args;
 105         CFIndex code = sql_code;
 106         CFErrorRef previousError = *error;
 107
 108         *error = NULL;
 109         va_start(args, format);
 110         SecCFCreateErrorWithFormatAndArguments(code, kSecDbErrorDomain, previousError, error, NULL, format, args);
 111         CFReleaseNull(previousError);
 112         va_end(args);
 113     }
 114     return false;
 115 }
 116
 117 bool SecDbErrorWithDb(int sql_code, sqlite3 *db, CFErrorRef *error, CFStringRef format, ...) {
 118     if (sql_code == SQLITE_OK) return true;
 119     if (error) {
 120         va_list args;
 121         va_start(args, format);
 122         CFStringRef message = CFStringCreateWithFormatAndArguments(kCFAllocatorDefault, NULL, format, args);
 123         va_end(args);
 124
 125         int extended_code = sqlite3_extended_errcode(db);
 126         if (sql_code == extended_code)
 127             SecDbError(sql_code, error, CFSTR("%@: [%d] %s"), message, sql_code, sqlite3_errmsg(db));
 128         else
 129             SecDbError(sql_code, error, CFSTR("%@: [%d->%d] %s"), message, sql_code, extended_code, sqlite3_errmsg(db));
 130         CFReleaseSafe(message);
 131     }
 132     return false;
 133 }
 134
 135 bool SecDbErrorWithStmt(int sql_code, sqlite3_stmt *stmt, CFErrorRef *error, CFStringRef format, ...) {
 136     if (sql_code == SQLITE_OK) return true;
 137     if (error) {
 138         va_list args;
 139         va_start(args, format);
 140         CFStringRef message = CFStringCreateWithFormatAndArguments(kCFAllocatorDefault, NULL, format, args);
 141         va_end(args);
 142
 143         sqlite3 *db = sqlite3_db_handle(stmt);
 144         const char *sql = sqlite3_sql(stmt);
 145         int extended_code = sqlite3_extended_errcode(db);
 146         if (sql_code == extended_code)
 147             SecDbError(sql_code, error, CFSTR("%@: [%d] %s sql: %s"), message, sql_code, sqlite3_errmsg(db), sql);
 148         else
 149             SecDbError(sql_code, error, CFSTR("%@: [%d->%d] %s sql: %s"), message, sql_code, extended_code, sqlite3_errmsg(db), sql);
 150         CFReleaseSafe(message);
 151     }
 152     return false;
 153 }
 154
 155
 156 // MARK: -
 157 // MARK: Static helper functions
 158
 159 static bool SecDbOpenHandle(SecDbConnectionRef dbconn, bool *created, CFErrorRef *error);
 160 static bool SecDbHandleCorrupt(SecDbConnectionRef dbconn, int rc, CFErrorRef *error);
 161
 162 #pragma mark -
 163 #pragma mark SecDbRef
 164
 165 static CFStringRef
 166 SecDbCopyFormatDescription(CFTypeRef value, CFDictionaryRef formatOptions)
 167 {
 168     SecDbRef db = (SecDbRef)value;
 169     return CFStringCreateWithFormat(kCFAllocatorDefault, NULL, CFSTR("<SecDb path:%@ connections: %@>"), db->db_path, db->connections);
 170 }
 171
 172
 173 static void
 174 SecDbDestroy(CFTypeRef value)
 175 {
 176     SecDbRef db = (SecDbRef)value;
 177     CFReleaseSafe(db->connections);
 178     CFReleaseSafe(db->db_path);
 179     dispatch_release(db->queue);
 180     dispatch_release(db->read_semaphore);
 181     dispatch_release(db->write_semaphore);
 182     if (db->opened)
 183         Block_release(db->opened);
 184 }
 185
 186 CFGiblisFor(SecDb)
 187
 188 SecDbRef
 189 SecDbCreate(CFStringRef dbName,
 190             bool (^opened)(SecDbConnectionRef dbconn, bool didCreate, bool *callMeAgainForNextConnection, CFErrorRef *error))
 191 {
 192     SecDbRef db = NULL;
 193
 194     db = CFTypeAllocate(SecDb, struct __OpaqueSecDb, kCFAllocatorDefault);
 195     require(db != NULL, done);
 196
 197     CFStringPerformWithCString(dbName, ^(const char *dbNameStr) {
 198         db->queue = dispatch_queue_create(dbNameStr, DISPATCH_QUEUE_SERIAL);
 199     });
 200     db->read_semaphore = dispatch_semaphore_create(kSecDbMaxReaders);
 201     db->write_semaphore = dispatch_semaphore_create(kSecDbMaxWriters);
 202     db->connections = CFArrayCreateMutableForCFTypes(kCFAllocatorDefault);
 203     db->opened = opened ? Block_copy(opened) : NULL;
 204     if (getenv("__OSINSTALL_ENVIRONMENT") != NULL) {
 205         // TODO: Move this code out of this layer
 206         LOGV("sqlDb: running from installer");
 207         db->db_path = CFSTR("file::memory:?cache=shared");
 208     } else {
 209         db->db_path = CFStringCreateCopy(kCFAllocatorDefault, dbName);
 210     }
 211 done:
 212     return db;
 213 }
 214
 215 CFIndex
 216 SecDbIdleConnectionCount(SecDbRef db) {
 217     __block CFIndex count = 0;
 218     dispatch_sync(db->queue, ^{
 219         count = CFArrayGetCount(db->connections);
 220     });
 221     return count;
 222 }
 223
 224 void SecDbSetNotifyPhaseBlock(SecDbRef db, dispatch_queue_t queue, SecDBNotifyBlock notifyPhase) {
 225     if (db->notifyQueue)
 226         dispatch_release(db->notifyQueue);
 227     if (db->notifyPhase)
 228         Block_release(db->notifyPhase);
 229
 230     if (queue) {
 231         db->notifyQueue = queue;
 232         dispatch_retain(db->notifyQueue);
 233     } else {
 234         db->notifyQueue = NULL;
 235     }
 236     if (notifyPhase)
 237         db->notifyPhase = Block_copy(notifyPhase);
 238     else
 239         db->notifyPhase = NULL;
 240 }
 241
 242 static void SecDbNotifyPhase(SecDbConnectionRef dbconn, SecDbTransactionPhase phase) {
 243     if (CFArrayGetCount(dbconn->changes)) {
 244         CFArrayRef changes = dbconn->changes;
 245         dbconn->changes = CFArrayCreateMutableForCFTypes(kCFAllocatorDefault);
 246         if (dbconn->db->notifyPhase)
 247             dbconn->db->notifyPhase(dbconn, phase, dbconn->source, changes);
 248         CFReleaseSafe(changes);
 249     }
 250 }
 251
 252 static void SecDbOnNotifyQueue(SecDbConnectionRef dbconn, void (^perform)()) {
 253     if (dbconn->db->notifyQueue) {
 254         dispatch_sync(dbconn->db->notifyQueue, ^{
 255             perform();
 256         });
 257     } else {
 258         perform();
 259     }
 260 }
 261
 262 CFStringRef SecDbGetPath(SecDbRef db) {
 263     return db->db_path;
 264 }
 265
 266
 267 #pragma mark -
 268 #pragma mark SecDbConnectionRef
 269
 270 static bool SecDbCheckCorrupted(SecDbConnectionRef dbconn)
 271 {
 272     __block bool isCorrupted = true;
 273     __block CFErrorRef error = NULL;
 274     SecDbPrepare(dbconn, CFSTR("PRAGMA integrity_check"), &error, ^(sqlite3_stmt *stmt) {
 275         SecDbStep(dbconn, stmt, &error, ^(bool *stop) {
 276             const char * result = (const char*)sqlite3_column_text(stmt, 0);
 277             if (result && strncasecmp(result, "ok", 3) == 0) {
 278                 isCorrupted = false;
 279             }
 280         });
 281     });
 282     if (error) {
 283         LOGV("sqlDb: warning error %@ when running integrity check", error);
 284         CFRelease(error);
 285     }
 286     return isCorrupted;
 287 }
 288
 289 static bool SecDbDidCreateFirstConnection(SecDbConnectionRef dbconn, bool didCreate, CFErrorRef *error)
 290 {
 291     LOGD("sqlDb: starting maintenance");
 292     bool ok = true;
 293
 294     if (!didCreate && !dbconn->isCorrupted) {
 295         dbconn->isCorrupted = SecDbCheckCorrupted(dbconn);
 296         if (dbconn->isCorrupted)
 297             secerror("integrity check=fail");
 298         else
 299             LOGD("sqlDb: integrity check=pass");
 300     }
 301
 302     if (!dbconn->isCorrupted && dbconn->db->opened) {
 303         CFErrorRef localError = NULL;
 304
 305         dbconn->db->callOpenedHandlerForNextConnection = false;
 306         ok = dbconn->db->opened(dbconn, didCreate, &dbconn->db->callOpenedHandlerForNextConnection, &localError);
 307
 308         if (!ok)
 309             secerror("opened block failed: %@", localError);
 310
 311         if (!dbconn->isCorrupted && error && *error == NULL) {
 312             *error = localError;
 313             localError = NULL;
 314         } else {
 315             secerror("opened block failed: error is released and lost");
 316             CFReleaseNull(localError);
 317         }
 318     }
 319
 320     if (dbconn->isCorrupted) {
 321         ok = SecDbHandleCorrupt(dbconn, 0, error);
 322     }
 323
 324     LOGD("sqlDb: finished maintenance");
 325     return ok;
 326 }
 327
 328 void SecDbCorrupt(SecDbConnectionRef dbconn)
 329 {
 330     dbconn->isCorrupted = true;
 331 }
 332
 333
 334 static uint8_t knownDbPathIndex(SecDbConnectionRef dbconn)
 335 {
 336
 337     if(CFEqual(dbconn->db->db_path, CFSTR("/Library/Keychains/keychain-2.db")))
 338         return 1;
 339     if(CFEqual(dbconn->db->db_path, CFSTR("/Library/Keychains/ocspcache.sqlite3")))
 340         return 2;
 341     if(CFEqual(dbconn->db->db_path, CFSTR("/Library/Keychains/TrustStore.sqlite3")))
 342         return 3;
 343     if(CFEqual(dbconn->db->db_path, CFSTR("/Library/Keychains/caissuercache.sqlite3")))
 344         return 4;
 345
 346     /* Unknown DB path */
 347     return 0;
 348 }
 349
 350
 351 // Return true if there was no error, returns false otherwise and set *error to an appropriate CFErrorRef.
 352 static bool SecDbConnectionCheckCode(SecDbConnectionRef dbconn, int code, CFErrorRef *error, CFStringRef desc, ...) {
 353     if (code == SQLITE_OK || code == SQLITE_DONE)
 354         return true;
 355
 356     if (error) {
 357         va_list args;
 358         va_start(args, desc);
 359         CFStringRef msg = CFStringCreateWithFormatAndArguments(kCFAllocatorDefault, NULL, desc, args);
 360         va_end(args);
 361         SecDbErrorWithDb(code, dbconn->handle, error, msg);
 362         CFRelease(msg);
 363     }
 364
 365     /* If it's already corrupted, don't try to recover */
 366     if (dbconn->isCorrupted) {
 367         CFStringRef reason = CFStringCreateWithFormat(kCFAllocatorDefault, NULL, CFSTR("SQL DB %@ is corrupted already. Not trying to recover"), dbconn->db->db_path);
 368         secerror("%@",reason);
 369         __security_simulatecrash(reason, __sec_exception_code_TwiceCorruptDb(knownDbPathIndex(dbconn)));
 370         CFReleaseSafe(reason);
 371         return false;
 372     }
 373
 374     dbconn->isCorrupted = (SQLITE_CORRUPT == code) || (SQLITE_NOTADB == code) || (SQLITE_IOERR == code) || (SQLITE_CANTOPEN == code);
 375     if (dbconn->isCorrupted) {
 376         /* Run integrity check and only make dbconn->isCorrupted true and
 377            run the corruption handler if the integrity check conclusively fails. */
 378         dbconn->isCorrupted = SecDbCheckCorrupted(dbconn);
 379         if (dbconn->isCorrupted) {
 380             secerror("operation returned code: %d integrity check=fail", code);
 381             SecDbHandleCorrupt(dbconn, code, error);
 382         } else {
 383             secerror("operation returned code: %d: integrity check=pass", code);
 384         }
 385     }
 386
 387     return false;
 388 }
 389
 390 #if HAVE_UNLOCK_NOTIFY
 391
 392 static void SecDbUnlockNotify(void **apArg, int nArg) {
 393     int i;
 394     for(i=0; i<nArg; i++) {
 395         dispatch_semaphore_t dsema = (dispatch_semaphore_t)apArg[i];
 396         dispatch_semaphore_signal(dsema);
 397     }
 398 }
 399
 400 static bool SecDbWaitForUnlockNotify(SecDbConnectionRef dbconn, sqlite3_stmt *stmt, CFErrorRef *error) {
 401     int rc;
 402     dispatch_semaphore_t dsema = dispatch_semaphore_create(0);
 403     rc = sqlite3_unlock_notify(dbconn->handle, SecDbUnlockNotify, dsema);
 404     assert(rc == SQLITE_LOCKED || rc == SQLITE_OK);
 405     if (rc == SQLITE_OK) {
 406         dispatch_semaphore_wait(dsema, DISPATCH_TIME_FOREVER);
 407     }
 408     dispatch_release(dsema);
 409     return (rc == SQLITE_OK
 410             ? true
 411             : (stmt
 412                ? SecDbErrorWithStmt(rc, stmt, error, CFSTR("sqlite3_unlock_notify"))
 413                : SecDbErrorWithDb(rc, dbconn->handle, error, CFSTR("sqlite3_unlock_notify"))));
 414 }
 415
 416 #endif
 417
 418 #if USE_BUSY_HANDLER
 419
 420 // Return 0 to stop retrying.
 421 static int SecDbHandleBusy(void *ctx, int retryCount) {
 422     SecDbConnectionRef dbconn __unused = ctx;
 423     struct timespec sleeptime = { .tv_sec = 0, .tv_nsec = 10000 };
 424     while (retryCount--) {
 425         // Double sleeptime until we hit one second then add one
 426         // second more every time we sleep.
 427         if (sleeptime.tv_sec) {
 428             sleeptime.tv_sec++;
 429         } else {
 430             sleeptime.tv_nsec *= 2;
 431             if (sleeptime.tv_nsec > NSEC_PER_SEC) {
 432                 sleeptime.tv_nsec = 0;
 433                 sleeptime.tv_sec++;
 434             }
 435         }
 436     }
 437     struct timespec unslept = {};
 438     nanosleep(&sleeptime, &unslept);
 439
 440     return 1;
 441 }
 442
 443 static bool SecDbBusyHandler(SecDbConnectionRef dbconn, CFErrorRef *error) {
 444     return SecDbErrorWithDb(sqlite3_busy_handler(dbconn->handle, SecDbHandleBusy, dbconn), dbconn->handle, error, CFSTR("busy_handler"));
 445 }
 446
 447 #endif // USE_BUSY_HANDLER
 448
 449 // Return true causes the operation to be tried again.
 450 static bool SecDbWaitIfNeeded(SecDbConnectionRef dbconn, int s3e, sqlite3_stmt *stmt, CFStringRef desc, struct timespec *sleeptime, CFErrorRef *error) {
 451 #if HAVE_UNLOCK_NOTIFY
 452     if (s3e == SQLITE_LOCKED) { // Optionally check for extended code being SQLITE_LOCKED_SHAREDCACHE
 453         return SecDbWaitForUnlockNotify(dbconn, stmt, error))
 454     }
 455 #endif
 456
 457 #if !USE_BUSY_HANDLER
 458     if (s3e == SQLITE_LOCKED || s3e == SQLITE_BUSY) {
 459         LOGV("sqlDb: %s", sqlite3_errmsg(dbconn->handle));
 460         while (s3e == SQLITE_LOCKED || s3e == SQLITE_BUSY) {
 461             struct timespec unslept = {};
 462             nanosleep(sleeptime, &unslept);
 463             s3e = SQLITE_OK;
 464             if (stmt)
 465                 s3e = sqlite3_reset(stmt);
 466
 467             // Double sleeptime until we hit one second the add one
 468             // second more every time we sleep.
 469             if (sleeptime->tv_sec) {
 470                 sleeptime->tv_sec++;
 471             } else {
 472                 sleeptime->tv_nsec *= 2;
 473                 if (sleeptime->tv_nsec > NSEC_PER_SEC) {
 474                     sleeptime->tv_nsec = 0;
 475                     sleeptime->tv_sec++;
 476                 }
 477             }
 478         }
 479         if (s3e)
 480             return SecDbErrorWithStmt(s3e, stmt, error, CFSTR("reset"));
 481     } else
 482 #endif // !USE_BUSY_HANDLER
 483     {
 484         return SecDbConnectionCheckCode(dbconn, s3e, error, desc);
 485     }
 486     return true;
 487 }
 488
 489 enum SecDbStepResult {
 490     kSecDbErrorStep = 0,
 491     kSecDbRowStep = 1,
 492     kSecDbDoneStep = 2,
 493 };
 494 typedef enum SecDbStepResult SecDbStepResult;
 495
 496 static SecDbStepResult _SecDbStep(SecDbConnectionRef dbconn, sqlite3_stmt *stmt, CFErrorRef *error) {
 497     assert(stmt != NULL);
 498     int s3e;
 499     struct timespec sleeptime = { .tv_sec = 0, .tv_nsec = 10000 };
 500     for (;;) {
 501         s3e = sqlite3_step(stmt);
 502         if (s3e == SQLITE_ROW)
 503             return kSecDbRowStep;
 504         else if (s3e == SQLITE_DONE)
 505             return kSecDbDoneStep;
 506         else if (!SecDbWaitIfNeeded(dbconn, s3e, stmt, CFSTR("step"), &sleeptime, error))
 507             return kSecDbErrorStep;
 508     };
 509 }
 510
 511 bool
 512 SecDbExec(SecDbConnectionRef dbconn, CFStringRef sql, CFErrorRef *error)
 513 {
 514     bool ok = true;
 515     CFRetain(sql);
 516     while (sql) {
 517         CFStringRef tail = NULL;
 518         if (ok) {
 519             sqlite3_stmt *stmt = SecDbCopyStmt(dbconn, sql, &tail, error);
 520             ok = stmt != NULL;
 521             if (stmt) {
 522                 SecDbStepResult sr;
 523                 while ((sr = _SecDbStep(dbconn, stmt, error)) == kSecDbRowStep);
 524                 if (sr == kSecDbErrorStep)
 525                     ok = false;
 526                 ok &= SecDbReleaseCachedStmt(dbconn, sql, stmt, error);
 527             }
 528         } else {
 529             // TODO We already have an error here we really just want the left over sql in it's userData
 530             ok = SecDbError(SQLITE_ERROR, error, CFSTR("Error with unexecuted sql remaining %@"), sql);
 531         }
 532         CFRelease(sql);
 533         sql = tail;
 534     }
 535     return ok;
 536 }
 537
 538 static bool SecDbBeginTransaction(SecDbConnectionRef dbconn, SecDbTransactionType type, CFErrorRef *error)
 539 {
 540     bool ok = true;
 541     CFStringRef query;
 542     switch (type) {
 543         case kSecDbImmediateTransactionType:
 544             query = CFSTR("BEGIN IMMEDATE");
 545             break;
 546         case kSecDbExclusiveRemoteTransactionType:
 547             dbconn->source = kSecDbSOSTransaction;
 548         case kSecDbExclusiveTransactionType:
 549             query = CFSTR("BEGIN EXCLUSIVE");
 550             break;
 551         case kSecDbNormalTransactionType:
 552             query = CFSTR("BEGIN");
 553             break;
 554         default:
 555             ok = SecDbError(SQLITE_ERROR, error, CFSTR("invalid transaction type %" PRIu32), type);
 556             query = NULL;
 557             break;
 558     }
 559
 560     if (query != NULL && sqlite3_get_autocommit(dbconn->handle) != 0) {
 561         ok = SecDbExec(dbconn, query, error);
 562     }
 563     if (ok)
 564         dbconn->inTransaction = true;
 565
 566     return ok;
 567 }
 568
 569 static bool SecDbEndTransaction(SecDbConnectionRef dbconn, bool commit, CFErrorRef *error)
 570 {
 571     __block bool ok = true;
 572     SecDbOnNotifyQueue(dbconn, ^{
 573         bool commited = false;
 574         if (commit) {
 575             SecDbNotifyPhase(dbconn, kSecDbTransactionWillCommit);
 576             commited = ok = SecDbExec(dbconn, CFSTR("END"), error);
 577         } else {
 578             ok = SecDbExec(dbconn, CFSTR("ROLLBACK"), error);
 579             commited = false;
 580         }
 581         dbconn->inTransaction = false;
 582         SecDbNotifyPhase(dbconn, commited ? kSecDbTransactionDidCommit : kSecDbTransactionDidRollback);
 583         dbconn->source = kSecDbAPITransaction;
 584     });
 585     return ok;
 586 }
 587
 588 bool SecDbTransaction(SecDbConnectionRef dbconn, SecDbTransactionType type,
 589                       CFErrorRef *error, void (^transaction)(bool *commit))
 590 {
 591     bool ok = true;
 592     bool commit = true;
 593
 594     if (dbconn->inTransaction) {
 595         transaction(&commit);
 596         if (!commit) {
 597             LOGV("sqlDb: nested transaction asked to not be committed");
 598         }
 599     } else {
 600         ok = SecDbBeginTransaction(dbconn, type, error);
 601         if (ok) {
 602             transaction(&commit);
 603             ok = SecDbEndTransaction(dbconn, commit, error);
 604         }
 605     }
 606
 607 done:
 608     return ok && commit;
 609 }
 610
 611 sqlite3 *SecDbHandle(SecDbConnectionRef dbconn) {
 612     return dbconn->handle;
 613 }
 614
 615 bool SecDbStep(SecDbConnectionRef dbconn, sqlite3_stmt *stmt, CFErrorRef *error, void (^row)(bool *stop)) {
 616     for (;;) {
 617         switch (_SecDbStep(dbconn, stmt, error)) {
 618             case kSecDbErrorStep:
 619                 return false;
 620             case kSecDbRowStep:
 621                 if (row) {
 622                     bool stop = false;
 623                     row(&stop);
 624                     if (stop)
 625                         return true;
 626                     break;
 627                 }
 628                 SecDbError(SQLITE_ERROR, error, CFSTR("SecDbStep SQLITE_ROW returned without a row handler"));
 629                 return false;
 630             case kSecDbDoneStep:
 631                 return true;
 632         }
 633     }
 634 }
 635
 636 bool SecDbCheckpoint(SecDbConnectionRef dbconn, CFErrorRef *error)
 637 {
 638     return SecDbConnectionCheckCode(dbconn, sqlite3_wal_checkpoint(dbconn->handle, NULL), error, CFSTR("wal_checkpoint"));
 639 }
 640
 641 static bool SecDbFileControl(SecDbConnectionRef dbconn, int op, void *arg, CFErrorRef *error) {
 642     return SecDbConnectionCheckCode(dbconn, sqlite3_file_control(dbconn->handle, NULL, op, arg), error, CFSTR("file_control"));
 643 }
 644
 645 static sqlite3 *_SecDbOpenV2(const char *path, int flags, CFErrorRef *error) {
 646 #if HAVE_UNLOCK_NOTIFY
 647     flags |= SQLITE_OPEN_SHAREDCACHE;
 648 #endif
 649     sqlite3 *handle = NULL;
 650     int s3e = sqlite3_open_v2(path, &handle, flags, NULL);
 651     if (s3e) {
 652         if (handle) {
 653             SecDbErrorWithDb(s3e, handle, error, CFSTR("open_v2 \"%s\" 0x%X"), path, flags);
 654             sqlite3_close(handle);
 655             handle = NULL;
 656         } else {
 657             SecDbError(s3e, error, CFSTR("open_v2 \"%s\" 0x%X"), path, flags);
 658         }
 659     }
 660     return handle;
 661 }
 662
 663 static bool SecDbOpenV2(SecDbConnectionRef dbconn, const char *path, int flags, CFErrorRef *error) {
 664     return (dbconn->handle = _SecDbOpenV2(path, flags, error)) != NULL;
 665 }
 666
 667 static bool SecDbTruncate(SecDbConnectionRef dbconn, CFErrorRef *error)
 668 {
 669     int flags = SQLITE_TRUNCATE_JOURNALMODE_WAL | SQLITE_TRUNCATE_AUTOVACUUM_FULL;
 670     __block bool ok = SecDbFileControl(dbconn, SQLITE_TRUNCATE_DATABASE, &flags, error);
 671     if (!ok) {
 672         sqlite3_close(dbconn->handle);
 673         dbconn->handle = NULL;
 674         CFStringPerformWithCString(dbconn->db->db_path, ^(const char *path) {
 675             if (error)
 676                 CFReleaseNull(*error);
 677             if (SecCheckErrno(unlink(path), error, CFSTR("unlink %s"), path)) {
 678                 ok = SecDbOpenHandle(dbconn, NULL, error);
 679             }
 680         });
 681         if (!ok) {
 682             secerror("Failed to delete db handle: %@", error ? *error : NULL);
 683             abort();
 684         }
 685     }
 686
 687     return ok;
 688 }
 689
 690 static bool SecDbHandleCorrupt(SecDbConnectionRef dbconn, int rc, CFErrorRef *error)
 691 {
 692     CFStringRef reason = CFStringCreateWithFormat(kCFAllocatorDefault, NULL, CFSTR("SQL DB %@ is corrupted, trying to recover (rc=%d)"), dbconn->db->db_path, rc);
 693     __security_simulatecrash(reason, __sec_exception_code_CorruptDb(knownDbPathIndex(dbconn), rc));
 694     CFReleaseSafe(reason);
 695
 696     // Backup current db.
 697     __block bool didRename = false;
 698     CFStringPerformWithCString(dbconn->db->db_path, ^(const char *db_path) {
 699         sqlite3 *corrupt_db = NULL;
 700         char buf[PATH_MAX+1];
 701         snprintf(buf, sizeof(buf), "%s-corrupt", db_path);
 702         if (dbconn->handle && (corrupt_db = _SecDbOpenV2(buf, SQLITE_OPEN_READWRITE, error))) {
 703             int on = 1;
 704             didRename =  SecDbErrorWithDb(sqlite3_file_control(corrupt_db, NULL, SQLITE_FCNTL_PERSIST_WAL, &on), corrupt_db, error, CFSTR("persist wal"));
 705             didRename &=  SecDbErrorWithDb(sqlite3_file_control(corrupt_db, NULL, SQLITE_REPLACE_DATABASE, (void *)dbconn->handle), corrupt_db, error, CFSTR("replace database"));
 706             sqlite3_close(corrupt_db);
 707         }
 708         if (!didRename) {
 709             if (dbconn->handle)
 710                 secerror("Tried to rename corrupt database at path %@, but we failed: %@, trying explicit rename", dbconn->db->db_path, error ? *error : NULL);
 711             if (error)
 712                 CFReleaseNull(*error);
 713
 714             didRename = SecCheckErrno(rename(db_path, buf), error, CFSTR("rename %s %s"), db_path, buf) &&
 715                 (!dbconn->handle || SecDbError(sqlite3_close(dbconn->handle), error, CFSTR("close"))) &&
 716                 SecDbOpenHandle(dbconn, NULL, error);
 717         }
 718         if (didRename) {
 719             secerror("Database at path %@ is corrupt. Copied it to %s for further investigation.", dbconn->db->db_path, buf);
 720         } else {
 721             seccritical("Tried to copy corrupt database at path %@, but we failed: %@", dbconn->db->db_path, error ? *error : NULL);
 722         }
 723     });
 724
 725     bool ok = (didRename &&
 726                (dbconn->handle || SecDbOpenHandle(dbconn, NULL, error)) &&
 727                SecDbTruncate(dbconn, error));
 728
 729     // Mark the db as not corrupted, even if something failed.
 730     // Always note we are no longer in the corruption handler
 731     dbconn->isCorrupted = false;
 732
 733     // Invoke our callers opened callback, since we just created a new database
 734     if (ok && dbconn->db->opened) {
 735         dbconn->db->callOpenedHandlerForNextConnection = false;
 736         ok = dbconn->db->opened(dbconn, true, &dbconn->db->callOpenedHandlerForNextConnection, error);
 737     }
 738
 739     return ok;
 740 }
 741
 742 static bool SecDbProfileEnabled(void)
 743 {
 744 #if 0
 745     static dispatch_once_t onceToken;
 746     static bool profile_enabled = false;
 747
 748 #if DEBUG
 749     //sudo defaults write /Library/Preferences/com.apple.security.auth profile -bool true
 750     dispatch_once(&onceToken, ^{
 751                 CFTypeRef profile = (CFNumberRef)CFPreferencesCopyValue(CFSTR("profile"), CFSTR(SECURITY_AUTH_NAME), kCFPreferencesAnyUser, kCFPreferencesCurrentHost);
 752
 753         if (profile && CFGetTypeID(profile) == CFBooleanGetTypeID()) {
 754             profile_enabled = CFBooleanGetValue((CFBooleanRef)profile);
 755         }
 756
 757         LOGV("sqlDb: sql profile: %s", profile_enabled ? "enabled" : "disabled");
 758
 759         CFReleaseSafe(profile);
 760     });
 761 #endif
 762
 763     return profile_enabled;
 764 #else
 765 #if DEBUG
 766     return true;
 767 #else
 768     return false;
 769 #endif
 770 #endif
 771 }
 772
 773 #if 0
 774 static void SecDbProfile(void *context __unused, const char *sql, sqlite3_uint64 ns) {
 775     LOGV("==\nsqlDb: %s\nTime: %llu ms\n", sql, ns >> 20);
 776 }
 777 #else
 778 static void SecDbProfile(void *context, const char *sql, sqlite3_uint64 ns) {
 779     sqlite3 *s3h = context;
 780     int code = sqlite3_extended_errcode(s3h);
 781     if (code == SQLITE_OK || code == SQLITE_DONE) {
 782         secdebug("profile", "==\nsqlDb: %s\nTime: %llu ms\n", sql, ns >> 20);
 783     } else {
 784         secdebug("profile", "==error[%d]: %s==\nsqlDb: %s\nTime: %llu ms \n", code, sqlite3_errmsg(s3h), sql, ns >> 20);
 785     }
 786 }
 787 #endif
 788
 789 static bool SecDbTraceEnabled(void)
 790 {
 791 #if DEBUG
 792     return true;
 793 #else
 794     return false;
 795 #endif
 796 }
 797
 798 static void SecDbTrace(void *ctx, const char *trace) {
 799     SecDbConnectionRef dbconn __unused = ctx;
 800     static dispatch_queue_t queue;
 801     static dispatch_once_t once;
 802     dispatch_once(&once, ^{
 803         queue = dispatch_queue_create("trace_queue", DISPATCH_QUEUE_SERIAL);
 804     });
 805     dispatch_sync(queue, ^{
 806         __security_debug(CFSTR("trace"), "", "", 0, CFSTR("%s"), trace);
 807     });
 808 }
 809
 810 static bool SecDbOpenHandle(SecDbConnectionRef dbconn, bool *created, CFErrorRef *error)
 811 {
 812     __block bool ok = true;
 813     CFStringPerformWithCString(dbconn->db->db_path, ^(const char *db_path) {
 814         ok = created && SecDbOpenV2(dbconn, db_path, SQLITE_OPEN_READWRITE, NULL);
 815         if (!ok) {
 816             ok = true;
 817             if (created) {
 818                 char *tmp = dirname((char *)db_path);
 819                 if (tmp) {
 820                     int errnum = mkpath_np(tmp, 0700);
 821                     if (errnum != 0 && errnum != EEXIST) {
 822                         SecCFCreateErrorWithFormat(errnum, kSecErrnoDomain, NULL, error, NULL,
 823                                                    CFSTR("mkpath_np %s: [%d] %s"), tmp, errnum, strerror(errnum));
 824                         ok = false;
 825                     }
 826                 }
 827             }
 828             ok = ok && SecDbOpenV2(dbconn, db_path, SQLITE_OPEN_READWRITE | SQLITE_OPEN_CREATE, error);
 829             if (ok) {
 830                 chmod(db_path, S_IRUSR | S_IWUSR);
 831                 if (created)
 832                     *created = true;
 833             }
 834         }
 835
 836         if (ok && SecDbProfileEnabled()) {
 837             sqlite3_profile(dbconn->handle, SecDbProfile, dbconn->handle);
 838         }
 839         if (ok && SecDbTraceEnabled()) {
 840             sqlite3_trace(dbconn->handle, SecDbTrace, dbconn);
 841         }
 842 #if USE_BUSY_HANDLER
 843         ok = ok && SecDbBusyHandler(dbconn, error);
 844 #endif
 845     });
 846
 847 done:
 848     return ok;
 849 }
 850
 851 static SecDbConnectionRef
 852 SecDbConnectionCreate(SecDbRef db, bool readOnly, CFErrorRef *error)
 853 {
 854     SecDbConnectionRef dbconn = NULL;
 855
 856     dbconn = CFTypeAllocate(SecDbConnection, struct __OpaqueSecDbConnection, kCFAllocatorDefault);
 857     require(dbconn != NULL, done);
 858
 859     dbconn->db = db;
 860     dbconn->readOnly = readOnly;
 861     dbconn->changes = CFArrayCreateMutableForCFTypes(kCFAllocatorDefault);
 862
 863 done:
 864     return dbconn;
 865 }
 866
 867 static bool SecDbConnectionIsReadOnly(SecDbConnectionRef dbconn) {
 868     return dbconn->readOnly;
 869 }
 870
 871 static void SecDbConectionSetReadOnly(SecDbConnectionRef dbconn, bool readOnly) {
 872     dbconn->readOnly = readOnly;
 873 }
 874
 875 /* Read only connections go to the end of the queue, writeable connections
 876  go to the start of the queue. */
 877 SecDbConnectionRef SecDbConnectionAquire(SecDbRef db, bool readOnly, CFErrorRef *error) {
 878     CFRetain(db);
 879     secdebug("dbconn", "acquire %s connection", readOnly ? "ro" : "rw");
 880     dispatch_semaphore_wait(readOnly ? db->read_semaphore : db->write_semaphore, DISPATCH_TIME_FOREVER);
 881     __block SecDbConnectionRef dbconn = NULL;
 882     __block bool ok = true;
 883     __block bool ranOpenedHandler = false;
 884     dispatch_sync(db->queue, ^{
 885         if (!db->didFirstOpen) {
 886             bool didCreate = false;
 887             ok = dbconn = SecDbConnectionCreate(db, false, error);
 888             CFErrorRef localError = NULL;
 889             if (ok && !SecDbOpenHandle(dbconn, &didCreate, &localError)) {
 890                 secerror("Unable to create database: %@", localError);
 891                 if (localError && CFEqual(CFErrorGetDomain(localError), kSecDbErrorDomain)) {
 892                     int code = (int)CFErrorGetCode(localError);
 893                     dbconn->isCorrupted = (SQLITE_CORRUPT == code) || (SQLITE_NOTADB == code) || (SQLITE_IOERR == code) || (SQLITE_CANTOPEN == code);
 894                 }
 895                 // If the open failure isn't due to corruption, propagte the error.
 896                 ok = dbconn->isCorrupted;
 897                 if (!ok && error && *error == NULL) {
 898                     *error = localError;
 899                     localError = NULL;
 900                 }
 901             }
 902             CFReleaseNull(localError);
 903
 904             if (ok) {
 905                 db->didFirstOpen = ok = SecDbDidCreateFirstConnection(dbconn, didCreate, error);
 906                 ranOpenedHandler = true;
 907             }
 908             if (!ok)
 909                 CFReleaseNull(dbconn);
 910         } else {
 911             /* Try to get one from the cache */
 912             CFIndex count = CFArrayGetCount(db->connections);
 913             while (count && !dbconn) {
 914                 CFIndex ix = readOnly ? count - 1 : 0;
 915                 dbconn = (SecDbConnectionRef)CFArrayGetValueAtIndex(db->connections, ix);
 916                 if (dbconn)
 917                     CFRetain(dbconn);
 918                 else
 919                     secerror("got NULL dbconn at index: %" PRIdCFIndex " skipping", ix);
 920                 CFArrayRemoveValueAtIndex(db->connections, ix);
 921             }
 922         }
 923     });
 924
 925     if (dbconn) {
 926         /* Make sure the connection we found has the right access */
 927         if (SecDbConnectionIsReadOnly(dbconn) != readOnly) {
 928             SecDbConectionSetReadOnly(dbconn, readOnly);
 929         }
 930     } else if (ok) {
 931         /* Nothing found in cache, create a new connection */
 932         bool created = false;
 933         dbconn = SecDbConnectionCreate(db, readOnly, error);
 934         if (dbconn && !SecDbOpenHandle(dbconn, &created, error)) {
 935             CFReleaseNull(dbconn);
 936         }
 937     }
 938
 939     if (dbconn && !ranOpenedHandler && dbconn->db->opened) {
 940         dispatch_sync(db->queue, ^{
 941             if (dbconn->db->callOpenedHandlerForNextConnection) {
 942                 dbconn->db->callOpenedHandlerForNextConnection = false;
 943                 if (!dbconn->db->opened(dbconn, false, &dbconn->db->callOpenedHandlerForNextConnection, error)) {
 944                     if (!dbconn->isCorrupted || !SecDbHandleCorrupt(dbconn, 0, error)) {
 945                         CFReleaseNull(dbconn);
 946                     }
 947                 }
 948             }
 949         });
 950     }
 951
 952     if (!dbconn) {
 953         // If aquire fails we need to signal the semaphore again.
 954         dispatch_semaphore_signal(readOnly ? db->read_semaphore : db->write_semaphore);
 955         CFRelease(db);
 956     }
 957
 958     return dbconn;
 959 }
 960
 961 void SecDbConnectionRelease(SecDbConnectionRef dbconn) {
 962     if (!dbconn) {
 963         secerror("called with NULL dbconn");
 964         return;
 965     }
 966     SecDbRef db = dbconn->db;
 967     secdebug("dbconn", "release %@", dbconn);
 968     dispatch_sync(db->queue, ^{
 969         CFIndex count = CFArrayGetCount(db->connections);
 970         // Add back possible writable dbconn to the pool.
 971         bool readOnly = SecDbConnectionIsReadOnly(dbconn);
 972         CFArrayInsertValueAtIndex(db->connections, readOnly ? count : 0, dbconn);
 973         // Remove the last (probably read-only) dbconn from the pool.
 974         if (count >= kSecDbMaxIdleHandles) {
 975             CFArrayRemoveValueAtIndex(db->connections, count);
 976         }
 977         // Signal after we have put the connection back in the pool of connections
 978         dispatch_semaphore_signal(readOnly ? db->read_semaphore : db->write_semaphore);
 979         CFRelease(dbconn);
 980         CFRelease(db);
 981     });
 982 }
 983
 984 bool SecDbPerformRead(SecDbRef db, CFErrorRef *error, void (^perform)(SecDbConnectionRef dbconn)) {
 985     SecDbConnectionRef dbconn = SecDbConnectionAquire(db, true, error);
 986     bool success = false;
 987     if (dbconn) {
 988         perform(dbconn);
 989         success = true;
 990         SecDbConnectionRelease(dbconn);
 991     }
 992     return success;
 993 }
 994
 995 bool SecDbPerformWrite(SecDbRef db, CFErrorRef *error, void (^perform)(SecDbConnectionRef dbconn)) {
 996     SecDbConnectionRef dbconn = SecDbConnectionAquire(db, false, error);
 997     bool success = false;
 998     if (dbconn) {
 999         perform(dbconn);
1000         success = true;
1001         SecDbConnectionRelease(dbconn);
1002     }
1003     return success;
1004 }
1005
1006 static CFStringRef
1007 SecDbConnectionCopyFormatDescription(CFTypeRef value, CFDictionaryRef formatOptions)
1008 {
1009     SecDbConnectionRef dbconn = (SecDbConnectionRef)value;
1010     return CFStringCreateWithFormat(kCFAllocatorDefault, NULL, CFSTR("<SecDbConnection %s %s>"),
1011                                     dbconn->readOnly ? "ro" : "rw", dbconn->handle ? "open" : "closed");
1012 }
1013
1014 static void
1015 SecDbConnectionDestroy(CFTypeRef value)
1016 {
1017     SecDbConnectionRef dbconn = (SecDbConnectionRef)value;
1018     if (dbconn->handle) {
1019         sqlite3_close(dbconn->handle);
1020     }
1021     dbconn->db = NULL;
1022     CFReleaseNull(dbconn->changes);
1023 }
1024
1025
1026 // MARK: -
1027 // MARK: Bind helpers
1028
1029 #if 0
1030 bool SecDbBindNull(sqlite3_stmt *stmt, int param, CFErrorRef *error) {
1031     bool ok = SecDbErrorWithStmt(sqlite3_bind_null(stmt, param),
1032                                  stmt, error, CFSTR("bind_null[%d]"), param);
1033     secdebug("bind", "bind_null[%d]: %@", param, error ? *error : NULL);
1034     return ok;
1035 }
1036 #endif
1037
1038 bool SecDbBindBlob(sqlite3_stmt *stmt, int param, const void *zData, size_t n, void(*xDel)(void*), CFErrorRef *error) {
1039     if (n > INT_MAX) {
1040         return SecDbErrorWithStmt(SQLITE_TOOBIG, stmt, error,
1041                                   CFSTR("bind_blob[%d]: blob bigger than INT_MAX"), param);
1042     }
1043     bool ok = SecDbErrorWithStmt(sqlite3_bind_blob(stmt, param, zData, (int)n, xDel),
1044                                  stmt, error, CFSTR("bind_blob[%d]"), param);
1045     secdebug("bind", "bind_blob[%d]: %.*s: %@", param, (int)n, zData, error ? *error : NULL);
1046     return ok;
1047 }
1048
1049 bool SecDbBindText(sqlite3_stmt *stmt, int param, const char *zData, size_t n, void(*xDel)(void*), CFErrorRef *error) {
1050     if (n > INT_MAX) {
1051         return SecDbErrorWithStmt(SQLITE_TOOBIG, stmt, error,
1052                                   CFSTR("bind_text[%d]: text bigger than INT_MAX"), param);
1053     }
1054     bool ok = SecDbErrorWithStmt(sqlite3_bind_text(stmt, param, zData, (int)n, xDel), stmt, error,
1055                                  CFSTR("bind_text[%d]"), param);
1056     secdebug("bind", "bind_text[%d]: \"%s\": %@", param, zData, error ? *error : NULL);
1057     return ok;
1058 }
1059
1060 bool SecDbBindDouble(sqlite3_stmt *stmt, int param, double value, CFErrorRef *error) {
1061     bool ok = SecDbErrorWithStmt(sqlite3_bind_double(stmt, param, value), stmt, error,
1062                                  CFSTR("bind_double[%d]"), param);
1063     secdebug("bind", "bind_double[%d]: %f: %@", param, value, error ? *error : NULL);
1064     return ok;
1065 }
1066
1067 bool SecDbBindInt(sqlite3_stmt *stmt, int param, int value, CFErrorRef *error) {
1068     bool ok = SecDbErrorWithStmt(sqlite3_bind_int(stmt, param, value), stmt, error,
1069                                  CFSTR("bind_int[%d]"), param);
1070     secdebug("bind", "bind_int[%d]: %d: %@", param, value, error ? *error : NULL);
1071     return ok;
1072 }
1073
1074 bool SecDbBindInt64(sqlite3_stmt *stmt, int param, sqlite3_int64 value, CFErrorRef *error) {
1075     bool ok = SecDbErrorWithStmt(sqlite3_bind_int64(stmt, param, value), stmt, error,
1076                                  CFSTR("bind_int64[%d]"), param);
1077     secdebug("bind", "bind_int64[%d]: %lld: %@", param, value, error ? *error : NULL);
1078     return ok;
1079 }
1080
1081
1082 /* AUDIT[securityd](done):
1083  value (ok) is a caller provided, non NULL CFTypeRef.
1084  */
1085 bool SecDbBindObject(sqlite3_stmt *stmt, int param, CFTypeRef value, CFErrorRef *error) {
1086     CFTypeID valueId;
1087     __block bool result = false;
1088
1089         /* TODO: Can we use SQLITE_STATIC below everwhere we currently use
1090      SQLITE_TRANSIENT since we finalize the statement before the value
1091      goes out of scope? */
1092     if (!value || (valueId = CFGetTypeID(value)) == CFNullGetTypeID()) {
1093         /* Skip bindings for NULL values.  sqlite3 will interpret unbound
1094          params as NULL which is exactly what we want. */
1095 #if 1
1096         result = true;
1097 #else
1098         result = SecDbBindNull(stmt, param, error);
1099 #endif
1100     } else if (valueId == CFStringGetTypeID()) {
1101         CFStringPerformWithCStringAndLength(value, ^(const char *cstr, size_t clen) {
1102             result = SecDbBindText(stmt, param, cstr, clen, SQLITE_TRANSIENT, error);
1103         });
1104     } else if (valueId == CFDataGetTypeID()) {
1105         CFIndex len = CFDataGetLength(value);
1106         if (len) {
1107             result = SecDbBindBlob(stmt, param, CFDataGetBytePtr(value),
1108                                    len, SQLITE_TRANSIENT, error);
1109         } else {
1110             result = SecDbBindText(stmt, param, "", 0, SQLITE_TRANSIENT, error);
1111         }
1112     } else if (valueId == CFDateGetTypeID()) {
1113         CFAbsoluteTime abs_time = CFDateGetAbsoluteTime(value);
1114         result = SecDbBindDouble(stmt, param, abs_time, error);
1115     } else if (valueId == CFBooleanGetTypeID()) {
1116         int bval = CFBooleanGetValue(value);
1117         result = SecDbBindInt(stmt, param, bval, error);
1118     } else if (valueId == CFNumberGetTypeID()) {
1119         Boolean convertOk;
1120         if (CFNumberIsFloatType(value)) {
1121             double nval;
1122             convertOk = CFNumberGetValue(value, kCFNumberDoubleType, &nval);
1123             result = SecDbBindDouble(stmt, param, nval, error);
1124         } else {
1125             int nval;
1126             convertOk = CFNumberGetValue(value, kCFNumberSInt32Type, &nval);
1127             if (convertOk) {
1128                 result = SecDbBindInt(stmt, param, nval, error);
1129             } else {
1130                 sqlite_int64 nval64;
1131                 convertOk = CFNumberGetValue(value, kCFNumberSInt64Type, &nval64);
1132                 if (convertOk)
1133                     result = SecDbBindInt64(stmt, param, nval64, error);
1134             }
1135         }
1136         if (!convertOk) {
1137             result = SecDbError(SQLITE_INTERNAL, error, CFSTR("bind CFNumberGetValue failed for %@"), value);
1138         }
1139     } else {
1140         if (error) {
1141             CFStringRef valueDesc = CFCopyTypeIDDescription(valueId);
1142             SecDbError(SQLITE_MISMATCH, error, CFSTR("bind unsupported type %@"), valueDesc);
1143             CFReleaseSafe(valueDesc);
1144         }
1145     }
1146
1147         return result;
1148 }
1149
1150 // MARK: -
1151 // MARK: SecDbStatementRef
1152
1153 bool SecDbReset(sqlite3_stmt *stmt, CFErrorRef *error) {
1154     return SecDbErrorWithStmt(sqlite3_reset(stmt), stmt, error, CFSTR("reset"));
1155 }
1156
1157 bool SecDbClearBindings(sqlite3_stmt *stmt, CFErrorRef *error) {
1158     return SecDbErrorWithStmt(sqlite3_clear_bindings(stmt), stmt, error, CFSTR("clear bindings"));
1159 }
1160
1161 bool SecDbFinalize(sqlite3_stmt *stmt, CFErrorRef *error) {
1162     sqlite3 *handle = sqlite3_db_handle(stmt);
1163     int s3e = sqlite3_finalize(stmt);
1164     return s3e == SQLITE_OK ? true : SecDbErrorWithDb(s3e, handle, error, CFSTR("finalize: %p"), stmt);
1165 }
1166
1167 sqlite3_stmt *SecDbPrepareV2(SecDbConnectionRef dbconn, const char *sql, size_t sqlLen, const char **sqlTail, CFErrorRef *error) {
1168     sqlite3 *db = SecDbHandle(dbconn);
1169     if (sqlLen > INT_MAX) {
1170         SecDbErrorWithDb(SQLITE_TOOBIG, db, error, CFSTR("prepare_v2: sql bigger than INT_MAX"));
1171         return NULL;
1172     }
1173     struct timespec sleeptime = { .tv_sec = 0, .tv_nsec = 10000 };
1174     for (;;) {
1175         sqlite3_stmt *stmt = NULL;
1176         int s3e = sqlite3_prepare_v2(db, sql, (int)sqlLen, &stmt, sqlTail);
1177         if (s3e == SQLITE_OK)
1178             return stmt;
1179         else if (!SecDbWaitIfNeeded(dbconn, s3e, NULL, CFSTR("preparev2"), &sleeptime, error))
1180             return NULL;
1181     }
1182 }
1183
1184 static sqlite3_stmt *SecDbCopyStatementWithTailRange(SecDbConnectionRef dbconn, CFStringRef sql, CFRange *sqlTail, CFErrorRef *error) {
1185     __block sqlite3_stmt *stmt = NULL;
1186     if (sql) CFStringPerformWithCStringAndLength(sql, ^(const char *sqlStr, size_t sqlLen) {
1187         const char *tail = NULL;
1188         stmt = SecDbPrepareV2(dbconn, sqlStr, sqlLen, &tail, error);
1189         if (sqlTail && sqlStr < tail && tail < sqlStr + sqlLen) {
1190             sqlTail->location = tail - sqlStr;
1191             sqlTail->length = sqlLen - sqlTail->location;
1192         }
1193     });
1194
1195     return stmt;
1196 }
1197
1198 sqlite3_stmt *SecDbCopyStmt(SecDbConnectionRef dbconn, CFStringRef sql, CFStringRef *tail, CFErrorRef *error) {
1199     // TODO: Add caching and cache lookup of statements
1200     CFRange sqlTail = {};
1201     sqlite3_stmt *stmt = SecDbCopyStatementWithTailRange(dbconn, sql, &sqlTail, error);
1202     if (sqlTail.length > 0) {
1203         CFStringRef excess = CFStringCreateWithSubstring(CFGetAllocator(sql), sql, sqlTail);
1204         if (tail) {
1205             *tail = excess;
1206         } else {
1207             SecDbError(SQLITE_INTERNAL, error,
1208                        CFSTR("prepare_v2: %@ unused sql: %@"),
1209                        sql, excess);
1210             CFReleaseSafe(excess);
1211             SecDbFinalize(stmt, error);
1212             stmt = NULL;
1213         }
1214     }
1215     return stmt;
1216 }
1217
1218 /*
1219  TODO: Could do a hack here with a custom kCFAllocatorNULL allocator for a second CFRuntimeBase inside a SecDbStatement,
1220  TODO: Better yet make a full blow SecDbStatement instance whenever SecDbCopyStmt is called.  Then, when the statement is released, in the Dispose method, we Reset and ClearBindings the sqlite3_stmt * and hand it back to the SecDb with the original CFStringRef for the sql (or hash thereof) as an argument. */
1221 bool SecDbReleaseCachedStmt(SecDbConnectionRef dbconn, CFStringRef sql, sqlite3_stmt *stmt, CFErrorRef *error) {
1222     if (stmt) {
1223         return SecDbFinalize(stmt, error);
1224     }
1225     return true;
1226 }
1227
1228 bool SecDbPrepare(SecDbConnectionRef dbconn, CFStringRef sql, CFErrorRef *error, void(^exec)(sqlite3_stmt *stmt)) {
1229     assert(sql != NULL);
1230     sqlite3_stmt *stmt = SecDbCopyStmt(dbconn, sql, NULL, error);
1231     if (!stmt)
1232         return false;
1233
1234     exec(stmt);
1235     return SecDbReleaseCachedStmt(dbconn, sql, stmt, error);
1236 }
1237
1238 bool SecDbWithSQL(SecDbConnectionRef dbconn, CFStringRef sql, CFErrorRef *error, bool(^perform)(sqlite3_stmt *stmt)) {
1239     bool ok = true;
1240     CFRetain(sql);
1241     while (sql) {
1242         CFStringRef tail = NULL;
1243         if (ok) {
1244             sqlite3_stmt *stmt = SecDbCopyStmt(dbconn, sql, &tail, error);
1245             ok = stmt != NULL;
1246             if (stmt) {
1247                 if (perform) {
1248                     ok = perform(stmt);
1249                 } else {
1250                     // TODO: Use a different error scope here.
1251                     ok = SecError(-50 /* errSecParam */, error, CFSTR("SecDbWithSQL perform block missing"));
1252                 }
1253                 ok &= SecDbReleaseCachedStmt(dbconn, sql, stmt, error);
1254             }
1255         } else {
1256             // TODO We already have an error here we really just want the left over sql in it's userData
1257             ok = SecDbError(SQLITE_ERROR, error, CFSTR("Error with unexecuted sql remaining %@"), sql);
1258         }
1259         CFRelease(sql);
1260         sql = tail;
1261     }
1262     return ok;
1263 }
1264
1265 #if 1
1266 /* SecDbForEach returns true if all SQLITE_ROW returns of sqlite3_step() return true from the row block.
1267  If the row block returns false and doesn't set an error (to indicate it has reached a limit),
1268  this entire function returns false. In that case no error will be set. */
1269 bool SecDbForEach(sqlite3_stmt *stmt, CFErrorRef *error, bool(^row)(int row_index)) {
1270     bool result = false;
1271     for (int row_ix = 0;;++row_ix) {
1272         int s3e = sqlite3_step(stmt);
1273         if (s3e == SQLITE_ROW) {
1274             if (row) {
1275                 if (!row(row_ix)) {
1276                     break;
1277                 }
1278             } else {
1279                 // If we have no row block then getting SQLITE_ROW is an error
1280                 SecDbError(s3e, error,
1281                            CFSTR("step[%d]: %s returned SQLITE_ROW with NULL row block"),
1282                            row_ix, sqlite3_sql(stmt));
1283             }
1284         } else {
1285             if (s3e == SQLITE_DONE) {
1286                 result = true;
1287             } else {
1288                 SecDbErrorWithStmt(s3e, stmt, error, CFSTR("step[%d]"), row_ix);
1289             }
1290             break;
1291         }
1292     }
1293     return result;
1294 }
1295 #else
1296 bool SecDbForEach(sqlite3_stmt *stmt, CFErrorRef *error, bool(^row)(int row_index)) {
1297     int row_ix = 0;
1298     for (;;) {
1299         switch (_SecDbStep(dbconn, stmt, error)) {
1300             case kSecDbErrorStep:
1301                 return false;
1302             case kSecDbRowStep:
1303                 if (row) {
1304                     if (row(row_ix++))
1305                         break;
1306                 } else {
1307                     SecDbError(SQLITE_ERROR, error, CFSTR("SecDbStep SQLITE_ROW returned without a row handler"));
1308                 }
1309                 return false;
1310             case kSecDbDoneStep:
1311                 return true;
1312         }
1313     }
1314 }
1315 #endif
1316
1317 void SecDbRecordChange(SecDbConnectionRef dbconn, CFTypeRef deleted, CFTypeRef inserted) {
1318     if (!dbconn->db->notifyPhase) return;
1319     CFTypeRef entry = SecDbEventCreateWithComponents(deleted, inserted);
1320     if (entry) {
1321         CFArrayAppendValue(dbconn->changes, entry);
1322         CFRelease(entry);
1323
1324         if (!dbconn->inTransaction) {
1325             secerror("db %@ changed outside txn", dbconn);
1326             // Only notify of DidCommit, since WillCommit code assumes
1327             // we are in a txn.
1328             SecDbOnNotifyQueue(dbconn, ^{
1329                 SecDbNotifyPhase(dbconn, kSecDbTransactionDidCommit);
1330             });
1331         }
1332     }
1333 }
1334
1335
1336 CFGiblisFor(SecDbConnection)
1337
1338 //
1339 // SecDbEvent Creation and consumption
1340 //
1341
1342 static SecDbEventRef SecDbEventCreateInsert(CFTypeRef inserted) {
1343     return CFRetainSafe(inserted);
1344 }
1345
1346 static SecDbEventRef SecDbEventCreateDelete(CFTypeRef deleted) {
1347     return CFArrayCreate(kCFAllocatorDefault, &deleted, 1, &kCFTypeArrayCallBacks);
1348 }
1349
1350 static SecDbEventRef SecDbEventCreateUpdate(CFTypeRef deleted, CFTypeRef inserted) {
1351     const void *values[2] = { deleted, inserted };
1352     return CFArrayCreate(kCFAllocatorDefault, values, 2, &kCFTypeArrayCallBacks);
1353 }
1354
1355 SecDbEventRef SecDbEventCreateWithComponents(CFTypeRef deleted, CFTypeRef inserted) {
1356     if (deleted && inserted)
1357         return SecDbEventCreateUpdate(deleted, inserted);
1358     else if (deleted)
1359         return SecDbEventCreateDelete(deleted);
1360     else if (inserted)
1361         return SecDbEventCreateInsert(inserted);
1362     else
1363         return NULL;
1364 }
1365
1366 bool SecDbEventGetComponents(SecDbEventRef event, CFTypeRef *deleted, CFTypeRef *inserted, CFErrorRef *error) {
1367     if (isArray(event)) {
1368         CFArrayRef array = event;
1369         switch (CFArrayGetCount(array)) {
1370             case 2:
1371                 *deleted = CFArrayGetValueAtIndex(array, 0);
1372                 *inserted = CFArrayGetValueAtIndex(array, 1);
1373                 break;
1374             case 1:
1375                 *deleted = CFArrayGetValueAtIndex(array, 0);
1376                 *inserted = NULL;
1377                 break;
1378             default:
1379                 SecError(errSecParam, error, NULL, CFSTR("invalid entry in changes array: %@"), array);
1380                 break;
1381         }
1382     } else {
1383         *deleted = NULL;
1384         *inserted = event;
1385     }
1386     return true;
1387 }