Security/utilities/src/SecDb.c

   1 /*
   2  * Copyright (c) 2012-2014 Apple Inc. All Rights Reserved.
   3  *
   4  * @APPLE_LICENSE_HEADER_START@
   5  *
   6  * This file contains Original Code and/or Modifications of Original Code
   7  * as defined in and that are subject to the Apple Public Source License
   8  * Version 2.0 (the 'License'). You may not use this file except in
   9  * compliance with the License. Please obtain a copy of the License at
  10  * http://www.opensource.apple.com/apsl/ and read it before using this
  11  * file.
  12  *
  13  * The Original Code and all software distributed under the License are
  14  * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  15  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  16  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
  17  * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
  18  * Please see the License for the specific language governing rights and
  19  * limitations under the License.
  20  *
  21  * @APPLE_LICENSE_HEADER_END@
  22  */
  23
  24
  25 #include "SecDb.h"
  26 #include "debugging.h"
  27
  28 #include <sqlite3.h>
  29 #include <sqlite3_private.h>
  30 #include <CoreFoundation/CoreFoundation.h>
  31 #include <libgen.h>
  32 #include <sys/stat.h>
  33 #include <AssertMacros.h>
  34 #include "SecCFWrappers.h"
  35 #include "SecCFError.h"
  36 #include "SecIOFormat.h"
  37 #include <stdio.h>
  38 #include "Security/SecBase.h"
  39 #include <SecureObjectSync/SOSDigestVector.h>
  40 #include <SecureObjectSync/SOSManifest.h>
  41
  42 #define LOGE(ARG,...) secerror(ARG, ## __VA_ARGS__)
  43 #define LOGV(ARG,...) secdebug("secdb", ARG, ## __VA_ARGS__)
  44 #define LOGD(ARG,...) secdebug("secdb", ARG, ## __VA_ARGS__)
  45
  46 #define HAVE_UNLOCK_NOTIFY  0
  47 #define USE_BUSY_HANDLER  1
  48
  49 struct __OpaqueSecDbStatement {
  50     CFRuntimeBase _base;
  51
  52     SecDbConnectionRef dbconn;
  53     sqlite3_stmt *stmt;
  54 };
  55
  56 struct __OpaqueSecDbConnection {
  57     CFRuntimeBase _base;
  58
  59     //CFMutableDictionaryRef statements;
  60
  61     SecDbRef db;     // NONRETAINED, since db or block retains us
  62     bool readOnly;
  63     bool inTransaction;
  64     SecDbTransactionSource source;
  65     bool isCorrupted;
  66     sqlite3 *handle;
  67     // Pending deletions and addtions to the manifest for the current transaction
  68     struct SOSDigestVector todel;
  69     struct SOSDigestVector toadd;
  70 };
  71
  72 struct __OpaqueSecDb {
  73     CFRuntimeBase _base;
  74
  75     CFStringRef db_path;
  76     dispatch_queue_t queue;
  77     CFMutableArrayRef connections;
  78     dispatch_semaphore_t write_semaphore;
  79     dispatch_semaphore_t read_semaphore;
  80     bool didFirstOpen;
  81     bool (^opened)(SecDbConnectionRef dbconn, bool did_create, CFErrorRef *error);
  82     SecDBNotifyBlock notifyPhase;
  83 };
  84
  85 // MARK: Error domains and error helper functions
  86
  87 CFStringRef kSecDbErrorDomain = CFSTR("com.apple.utilities.sqlite3");
  88
  89 bool SecDbError(int sql_code, CFErrorRef *error, CFStringRef format, ...) {
  90     if (sql_code == SQLITE_OK) return true;
  91     if (error) {
  92         va_list args;
  93         CFIndex code = sql_code;
  94         CFErrorRef previousError = *error;
  95
  96         *error = NULL;
  97         va_start(args, format);
  98         SecCFCreateErrorWithFormatAndArguments(code, kSecDbErrorDomain, previousError, error, NULL, format, args);
  99         CFReleaseNull(previousError);
 100         va_end(args);
 101     }
 102     return false;
 103 }
 104
 105 bool SecDbErrorWithDb(int sql_code, sqlite3 *db, CFErrorRef *error, CFStringRef format, ...) {
 106     if (sql_code == SQLITE_OK) return true;
 107     if (error) {
 108         va_list args;
 109         va_start(args, format);
 110         CFStringRef message = CFStringCreateWithFormatAndArguments(kCFAllocatorDefault, NULL, format, args);
 111         va_end(args);
 112
 113         int extended_code = sqlite3_extended_errcode(db);
 114         if (sql_code == extended_code)
 115             SecDbError(sql_code, error, CFSTR("%@: [%d] %s"), message, sql_code, sqlite3_errmsg(db));
 116         else
 117             SecDbError(sql_code, error, CFSTR("%@: [%d->%d] %s"), message, sql_code, extended_code, sqlite3_errmsg(db));
 118         CFReleaseSafe(message);
 119     }
 120     return false;
 121 }
 122
 123 bool SecDbErrorWithStmt(int sql_code, sqlite3_stmt *stmt, CFErrorRef *error, CFStringRef format, ...) {
 124     if (sql_code == SQLITE_OK) return true;
 125     if (error) {
 126         va_list args;
 127         va_start(args, format);
 128         CFStringRef message = CFStringCreateWithFormatAndArguments(kCFAllocatorDefault, NULL, format, args);
 129         va_end(args);
 130
 131         sqlite3 *db = sqlite3_db_handle(stmt);
 132         const char *sql = sqlite3_sql(stmt);
 133         int extended_code = sqlite3_extended_errcode(db);
 134         if (sql_code == extended_code)
 135             SecDbError(sql_code, error, CFSTR("%@: [%d] %s sql: %s"), message, sql_code, sqlite3_errmsg(db), sql);
 136         else
 137             SecDbError(sql_code, error, CFSTR("%@: [%d->%d] %s sql: %s"), message, sql_code, extended_code, sqlite3_errmsg(db), sql);
 138         CFReleaseSafe(message);
 139     }
 140     return false;
 141 }
 142
 143
 144 // MARK: -
 145 // MARK: Static helper functions
 146
 147 static bool SecDbOpenHandle(SecDbConnectionRef dbconn, bool *created, CFErrorRef *error);
 148 static bool SecDbHandleCorrupt(SecDbConnectionRef dbconn, int rc, CFErrorRef *error);
 149
 150 #pragma mark -
 151 #pragma mark SecDbRef
 152
 153 static CFStringRef
 154 SecDbCopyFormatDescription(CFTypeRef value, CFDictionaryRef formatOptions)
 155 {
 156     SecDbRef db = (SecDbRef)value;
 157     return CFStringCreateWithFormat(kCFAllocatorDefault, NULL, CFSTR("<SecDb path:%@ connections: %@>"), db->db_path, db->connections);
 158 }
 159
 160
 161 static void
 162 SecDbDestroy(CFTypeRef value)
 163 {
 164     SecDbRef db = (SecDbRef)value;
 165     CFReleaseSafe(db->connections);
 166     CFReleaseSafe(db->db_path);
 167     dispatch_release(db->queue);
 168     dispatch_release(db->read_semaphore);
 169     dispatch_release(db->write_semaphore);
 170 }
 171
 172 CFGiblisFor(SecDb)
 173
 174 SecDbRef
 175 SecDbCreate(CFStringRef dbName,
 176             bool (^opened)(SecDbConnectionRef dbconn, bool did_create, CFErrorRef *error))
 177 {
 178     SecDbRef db = NULL;
 179
 180     db = CFTypeAllocate(SecDb, struct __OpaqueSecDb, kCFAllocatorDefault);
 181     require(db != NULL, done);
 182
 183     CFStringPerformWithCString(dbName, ^(const char *dbNameStr) {
 184         db->queue = dispatch_queue_create(dbNameStr, DISPATCH_QUEUE_SERIAL);
 185     });
 186     db->read_semaphore = dispatch_semaphore_create(kSecDbMaxReaders);
 187     db->write_semaphore = dispatch_semaphore_create(kSecDbMaxWriters);
 188     db->connections = CFArrayCreateMutableForCFTypes(kCFAllocatorDefault);
 189     db->opened = opened;
 190     if (getenv("__OSINSTALL_ENVIRONMENT") != NULL) {
 191         // TODO: Move this code out of this layer
 192         LOGV("sqlDb: running from installer");
 193         db->db_path = CFSTR("file::memory:?cache=shared");
 194     } else {
 195         db->db_path = CFStringCreateCopy(kCFAllocatorDefault, dbName);
 196     }
 197 done:
 198     return db;
 199 }
 200
 201 CFIndex
 202 SecDbIdleConnectionCount(SecDbRef db) {
 203     __block CFIndex count = 0;
 204     dispatch_sync(db->queue, ^{
 205         count = CFArrayGetCount(db->connections);
 206     });
 207     return count;
 208 }
 209
 210 void SecDbSetNotifyPhaseBlock(SecDbRef db, SecDBNotifyBlock notifyPhase) {
 211     if (db->notifyPhase)
 212         Block_release(db->notifyPhase);
 213     db->notifyPhase = Block_copy(notifyPhase);
 214 }
 215
 216 static void SecDbNotifyPhase(SecDbConnectionRef dbconn, SecDbTransactionPhase phase) {
 217     if (dbconn->todel.count || dbconn->toadd.count) {
 218         if (dbconn->db->notifyPhase)
 219             dbconn->db->notifyPhase(dbconn, phase, dbconn->source, &dbconn->todel, &dbconn->toadd);
 220         SOSDigestVectorFree(&dbconn->todel);
 221         SOSDigestVectorFree(&dbconn->toadd);
 222     }
 223 }
 224
 225 CFStringRef SecDbGetPath(SecDbRef db) {
 226     return db->db_path;
 227 }
 228
 229
 230 #pragma mark -
 231 #pragma mark SecDbConnectionRef
 232
 233 static bool SecDbCheckCorrupted(SecDbConnectionRef dbconn)
 234 {
 235     __block bool isCorrupted = true;
 236     __block CFErrorRef error = NULL;
 237     SecDbPrepare(dbconn, CFSTR("PRAGMA integrity_check"), &error, ^(sqlite3_stmt *stmt) {
 238         SecDbStep(dbconn, stmt, &error, ^(bool *stop) {
 239             const char * result = (const char*)sqlite3_column_text(stmt, 0);
 240             if (result && strncasecmp(result, "ok", 3) == 0) {
 241                 isCorrupted = false;
 242             }
 243         });
 244     });
 245     if (error) {
 246         LOGV("sqlDb: warning error %@ when running integrity check", error);
 247         CFRelease(error);
 248     }
 249     return isCorrupted;
 250 }
 251
 252 static bool SecDbDidCreateFirstConnection(SecDbConnectionRef dbconn, bool didCreate, CFErrorRef *error)
 253 {
 254     LOGD("sqlDb: starting maintenance");
 255     bool ok = true;
 256
 257     if (!didCreate && !dbconn->isCorrupted) {
 258         dbconn->isCorrupted = SecDbCheckCorrupted(dbconn);
 259         if (dbconn->isCorrupted)
 260             secerror("integrity check=fail");
 261         else
 262             LOGD("sqlDb: integrity check=pass");
 263     }
 264
 265     if (!dbconn->isCorrupted && dbconn->db->opened) {
 266         CFErrorRef localError = NULL;
 267
 268         ok = dbconn->db->opened(dbconn, didCreate, &localError);
 269
 270         if (!ok)
 271             secerror("opened block failed: %@", localError);
 272
 273         if (!dbconn->isCorrupted && error && *error == NULL) {
 274             *error = localError;
 275             localError = NULL;
 276         } else {
 277             secerror("opened block failed: error is released and lost");
 278             CFReleaseNull(localError);
 279         }
 280     }
 281
 282     if (dbconn->isCorrupted) {
 283         ok = SecDbHandleCorrupt(dbconn, 0, error);
 284     }
 285
 286     LOGD("sqlDb: finished maintenance");
 287     return ok;
 288 }
 289
 290 void SecDbCorrupt(SecDbConnectionRef dbconn)
 291 {
 292     dbconn->isCorrupted = true;
 293 }
 294
 295
 296 static uint8_t knownDbPathIndex(SecDbConnectionRef dbconn)
 297 {
 298
 299     if(CFEqual(dbconn->db->db_path, CFSTR("/Library/Keychains/keychain-2.db")))
 300         return 1;
 301     if(CFEqual(dbconn->db->db_path, CFSTR("/Library/Keychains/ocspcache.sqlite3")))
 302         return 2;
 303     if(CFEqual(dbconn->db->db_path, CFSTR("/Library/Keychains/TrustStore.sqlite3")))
 304         return 3;
 305     if(CFEqual(dbconn->db->db_path, CFSTR("/Library/Keychains/caissuercache.sqlite3")))
 306         return 4;
 307
 308     /* Unknown DB path */
 309     return 0;
 310 }
 311
 312
 313 // Return true if there was no error, returns false otherwise and set *error to an appropriate CFErrorRef.
 314 static bool SecDbConnectionCheckCode(SecDbConnectionRef dbconn, int code, CFErrorRef *error, CFStringRef desc, ...) {
 315     if (code == SQLITE_OK || code == SQLITE_DONE)
 316         return true;
 317
 318     if (error) {
 319         va_list args;
 320         va_start(args, desc);
 321         CFStringRef msg = CFStringCreateWithFormatAndArguments(kCFAllocatorDefault, NULL, desc, args);
 322         va_end(args);
 323         SecDbErrorWithDb(code, dbconn->handle, error, msg);
 324         CFRelease(msg);
 325     }
 326
 327     /* If it's already corrupted, don't try to recover */
 328     if (dbconn->isCorrupted) {
 329         CFStringRef reason = CFStringCreateWithFormat(kCFAllocatorDefault, NULL, CFSTR("SQL DB %@ is corrupted already. Not trying to recover"), dbconn->db->db_path);
 330         secerror("%@",reason);
 331         __security_simulatecrash(reason, __sec_exception_code_TwiceCorruptDb(knownDbPathIndex(dbconn)));
 332         CFReleaseSafe(reason);
 333         return false;
 334     }
 335
 336     dbconn->isCorrupted = (SQLITE_CORRUPT == code) || (SQLITE_NOTADB == code) || (SQLITE_IOERR == code) || (SQLITE_CANTOPEN == code);
 337     if (dbconn->isCorrupted) {
 338         /* Run integrity check and only make dbconn->isCorrupted true and
 339            run the corruption handler if the integrity check conclusively fails. */
 340         dbconn->isCorrupted = SecDbCheckCorrupted(dbconn);
 341         if (dbconn->isCorrupted) {
 342             secerror("operation returned code: %d integrity check=fail", code);
 343             SecDbHandleCorrupt(dbconn, code, error);
 344         } else {
 345             secerror("operation returned code: %d: integrity check=pass", code);
 346         }
 347     }
 348
 349     return false;
 350 }
 351
 352 #if HAVE_UNLOCK_NOTIFY
 353
 354 static void SecDbUnlockNotify(void **apArg, int nArg) {
 355     int i;
 356     for(i=0; i<nArg; i++) {
 357         dispatch_semaphore_t dsema = (dispatch_semaphore_t)apArg[i];
 358         dispatch_semaphore_signal(dsema);
 359     }
 360 }
 361
 362 static bool SecDbWaitForUnlockNotify(SecDbConnectionRef dbconn, sqlite3_stmt *stmt, CFErrorRef *error) {
 363     int rc;
 364     dispatch_semaphore_t dsema = dispatch_semaphore_create(0);
 365     rc = sqlite3_unlock_notify(dbconn->handle, SecDbUnlockNotify, dsema);
 366     assert(rc == SQLITE_LOCKED || rc == SQLITE_OK);
 367     if (rc == SQLITE_OK) {
 368         dispatch_semaphore_wait(dsema, DISPATCH_TIME_FOREVER);
 369     }
 370     dispatch_release(dsema);
 371     return (rc == SQLITE_OK
 372             ? true
 373             : (stmt
 374                ? SecDbErrorWithStmt(rc, stmt, error, CFSTR("sqlite3_unlock_notify"))
 375                : SecDbErrorWithDb(rc, dbconn->handle, error, CFSTR("sqlite3_unlock_notify"))));
 376 }
 377
 378 #endif
 379
 380 #if USE_BUSY_HANDLER
 381
 382 // Return 0 to stop retrying.
 383 static int SecDbHandleBusy(void *ctx, int retryCount) {
 384     SecDbConnectionRef dbconn __unused = ctx;
 385     struct timespec sleeptime = { .tv_sec = 0, .tv_nsec = 10000 };
 386     while (retryCount--) {
 387         // Double sleeptime until we hit one second then add one
 388         // second more every time we sleep.
 389         if (sleeptime.tv_sec) {
 390             sleeptime.tv_sec++;
 391         } else {
 392             sleeptime.tv_nsec *= 2;
 393             if (sleeptime.tv_nsec > NSEC_PER_SEC) {
 394                 sleeptime.tv_nsec = 0;
 395                 sleeptime.tv_sec++;
 396             }
 397         }
 398     }
 399     struct timespec unslept = {};
 400     nanosleep(&sleeptime, &unslept);
 401
 402     return 1;
 403 }
 404
 405 static bool SecDbBusyHandler(SecDbConnectionRef dbconn, CFErrorRef *error) {
 406     return SecDbErrorWithDb(sqlite3_busy_handler(dbconn->handle, SecDbHandleBusy, dbconn), dbconn->handle, error, CFSTR("busy_handler"));
 407 }
 408
 409 #endif // USE_BUSY_HANDLER
 410
 411 // Return true causes the operation to be tried again.
 412 static bool SecDbWaitIfNeeded(SecDbConnectionRef dbconn, int s3e, sqlite3_stmt *stmt, CFStringRef desc, struct timespec *sleeptime, CFErrorRef *error) {
 413 #if HAVE_UNLOCK_NOTIFY
 414     if (s3e == SQLITE_LOCKED) { // Optionally check for extended code being SQLITE_LOCKED_SHAREDCACHE
 415         return SecDbWaitForUnlockNotify(dbconn, stmt, error))
 416     }
 417 #endif
 418
 419 #if !USE_BUSY_HANDLER
 420     if (s3e == SQLITE_LOCKED || s3e == SQLITE_BUSY) {
 421         LOGV("sqlDb: %s", sqlite3_errmsg(dbconn->handle));
 422         while (s3e == SQLITE_LOCKED || s3e == SQLITE_BUSY) {
 423             struct timespec unslept = {};
 424             nanosleep(sleeptime, &unslept);
 425             s3e = SQLITE_OK;
 426             if (stmt)
 427                 s3e = sqlite3_reset(stmt);
 428
 429             // Double sleeptime until we hit one second the add one
 430             // second more every time we sleep.
 431             if (sleeptime->tv_sec) {
 432                 sleeptime->tv_sec++;
 433             } else {
 434                 sleeptime->tv_nsec *= 2;
 435                 if (sleeptime->tv_nsec > NSEC_PER_SEC) {
 436                     sleeptime->tv_nsec = 0;
 437                     sleeptime->tv_sec++;
 438                 }
 439             }
 440         }
 441         if (s3e)
 442             return SecDbErrorWithStmt(s3e, stmt, error, CFSTR("reset"));
 443     } else
 444 #endif // !USE_BUSY_HANDLER
 445     {
 446         return SecDbConnectionCheckCode(dbconn, s3e, error, desc);
 447     }
 448     return true;
 449 }
 450
 451 enum SecDbStepResult {
 452     kSecDbErrorStep = 0,
 453     kSecDbRowStep = 1,
 454     kSecDbDoneStep = 2,
 455 };
 456 typedef enum SecDbStepResult SecDbStepResult;
 457
 458 static SecDbStepResult _SecDbStep(SecDbConnectionRef dbconn, sqlite3_stmt *stmt, CFErrorRef *error) {
 459     assert(stmt != NULL);
 460     int s3e;
 461     struct timespec sleeptime = { .tv_sec = 0, .tv_nsec = 10000 };
 462     for (;;) {
 463         s3e = sqlite3_step(stmt);
 464         if (s3e == SQLITE_ROW)
 465             return kSecDbRowStep;
 466         else if (s3e == SQLITE_DONE)
 467             return kSecDbDoneStep;
 468         else if (!SecDbWaitIfNeeded(dbconn, s3e, stmt, CFSTR("step"), &sleeptime, error))
 469             return kSecDbErrorStep;
 470     };
 471 }
 472
 473 bool
 474 SecDbExec(SecDbConnectionRef dbconn, CFStringRef sql, CFErrorRef *error)
 475 {
 476     bool ok = true;
 477     CFRetain(sql);
 478     while (sql) {
 479         CFStringRef tail = NULL;
 480         if (ok) {
 481             sqlite3_stmt *stmt = SecDbCopyStmt(dbconn, sql, &tail, error);
 482             ok = stmt != NULL;
 483             if (stmt) {
 484                 SecDbStepResult sr;
 485                 while ((sr = _SecDbStep(dbconn, stmt, error)) == kSecDbRowStep);
 486                 if (sr == kSecDbErrorStep)
 487                     ok = false;
 488                 ok &= SecDbReleaseCachedStmt(dbconn, sql, stmt, error);
 489             }
 490         } else {
 491             // TODO We already have an error here we really just want the left over sql in it's userData
 492             ok = SecDbError(SQLITE_ERROR, error, CFSTR("Error with unexecuted sql remaining %@"), sql);
 493         }
 494         CFRelease(sql);
 495         sql = tail;
 496     }
 497     return ok;
 498 }
 499
 500 static bool SecDbBeginTransaction(SecDbConnectionRef dbconn, SecDbTransactionType type, CFErrorRef *error)
 501 {
 502     bool ok = true;
 503     CFStringRef query;
 504     switch (type) {
 505         case kSecDbImmediateTransactionType:
 506             query = CFSTR("BEGIN IMMEDATE");
 507             break;
 508         case kSecDbExclusiveRemoteTransactionType:
 509             dbconn->source = kSecDbSOSTransaction;
 510         case kSecDbExclusiveTransactionType:
 511             query = CFSTR("BEGIN EXCLUSIVE");
 512             break;
 513         case kSecDbNormalTransactionType:
 514             query = CFSTR("BEGIN");
 515             break;
 516         default:
 517             ok = SecDbError(SQLITE_ERROR, error, CFSTR("invalid transaction type %" PRIu32), type);
 518             query = NULL;
 519             break;
 520     }
 521
 522     if (query != NULL && sqlite3_get_autocommit(dbconn->handle) != 0) {
 523         ok = SecDbExec(dbconn, query, error);
 524     }
 525     if (ok)
 526         dbconn->inTransaction = true;
 527
 528     return ok;
 529 }
 530
 531 static bool SecDbEndTransaction(SecDbConnectionRef dbconn, bool commit, CFErrorRef *error)
 532 {
 533     bool ok = true, commited = false;
 534     if (commit) {
 535         SecDbNotifyPhase(dbconn, kSecDbTransactionWillCommit);
 536         commited = ok = SecDbExec(dbconn, CFSTR("END"), error);
 537     } else {
 538         ok = SecDbExec(dbconn, CFSTR("ROLLBACK"), error);
 539         commited = false;
 540     }
 541     dbconn->inTransaction = false;
 542     SecDbNotifyPhase(dbconn, commited ? kSecDbTransactionDidCommit : kSecDbTransactionDidRollback);
 543     dbconn->source = kSecDbAPITransaction;
 544     return ok;
 545 }
 546
 547 bool SecDbTransaction(SecDbConnectionRef dbconn, SecDbTransactionType type,
 548                       CFErrorRef *error, void (^transaction)(bool *commit))
 549 {
 550     bool ok = true;
 551     bool commit = true;
 552
 553     if (dbconn->inTransaction) {
 554         transaction(&commit);
 555         if (!commit) {
 556             LOGV("sqlDb: nested transaction asked to not be committed");
 557         }
 558     } else {
 559         ok = SecDbBeginTransaction(dbconn, type, error);
 560         if (ok) {
 561             transaction(&commit);
 562             ok = SecDbEndTransaction(dbconn, commit, error);
 563         }
 564     }
 565
 566 done:
 567     return ok && commit;
 568 }
 569
 570 sqlite3 *SecDbHandle(SecDbConnectionRef dbconn) {
 571     return dbconn->handle;
 572 }
 573
 574 bool SecDbStep(SecDbConnectionRef dbconn, sqlite3_stmt *stmt, CFErrorRef *error, void (^row)(bool *stop)) {
 575     for (;;) {
 576         switch (_SecDbStep(dbconn, stmt, error)) {
 577             case kSecDbErrorStep:
 578                 return false;
 579             case kSecDbRowStep:
 580                 if (row) {
 581                     bool stop = false;
 582                     row(&stop);
 583                     if (stop)
 584                         return true;
 585                     break;
 586                 }
 587                 SecDbError(SQLITE_ERROR, error, CFSTR("SecDbStep SQLITE_ROW returned without a row handler"));
 588                 return false;
 589             case kSecDbDoneStep:
 590                 return true;
 591         }
 592     }
 593 }
 594
 595 bool SecDbCheckpoint(SecDbConnectionRef dbconn, CFErrorRef *error)
 596 {
 597     return SecDbConnectionCheckCode(dbconn, sqlite3_wal_checkpoint(dbconn->handle, NULL), error, CFSTR("wal_checkpoint"));
 598 }
 599
 600 static bool SecDbFileControl(SecDbConnectionRef dbconn, int op, void *arg, CFErrorRef *error) {
 601     return SecDbConnectionCheckCode(dbconn, sqlite3_file_control(dbconn->handle, NULL, op, arg), error, CFSTR("file_control"));
 602 }
 603
 604 static sqlite3 *_SecDbOpenV2(const char *path, int flags, CFErrorRef *error) {
 605 #if HAVE_UNLOCK_NOTIFY
 606     flags |= SQLITE_OPEN_SHAREDCACHE;
 607 #endif
 608     sqlite3 *handle = NULL;
 609     int s3e = sqlite3_open_v2(path, &handle, flags, NULL);
 610     if (s3e) {
 611         if (handle) {
 612             SecDbErrorWithDb(s3e, handle, error, CFSTR("open_v2 \"%s\" 0x%X"), path, flags);
 613             sqlite3_close(handle);
 614             handle = NULL;
 615         } else {
 616             SecDbError(s3e, error, CFSTR("open_v2 \"%s\" 0x%X"), path, flags);
 617         }
 618     }
 619     return handle;
 620 }
 621
 622 static bool SecDbOpenV2(SecDbConnectionRef dbconn, const char *path, int flags, CFErrorRef *error) {
 623     return (dbconn->handle = _SecDbOpenV2(path, flags, error)) != NULL;
 624 }
 625
 626 static bool SecDbTruncate(SecDbConnectionRef dbconn, CFErrorRef *error)
 627 {
 628     int flags = SQLITE_TRUNCATE_JOURNALMODE_WAL | SQLITE_TRUNCATE_AUTOVACUUM_FULL;
 629     __block bool ok = SecDbFileControl(dbconn, SQLITE_TRUNCATE_DATABASE, &flags, error);
 630     if (!ok) {
 631         sqlite3_close(dbconn->handle);
 632         dbconn->handle = NULL;
 633         CFStringPerformWithCString(dbconn->db->db_path, ^(const char *path) {
 634             if (error)
 635                 CFReleaseNull(*error);
 636             if (SecCheckErrno(unlink(path), error, CFSTR("unlink %s"), path)) {
 637                 ok = SecDbOpenHandle(dbconn, NULL, error);
 638             }
 639         });
 640         if (!ok) {
 641             secerror("Failed to delete db handle: %@", error ? *error : NULL);
 642             abort();
 643         }
 644     }
 645
 646     return ok;
 647 }
 648
 649 static bool SecDbHandleCorrupt(SecDbConnectionRef dbconn, int rc, CFErrorRef *error)
 650 {
 651     CFStringRef reason = CFStringCreateWithFormat(kCFAllocatorDefault, NULL, CFSTR("SQL DB %@ is corrupted, trying to recover (rc=%d)"), dbconn->db->db_path, rc);
 652     __security_simulatecrash(reason, __sec_exception_code_CorruptDb(knownDbPathIndex(dbconn), rc));
 653     CFReleaseSafe(reason);
 654
 655     // Backup current db.
 656     __block bool didRename = false;
 657     CFStringPerformWithCString(dbconn->db->db_path, ^(const char *db_path) {
 658         sqlite3 *corrupt_db = NULL;
 659         char buf[PATH_MAX+1];
 660         snprintf(buf, sizeof(buf), "%s-corrupt", db_path);
 661         if (dbconn->handle && (corrupt_db = _SecDbOpenV2(buf, SQLITE_OPEN_READWRITE, error))) {
 662             int on = 1;
 663             didRename =  SecDbErrorWithDb(sqlite3_file_control(corrupt_db, NULL, SQLITE_FCNTL_PERSIST_WAL, &on), corrupt_db, error, CFSTR("persist wal"));
 664             didRename &=  SecDbErrorWithDb(sqlite3_file_control(corrupt_db, NULL, SQLITE_REPLACE_DATABASE, (void *)dbconn->handle), corrupt_db, error, CFSTR("replace database"));
 665             sqlite3_close(corrupt_db);
 666         }
 667         if (!didRename) {
 668             if (dbconn->handle)
 669                 secerror("Tried to rename corrupt database at path %@, but we failed: %@, trying explicit rename", dbconn->db->db_path, error ? *error : NULL);
 670             if (error)
 671                 CFReleaseNull(*error);
 672
 673             didRename = SecCheckErrno(rename(db_path, buf), error, CFSTR("rename %s %s"), db_path, buf) &&
 674                 (!dbconn->handle || SecDbError(sqlite3_close(dbconn->handle), error, CFSTR("close"))) &&
 675                 SecDbOpenHandle(dbconn, NULL, error);
 676         }
 677         if (didRename) {
 678             secerror("Database at path %@ is corrupt. Copied it to %s for further investigation.", dbconn->db->db_path, buf);
 679         } else {
 680             seccritical("Tried to copy corrupt database at path %@, but we failed: %@", dbconn->db->db_path, error ? *error : NULL);
 681         }
 682     });
 683
 684     bool ok = (didRename &&
 685                (dbconn->handle || SecDbOpenHandle(dbconn, NULL, error)) &&
 686                SecDbTruncate(dbconn, error));
 687
 688     // Mark the db as not corrupted, even if something failed.
 689     // Always note we are no longer in the corruption handler
 690     dbconn->isCorrupted = false;
 691
 692     // Invoke our callers opened callback, since we just created a new database
 693     if (ok && dbconn->db->opened)
 694         ok = dbconn->db->opened(dbconn, true, error);
 695
 696     return ok;
 697 }
 698
 699 static bool SecDbProfileEnabled(void)
 700 {
 701 #if 0
 702     static dispatch_once_t onceToken;
 703     static bool profile_enabled = false;
 704
 705 #if DEBUG
 706     //sudo defaults write /Library/Preferences/com.apple.security.auth profile -bool true
 707     dispatch_once(&onceToken, ^{
 708                 CFTypeRef profile = (CFNumberRef)CFPreferencesCopyValue(CFSTR("profile"), CFSTR(SECURITY_AUTH_NAME), kCFPreferencesAnyUser, kCFPreferencesCurrentHost);
 709
 710         if (profile && CFGetTypeID(profile) == CFBooleanGetTypeID()) {
 711             profile_enabled = CFBooleanGetValue((CFBooleanRef)profile);
 712         }
 713
 714         LOGV("sqlDb: sql profile: %s", profile_enabled ? "enabled" : "disabled");
 715
 716         CFReleaseSafe(profile);
 717     });
 718 #endif
 719
 720     return profile_enabled;
 721 #else
 722 #if DEBUG
 723     return true;
 724 #else
 725     return false;
 726 #endif
 727 #endif
 728 }
 729
 730 #if 0
 731 static void SecDbProfile(void *context __unused, const char *sql, sqlite3_uint64 ns) {
 732     LOGV("==\nsqlDb: %s\nTime: %llu ms\n", sql, ns >> 20);
 733 }
 734 #else
 735 static void SecDbProfile(void *context, const char *sql, sqlite3_uint64 ns) {
 736     sqlite3 *s3h = context;
 737     int code = sqlite3_extended_errcode(s3h);
 738     if (code == SQLITE_OK || code == SQLITE_DONE) {
 739         secdebug("profile", "==\nsqlDb: %s\nTime: %llu ms\n", sql, ns >> 20);
 740     } else {
 741         secdebug("profile", "==error[%d]: %s==\nsqlDb: %s\nTime: %llu ms \n", code, sqlite3_errmsg(s3h), sql, ns >> 20);
 742     }
 743 }
 744 #endif
 745
 746 static bool SecDbTraceEnabled(void)
 747 {
 748 #if DEBUG
 749     return true;
 750 #else
 751     return false;
 752 #endif
 753 }
 754
 755 static void SecDbTrace(void *ctx, const char *trace) {
 756     SecDbConnectionRef dbconn __unused = ctx;
 757     static dispatch_queue_t queue;
 758     static dispatch_once_t once;
 759     dispatch_once(&once, ^{
 760         queue = dispatch_queue_create("trace_queue", DISPATCH_QUEUE_SERIAL);
 761     });
 762     dispatch_sync(queue, ^{
 763         __security_debug(CFSTR("trace"), "", "", 0, CFSTR("%s"), trace);
 764     });
 765 }
 766
 767 static bool SecDbOpenHandle(SecDbConnectionRef dbconn, bool *created, CFErrorRef *error)
 768 {
 769     __block bool ok = true;
 770     CFStringPerformWithCString(dbconn->db->db_path, ^(const char *db_path) {
 771         ok = created && SecDbOpenV2(dbconn, db_path, SQLITE_OPEN_READWRITE, NULL);
 772         if (!ok) {
 773             ok = true;
 774             if (created) {
 775                 char *tmp = dirname((char *)db_path);
 776                 if (tmp) {
 777                     int errnum = mkpath_np(tmp, 0700);
 778                     if (errnum != 0 && errnum != EEXIST) {
 779                         SecCFCreateErrorWithFormat(errnum, kSecErrnoDomain, NULL, error, NULL,
 780                                                    CFSTR("mkpath_np %s: [%d] %s"), tmp, errnum, strerror(errnum));
 781                         ok = false;
 782                     }
 783                 }
 784             }
 785             ok = ok && SecDbOpenV2(dbconn, db_path, SQLITE_OPEN_READWRITE | SQLITE_OPEN_CREATE, error);
 786             if (ok) {
 787                 chmod(db_path, S_IRUSR | S_IWUSR);
 788                 if (created)
 789                     *created = true;
 790             }
 791         }
 792
 793         if (ok && SecDbProfileEnabled()) {
 794             sqlite3_profile(dbconn->handle, SecDbProfile, dbconn->handle);
 795         }
 796         if (ok && SecDbTraceEnabled()) {
 797             sqlite3_trace(dbconn->handle, SecDbTrace, dbconn);
 798         }
 799 #if USE_BUSY_HANDLER
 800         ok = ok && SecDbBusyHandler(dbconn, error);
 801 #endif
 802     });
 803
 804 done:
 805     return ok;
 806 }
 807
 808 static SecDbConnectionRef
 809 SecDbConnectionCreate(SecDbRef db, bool readOnly, CFErrorRef *error)
 810 {
 811     SecDbConnectionRef dbconn = NULL;
 812
 813     dbconn = CFTypeAllocate(SecDbConnection, struct __OpaqueSecDbConnection, kCFAllocatorDefault);
 814     require(dbconn != NULL, done);
 815
 816     dbconn->db = db;
 817     dbconn->readOnly = readOnly;
 818
 819 done:
 820     return dbconn;
 821 }
 822
 823 static bool SecDbConnectionIsReadOnly(SecDbConnectionRef dbconn) {
 824     return dbconn->readOnly;
 825 }
 826
 827 static void SecDbConectionSetReadOnly(SecDbConnectionRef dbconn, bool readOnly) {
 828     dbconn->readOnly = readOnly;
 829 }
 830
 831 /* Read only connections go to the end of the queue, writeable connections
 832  go to the start of the queue. */
 833 SecDbConnectionRef SecDbConnectionAquire(SecDbRef db, bool readOnly, CFErrorRef *error) {
 834     CFRetain(db);
 835     secdebug("dbconn", "acquire %s connection", readOnly ? "ro" : "rw");
 836     dispatch_semaphore_wait(readOnly ? db->read_semaphore : db->write_semaphore, DISPATCH_TIME_FOREVER);
 837     __block SecDbConnectionRef dbconn = NULL;
 838     __block bool ok = true;
 839     dispatch_sync(db->queue, ^{
 840         if (!db->didFirstOpen) {
 841             bool didCreate = false;
 842             ok = dbconn = SecDbConnectionCreate(db, false, error);
 843             CFErrorRef localError = NULL;
 844             if (ok && !SecDbOpenHandle(dbconn, &didCreate, &localError)) {
 845                 secerror("Unable to create database: %@", localError);
 846                 if (localError && CFEqual(CFErrorGetDomain(localError), kSecDbErrorDomain)) {
 847                     int code = (int)CFErrorGetCode(localError);
 848                     dbconn->isCorrupted = (SQLITE_CORRUPT == code) || (SQLITE_NOTADB == code) || (SQLITE_IOERR == code) || (SQLITE_CANTOPEN == code);
 849                 }
 850                 // If the open failure isn't due to corruption, propagte the error.
 851                 ok = dbconn->isCorrupted;
 852                 if (!ok && error && *error == NULL) {
 853                     *error = localError;
 854                     localError = NULL;
 855                 }
 856             }
 857             CFReleaseNull(localError);
 858
 859             if (ok)
 860                 db->didFirstOpen = ok = SecDbDidCreateFirstConnection(dbconn, didCreate, error);
 861             if (!ok)
 862                 CFReleaseNull(dbconn);
 863         } else {
 864             /* Try to get one from the cache */
 865             CFIndex count = CFArrayGetCount(db->connections);
 866             while (count && !dbconn) {
 867                 CFIndex ix = readOnly ? count - 1 : 0;
 868                 dbconn = (SecDbConnectionRef)CFArrayGetValueAtIndex(db->connections, ix);
 869                 if (dbconn)
 870                     CFRetain(dbconn);
 871                 else
 872                     secerror("got NULL dbconn at index: %" PRIdCFIndex " skipping", ix);
 873                 CFArrayRemoveValueAtIndex(db->connections, ix);
 874             }
 875         }
 876     });
 877
 878     if (dbconn) {
 879         /* Make sure the connection we found has the right access */
 880         if (SecDbConnectionIsReadOnly(dbconn) != readOnly) {
 881             SecDbConectionSetReadOnly(dbconn, readOnly);
 882         }
 883     } else if (ok) {
 884         /* Nothing found in cache, create a new connection */
 885         bool created = false;
 886         dbconn = SecDbConnectionCreate(db, readOnly, error);
 887         if (dbconn && !SecDbOpenHandle(dbconn, &created, error)) {
 888             CFReleaseNull(dbconn);
 889         }
 890     }
 891
 892     if (!dbconn) {
 893         // If aquire fails we need to signal the semaphore again.
 894         dispatch_semaphore_signal(readOnly ? db->read_semaphore : db->write_semaphore);
 895         CFRelease(db);
 896     }
 897
 898     return dbconn;
 899 }
 900
 901 void SecDbConnectionRelease(SecDbConnectionRef dbconn) {
 902     if (!dbconn) {
 903         secerror("called with NULL dbconn");
 904         return;
 905     }
 906     SecDbRef db = dbconn->db;
 907     secdebug("dbconn", "release %@", dbconn);
 908     dispatch_sync(db->queue, ^{
 909         CFIndex count = CFArrayGetCount(db->connections);
 910         // Add back possible writable dbconn to the pool.
 911         bool readOnly = SecDbConnectionIsReadOnly(dbconn);
 912         CFArrayInsertValueAtIndex(db->connections, readOnly ? count : 0, dbconn);
 913         // Remove the last (probably read-only) dbconn from the pool.
 914         if (count >= kSecDbMaxIdleHandles) {
 915             CFArrayRemoveValueAtIndex(db->connections, count);
 916         }
 917         // Signal after we have put the connection back in the pool of connections
 918         dispatch_semaphore_signal(readOnly ? db->read_semaphore : db->write_semaphore);
 919         CFRelease(dbconn);
 920         CFRelease(db);
 921     });
 922 }
 923
 924 bool SecDbPerformRead(SecDbRef db, CFErrorRef *error, void (^perform)(SecDbConnectionRef dbconn)) {
 925     SecDbConnectionRef dbconn = SecDbConnectionAquire(db, true, error);
 926     bool success = false;
 927     if (dbconn) {
 928         perform(dbconn);
 929         success = true;
 930         SecDbConnectionRelease(dbconn);
 931     }
 932     return success;
 933 }
 934
 935 bool SecDbPerformWrite(SecDbRef db, CFErrorRef *error, void (^perform)(SecDbConnectionRef dbconn)) {
 936     SecDbConnectionRef dbconn = SecDbConnectionAquire(db, false, error);
 937     bool success = false;
 938     if (dbconn) {
 939         perform(dbconn);
 940         success = true;
 941         SecDbConnectionRelease(dbconn);
 942     }
 943     return success;
 944 }
 945
 946 static CFStringRef
 947 SecDbConnectionCopyFormatDescription(CFTypeRef value, CFDictionaryRef formatOptions)
 948 {
 949     SecDbConnectionRef dbconn = (SecDbConnectionRef)value;
 950     return CFStringCreateWithFormat(kCFAllocatorDefault, NULL, CFSTR("<SecDbConnection %s %s>"),
 951                                     dbconn->readOnly ? "ro" : "rw", dbconn->handle ? "open" : "closed");
 952 }
 953
 954 static void
 955 SecDbConnectionDestroy(CFTypeRef value)
 956 {
 957     SecDbConnectionRef dbconn = (SecDbConnectionRef)value;
 958     if (dbconn->handle) {
 959         sqlite3_close(dbconn->handle);
 960     }
 961     dbconn->db = NULL;
 962 }
 963
 964
 965 // MARK: -
 966 // MARK: Bind helpers
 967
 968 #if 0
 969 bool SecDbBindNull(sqlite3_stmt *stmt, int param, CFErrorRef *error) {
 970     bool ok = SecDbErrorWithStmt(sqlite3_bind_null(stmt, param),
 971                                  stmt, error, CFSTR("bind_null[%d]"), param);
 972     secdebug("bind", "bind_null[%d]: %@", param, error ? *error : NULL);
 973     return ok;
 974 }
 975 #endif
 976
 977 bool SecDbBindBlob(sqlite3_stmt *stmt, int param, const void *zData, size_t n, void(*xDel)(void*), CFErrorRef *error) {
 978     if (n > INT_MAX) {
 979         return SecDbErrorWithStmt(SQLITE_TOOBIG, stmt, error,
 980                                   CFSTR("bind_blob[%d]: blob bigger than INT_MAX"), param);
 981     }
 982     bool ok = SecDbErrorWithStmt(sqlite3_bind_blob(stmt, param, zData, (int)n, xDel),
 983                                  stmt, error, CFSTR("bind_blob[%d]"), param);
 984     secdebug("bind", "bind_blob[%d]: %.*s: %@", param, (int)n, zData, error ? *error : NULL);
 985     return ok;
 986 }
 987
 988 bool SecDbBindText(sqlite3_stmt *stmt, int param, const char *zData, size_t n, void(*xDel)(void*), CFErrorRef *error) {
 989     if (n > INT_MAX) {
 990         return SecDbErrorWithStmt(SQLITE_TOOBIG, stmt, error,
 991                                   CFSTR("bind_text[%d]: text bigger than INT_MAX"), param);
 992     }
 993     bool ok = SecDbErrorWithStmt(sqlite3_bind_text(stmt, param, zData, (int)n, xDel), stmt, error,
 994                                  CFSTR("bind_text[%d]"), param);
 995     secdebug("bind", "bind_text[%d]: \"%s\": %@", param, zData, error ? *error : NULL);
 996     return ok;
 997 }
 998
 999 bool SecDbBindDouble(sqlite3_stmt *stmt, int param, double value, CFErrorRef *error) {
1000     bool ok = SecDbErrorWithStmt(sqlite3_bind_double(stmt, param, value), stmt, error,
1001                                  CFSTR("bind_double[%d]"), param);
1002     secdebug("bind", "bind_double[%d]: %f: %@", param, value, error ? *error : NULL);
1003     return ok;
1004 }
1005
1006 bool SecDbBindInt(sqlite3_stmt *stmt, int param, int value, CFErrorRef *error) {
1007     bool ok = SecDbErrorWithStmt(sqlite3_bind_int(stmt, param, value), stmt, error,
1008                                  CFSTR("bind_int[%d]"), param);
1009     secdebug("bind", "bind_int[%d]: %d: %@", param, value, error ? *error : NULL);
1010     return ok;
1011 }
1012
1013 bool SecDbBindInt64(sqlite3_stmt *stmt, int param, sqlite3_int64 value, CFErrorRef *error) {
1014     bool ok = SecDbErrorWithStmt(sqlite3_bind_int64(stmt, param, value), stmt, error,
1015                                  CFSTR("bind_int64[%d]"), param);
1016     secdebug("bind", "bind_int64[%d]: %lld: %@", param, value, error ? *error : NULL);
1017     return ok;
1018 }
1019
1020
1021 /* AUDIT[securityd](done):
1022  value (ok) is a caller provided, non NULL CFTypeRef.
1023  */
1024 bool SecDbBindObject(sqlite3_stmt *stmt, int param, CFTypeRef value, CFErrorRef *error) {
1025     CFTypeID valueId;
1026     __block bool result = false;
1027
1028         /* TODO: Can we use SQLITE_STATIC below everwhere we currently use
1029      SQLITE_TRANSIENT since we finalize the statement before the value
1030      goes out of scope? */
1031     if (!value || (valueId = CFGetTypeID(value)) == CFNullGetTypeID()) {
1032         /* Skip bindings for NULL values.  sqlite3 will interpret unbound
1033          params as NULL which is exactly what we want. */
1034 #if 1
1035         result = true;
1036 #else
1037         result = SecDbBindNull(stmt, param, error);
1038 #endif
1039     } else if (valueId == CFStringGetTypeID()) {
1040         CFStringPerformWithCStringAndLength(value, ^(const char *cstr, size_t clen) {
1041             result = SecDbBindText(stmt, param, cstr, clen, SQLITE_TRANSIENT, error);
1042         });
1043     } else if (valueId == CFDataGetTypeID()) {
1044         CFIndex len = CFDataGetLength(value);
1045         if (len) {
1046             result = SecDbBindBlob(stmt, param, CFDataGetBytePtr(value),
1047                                    len, SQLITE_TRANSIENT, error);
1048         } else {
1049             result = SecDbBindText(stmt, param, "", 0, SQLITE_TRANSIENT, error);
1050         }
1051     } else if (valueId == CFDateGetTypeID()) {
1052         CFAbsoluteTime abs_time = CFDateGetAbsoluteTime(value);
1053         result = SecDbBindDouble(stmt, param, abs_time, error);
1054     } else if (valueId == CFBooleanGetTypeID()) {
1055         int bval = CFBooleanGetValue(value);
1056         result = SecDbBindInt(stmt, param, bval, error);
1057     } else if (valueId == CFNumberGetTypeID()) {
1058         Boolean convertOk;
1059         if (CFNumberIsFloatType(value)) {
1060             double nval;
1061             convertOk = CFNumberGetValue(value, kCFNumberDoubleType, &nval);
1062             result = SecDbBindDouble(stmt, param, nval, error);
1063         } else {
1064             int nval;
1065             convertOk = CFNumberGetValue(value, kCFNumberSInt32Type, &nval);
1066             if (convertOk) {
1067                 result = SecDbBindInt(stmt, param, nval, error);
1068             } else {
1069                 sqlite_int64 nval64;
1070                 convertOk = CFNumberGetValue(value, kCFNumberSInt64Type, &nval64);
1071                 if (convertOk)
1072                     result = SecDbBindInt64(stmt, param, nval64, error);
1073             }
1074         }
1075         if (!convertOk) {
1076             result = SecDbError(SQLITE_INTERNAL, error, CFSTR("bind CFNumberGetValue failed for %@"), value);
1077         }
1078     } else {
1079         if (error) {
1080             CFStringRef valueDesc = CFCopyTypeIDDescription(valueId);
1081             SecDbError(SQLITE_MISMATCH, error, CFSTR("bind unsupported type %@"), valueDesc);
1082             CFReleaseSafe(valueDesc);
1083         }
1084     }
1085
1086         return result;
1087 }
1088
1089 // MARK: -
1090 // MARK: SecDbStatementRef
1091
1092 bool SecDbReset(sqlite3_stmt *stmt, CFErrorRef *error) {
1093     return SecDbErrorWithStmt(sqlite3_reset(stmt), stmt, error, CFSTR("reset"));
1094 }
1095
1096 bool SecDbClearBindings(sqlite3_stmt *stmt, CFErrorRef *error) {
1097     return SecDbErrorWithStmt(sqlite3_clear_bindings(stmt), stmt, error, CFSTR("clear bindings"));
1098 }
1099
1100 bool SecDbFinalize(sqlite3_stmt *stmt, CFErrorRef *error) {
1101     sqlite3 *handle = sqlite3_db_handle(stmt);
1102     int s3e = sqlite3_finalize(stmt);
1103     return s3e == SQLITE_OK ? true : SecDbErrorWithDb(s3e, handle, error, CFSTR("finalize: %p"), stmt);
1104 }
1105
1106 sqlite3_stmt *SecDbPrepareV2(SecDbConnectionRef dbconn, const char *sql, size_t sqlLen, const char **sqlTail, CFErrorRef *error) {
1107     sqlite3 *db = SecDbHandle(dbconn);
1108     if (sqlLen > INT_MAX) {
1109         SecDbErrorWithDb(SQLITE_TOOBIG, db, error, CFSTR("prepare_v2: sql bigger than INT_MAX"));
1110         return NULL;
1111     }
1112     struct timespec sleeptime = { .tv_sec = 0, .tv_nsec = 10000 };
1113     for (;;) {
1114         sqlite3_stmt *stmt = NULL;
1115         int s3e = sqlite3_prepare_v2(db, sql, (int)sqlLen, &stmt, sqlTail);
1116         if (s3e == SQLITE_OK)
1117             return stmt;
1118         else if (!SecDbWaitIfNeeded(dbconn, s3e, NULL, CFSTR("preparev2"), &sleeptime, error))
1119             return NULL;
1120     }
1121 }
1122
1123 static sqlite3_stmt *SecDbCopyStatementWithTailRange(SecDbConnectionRef dbconn, CFStringRef sql, CFRange *sqlTail, CFErrorRef *error) {
1124     __block sqlite3_stmt *stmt = NULL;
1125     if (sql) CFStringPerformWithCStringAndLength(sql, ^(const char *sqlStr, size_t sqlLen) {
1126         const char *tail = NULL;
1127         stmt = SecDbPrepareV2(dbconn, sqlStr, sqlLen, &tail, error);
1128         if (sqlTail && sqlStr < tail && tail < sqlStr + sqlLen) {
1129             sqlTail->location = tail - sqlStr;
1130             sqlTail->length = sqlLen - sqlTail->location;
1131         }
1132     });
1133
1134     return stmt;
1135 }
1136
1137 sqlite3_stmt *SecDbCopyStmt(SecDbConnectionRef dbconn, CFStringRef sql, CFStringRef *tail, CFErrorRef *error) {
1138     // TODO: Add caching and cache lookup of statements
1139     CFRange sqlTail = {};
1140     sqlite3_stmt *stmt = SecDbCopyStatementWithTailRange(dbconn, sql, &sqlTail, error);
1141     if (sqlTail.length > 0) {
1142         CFStringRef excess = CFStringCreateWithSubstring(CFGetAllocator(sql), sql, sqlTail);
1143         if (tail) {
1144             *tail = excess;
1145         } else {
1146             SecDbError(SQLITE_INTERNAL, error,
1147                        CFSTR("prepare_v2: %@ unused sql: %@"),
1148                        sql, excess);
1149             CFReleaseSafe(excess);
1150             SecDbFinalize(stmt, error);
1151             stmt = NULL;
1152         }
1153     }
1154     return stmt;
1155 }
1156
1157 /*
1158  TODO: Could do a hack here with a custom kCFAllocatorNULL allocator for a second CFRuntimeBase inside a SecDbStatement,
1159  TODO: Better yet make a full blow SecDbStatement instance whenever SecDbCopyStmt is called.  Then, when the statement is released, in the Dispose method, we Reset and ClearBindings the sqlite3_stmt * and hand it back to the SecDb with the original CFStringRef for the sql (or hash thereof) as an argument. */
1160 bool SecDbReleaseCachedStmt(SecDbConnectionRef dbconn, CFStringRef sql, sqlite3_stmt *stmt, CFErrorRef *error) {
1161     if (stmt) {
1162         return SecDbFinalize(stmt, error);
1163     }
1164     return true;
1165 }
1166
1167 bool SecDbPrepare(SecDbConnectionRef dbconn, CFStringRef sql, CFErrorRef *error, void(^exec)(sqlite3_stmt *stmt)) {
1168     assert(sql != NULL);
1169     sqlite3_stmt *stmt = SecDbCopyStmt(dbconn, sql, NULL, error);
1170     if (!stmt)
1171         return false;
1172
1173     exec(stmt);
1174     return SecDbReleaseCachedStmt(dbconn, sql, stmt, error);
1175 }
1176
1177 bool SecDbWithSQL(SecDbConnectionRef dbconn, CFStringRef sql, CFErrorRef *error, bool(^perform)(sqlite3_stmt *stmt)) {
1178     bool ok = true;
1179     CFRetain(sql);
1180     while (sql) {
1181         CFStringRef tail = NULL;
1182         if (ok) {
1183             sqlite3_stmt *stmt = SecDbCopyStmt(dbconn, sql, &tail, error);
1184             ok = stmt != NULL;
1185             if (stmt) {
1186                 if (perform) {
1187                     ok = perform(stmt);
1188                 } else {
1189                     // TODO: Use a different error scope here.
1190                     ok = SecError(-50 /* errSecParam */, error, CFSTR("SecDbWithSQL perform block missing"));
1191                 }
1192                 ok &= SecDbReleaseCachedStmt(dbconn, sql, stmt, error);
1193             }
1194         } else {
1195             // TODO We already have an error here we really just want the left over sql in it's userData
1196             ok = SecDbError(SQLITE_ERROR, error, CFSTR("Error with unexecuted sql remaining %@"), sql);
1197         }
1198         CFRelease(sql);
1199         sql = tail;
1200     }
1201     return ok;
1202 }
1203
1204 #if 1
1205 /* SecDbForEach returns true if all SQLITE_ROW returns of sqlite3_step() return true from the row block.
1206  If the row block returns false and doesn't set an error (to indicate it has reached a limit),
1207  this entire function returns false. In that case no error will be set. */
1208 bool SecDbForEach(sqlite3_stmt *stmt, CFErrorRef *error, bool(^row)(int row_index)) {
1209     bool result = false;
1210     for (int row_ix = 0;;++row_ix) {
1211         int s3e = sqlite3_step(stmt);
1212         if (s3e == SQLITE_ROW) {
1213             if (row) {
1214                 if (!row(row_ix)) {
1215                     break;
1216                 }
1217             } else {
1218                 // If we have no row block then getting SQLITE_ROW is an error
1219                 SecDbError(s3e, error,
1220                            CFSTR("step[%d]: %s returned SQLITE_ROW with NULL row block"),
1221                            row_ix, sqlite3_sql(stmt));
1222             }
1223         } else {
1224             if (s3e == SQLITE_DONE) {
1225                 result = true;
1226             } else {
1227                 SecDbErrorWithStmt(s3e, stmt, error, CFSTR("step[%d]"), row_ix);
1228             }
1229             break;
1230         }
1231     }
1232     return result;
1233 }
1234 #else
1235 bool SecDbForEach(sqlite3_stmt *stmt, CFErrorRef *error, bool(^row)(int row_index)) {
1236     int row_ix = 0;
1237     for (;;) {
1238         switch (_SecDbStep(dbconn, stmt, error)) {
1239             case kSecDbErrorStep:
1240                 return false;
1241             case kSecDbRowStep:
1242                 if (row) {
1243                     if (row(row_ix++))
1244                         break;
1245                 } else {
1246                     SecDbError(SQLITE_ERROR, error, CFSTR("SecDbStep SQLITE_ROW returned without a row handler"));
1247                 }
1248                 return false;
1249             case kSecDbDoneStep:
1250                 return true;
1251         }
1252     }
1253 }
1254 #endif
1255
1256 void SecDbRecordChange(SecDbConnectionRef dbconn, CFDataRef deleted, CFDataRef inserted) {
1257     if (!dbconn->db->notifyPhase) return;
1258     if (deleted)
1259         SOSDigestVectorAppend(&dbconn->todel, CFDataGetBytePtr(deleted));
1260     if (inserted)
1261         SOSDigestVectorAppend(&dbconn->toadd, CFDataGetBytePtr(inserted));
1262     if (!dbconn->inTransaction) {
1263         secerror("db %@ changed outside txn", dbconn);
1264         SecDbNotifyPhase(dbconn, kSecDbTransactionWillCommit);
1265         SecDbNotifyPhase(dbconn, kSecDbTransactionDidCommit);
1266     }
1267 }
1268
1269
1270 CFGiblisFor(SecDbConnection)