OSX/utilities/src/SecDb.c

   1 /*
   2  * Copyright (c) 2012-2014 Apple Inc. All Rights Reserved.
   3  *
   4  * @APPLE_LICENSE_HEADER_START@
   5  *
   6  * This file contains Original Code and/or Modifications of Original Code
   7  * as defined in and that are subject to the Apple Public Source License
   8  * Version 2.0 (the 'License'). You may not use this file except in
   9  * compliance with the License. Please obtain a copy of the License at
  10  * http://www.opensource.apple.com/apsl/ and read it before using this
  11  * file.
  12  *
  13  * The Original Code and all software distributed under the License are
  14  * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  15  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  16  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
  17  * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
  18  * Please see the License for the specific language governing rights and
  19  * limitations under the License.
  20  *
  21  * @APPLE_LICENSE_HEADER_END@
  22  */
  23
  24
  25 #include "SecDb.h"
  26 #include "debugging.h"
  27
  28 #include <sqlite3.h>
  29 #include <sqlite3_private.h>
  30 #include <CoreFoundation/CoreFoundation.h>
  31 #include <libgen.h>
  32 #include <sys/stat.h>
  33 #include <AssertMacros.h>
  34 #include "SecCFWrappers.h"
  35 #include "SecCFError.h"
  36 #include "SecIOFormat.h"
  37 #include <stdio.h>
  38 #include "Security/SecBase.h"
  39
  40
  41 //
  42 // Architecturally inverted files
  43 // These are in SecureObjectSync but utilities depends on them
  44 // <rdar://problem/20802079> Fix layer violation (SOSDigestVector, SOSManifest, SecDB.c)
  45 //
  46 #include <Security/SecureObjectSync/SOSDigestVector.h>
  47 #include <Security/SecureObjectSync/SOSManifest.h>
  48
  49 #define LOGE(ARG,...) secerror(ARG, ## __VA_ARGS__)
  50 #define LOGV(ARG,...) secdebug("secdb", ARG, ## __VA_ARGS__)
  51 #define LOGD(ARG,...) secdebug("secdb", ARG, ## __VA_ARGS__)
  52
  53 #define HAVE_UNLOCK_NOTIFY  0
  54
  55 struct __OpaqueSecDbStatement {
  56     CFRuntimeBase _base;
  57
  58     SecDbConnectionRef dbconn;
  59     sqlite3_stmt *stmt;
  60 };
  61
  62 struct __OpaqueSecDbConnection {
  63     CFRuntimeBase _base;
  64
  65     //CFMutableDictionaryRef statements;
  66
  67     SecDbRef db;     // NONRETAINED, since db or block retains us
  68     bool readOnly;
  69     bool inTransaction;
  70     SecDbTransactionSource source;
  71     bool isCorrupted;
  72     CFErrorRef corruptionError;
  73     sqlite3 *handle;
  74     // Pending deletions and additions for the current transaction
  75     // Entires are either:
  76     // 1) a CFArrayRef of 1 element representing a deletion,
  77     // 2) a CFArrayRef of 2 elements representing the element 0 having been replaced with element 1
  78     // 3) a CFTypeRef that is not a CFArrayRef, representing an add of the element in question.
  79     CFMutableArrayRef changes;
  80 };
  81
  82 struct __OpaqueSecDb {
  83     CFRuntimeBase _base;
  84
  85     CFStringRef db_path;
  86     dispatch_queue_t queue;
  87     CFMutableArrayRef connections;
  88     dispatch_semaphore_t write_semaphore;
  89     dispatch_semaphore_t read_semaphore;
  90     bool didFirstOpen;
  91     bool (^opened)(SecDbConnectionRef dbconn, bool didCreate, bool *callMeAgainForNextConnection, CFErrorRef *error);
  92     bool callOpenedHandlerForNextConnection;
  93     dispatch_queue_t notifyQueue;
  94     SecDBNotifyBlock notifyPhase;
  95 };
  96
  97 // MARK: Error domains and error helper functions
  98
  99 CFStringRef kSecDbErrorDomain = CFSTR("com.apple.utilities.sqlite3");
 100
 101 bool SecDbError(int sql_code, CFErrorRef *error, CFStringRef format, ...) {
 102     if (sql_code == SQLITE_OK) return true;
 103     if (error) {
 104         va_list args;
 105         CFIndex code = sql_code;
 106         CFErrorRef previousError = *error;
 107
 108         *error = NULL;
 109         va_start(args, format);
 110         SecCFCreateErrorWithFormatAndArguments(code, kSecDbErrorDomain, previousError, error, NULL, format, args);
 111         CFReleaseNull(previousError);
 112         va_end(args);
 113     }
 114     return false;
 115 }
 116
 117 bool SecDbErrorWithDb(int sql_code, sqlite3 *db, CFErrorRef *error, CFStringRef format, ...) {
 118     if (sql_code == SQLITE_OK) return true;
 119     if (error) {
 120         va_list args;
 121         va_start(args, format);
 122         CFStringRef message = CFStringCreateWithFormatAndArguments(kCFAllocatorDefault, NULL, format, args);
 123         va_end(args);
 124
 125         int extended_code = sqlite3_extended_errcode(db);
 126         if (sql_code == extended_code)
 127             SecDbError(sql_code, error, CFSTR("%@: [%d] %s"), message, sql_code, sqlite3_errmsg(db));
 128         else
 129             SecDbError(sql_code, error, CFSTR("%@: [%d->%d] %s"), message, sql_code, extended_code, sqlite3_errmsg(db));
 130         CFReleaseSafe(message);
 131     }
 132     return false;
 133 }
 134
 135 bool SecDbErrorWithStmt(int sql_code, sqlite3_stmt *stmt, CFErrorRef *error, CFStringRef format, ...) {
 136     if (sql_code == SQLITE_OK) return true;
 137     if (error) {
 138         va_list args;
 139         va_start(args, format);
 140         CFStringRef message = CFStringCreateWithFormatAndArguments(kCFAllocatorDefault, NULL, format, args);
 141         va_end(args);
 142
 143         sqlite3 *db = sqlite3_db_handle(stmt);
 144         const char *sql = sqlite3_sql(stmt);
 145         int extended_code = sqlite3_extended_errcode(db);
 146         if (sql_code == extended_code)
 147             SecDbError(sql_code, error, CFSTR("%@: [%d] %s sql: %s"), message, sql_code, sqlite3_errmsg(db), sql);
 148         else
 149             SecDbError(sql_code, error, CFSTR("%@: [%d->%d] %s sql: %s"), message, sql_code, extended_code, sqlite3_errmsg(db), sql);
 150         CFReleaseSafe(message);
 151     }
 152     return false;
 153 }
 154
 155
 156 // MARK: -
 157 // MARK: Static helper functions
 158
 159 static bool SecDbOpenHandle(SecDbConnectionRef dbconn, bool *created, CFErrorRef *error);
 160 static bool SecDbHandleCorrupt(SecDbConnectionRef dbconn, int rc, CFErrorRef *error);
 161
 162 #pragma mark -
 163 #pragma mark SecDbRef
 164
 165 static CFStringRef
 166 SecDbCopyFormatDescription(CFTypeRef value, CFDictionaryRef formatOptions)
 167 {
 168     SecDbRef db = (SecDbRef)value;
 169     return CFStringCreateWithFormat(kCFAllocatorDefault, NULL, CFSTR("<SecDb path:%@ connections: %@>"), db->db_path, db->connections);
 170 }
 171
 172
 173 static void
 174 SecDbDestroy(CFTypeRef value)
 175 {
 176     SecDbRef db = (SecDbRef)value;
 177     CFReleaseSafe(db->connections);
 178     CFReleaseSafe(db->db_path);
 179     dispatch_release(db->queue);
 180     dispatch_release(db->read_semaphore);
 181     dispatch_release(db->write_semaphore);
 182     if (db->opened)
 183         Block_release(db->opened);
 184 }
 185
 186 CFGiblisFor(SecDb)
 187
 188 SecDbRef
 189 SecDbCreate(CFStringRef dbName,
 190             bool (^opened)(SecDbConnectionRef dbconn, bool didCreate, bool *callMeAgainForNextConnection, CFErrorRef *error))
 191 {
 192     SecDbRef db = NULL;
 193
 194     db = CFTypeAllocate(SecDb, struct __OpaqueSecDb, kCFAllocatorDefault);
 195     require(db != NULL, done);
 196
 197     CFStringPerformWithCString(dbName, ^(const char *dbNameStr) {
 198         db->queue = dispatch_queue_create(dbNameStr, DISPATCH_QUEUE_SERIAL);
 199     });
 200     db->read_semaphore = dispatch_semaphore_create(kSecDbMaxReaders);
 201     db->write_semaphore = dispatch_semaphore_create(kSecDbMaxWriters);
 202     db->connections = CFArrayCreateMutableForCFTypes(kCFAllocatorDefault);
 203     db->opened = opened ? Block_copy(opened) : NULL;
 204     if (getenv("__OSINSTALL_ENVIRONMENT") != NULL) {
 205         // TODO: Move this code out of this layer
 206         LOGV("sqlDb: running from installer");
 207         db->db_path = CFSTR("file::memory:?cache=shared");
 208     } else {
 209         db->db_path = CFStringCreateCopy(kCFAllocatorDefault, dbName);
 210     }
 211 done:
 212     return db;
 213 }
 214
 215 CFIndex
 216 SecDbIdleConnectionCount(SecDbRef db) {
 217     __block CFIndex count = 0;
 218     dispatch_sync(db->queue, ^{
 219         count = CFArrayGetCount(db->connections);
 220     });
 221     return count;
 222 }
 223
 224 void SecDbSetNotifyPhaseBlock(SecDbRef db, dispatch_queue_t queue, SecDBNotifyBlock notifyPhase) {
 225     if (db->notifyQueue)
 226         dispatch_release(db->notifyQueue);
 227     if (db->notifyPhase)
 228         Block_release(db->notifyPhase);
 229
 230     if (queue) {
 231         db->notifyQueue = queue;
 232         dispatch_retain(db->notifyQueue);
 233     } else {
 234         db->notifyQueue = NULL;
 235     }
 236     if (notifyPhase)
 237         db->notifyPhase = Block_copy(notifyPhase);
 238     else
 239         db->notifyPhase = NULL;
 240 }
 241
 242 static void SecDbNotifyPhase(SecDbConnectionRef dbconn, SecDbTransactionPhase phase) {
 243     if (CFArrayGetCount(dbconn->changes)) {
 244         CFArrayRef changes = dbconn->changes;
 245         dbconn->changes = CFArrayCreateMutableForCFTypes(kCFAllocatorDefault);
 246         if (dbconn->db->notifyPhase)
 247             dbconn->db->notifyPhase(dbconn, phase, dbconn->source, changes);
 248         CFReleaseSafe(changes);
 249     }
 250 }
 251
 252 static void SecDbOnNotifyQueue(SecDbConnectionRef dbconn, void (^perform)()) {
 253     if (dbconn->db->notifyQueue) {
 254         dispatch_sync(dbconn->db->notifyQueue, ^{
 255             perform();
 256         });
 257     } else {
 258         perform();
 259     }
 260 }
 261
 262 CFStringRef SecDbGetPath(SecDbRef db) {
 263     return db->db_path;
 264 }
 265
 266
 267 #pragma mark -
 268 #pragma mark SecDbConnectionRef
 269
 270 static bool SecDbCheckCorrupted(SecDbConnectionRef dbconn)
 271 {
 272     __block bool isCorrupted = true;
 273     __block CFErrorRef error = NULL;
 274     SecDbPrepare(dbconn, CFSTR("PRAGMA integrity_check"), &error, ^(sqlite3_stmt *stmt) {
 275         SecDbStep(dbconn, stmt, &error, ^(bool *stop) {
 276             const char * result = (const char*)sqlite3_column_text(stmt, 0);
 277             if (result && strncasecmp(result, "ok", 3) == 0) {
 278                 isCorrupted = false;
 279             }
 280         });
 281     });
 282     if (error) {
 283         LOGV("sqlDb: warning error %@ when running integrity check", error);
 284         CFRelease(error);
 285     }
 286     return isCorrupted;
 287 }
 288
 289 static bool SecDbDidCreateFirstConnection(SecDbConnectionRef dbconn, bool didCreate, CFErrorRef *error)
 290 {
 291     LOGD("sqlDb: starting maintenance");
 292     bool ok = true;
 293
 294     if (!didCreate && !dbconn->isCorrupted) {
 295         dbconn->isCorrupted = SecDbCheckCorrupted(dbconn);
 296         if (dbconn->isCorrupted)
 297             secerror("integrity check=fail");
 298         else
 299             LOGD("sqlDb: integrity check=pass");
 300     }
 301
 302     if (!dbconn->isCorrupted && dbconn->db->opened) {
 303         CFErrorRef localError = NULL;
 304
 305         dbconn->db->callOpenedHandlerForNextConnection = false;
 306         ok = dbconn->db->opened(dbconn, didCreate, &dbconn->db->callOpenedHandlerForNextConnection, &localError);
 307
 308         if (!ok)
 309             secerror("opened block failed: %@", localError);
 310
 311         if (!dbconn->isCorrupted && error && *error == NULL) {
 312             *error = localError;
 313             localError = NULL;
 314         } else {
 315             secerror("opened block failed: error is released and lost");
 316             CFReleaseNull(localError);
 317         }
 318     }
 319
 320     if (dbconn->isCorrupted) {
 321         ok = SecDbHandleCorrupt(dbconn, 0, error);
 322     }
 323
 324     LOGD("sqlDb: finished maintenance");
 325     return ok;
 326 }
 327
 328 void SecDbCorrupt(SecDbConnectionRef dbconn, CFErrorRef error)
 329 {
 330     dbconn->isCorrupted = true;
 331     CFAssignRetained(dbconn->corruptionError, error);
 332 }
 333
 334
 335 static uint8_t knownDbPathIndex(SecDbConnectionRef dbconn)
 336 {
 337
 338     if(CFEqual(dbconn->db->db_path, CFSTR("/Library/Keychains/keychain-2.db")))
 339         return 1;
 340     if(CFEqual(dbconn->db->db_path, CFSTR("/Library/Keychains/ocspcache.sqlite3")))
 341         return 2;
 342     if(CFEqual(dbconn->db->db_path, CFSTR("/Library/Keychains/TrustStore.sqlite3")))
 343         return 3;
 344     if(CFEqual(dbconn->db->db_path, CFSTR("/Library/Keychains/caissuercache.sqlite3")))
 345         return 4;
 346
 347     /* Unknown DB path */
 348     return 0;
 349 }
 350
 351
 352 // Return true if there was no error, returns false otherwise and set *error to an appropriate CFErrorRef.
 353 static bool SecDbConnectionCheckCode(SecDbConnectionRef dbconn, int code, CFErrorRef *error, CFStringRef desc, ...) {
 354     if (code == SQLITE_OK || code == SQLITE_DONE)
 355         return true;
 356
 357     if (error) {
 358         va_list args;
 359         va_start(args, desc);
 360         CFStringRef msg = CFStringCreateWithFormatAndArguments(kCFAllocatorDefault, NULL, desc, args);
 361         va_end(args);
 362         SecDbErrorWithDb(code, dbconn->handle, error, msg);
 363         CFRelease(msg);
 364     }
 365
 366     /* If it's already corrupted, don't try to recover */
 367     if (dbconn->isCorrupted) {
 368         CFStringRef reason = CFStringCreateWithFormat(kCFAllocatorDefault, NULL, CFSTR("SQL DB %@ is corrupted already. Not trying to recover"), dbconn->db->db_path);
 369         secerror("%@",reason);
 370         __security_simulatecrash(reason, __sec_exception_code_TwiceCorruptDb(knownDbPathIndex(dbconn)));
 371         CFReleaseSafe(reason);
 372         return false;
 373     }
 374
 375     dbconn->isCorrupted = (SQLITE_CORRUPT == code) || (SQLITE_NOTADB == code) || (SQLITE_IOERR == code) || (SQLITE_CANTOPEN == code);
 376     if (dbconn->isCorrupted) {
 377         /* Run integrity check and only make dbconn->isCorrupted true and
 378            run the corruption handler if the integrity check conclusively fails. */
 379         dbconn->isCorrupted = SecDbCheckCorrupted(dbconn);
 380         if (dbconn->isCorrupted) {
 381             secerror("operation returned code: %d integrity check=fail", code);
 382             SecDbHandleCorrupt(dbconn, code, error);
 383         } else {
 384             secerror("operation returned code: %d: integrity check=pass", code);
 385         }
 386     }
 387
 388     return false;
 389 }
 390
 391 #if HAVE_UNLOCK_NOTIFY
 392
 393 static void SecDbUnlockNotify(void **apArg, int nArg) {
 394     int i;
 395     for(i=0; i<nArg; i++) {
 396         dispatch_semaphore_t dsema = (dispatch_semaphore_t)apArg[i];
 397         dispatch_semaphore_signal(dsema);
 398     }
 399 }
 400
 401 static bool SecDbWaitForUnlockNotify(SecDbConnectionRef dbconn, sqlite3_stmt *stmt, CFErrorRef *error) {
 402     int rc;
 403     dispatch_semaphore_t dsema = dispatch_semaphore_create(0);
 404     rc = sqlite3_unlock_notify(dbconn->handle, SecDbUnlockNotify, dsema);
 405     assert(rc == SQLITE_LOCKED || rc == SQLITE_OK);
 406     if (rc == SQLITE_OK) {
 407         dispatch_semaphore_wait(dsema, DISPATCH_TIME_FOREVER);
 408     }
 409     dispatch_release(dsema);
 410     return (rc == SQLITE_OK
 411             ? true
 412             : (stmt
 413                ? SecDbErrorWithStmt(rc, stmt, error, CFSTR("sqlite3_unlock_notify"))
 414                : SecDbErrorWithDb(rc, dbconn->handle, error, CFSTR("sqlite3_unlock_notify"))));
 415 }
 416
 417 #endif
 418
 419 #define BUSY_TIMEOUT_MS (5 * 60 * 1000)  /* 5 minutes */
 420
 421 static bool SecDbBusyHandler(SecDbConnectionRef dbconn, CFErrorRef *error) {
 422     return SecDbErrorWithDb(sqlite3_busy_timeout(dbconn->handle, BUSY_TIMEOUT_MS), dbconn->handle, error, CFSTR("busy_handler"));
 423 }
 424
 425 static int sleepBackoff[] = { 10, 20, 50, 100, 250 };
 426 static int sumBackoff[]   = { 10, 30, 80, 180, 430 };
 427 static int numEntries = sizeof(sleepBackoff)/sizeof(sleepBackoff[0]);
 428
 429 // Return true causes the operation to be tried again.
 430 static bool SecDbWaitIfNeeded(SecDbConnectionRef dbconn, int s3e, sqlite3_stmt *stmt, CFStringRef desc, int nTries, CFErrorRef *error) {
 431 #if HAVE_UNLOCK_NOTIFY
 432     if (s3e == SQLITE_LOCKED) { // Optionally check for extended code being SQLITE_LOCKED_SHAREDCACHE
 433         return SecDbWaitForUnlockNotify(dbconn, stmt, error))
 434     }
 435 #endif
 436     if (((0xFF & s3e) == SQLITE_BUSY) || ((0xFF & s3e) == SQLITE_LOCKED)) {
 437         int totaltimeout, timeout;
 438
 439         _Static_assert(sizeof(sumBackoff) == sizeof(sleepBackoff), "matching arrays not matching");
 440         _Static_assert(sizeof(sumBackoff[0]) == sizeof(sleepBackoff[0]), "matching arrays not matching");
 441
 442         if (nTries < numEntries) {
 443             timeout = sleepBackoff[nTries];
 444             totaltimeout = sumBackoff[nTries];
 445         } else {
 446             timeout = sleepBackoff[numEntries - 1];
 447             totaltimeout = sumBackoff[numEntries - 1] + (timeout * (nTries - numEntries));
 448         }
 449         if (totaltimeout < BUSY_TIMEOUT_MS) {
 450             LOGE("sqlite busy/locked: %d ntries: %d totaltimeout: %d", s3e, nTries, totaltimeout);
 451             sqlite3_sleep(timeout);
 452             return true;
 453         } else {
 454             LOGE("sqlite busy/locked: too long: %d ms, giving up", totaltimeout);
 455         }
 456     }
 457
 458     return SecDbConnectionCheckCode(dbconn, s3e, error, desc);
 459 }
 460
 461 enum SecDbStepResult {
 462     kSecDbErrorStep = 0,
 463     kSecDbRowStep = 1,
 464     kSecDbDoneStep = 2,
 465 };
 466 typedef enum SecDbStepResult SecDbStepResult;
 467
 468 static SecDbStepResult _SecDbStep(SecDbConnectionRef dbconn, sqlite3_stmt *stmt, CFErrorRef *error) {
 469     assert(stmt != NULL);
 470     int s3e;
 471     int ntries = 0;
 472     for (;;) {
 473         s3e = sqlite3_step(stmt);
 474         if (s3e == SQLITE_ROW) {
 475             ntries = 0;
 476             return kSecDbRowStep;
 477         } else if (s3e == SQLITE_DONE)
 478             return kSecDbDoneStep;
 479         else if (!SecDbWaitIfNeeded(dbconn, s3e, stmt, CFSTR("step"), ntries, error)) {
 480             return kSecDbErrorStep;
 481         }
 482         ntries++;
 483     };
 484 }
 485
 486 bool
 487 SecDbExec(SecDbConnectionRef dbconn, CFStringRef sql, CFErrorRef *error)
 488 {
 489     bool ok = true;
 490     CFRetain(sql);
 491     while (sql) {
 492         CFStringRef tail = NULL;
 493         if (ok) {
 494             sqlite3_stmt *stmt = SecDbCopyStmt(dbconn, sql, &tail, error);
 495             ok = stmt != NULL;
 496             if (stmt) {
 497                 SecDbStepResult sr;
 498                 while ((sr = _SecDbStep(dbconn, stmt, error)) == kSecDbRowStep);
 499                 if (sr == kSecDbErrorStep)
 500                     ok = false;
 501                 ok &= SecDbReleaseCachedStmt(dbconn, sql, stmt, error);
 502             }
 503         } else {
 504             // TODO We already have an error here we really just want the left over sql in it's userData
 505             ok = SecDbError(SQLITE_ERROR, error, CFSTR("Error with unexecuted sql remaining %@"), sql);
 506         }
 507         CFRelease(sql);
 508         sql = tail;
 509     }
 510     return ok;
 511 }
 512
 513 static bool SecDbBeginTransaction(SecDbConnectionRef dbconn, SecDbTransactionType type, CFErrorRef *error)
 514 {
 515     bool ok = true;
 516     CFStringRef query;
 517     switch (type) {
 518         case kSecDbImmediateTransactionType:
 519             query = CFSTR("BEGIN IMMEDATE");
 520             break;
 521         case kSecDbExclusiveRemoteTransactionType:
 522             dbconn->source = kSecDbSOSTransaction;
 523         case kSecDbExclusiveTransactionType:
 524             query = CFSTR("BEGIN EXCLUSIVE");
 525             break;
 526         case kSecDbNormalTransactionType:
 527             query = CFSTR("BEGIN");
 528             break;
 529         default:
 530             ok = SecDbError(SQLITE_ERROR, error, CFSTR("invalid transaction type %" PRIu32), type);
 531             query = NULL;
 532             break;
 533     }
 534
 535     if (query != NULL && sqlite3_get_autocommit(dbconn->handle) != 0) {
 536         ok = SecDbExec(dbconn, query, error);
 537     }
 538     if (ok)
 539         dbconn->inTransaction = true;
 540
 541     return ok;
 542 }
 543
 544 static bool SecDbEndTransaction(SecDbConnectionRef dbconn, bool commit, CFErrorRef *error)
 545 {
 546     __block bool ok = true;
 547     SecDbOnNotifyQueue(dbconn, ^{
 548         bool commited = false;
 549         if (commit) {
 550             SecDbNotifyPhase(dbconn, kSecDbTransactionWillCommit);
 551             commited = ok = SecDbExec(dbconn, CFSTR("END"), error);
 552         } else {
 553             ok = SecDbExec(dbconn, CFSTR("ROLLBACK"), error);
 554             commited = false;
 555         }
 556         dbconn->inTransaction = false;
 557         SecDbNotifyPhase(dbconn, commited ? kSecDbTransactionDidCommit : kSecDbTransactionDidRollback);
 558         dbconn->source = kSecDbAPITransaction;
 559     });
 560     return ok;
 561 }
 562
 563 bool SecDbTransaction(SecDbConnectionRef dbconn, SecDbTransactionType type,
 564                       CFErrorRef *error, void (^transaction)(bool *commit))
 565 {
 566     bool ok = true;
 567     bool commit = true;
 568
 569     if (dbconn->inTransaction) {
 570         transaction(&commit);
 571         if (!commit) {
 572             LOGV("sqlDb: nested transaction asked to not be committed");
 573         }
 574     } else {
 575         ok = SecDbBeginTransaction(dbconn, type, error);
 576         if (ok) {
 577             transaction(&commit);
 578             ok = SecDbEndTransaction(dbconn, commit, error);
 579         }
 580     }
 581
 582 done:
 583     return ok && commit;
 584 }
 585
 586 sqlite3 *SecDbHandle(SecDbConnectionRef dbconn) {
 587     return dbconn->handle;
 588 }
 589
 590 bool SecDbStep(SecDbConnectionRef dbconn, sqlite3_stmt *stmt, CFErrorRef *error, void (^row)(bool *stop)) {
 591     for (;;) {
 592         switch (_SecDbStep(dbconn, stmt, error)) {
 593             case kSecDbErrorStep:
 594                 return false;
 595             case kSecDbRowStep:
 596                 if (row) {
 597                     bool stop = false;
 598                     row(&stop);
 599                     if (stop)
 600                         return true;
 601                     break;
 602                 }
 603                 SecDbError(SQLITE_ERROR, error, CFSTR("SecDbStep SQLITE_ROW returned without a row handler"));
 604                 return false;
 605             case kSecDbDoneStep:
 606                 return true;
 607         }
 608     }
 609 }
 610
 611 bool SecDbCheckpoint(SecDbConnectionRef dbconn, CFErrorRef *error)
 612 {
 613     return SecDbConnectionCheckCode(dbconn, sqlite3_wal_checkpoint(dbconn->handle, NULL), error, CFSTR("wal_checkpoint"));
 614 }
 615
 616 static bool SecDbFileControl(SecDbConnectionRef dbconn, int op, void *arg, CFErrorRef *error) {
 617     return SecDbConnectionCheckCode(dbconn, sqlite3_file_control(dbconn->handle, NULL, op, arg), error, CFSTR("file_control"));
 618 }
 619
 620 static sqlite3 *_SecDbOpenV2(const char *path, int flags, CFErrorRef *error) {
 621 #if HAVE_UNLOCK_NOTIFY
 622     flags |= SQLITE_OPEN_SHAREDCACHE;
 623 #endif
 624     sqlite3 *handle = NULL;
 625     int s3e = sqlite3_open_v2(path, &handle, flags, NULL);
 626     if (s3e) {
 627         if (handle) {
 628             SecDbErrorWithDb(s3e, handle, error, CFSTR("open_v2 \"%s\" 0x%X"), path, flags);
 629             sqlite3_close(handle);
 630             handle = NULL;
 631         } else {
 632             SecDbError(s3e, error, CFSTR("open_v2 \"%s\" 0x%X"), path, flags);
 633         }
 634     }
 635     return handle;
 636 }
 637
 638 static bool SecDbOpenV2(SecDbConnectionRef dbconn, const char *path, int flags, CFErrorRef *error) {
 639     return (dbconn->handle = _SecDbOpenV2(path, flags, error)) != NULL;
 640 }
 641
 642 static bool SecDbTruncate(SecDbConnectionRef dbconn, CFErrorRef *error)
 643 {
 644     int flags = SQLITE_TRUNCATE_JOURNALMODE_WAL | SQLITE_TRUNCATE_AUTOVACUUM_FULL;
 645     __block bool ok = SecDbFileControl(dbconn, SQLITE_TRUNCATE_DATABASE, &flags, error);
 646     if (!ok) {
 647         sqlite3_close(dbconn->handle);
 648         dbconn->handle = NULL;
 649         CFStringPerformWithCString(dbconn->db->db_path, ^(const char *path) {
 650             if (error)
 651                 CFReleaseNull(*error);
 652             if (SecCheckErrno(unlink(path), error, CFSTR("unlink %s"), path)) {
 653                 ok = SecDbOpenHandle(dbconn, NULL, error);
 654             }
 655         });
 656         if (!ok) {
 657             secerror("Failed to delete db handle: %@", error ? *error : NULL);
 658             abort();
 659         }
 660     }
 661
 662     return ok;
 663 }
 664
 665 static bool SecDbHandleCorrupt(SecDbConnectionRef dbconn, int rc, CFErrorRef *error)
 666 {
 667     CFStringRef reason = CFStringCreateWithFormat(kCFAllocatorDefault, NULL, CFSTR("SQL DB %@ is corrupted, trying to recover (rc=%d) %@"), dbconn->db->db_path, rc, dbconn->corruptionError);
 668     __security_simulatecrash(reason, __sec_exception_code_CorruptDb(knownDbPathIndex(dbconn), rc));
 669     CFReleaseSafe(reason);
 670
 671     // Backup current db.
 672     __block bool didRename = false;
 673     CFStringPerformWithCString(dbconn->db->db_path, ^(const char *db_path) {
 674         sqlite3 *corrupt_db = NULL;
 675         char buf[PATH_MAX+1];
 676         snprintf(buf, sizeof(buf), "%s-corrupt", db_path);
 677         if (dbconn->handle && (corrupt_db = _SecDbOpenV2(buf, SQLITE_OPEN_READWRITE, error))) {
 678             int on = 1;
 679             didRename =  SecDbErrorWithDb(sqlite3_file_control(corrupt_db, NULL, SQLITE_FCNTL_PERSIST_WAL, &on), corrupt_db, error, CFSTR("persist wal"));
 680             didRename &=  SecDbErrorWithDb(sqlite3_file_control(corrupt_db, NULL, SQLITE_REPLACE_DATABASE, (void *)dbconn->handle), corrupt_db, error, CFSTR("replace database"));
 681             sqlite3_close(corrupt_db);
 682         }
 683         if (!didRename) {
 684             if (dbconn->handle)
 685                 secerror("Tried to rename corrupt database at path %@, but we failed: %@, trying explicit rename", dbconn->db->db_path, error ? *error : NULL);
 686             if (error)
 687                 CFReleaseNull(*error);
 688
 689             didRename = SecCheckErrno(rename(db_path, buf), error, CFSTR("rename %s %s"), db_path, buf) &&
 690                 (!dbconn->handle || SecDbError(sqlite3_close(dbconn->handle), error, CFSTR("close"))) &&
 691                 SecDbOpenHandle(dbconn, NULL, error);
 692         }
 693         if (didRename) {
 694             secerror("Database at path %@ is corrupt. Copied it to %s for further investigation.", dbconn->db->db_path, buf);
 695         } else {
 696             seccritical("Tried to copy corrupt database at path %@, but we failed: %@", dbconn->db->db_path, error ? *error : NULL);
 697         }
 698     });
 699
 700     bool ok = (didRename &&
 701                (dbconn->handle || SecDbOpenHandle(dbconn, NULL, error)) &&
 702                SecDbTruncate(dbconn, error));
 703
 704     // Mark the db as not corrupted, even if something failed.
 705     // Always note we are no longer in the corruption handler
 706     dbconn->isCorrupted = false;
 707
 708     // Invoke our callers opened callback, since we just created a new database
 709     if (ok && dbconn->db->opened) {
 710         dbconn->db->callOpenedHandlerForNextConnection = false;
 711         ok = dbconn->db->opened(dbconn, true, &dbconn->db->callOpenedHandlerForNextConnection, error);
 712     }
 713
 714     return ok;
 715 }
 716
 717 static bool SecDbProfileEnabled(void)
 718 {
 719 #if 0
 720     static dispatch_once_t onceToken;
 721     static bool profile_enabled = false;
 722
 723 #if DEBUG
 724     //sudo defaults write /Library/Preferences/com.apple.security.auth profile -bool true
 725     dispatch_once(&onceToken, ^{
 726                 CFTypeRef profile = (CFNumberRef)CFPreferencesCopyValue(CFSTR("profile"), CFSTR(SECURITY_AUTH_NAME), kCFPreferencesAnyUser, kCFPreferencesCurrentHost);
 727
 728         if (profile && CFGetTypeID(profile) == CFBooleanGetTypeID()) {
 729             profile_enabled = CFBooleanGetValue((CFBooleanRef)profile);
 730         }
 731
 732         LOGV("sqlDb: sql profile: %s", profile_enabled ? "enabled" : "disabled");
 733
 734         CFReleaseSafe(profile);
 735     });
 736 #endif
 737
 738     return profile_enabled;
 739 #else
 740 #if DEBUG
 741     return true;
 742 #else
 743     return false;
 744 #endif
 745 #endif
 746 }
 747
 748 #if 0
 749 static void SecDbProfile(void *context __unused, const char *sql, sqlite3_uint64 ns) {
 750     LOGV("==\nsqlDb: %s\nTime: %llu ms\n", sql, ns >> 20);
 751 }
 752 #else
 753 static void SecDbProfile(void *context, const char *sql, sqlite3_uint64 ns) {
 754     sqlite3 *s3h = context;
 755     int code = sqlite3_extended_errcode(s3h);
 756     if (code == SQLITE_OK || code == SQLITE_DONE) {
 757         secdebug("profile", "==\nsqlDb: %s\nTime: %llu ms\n", sql, ns >> 20);
 758     } else {
 759         secdebug("profile", "==error[%d]: %s==\nsqlDb: %s\nTime: %llu ms \n", code, sqlite3_errmsg(s3h), sql, ns >> 20);
 760     }
 761 }
 762 #endif
 763
 764 static bool SecDbTraceEnabled(void)
 765 {
 766 #if DEBUG
 767     return true;
 768 #else
 769     return false;
 770 #endif
 771 }
 772
 773 static void SecDbTrace(void *ctx, const char *trace) {
 774     SecDbConnectionRef dbconn __unused = ctx;
 775     static dispatch_queue_t queue;
 776     static dispatch_once_t once;
 777     dispatch_once(&once, ^{
 778         queue = dispatch_queue_create("trace_queue", DISPATCH_QUEUE_SERIAL);
 779     });
 780     dispatch_sync(queue, ^{
 781         __security_debug(CFSTR("trace"), "", "", 0, CFSTR("%s"), trace);
 782     });
 783 }
 784
 785 static bool SecDbOpenHandle(SecDbConnectionRef dbconn, bool *created, CFErrorRef *error)
 786 {
 787     __block bool ok = true;
 788     CFStringPerformWithCString(dbconn->db->db_path, ^(const char *db_path) {
 789         ok = created && SecDbOpenV2(dbconn, db_path, SQLITE_OPEN_READWRITE, NULL);
 790         if (!ok) {
 791             ok = true;
 792             if (created) {
 793                 char *tmp = dirname((char *)db_path);
 794                 if (tmp) {
 795                     int errnum = mkpath_np(tmp, 0700);
 796                     if (errnum != 0 && errnum != EEXIST) {
 797                         SecCFCreateErrorWithFormat(errnum, kSecErrnoDomain, NULL, error, NULL,
 798                                                    CFSTR("mkpath_np %s: [%d] %s"), tmp, errnum, strerror(errnum));
 799                         ok = false;
 800                     }
 801                 }
 802             }
 803             ok = ok && SecDbOpenV2(dbconn, db_path, SQLITE_OPEN_READWRITE | SQLITE_OPEN_CREATE, error);
 804             if (ok) {
 805                 chmod(db_path, S_IRUSR | S_IWUSR);
 806                 if (created)
 807                     *created = true;
 808             }
 809         }
 810
 811         if (ok && SecDbProfileEnabled()) {
 812             sqlite3_profile(dbconn->handle, SecDbProfile, dbconn->handle);
 813         }
 814         if (ok && SecDbTraceEnabled()) {
 815             sqlite3_trace(dbconn->handle, SecDbTrace, dbconn);
 816         }
 817         ok = ok && SecDbBusyHandler(dbconn, error);
 818     });
 819
 820 done:
 821     return ok;
 822 }
 823
 824 static SecDbConnectionRef
 825 SecDbConnectionCreate(SecDbRef db, bool readOnly, CFErrorRef *error)
 826 {
 827     SecDbConnectionRef dbconn = NULL;
 828
 829     dbconn = CFTypeAllocate(SecDbConnection, struct __OpaqueSecDbConnection, kCFAllocatorDefault);
 830     require(dbconn != NULL, done);
 831
 832     dbconn->db = db;
 833     dbconn->readOnly = readOnly;
 834     dbconn->inTransaction = false;
 835     dbconn->source = NULL;
 836     dbconn->isCorrupted = false;
 837     dbconn->corruptionError = NULL;
 838     dbconn->handle = NULL;
 839     dbconn->changes = CFArrayCreateMutableForCFTypes(kCFAllocatorDefault);
 840
 841 done:
 842     return dbconn;
 843 }
 844
 845 static bool SecDbConnectionIsReadOnly(SecDbConnectionRef dbconn) {
 846     return dbconn->readOnly;
 847 }
 848
 849 static void SecDbConectionSetReadOnly(SecDbConnectionRef dbconn, bool readOnly) {
 850     dbconn->readOnly = readOnly;
 851 }
 852
 853 /* Read only connections go to the end of the queue, writeable connections
 854  go to the start of the queue. */
 855 SecDbConnectionRef SecDbConnectionAquire(SecDbRef db, bool readOnly, CFErrorRef *error) {
 856     CFRetain(db);
 857     secdebug("dbconn", "acquire %s connection", readOnly ? "ro" : "rw");
 858     dispatch_semaphore_wait(readOnly ? db->read_semaphore : db->write_semaphore, DISPATCH_TIME_FOREVER);
 859     __block SecDbConnectionRef dbconn = NULL;
 860     __block bool ok = true;
 861     __block bool ranOpenedHandler = false;
 862     dispatch_sync(db->queue, ^{
 863         if (!db->didFirstOpen) {
 864             bool didCreate = false;
 865             ok = dbconn = SecDbConnectionCreate(db, false, error);
 866             CFErrorRef localError = NULL;
 867             if (ok && !SecDbOpenHandle(dbconn, &didCreate, &localError)) {
 868                 secerror("Unable to create database: %@", localError);
 869                 if (localError && CFEqual(CFErrorGetDomain(localError), kSecDbErrorDomain)) {
 870                     int code = (int)CFErrorGetCode(localError);
 871                     dbconn->isCorrupted = (SQLITE_CORRUPT == code) || (SQLITE_NOTADB == code) || (SQLITE_IOERR == code) || (SQLITE_CANTOPEN == code);
 872                 }
 873                 // If the open failure isn't due to corruption, propagte the error.
 874                 ok = dbconn->isCorrupted;
 875                 if (!ok && error && *error == NULL) {
 876                     *error = localError;
 877                     localError = NULL;
 878                 }
 879             }
 880             CFReleaseNull(localError);
 881
 882             if (ok) {
 883                 db->didFirstOpen = ok = SecDbDidCreateFirstConnection(dbconn, didCreate, error);
 884                 ranOpenedHandler = true;
 885             }
 886             if (!ok)
 887                 CFReleaseNull(dbconn);
 888         } else {
 889             /* Try to get one from the cache */
 890             CFIndex count = CFArrayGetCount(db->connections);
 891             while (count && !dbconn) {
 892                 CFIndex ix = readOnly ? count - 1 : 0;
 893                 dbconn = (SecDbConnectionRef)CFArrayGetValueAtIndex(db->connections, ix);
 894                 if (dbconn)
 895                     CFRetain(dbconn);
 896                 else
 897                     secerror("got NULL dbconn at index: %" PRIdCFIndex " skipping", ix);
 898                 CFArrayRemoveValueAtIndex(db->connections, ix);
 899             }
 900         }
 901     });
 902
 903     if (dbconn) {
 904         /* Make sure the connection we found has the right access */
 905         if (SecDbConnectionIsReadOnly(dbconn) != readOnly) {
 906             SecDbConectionSetReadOnly(dbconn, readOnly);
 907         }
 908     } else if (ok) {
 909         /* Nothing found in cache, create a new connection */
 910         bool created = false;
 911         dbconn = SecDbConnectionCreate(db, readOnly, error);
 912         if (dbconn && !SecDbOpenHandle(dbconn, &created, error)) {
 913             CFReleaseNull(dbconn);
 914         }
 915     }
 916
 917     if (dbconn && !ranOpenedHandler && dbconn->db->opened) {
 918         dispatch_sync(db->queue, ^{
 919             if (dbconn->db->callOpenedHandlerForNextConnection) {
 920                 dbconn->db->callOpenedHandlerForNextConnection = false;
 921                 if (!dbconn->db->opened(dbconn, false, &dbconn->db->callOpenedHandlerForNextConnection, error)) {
 922                     if (!dbconn->isCorrupted || !SecDbHandleCorrupt(dbconn, 0, error)) {
 923                         CFReleaseNull(dbconn);
 924                     }
 925                 }
 926             }
 927         });
 928     }
 929
 930     if (!dbconn) {
 931         // If aquire fails we need to signal the semaphore again.
 932         dispatch_semaphore_signal(readOnly ? db->read_semaphore : db->write_semaphore);
 933         CFRelease(db);
 934     }
 935
 936     return dbconn;
 937 }
 938
 939 void SecDbConnectionRelease(SecDbConnectionRef dbconn) {
 940     if (!dbconn) {
 941         secerror("called with NULL dbconn");
 942         return;
 943     }
 944     SecDbRef db = dbconn->db;
 945     secdebug("dbconn", "release %@", dbconn);
 946     dispatch_sync(db->queue, ^{
 947         CFIndex count = CFArrayGetCount(db->connections);
 948         // Add back possible writable dbconn to the pool.
 949         bool readOnly = SecDbConnectionIsReadOnly(dbconn);
 950         CFArrayInsertValueAtIndex(db->connections, readOnly ? count : 0, dbconn);
 951         // Remove the last (probably read-only) dbconn from the pool.
 952         if (count >= kSecDbMaxIdleHandles) {
 953             CFArrayRemoveValueAtIndex(db->connections, count);
 954         }
 955         // Signal after we have put the connection back in the pool of connections
 956         dispatch_semaphore_signal(readOnly ? db->read_semaphore : db->write_semaphore);
 957         CFRelease(dbconn);
 958         CFRelease(db);
 959     });
 960 }
 961
 962 bool SecDbPerformRead(SecDbRef db, CFErrorRef *error, void (^perform)(SecDbConnectionRef dbconn)) {
 963     SecDbConnectionRef dbconn = SecDbConnectionAquire(db, true, error);
 964     bool success = false;
 965     if (dbconn) {
 966         perform(dbconn);
 967         success = true;
 968         SecDbConnectionRelease(dbconn);
 969     }
 970     return success;
 971 }
 972
 973 bool SecDbPerformWrite(SecDbRef db, CFErrorRef *error, void (^perform)(SecDbConnectionRef dbconn)) {
 974     SecDbConnectionRef dbconn = SecDbConnectionAquire(db, false, error);
 975     bool success = false;
 976     if (dbconn) {
 977         perform(dbconn);
 978         success = true;
 979         SecDbConnectionRelease(dbconn);
 980     }
 981     return success;
 982 }
 983
 984 static CFStringRef
 985 SecDbConnectionCopyFormatDescription(CFTypeRef value, CFDictionaryRef formatOptions)
 986 {
 987     SecDbConnectionRef dbconn = (SecDbConnectionRef)value;
 988     return CFStringCreateWithFormat(kCFAllocatorDefault, NULL, CFSTR("<SecDbConnection %s %s>"),
 989                                     dbconn->readOnly ? "ro" : "rw", dbconn->handle ? "open" : "closed");
 990 }
 991
 992 static void
 993 SecDbConnectionDestroy(CFTypeRef value)
 994 {
 995     SecDbConnectionRef dbconn = (SecDbConnectionRef)value;
 996     if (dbconn->handle) {
 997         sqlite3_close(dbconn->handle);
 998     }
 999     dbconn->db = NULL;
1000     CFReleaseNull(dbconn->changes);
1001     CFReleaseNull(dbconn->corruptionError);
1002
1003 }
1004
1005
1006 // MARK: -
1007 // MARK: Bind helpers
1008
1009 #if 0
1010 bool SecDbBindNull(sqlite3_stmt *stmt, int param, CFErrorRef *error) {
1011     bool ok = SecDbErrorWithStmt(sqlite3_bind_null(stmt, param),
1012                                  stmt, error, CFSTR("bind_null[%d]"), param);
1013     secdebug("bind", "bind_null[%d]: %@", param, error ? *error : NULL);
1014     return ok;
1015 }
1016 #endif
1017
1018 bool SecDbBindBlob(sqlite3_stmt *stmt, int param, const void *zData, size_t n, void(*xDel)(void*), CFErrorRef *error) {
1019     if (n > INT_MAX) {
1020         return SecDbErrorWithStmt(SQLITE_TOOBIG, stmt, error,
1021                                   CFSTR("bind_blob[%d]: blob bigger than INT_MAX"), param);
1022     }
1023     bool ok = SecDbErrorWithStmt(sqlite3_bind_blob(stmt, param, zData, (int)n, xDel),
1024                                  stmt, error, CFSTR("bind_blob[%d]"), param);
1025     secdebug("bind", "bind_blob[%d]: %.*s: %@", param, (int)n, zData, error ? *error : NULL);
1026     return ok;
1027 }
1028
1029 bool SecDbBindText(sqlite3_stmt *stmt, int param, const char *zData, size_t n, void(*xDel)(void*), CFErrorRef *error) {
1030     if (n > INT_MAX) {
1031         return SecDbErrorWithStmt(SQLITE_TOOBIG, stmt, error,
1032                                   CFSTR("bind_text[%d]: text bigger than INT_MAX"), param);
1033     }
1034     bool ok = SecDbErrorWithStmt(sqlite3_bind_text(stmt, param, zData, (int)n, xDel), stmt, error,
1035                                  CFSTR("bind_text[%d]"), param);
1036     secdebug("bind", "bind_text[%d]: \"%s\": %@", param, zData, error ? *error : NULL);
1037     return ok;
1038 }
1039
1040 bool SecDbBindDouble(sqlite3_stmt *stmt, int param, double value, CFErrorRef *error) {
1041     bool ok = SecDbErrorWithStmt(sqlite3_bind_double(stmt, param, value), stmt, error,
1042                                  CFSTR("bind_double[%d]"), param);
1043     secdebug("bind", "bind_double[%d]: %f: %@", param, value, error ? *error : NULL);
1044     return ok;
1045 }
1046
1047 bool SecDbBindInt(sqlite3_stmt *stmt, int param, int value, CFErrorRef *error) {
1048     bool ok = SecDbErrorWithStmt(sqlite3_bind_int(stmt, param, value), stmt, error,
1049                                  CFSTR("bind_int[%d]"), param);
1050     secdebug("bind", "bind_int[%d]: %d: %@", param, value, error ? *error : NULL);
1051     return ok;
1052 }
1053
1054 bool SecDbBindInt64(sqlite3_stmt *stmt, int param, sqlite3_int64 value, CFErrorRef *error) {
1055     bool ok = SecDbErrorWithStmt(sqlite3_bind_int64(stmt, param, value), stmt, error,
1056                                  CFSTR("bind_int64[%d]"), param);
1057     secdebug("bind", "bind_int64[%d]: %lld: %@", param, value, error ? *error : NULL);
1058     return ok;
1059 }
1060
1061
1062 /* AUDIT[securityd](done):
1063  value (ok) is a caller provided, non NULL CFTypeRef.
1064  */
1065 bool SecDbBindObject(sqlite3_stmt *stmt, int param, CFTypeRef value, CFErrorRef *error) {
1066     CFTypeID valueId;
1067     __block bool result = false;
1068
1069         /* TODO: Can we use SQLITE_STATIC below everwhere we currently use
1070      SQLITE_TRANSIENT since we finalize the statement before the value
1071      goes out of scope? */
1072     if (!value || (valueId = CFGetTypeID(value)) == CFNullGetTypeID()) {
1073         /* Skip bindings for NULL values.  sqlite3 will interpret unbound
1074          params as NULL which is exactly what we want. */
1075 #if 1
1076         result = true;
1077 #else
1078         result = SecDbBindNull(stmt, param, error);
1079 #endif
1080     } else if (valueId == CFStringGetTypeID()) {
1081         CFStringPerformWithCStringAndLength(value, ^(const char *cstr, size_t clen) {
1082             result = SecDbBindText(stmt, param, cstr, clen, SQLITE_TRANSIENT, error);
1083         });
1084     } else if (valueId == CFDataGetTypeID()) {
1085         CFIndex len = CFDataGetLength(value);
1086         if (len) {
1087             result = SecDbBindBlob(stmt, param, CFDataGetBytePtr(value),
1088                                    len, SQLITE_TRANSIENT, error);
1089         } else {
1090             result = SecDbBindText(stmt, param, "", 0, SQLITE_TRANSIENT, error);
1091         }
1092     } else if (valueId == CFDateGetTypeID()) {
1093         CFAbsoluteTime abs_time = CFDateGetAbsoluteTime(value);
1094         result = SecDbBindDouble(stmt, param, abs_time, error);
1095     } else if (valueId == CFBooleanGetTypeID()) {
1096         int bval = CFBooleanGetValue(value);
1097         result = SecDbBindInt(stmt, param, bval, error);
1098     } else if (valueId == CFNumberGetTypeID()) {
1099         Boolean convertOk;
1100         if (CFNumberIsFloatType(value)) {
1101             double nval;
1102             convertOk = CFNumberGetValue(value, kCFNumberDoubleType, &nval);
1103             result = SecDbBindDouble(stmt, param, nval, error);
1104         } else {
1105             int nval;
1106             convertOk = CFNumberGetValue(value, kCFNumberSInt32Type, &nval);
1107             if (convertOk) {
1108                 result = SecDbBindInt(stmt, param, nval, error);
1109             } else {
1110                 sqlite_int64 nval64;
1111                 convertOk = CFNumberGetValue(value, kCFNumberSInt64Type, &nval64);
1112                 if (convertOk)
1113                     result = SecDbBindInt64(stmt, param, nval64, error);
1114             }
1115         }
1116         if (!convertOk) {
1117             result = SecDbError(SQLITE_INTERNAL, error, CFSTR("bind CFNumberGetValue failed for %@"), value);
1118         }
1119     } else {
1120         if (error) {
1121             CFStringRef valueDesc = CFCopyTypeIDDescription(valueId);
1122             SecDbError(SQLITE_MISMATCH, error, CFSTR("bind unsupported type %@"), valueDesc);
1123             CFReleaseSafe(valueDesc);
1124         }
1125     }
1126
1127         return result;
1128 }
1129
1130 // MARK: -
1131 // MARK: SecDbStatementRef
1132
1133 bool SecDbReset(sqlite3_stmt *stmt, CFErrorRef *error) {
1134     return SecDbErrorWithStmt(sqlite3_reset(stmt), stmt, error, CFSTR("reset"));
1135 }
1136
1137 bool SecDbClearBindings(sqlite3_stmt *stmt, CFErrorRef *error) {
1138     return SecDbErrorWithStmt(sqlite3_clear_bindings(stmt), stmt, error, CFSTR("clear bindings"));
1139 }
1140
1141 bool SecDbFinalize(sqlite3_stmt *stmt, CFErrorRef *error) {
1142     sqlite3 *handle = sqlite3_db_handle(stmt);
1143     int s3e = sqlite3_finalize(stmt);
1144     return s3e == SQLITE_OK ? true : SecDbErrorWithDb(s3e, handle, error, CFSTR("finalize: %p"), stmt);
1145 }
1146
1147 sqlite3_stmt *SecDbPrepareV2(SecDbConnectionRef dbconn, const char *sql, size_t sqlLen, const char **sqlTail, CFErrorRef *error) {
1148     sqlite3 *db = SecDbHandle(dbconn);
1149     if (sqlLen > INT_MAX) {
1150         SecDbErrorWithDb(SQLITE_TOOBIG, db, error, CFSTR("prepare_v2: sql bigger than INT_MAX"));
1151         return NULL;
1152     }
1153     int ntries = 0;
1154     for (;;) {
1155         sqlite3_stmt *stmt = NULL;
1156         int s3e = sqlite3_prepare_v2(db, sql, (int)sqlLen, &stmt, sqlTail);
1157         if (s3e == SQLITE_OK)
1158             return stmt;
1159         else if (!SecDbWaitIfNeeded(dbconn, s3e, NULL, CFSTR("preparev2"), ntries, error))
1160             return NULL;
1161         ntries++;
1162     }
1163 }
1164
1165 static sqlite3_stmt *SecDbCopyStatementWithTailRange(SecDbConnectionRef dbconn, CFStringRef sql, CFRange *sqlTail, CFErrorRef *error) {
1166     __block sqlite3_stmt *stmt = NULL;
1167     if (sql) CFStringPerformWithCStringAndLength(sql, ^(const char *sqlStr, size_t sqlLen) {
1168         const char *tail = NULL;
1169         stmt = SecDbPrepareV2(dbconn, sqlStr, sqlLen, &tail, error);
1170         if (sqlTail && sqlStr < tail && tail < sqlStr + sqlLen) {
1171             sqlTail->location = tail - sqlStr;
1172             sqlTail->length = sqlLen - sqlTail->location;
1173         }
1174     });
1175
1176     return stmt;
1177 }
1178
1179 sqlite3_stmt *SecDbCopyStmt(SecDbConnectionRef dbconn, CFStringRef sql, CFStringRef *tail, CFErrorRef *error) {
1180     // TODO: Add caching and cache lookup of statements
1181     CFRange sqlTail = {};
1182     sqlite3_stmt *stmt = SecDbCopyStatementWithTailRange(dbconn, sql, &sqlTail, error);
1183     if (sqlTail.length > 0) {
1184         CFStringRef excess = CFStringCreateWithSubstring(CFGetAllocator(sql), sql, sqlTail);
1185         if (tail) {
1186             *tail = excess;
1187         } else {
1188             SecDbError(SQLITE_INTERNAL, error,
1189                        CFSTR("prepare_v2: %@ unused sql: %@"),
1190                        sql, excess);
1191             CFReleaseSafe(excess);
1192             SecDbFinalize(stmt, error);
1193             stmt = NULL;
1194         }
1195     }
1196     return stmt;
1197 }
1198
1199 /*
1200  TODO: Could do a hack here with a custom kCFAllocatorNULL allocator for a second CFRuntimeBase inside a SecDbStatement,
1201  TODO: Better yet make a full blow SecDbStatement instance whenever SecDbCopyStmt is called.  Then, when the statement is released, in the Dispose method, we Reset and ClearBindings the sqlite3_stmt * and hand it back to the SecDb with the original CFStringRef for the sql (or hash thereof) as an argument. */
1202 bool SecDbReleaseCachedStmt(SecDbConnectionRef dbconn, CFStringRef sql, sqlite3_stmt *stmt, CFErrorRef *error) {
1203     if (stmt) {
1204         return SecDbFinalize(stmt, error);
1205     }
1206     return true;
1207 }
1208
1209 bool SecDbPrepare(SecDbConnectionRef dbconn, CFStringRef sql, CFErrorRef *error, void(^exec)(sqlite3_stmt *stmt)) {
1210     assert(sql != NULL);
1211     sqlite3_stmt *stmt = SecDbCopyStmt(dbconn, sql, NULL, error);
1212     if (!stmt)
1213         return false;
1214
1215     exec(stmt);
1216     return SecDbReleaseCachedStmt(dbconn, sql, stmt, error);
1217 }
1218
1219 bool SecDbWithSQL(SecDbConnectionRef dbconn, CFStringRef sql, CFErrorRef *error, bool(^perform)(sqlite3_stmt *stmt)) {
1220     bool ok = true;
1221     CFRetain(sql);
1222     while (sql) {
1223         CFStringRef tail = NULL;
1224         if (ok) {
1225             sqlite3_stmt *stmt = SecDbCopyStmt(dbconn, sql, &tail, error);
1226             ok = stmt != NULL;
1227             if (stmt) {
1228                 if (perform) {
1229                     ok = perform(stmt);
1230                 } else {
1231                     // TODO: Use a different error scope here.
1232                     ok = SecError(-50 /* errSecParam */, error, CFSTR("SecDbWithSQL perform block missing"));
1233                 }
1234                 ok &= SecDbReleaseCachedStmt(dbconn, sql, stmt, error);
1235             }
1236         } else {
1237             // TODO We already have an error here we really just want the left over sql in it's userData
1238             ok = SecDbError(SQLITE_ERROR, error, CFSTR("Error with unexecuted sql remaining %@"), sql);
1239         }
1240         CFRelease(sql);
1241         sql = tail;
1242     }
1243     return ok;
1244 }
1245
1246 #if 1
1247 /* SecDbForEach returns true if all SQLITE_ROW returns of sqlite3_step() return true from the row block.
1248  If the row block returns false and doesn't set an error (to indicate it has reached a limit),
1249  this entire function returns false. In that case no error will be set. */
1250 bool SecDbForEach(sqlite3_stmt *stmt, CFErrorRef *error, bool(^row)(int row_index)) {
1251     bool result = false;
1252     for (int row_ix = 0;;++row_ix) {
1253         int s3e = sqlite3_step(stmt);
1254         if (s3e == SQLITE_ROW) {
1255             if (row) {
1256                 if (!row(row_ix)) {
1257                     break;
1258                 }
1259             } else {
1260                 // If we have no row block then getting SQLITE_ROW is an error
1261                 SecDbError(s3e, error,
1262                            CFSTR("step[%d]: %s returned SQLITE_ROW with NULL row block"),
1263                            row_ix, sqlite3_sql(stmt));
1264             }
1265         } else {
1266             if (s3e == SQLITE_DONE) {
1267                 result = true;
1268             } else {
1269                 SecDbErrorWithStmt(s3e, stmt, error, CFSTR("step[%d]"), row_ix);
1270             }
1271             break;
1272         }
1273     }
1274     return result;
1275 }
1276 #else
1277 bool SecDbForEach(sqlite3_stmt *stmt, CFErrorRef *error, bool(^row)(int row_index)) {
1278     int row_ix = 0;
1279     for (;;) {
1280         switch (_SecDbStep(dbconn, stmt, error)) {
1281             case kSecDbErrorStep:
1282                 return false;
1283             case kSecDbRowStep:
1284                 if (row) {
1285                     if (row(row_ix++))
1286                         break;
1287                 } else {
1288                     SecDbError(SQLITE_ERROR, error, CFSTR("SecDbStep SQLITE_ROW returned without a row handler"));
1289                 }
1290                 return false;
1291             case kSecDbDoneStep:
1292                 return true;
1293         }
1294     }
1295 }
1296 #endif
1297
1298 void SecDbRecordChange(SecDbConnectionRef dbconn, CFTypeRef deleted, CFTypeRef inserted) {
1299     if (!dbconn->db->notifyPhase) return;
1300     CFTypeRef entry = SecDbEventCreateWithComponents(deleted, inserted);
1301     if (entry) {
1302         CFArrayAppendValue(dbconn->changes, entry);
1303         CFRelease(entry);
1304
1305         if (!dbconn->inTransaction) {
1306             secerror("db %@ changed outside txn", dbconn);
1307             // Only notify of DidCommit, since WillCommit code assumes
1308             // we are in a txn.
1309             SecDbOnNotifyQueue(dbconn, ^{
1310                 SecDbNotifyPhase(dbconn, kSecDbTransactionDidCommit);
1311             });
1312         }
1313     }
1314 }
1315
1316
1317 CFGiblisFor(SecDbConnection)
1318
1319 //
1320 // SecDbEvent Creation and consumption
1321 //
1322
1323 static SecDbEventRef SecDbEventCreateInsert(CFTypeRef inserted) {
1324     return CFRetainSafe(inserted);
1325 }
1326
1327 static SecDbEventRef SecDbEventCreateDelete(CFTypeRef deleted) {
1328     return CFArrayCreate(kCFAllocatorDefault, &deleted, 1, &kCFTypeArrayCallBacks);
1329 }
1330
1331 static SecDbEventRef SecDbEventCreateUpdate(CFTypeRef deleted, CFTypeRef inserted) {
1332     const void *values[2] = { deleted, inserted };
1333     return CFArrayCreate(kCFAllocatorDefault, values, 2, &kCFTypeArrayCallBacks);
1334 }
1335
1336 SecDbEventRef SecDbEventCreateWithComponents(CFTypeRef deleted, CFTypeRef inserted) {
1337     if (deleted && inserted)
1338         return SecDbEventCreateUpdate(deleted, inserted);
1339     else if (deleted)
1340         return SecDbEventCreateDelete(deleted);
1341     else if (inserted)
1342         return SecDbEventCreateInsert(inserted);
1343     else
1344         return NULL;
1345 }
1346
1347 bool SecDbEventGetComponents(SecDbEventRef event, CFTypeRef *deleted, CFTypeRef *inserted, CFErrorRef *error) {
1348     if (isArray(event)) {
1349         CFArrayRef array = event;
1350         switch (CFArrayGetCount(array)) {
1351             case 2:
1352                 *deleted = CFArrayGetValueAtIndex(array, 0);
1353                 *inserted = CFArrayGetValueAtIndex(array, 1);
1354                 break;
1355             case 1:
1356                 *deleted = CFArrayGetValueAtIndex(array, 0);
1357                 *inserted = NULL;
1358                 break;
1359             default:
1360                 SecError(errSecParam, error, NULL, CFSTR("invalid entry in changes array: %@"), array);
1361                 break;
1362         }
1363     } else {
1364         *deleted = NULL;
1365         *inserted = event;
1366     }
1367     return true;
1368 }