OSX/utilities/src/SecDb.c

   1 /*
   2  * Copyright (c) 2012-2014 Apple Inc. All Rights Reserved.
   3  *
   4  * @APPLE_LICENSE_HEADER_START@
   5  *
   6  * This file contains Original Code and/or Modifications of Original Code
   7  * as defined in and that are subject to the Apple Public Source License
   8  * Version 2.0 (the 'License'). You may not use this file except in
   9  * compliance with the License. Please obtain a copy of the License at
  10  * http://www.opensource.apple.com/apsl/ and read it before using this
  11  * file.
  12  *
  13  * The Original Code and all software distributed under the License are
  14  * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  15  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  16  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
  17  * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
  18  * Please see the License for the specific language governing rights and
  19  * limitations under the License.
  20  *
  21  * @APPLE_LICENSE_HEADER_END@
  22  */
  23
  24
  25 #include "SecDb.h"
  26 #include "debugging.h"
  27
  28 #include <sqlite3.h>
  29 #include <sqlite3_private.h>
  30 #include <CoreFoundation/CoreFoundation.h>
  31 #include <libgen.h>
  32 #include <sys/csr.h>
  33 #include <sys/stat.h>
  34 #include <AssertMacros.h>
  35 #include "SecCFWrappers.h"
  36 #include "SecCFError.h"
  37 #include "SecIOFormat.h"
  38 #include <stdio.h>
  39 #include "Security/SecBase.h"
  40
  41
  42 //
  43 // Architecturally inverted files
  44 // These are in SecureObjectSync but utilities depends on them
  45 // <rdar://problem/20802079> Fix layer violation (SOSDigestVector, SOSManifest, SecDB.c)
  46 //
  47 #include <Security/SecureObjectSync/SOSDigestVector.h>
  48 #include <Security/SecureObjectSync/SOSManifest.h>
  49
  50 #define HAVE_UNLOCK_NOTIFY  0
  51
  52 struct __OpaqueSecDbStatement {
  53     CFRuntimeBase _base;
  54
  55     SecDbConnectionRef dbconn;
  56     sqlite3_stmt *stmt;
  57 };
  58
  59 struct __OpaqueSecDbConnection {
  60     CFRuntimeBase _base;
  61
  62     //CFMutableDictionaryRef statements;
  63
  64     SecDbRef db;     // NONRETAINED, since db or block retains us
  65     bool readOnly;
  66     bool inTransaction;
  67     SecDbTransactionSource source;
  68     bool isCorrupted;
  69     int maybeCorruptedCode;
  70     CFErrorRef corruptionError;
  71     sqlite3 *handle;
  72     // Pending deletions and additions for the current transaction
  73     // Entires are either:
  74     // 1) a CFArrayRef of 1 element representing a deletion,
  75     // 2) a CFArrayRef of 2 elements representing the element 0 having been replaced with element 1
  76     // 3) a CFTypeRef that is not a CFArrayRef, representing an add of the element in question.
  77     CFMutableArrayRef changes;
  78 };
  79
  80 struct __OpaqueSecDb {
  81     CFRuntimeBase _base;
  82
  83     CFStringRef db_path;
  84     dispatch_queue_t queue;
  85     dispatch_queue_t commitQueue;
  86     CFMutableArrayRef connections;
  87     dispatch_semaphore_t write_semaphore;
  88     dispatch_semaphore_t read_semaphore;
  89     bool didFirstOpen;
  90     bool (^opened)(SecDbConnectionRef dbconn, bool didCreate, bool *callMeAgainForNextConnection, CFErrorRef *error);
  91     bool callOpenedHandlerForNextConnection;
  92     CFMutableArrayRef notifyPhase; /* array of SecDBNotifyBlock */
  93 };
  94
  95 // MARK: Error domains and error helper functions
  96
  97 CFStringRef kSecDbErrorDomain = CFSTR("com.apple.utilities.sqlite3");
  98
  99 bool SecDbError(int sql_code, CFErrorRef *error, CFStringRef format, ...) {
 100     if (sql_code == SQLITE_OK) return true;
 101
 102     if (error) {
 103         va_list args;
 104         CFIndex code = sql_code;
 105         CFErrorRef previousError = *error;
 106
 107         *error = NULL;
 108         va_start(args, format);
 109         SecCFCreateErrorWithFormatAndArguments(code, kSecDbErrorDomain, previousError, error, NULL, format, args);
 110         va_end(args);
 111     }
 112     return false;
 113 }
 114
 115 bool SecDbErrorWithDb(int sql_code, sqlite3 *db, CFErrorRef *error, CFStringRef format, ...) {
 116     if (sql_code == SQLITE_OK) return true;
 117     if (error) {
 118         va_list args;
 119         va_start(args, format);
 120         CFStringRef message = CFStringCreateWithFormatAndArguments(kCFAllocatorDefault, NULL, format, args);
 121         va_end(args);
 122
 123         int extended_code = sqlite3_extended_errcode(db);
 124         if (sql_code == extended_code)
 125             SecDbError(sql_code, error, CFSTR("%@: [%d] %s"), message, sql_code, sqlite3_errmsg(db));
 126         else
 127             SecDbError(sql_code, error, CFSTR("%@: [%d->%d] %s"), message, sql_code, extended_code, sqlite3_errmsg(db));
 128         CFReleaseSafe(message);
 129     }
 130     return false;
 131 }
 132
 133 bool SecDbErrorWithStmt(int sql_code, sqlite3_stmt *stmt, CFErrorRef *error, CFStringRef format, ...) {
 134     if (sql_code == SQLITE_OK) return true;
 135     if (error) {
 136         va_list args;
 137         va_start(args, format);
 138         CFStringRef message = CFStringCreateWithFormatAndArguments(kCFAllocatorDefault, NULL, format, args);
 139         va_end(args);
 140
 141         sqlite3 *db = sqlite3_db_handle(stmt);
 142         const char *sql = sqlite3_sql(stmt);
 143         int extended_code = sqlite3_extended_errcode(db);
 144         if (sql_code == extended_code)
 145             SecDbError(sql_code, error, CFSTR("%@: [%d] %s sql: %s"), message, sql_code, sqlite3_errmsg(db), sql);
 146         else
 147             SecDbError(sql_code, error, CFSTR("%@: [%d->%d] %s sql: %s"), message, sql_code, extended_code, sqlite3_errmsg(db), sql);
 148         CFReleaseSafe(message);
 149     }
 150     return false;
 151 }
 152
 153
 154 // MARK: -
 155 // MARK: Static helper functions
 156
 157 static bool SecDbOpenHandle(SecDbConnectionRef dbconn, bool *created, CFErrorRef *error);
 158 static bool SecDbHandleCorrupt(SecDbConnectionRef dbconn, int rc, CFErrorRef *error);
 159
 160 #pragma mark -
 161 #pragma mark SecDbRef
 162
 163 static CFStringRef
 164 SecDbCopyFormatDescription(CFTypeRef value, CFDictionaryRef formatOptions)
 165 {
 166     SecDbRef db = (SecDbRef)value;
 167     return CFStringCreateWithFormat(kCFAllocatorDefault, NULL, CFSTR("<SecDb path:%@ connections: %@>"), db->db_path, db->connections);
 168 }
 169
 170
 171 static void
 172 SecDbDestroy(CFTypeRef value)
 173 {
 174     SecDbRef db = (SecDbRef)value;
 175     CFReleaseSafe(db->connections);
 176     CFReleaseSafe(db->db_path);
 177     dispatch_release(db->queue);
 178     dispatch_release(db->commitQueue);
 179     dispatch_release(db->read_semaphore);
 180     dispatch_release(db->write_semaphore);
 181     if (db->opened)
 182         Block_release(db->opened);
 183     CFReleaseNull(db->notifyPhase);
 184 }
 185
 186 CFGiblisFor(SecDb)
 187
 188 SecDbRef
 189 SecDbCreate(CFStringRef dbName,
 190             bool (^opened)(SecDbConnectionRef dbconn, bool didCreate, bool *callMeAgainForNextConnection, CFErrorRef *error))
 191 {
 192     SecDbRef db = NULL;
 193
 194     db = CFTypeAllocate(SecDb, struct __OpaqueSecDb, kCFAllocatorDefault);
 195     require(db != NULL, done);
 196
 197     CFStringPerformWithCString(dbName, ^(const char *dbNameStr) {
 198         db->queue = dispatch_queue_create(dbNameStr, DISPATCH_QUEUE_SERIAL);
 199     });
 200     CFStringRef commitQueueStr = CFStringCreateWithFormat(kCFAllocatorDefault, NULL, CFSTR("%@-commit"), dbName);
 201     CFStringPerformWithCString(commitQueueStr, ^(const char *cqNameStr) {
 202         db->commitQueue = dispatch_queue_create(cqNameStr, DISPATCH_QUEUE_CONCURRENT);
 203     });
 204     CFReleaseNull(commitQueueStr);
 205     db->read_semaphore = dispatch_semaphore_create(kSecDbMaxReaders);
 206     db->write_semaphore = dispatch_semaphore_create(kSecDbMaxWriters);
 207     db->connections = CFArrayCreateMutableForCFTypes(kCFAllocatorDefault);
 208     db->opened = opened ? Block_copy(opened) : NULL;
 209     if (getenv("__OSINSTALL_ENVIRONMENT") != NULL) {
 210         // TODO: Move this code out of this layer
 211         secinfo("#SecDB", "SecDB: running from installer");
 212
 213         db->db_path = CFSTR("file::memory:?cache=shared");
 214     } else {
 215         db->db_path = CFStringCreateCopy(kCFAllocatorDefault, dbName);
 216     }
 217
 218 done:
 219     return db;
 220 }
 221
 222 CFIndex
 223 SecDbIdleConnectionCount(SecDbRef db) {
 224     __block CFIndex count = 0;
 225     dispatch_sync(db->queue, ^{
 226         count = CFArrayGetCount(db->connections);
 227     });
 228     return count;
 229 }
 230
 231 void SecDbAddNotifyPhaseBlock(SecDbRef db, SecDBNotifyBlock notifyPhase)
 232 {
 233     SecDBNotifyBlock block = Block_copy(notifyPhase); /* Force the block off the stack */
 234     if (db->notifyPhase == NULL) {
 235         db->notifyPhase = CFArrayCreateMutableForCFTypes(NULL);
 236     }
 237     CFArrayAppendValue(db->notifyPhase, block);
 238     Block_release(block);
 239 }
 240
 241 static void SecDbNotifyPhase(SecDbConnectionRef dbconn, SecDbTransactionPhase phase) {
 242     if (CFArrayGetCount(dbconn->changes)) {
 243         CFArrayRef changes = dbconn->changes;
 244         dbconn->changes = CFArrayCreateMutableForCFTypes(kCFAllocatorDefault);
 245         if (dbconn->db->notifyPhase) {
 246             CFArrayForEach(dbconn->db->notifyPhase, ^(const void *value) {
 247                 SecDBNotifyBlock notifyBlock = (SecDBNotifyBlock)value;
 248                 notifyBlock(dbconn, phase, dbconn->source, changes);
 249             });
 250         }
 251         CFReleaseSafe(changes);
 252     }
 253 }
 254
 255 static void SecDbOnNotify(SecDbConnectionRef dbconn, void (^perform)()) {
 256     perform();
 257 }
 258
 259 CFStringRef SecDbGetPath(SecDbRef db) {
 260     return db->db_path;
 261 }
 262
 263
 264 #pragma mark -
 265 #pragma mark SecDbConnectionRef
 266
 267 static bool SecDbCheckCorrupted(SecDbConnectionRef dbconn)
 268 {
 269     __block bool isCorrupted = true;
 270     __block CFErrorRef error = NULL;
 271     SecDbPrepare(dbconn, CFSTR("PRAGMA integrity_check"), &error, ^(sqlite3_stmt *stmt) {
 272         SecDbStep(dbconn, stmt, &error, ^(bool *stop) {
 273             const char * result = (const char*)sqlite3_column_text(stmt, 0);
 274             if (result && strncasecmp(result, "ok", 3) == 0) {
 275                 isCorrupted = false;
 276             }
 277         });
 278     });
 279     if (error) {
 280         secinfo("#SecDB", "#SecDB warning error %{public}@ when running integrity check", error);
 281         CFRelease(error);
 282     }
 283     return isCorrupted;
 284 }
 285
 286 static bool SecDbDidCreateFirstConnection(SecDbConnectionRef dbconn, bool didCreate, CFErrorRef *error)
 287 {
 288     secinfo("#SecDB", "#SecDB starting maintenance");
 289     bool ok = true;
 290
 291     if (!didCreate && !dbconn->isCorrupted) {
 292         dbconn->isCorrupted = SecDbCheckCorrupted(dbconn);
 293         if (dbconn->isCorrupted) {
 294             secinfo("#SecDB", "#SecDB integrity check=fail");
 295         } else {
 296             secinfo("#SecDB", "#SecDB starting maintenance");
 297         }
 298     }
 299
 300     if (!dbconn->isCorrupted && dbconn->db->opened) {
 301         CFErrorRef localError = NULL;
 302
 303         dbconn->db->callOpenedHandlerForNextConnection = false;
 304         ok = dbconn->db->opened(dbconn, didCreate, &dbconn->db->callOpenedHandlerForNextConnection, &localError);
 305
 306         if (!ok)
 307             secerror("opened block failed: %@", localError);
 308
 309         if (!dbconn->isCorrupted && error && *error == NULL) {
 310             *error = localError;
 311             localError = NULL;
 312         } else {
 313             if (localError)
 314                 secerror("opened block failed: error is released and lost");
 315             CFReleaseNull(localError);
 316         }
 317     }
 318
 319     if (dbconn->isCorrupted) {
 320         ok = SecDbHandleCorrupt(dbconn, 0, error);
 321     }
 322
 323     secinfo("#SecDB", "#SecDB starting maintenance");
 324     return ok;
 325 }
 326
 327 void SecDbCorrupt(SecDbConnectionRef dbconn, CFErrorRef error)
 328 {
 329     CFStringRef str = CFStringCreateWithFormat(NULL, NULL, CFSTR("SecDBCorrupt: %@"), error);
 330     if (str) {
 331         char buffer[1000] = "?";
 332         uint32_t errorCode = 0;
 333         CFStringGetCString(str, buffer, sizeof(buffer), kCFStringEncodingUTF8);
 334         os_log_fault(logObjForScope("SecEmergency"), "%s", buffer);
 335         if (error)
 336             errorCode = (uint32_t)CFErrorGetCode(error);
 337         __security_simulatecrash(str, __sec_exception_code_CorruptDb(errorCode));
 338         CFRelease(str);
 339     }
 340     dbconn->isCorrupted = true;
 341     CFRetainAssign(dbconn->corruptionError, error);
 342 }
 343
 344
 345 static uint8_t knownDbPathIndex(SecDbConnectionRef dbconn)
 346 {
 347
 348     if(CFEqual(dbconn->db->db_path, CFSTR("/Library/Keychains/keychain-2.db")))
 349         return 1;
 350     if(CFEqual(dbconn->db->db_path, CFSTR("/Library/Keychains/ocspcache.sqlite3")))
 351         return 2;
 352     if(CFEqual(dbconn->db->db_path, CFSTR("/Library/Keychains/TrustStore.sqlite3")))
 353         return 3;
 354     if(CFEqual(dbconn->db->db_path, CFSTR("/Library/Keychains/caissuercache.sqlite3")))
 355         return 4;
 356
 357     /* Unknown DB path */
 358     return 0;
 359 }
 360
 361
 362 // Return true if there was no error, returns false otherwise and set *error to an appropriate CFErrorRef.
 363 static bool SecDbConnectionCheckCode(SecDbConnectionRef dbconn, int code, CFErrorRef *error, CFStringRef desc, ...) {
 364     if (code == SQLITE_OK || code == SQLITE_DONE)
 365         return true;
 366
 367     if (error) {
 368         va_list args;
 369         va_start(args, desc);
 370         CFStringRef msg = CFStringCreateWithFormatAndArguments(kCFAllocatorDefault, NULL, desc, args);
 371         va_end(args);
 372         SecDbErrorWithDb(code, dbconn->handle, error, msg);
 373         CFRelease(msg);
 374     }
 375
 376     /* If it's already corrupted, don't try to recover */
 377     if (dbconn->isCorrupted) {
 378         CFStringRef reason = CFStringCreateWithFormat(kCFAllocatorDefault, NULL,
 379                                                       CFSTR("SQL DB %@ is corrupted already. Not trying to recover, corrution error was: %d (previously %d)"),
 380                                                       dbconn->db->db_path, code, dbconn->maybeCorruptedCode);
 381         secerror("%@",reason);
 382         __security_simulatecrash(reason, __sec_exception_code_TwiceCorruptDb(knownDbPathIndex(dbconn)));
 383         CFReleaseSafe(reason);
 384         return false;
 385     }
 386
 387     dbconn->isCorrupted = (SQLITE_CORRUPT == code) || (SQLITE_NOTADB == code) || (SQLITE_IOERR == code) || (SQLITE_CANTOPEN == code);
 388     if (dbconn->isCorrupted) {
 389         /* Run integrity check and only make dbconn->isCorrupted true and
 390            run the corruption handler if the integrity check conclusively fails. */
 391         dbconn->maybeCorruptedCode = code;
 392         dbconn->isCorrupted = SecDbCheckCorrupted(dbconn);
 393         if (dbconn->isCorrupted) {
 394             secerror("operation returned code: %d integrity check=fail", code);
 395             SecDbHandleCorrupt(dbconn, code, error);
 396         } else {
 397             secerror("operation returned code: %d: integrity check=pass", code);
 398         }
 399     }
 400
 401     return false;
 402 }
 403
 404 #if HAVE_UNLOCK_NOTIFY
 405
 406 static void SecDbUnlockNotify(void **apArg, int nArg) {
 407     int i;
 408     for(i=0; i<nArg; i++) {
 409         dispatch_semaphore_t dsema = (dispatch_semaphore_t)apArg[i];
 410         dispatch_semaphore_signal(dsema);
 411     }
 412 }
 413
 414 static bool SecDbWaitForUnlockNotify(SecDbConnectionRef dbconn, sqlite3_stmt *stmt, CFErrorRef *error) {
 415     int rc;
 416     dispatch_semaphore_t dsema = dispatch_semaphore_create(0);
 417     rc = sqlite3_unlock_notify(dbconn->handle, SecDbUnlockNotify, dsema);
 418     assert(rc == SQLITE_LOCKED || rc == SQLITE_OK);
 419     if (rc == SQLITE_OK) {
 420         dispatch_semaphore_wait(dsema, DISPATCH_TIME_FOREVER);
 421     }
 422     dispatch_release(dsema);
 423     return (rc == SQLITE_OK
 424             ? true
 425             : (stmt
 426                ? SecDbErrorWithStmt(rc, stmt, error, CFSTR("sqlite3_unlock_notify"))
 427                : SecDbErrorWithDb(rc, dbconn->handle, error, CFSTR("sqlite3_unlock_notify"))));
 428 }
 429
 430 #endif
 431
 432 #define BUSY_TIMEOUT_MS (5 * 60 * 1000)  /* 5 minutes */
 433
 434 static bool SecDbBusyHandler(SecDbConnectionRef dbconn, CFErrorRef *error) {
 435     return SecDbErrorWithDb(sqlite3_busy_timeout(dbconn->handle, BUSY_TIMEOUT_MS), dbconn->handle, error, CFSTR("busy_handler"));
 436 }
 437
 438 static int sleepBackoff[] = { 10, 20, 50, 100, 250 };
 439 static int sumBackoff[]   = { 10, 30, 80, 180, 430 };
 440 static int NumberOfSleepBackoff = sizeof(sleepBackoff)/sizeof(sleepBackoff[0]);
 441
 442 // Return true causes the operation to be tried again.
 443 static bool SecDbWaitIfNeeded(SecDbConnectionRef dbconn, int s3e, sqlite3_stmt *stmt, CFStringRef desc, int nTries, CFErrorRef *error) {
 444 #if HAVE_UNLOCK_NOTIFY
 445     if (s3e == SQLITE_LOCKED) { // Optionally check for extended code being SQLITE_LOCKED_SHAREDCACHE
 446         return SecDbWaitForUnlockNotify(dbconn, stmt, error))
 447     }
 448 #endif
 449     if (((0xFF & s3e) == SQLITE_BUSY) || ((0xFF & s3e) == SQLITE_LOCKED)) {
 450         int totaltimeout, timeout;
 451
 452         _Static_assert(sizeof(sumBackoff) == sizeof(sleepBackoff), "matching arrays not matching");
 453         _Static_assert(sizeof(sumBackoff[0]) == sizeof(sleepBackoff[0]), "matching arrays not matching");
 454
 455         if (nTries < NumberOfSleepBackoff) {
 456             timeout = sleepBackoff[nTries];
 457             totaltimeout = sumBackoff[nTries];
 458         } else {
 459             timeout = sleepBackoff[NumberOfSleepBackoff - 1];
 460             totaltimeout = sumBackoff[NumberOfSleepBackoff - 1] + (timeout * (nTries - NumberOfSleepBackoff));
 461         }
 462         if (totaltimeout < BUSY_TIMEOUT_MS) {
 463             secinfo("#SecDB", "sqlite busy/locked: %d ntries: %d totaltimeout: %d", s3e, nTries, totaltimeout);
 464             sqlite3_sleep(timeout);
 465             return true;
 466         } else {
 467             secinfo("#SecDB", "sqlite busy/locked: too long: %d ms, giving up", totaltimeout);
 468         }
 469     }
 470
 471     return SecDbConnectionCheckCode(dbconn, s3e, error, desc);
 472 }
 473
 474 enum SecDbStepResult {
 475     kSecDbErrorStep = 0,
 476     kSecDbRowStep = 1,
 477     kSecDbDoneStep = 2,
 478 };
 479 typedef enum SecDbStepResult SecDbStepResult;
 480
 481 static SecDbStepResult _SecDbStep(SecDbConnectionRef dbconn, sqlite3_stmt *stmt, CFErrorRef *error) {
 482     assert(stmt != NULL);
 483     int s3e;
 484     int ntries = 0;
 485     for (;;) {
 486         s3e = sqlite3_step(stmt);
 487         if (s3e == SQLITE_ROW) {
 488             return kSecDbRowStep;
 489         } else if (s3e == SQLITE_DONE) {
 490             /*
 491              ** ^[SQLITE_DONE] means that the statement has finished executing
 492              ** successfully.  sqlite3_step() should not be called again on this virtual
 493              ** machine without first calling [] to reset the virtual
 494              ** machine back to its initial state.
 495              */
 496             sqlite3_reset(stmt);
 497             return kSecDbDoneStep;
 498         } else if (!SecDbWaitIfNeeded(dbconn, s3e, stmt, CFSTR("step"), ntries, error)) {
 499             return kSecDbErrorStep;
 500         }
 501         ntries++;
 502     };
 503 }
 504
 505 bool
 506 SecDbExec(SecDbConnectionRef dbconn, CFStringRef sql, CFErrorRef *error)
 507 {
 508     bool ok = true;
 509     CFRetain(sql);
 510     while (sql) {
 511         CFStringRef tail = NULL;
 512         if (ok) {
 513             sqlite3_stmt *stmt = SecDbCopyStmt(dbconn, sql, &tail, error);
 514             ok = stmt != NULL;
 515             if (stmt) {
 516                 SecDbStepResult sr;
 517                 while ((sr = _SecDbStep(dbconn, stmt, error)) == kSecDbRowStep);
 518                 if (sr == kSecDbErrorStep)
 519                     ok = false;
 520                 ok &= SecDbReleaseCachedStmt(dbconn, sql, stmt, error);
 521             }
 522         } else {
 523             // TODO We already have an error here we really just want the left over sql in it's userData
 524             ok = SecDbError(SQLITE_ERROR, error, CFSTR("Error with unexecuted sql remaining %@"), sql);
 525         }
 526         CFRelease(sql);
 527         sql = tail;
 528     }
 529     return ok;
 530 }
 531
 532 static bool SecDbBeginTransaction(SecDbConnectionRef dbconn, SecDbTransactionType type, CFErrorRef *error)
 533 {
 534     bool ok = true;
 535     CFStringRef query;
 536     switch (type) {
 537         case kSecDbImmediateTransactionType:
 538             secnoticeq("db", "SecDbBeginTransaction SecDbBeginTransaction %p", dbconn);
 539             query = CFSTR("BEGIN IMMEDATE");
 540             break;
 541         case kSecDbExclusiveRemoteTransactionType:
 542             secnoticeq("db", "SecDbBeginTransaction kSecDbExclusiveRemoteTransactionType %p", dbconn);
 543             dbconn->source = kSecDbSOSTransaction;
 544             // FALL THROUGH
 545         case kSecDbExclusiveTransactionType:
 546             if (type==kSecDbExclusiveTransactionType)
 547                 secnoticeq("db", "SecDbBeginTransaction kSecDbExclusiveTransactionType %p", dbconn);
 548             query = CFSTR("BEGIN EXCLUSIVE");
 549             break;
 550         case kSecDbNormalTransactionType:
 551             secnoticeq("db", "SecDbBeginTransaction kSecDbNormalTransactionType %p", dbconn);
 552             query = CFSTR("BEGIN");
 553             break;
 554         default:
 555             secnoticeq("db", "SecDbBeginTransaction invalid transaction type %lu", type);
 556             ok = SecDbError(SQLITE_ERROR, error, CFSTR("invalid transaction type %" PRIu32), type);
 557             query = NULL;
 558             break;
 559     }
 560
 561     if (query != NULL && sqlite3_get_autocommit(dbconn->handle) != 0) {
 562         ok = SecDbExec(dbconn, query, error);
 563     }
 564     if (ok)
 565         dbconn->inTransaction = true;
 566
 567     return ok;
 568 }
 569
 570 static bool SecDbEndTransaction(SecDbConnectionRef dbconn, bool commit, CFErrorRef *error)
 571 {
 572     __block bool ok = true;
 573     __block bool commited = false;
 574
 575     dispatch_block_t notifyAndExec = ^{
 576         if (commit) {
 577             secnoticeq("db", "SecDbEndTransaction kSecDbTransactionWillCommit %p", dbconn);
 578             SecDbNotifyPhase(dbconn, kSecDbTransactionWillCommit);
 579             commited = ok = SecDbExec(dbconn, CFSTR("END"), error);
 580             secnoticeq("db", "SecDbEndTransaction kSecDbTransactionWillCommit %p (after notify)", dbconn);
 581         } else {
 582             ok = SecDbExec(dbconn, CFSTR("ROLLBACK"), error);
 583             commited = false;
 584         }
 585         dbconn->inTransaction = false;
 586         SecDbNotifyPhase(dbconn, commited ? kSecDbTransactionDidCommit : kSecDbTransactionDidRollback);
 587         secnoticeq("db", "SecDbEndTransaction %s %p", commited ? "kSecDbTransactionDidCommit" : "kSecDbTransactionDidRollback", dbconn);
 588         dbconn->source = kSecDbAPITransaction;
 589     };
 590
 591     SecDbPerformOnCommitQueue(dbconn, true, notifyAndExec);
 592
 593     return ok;
 594 }
 595
 596 bool SecDbTransaction(SecDbConnectionRef dbconn, SecDbTransactionType type,
 597                       CFErrorRef *error, void (^transaction)(bool *commit))
 598 {
 599     bool ok = true;
 600     bool commit = true;
 601
 602     if (dbconn->inTransaction) {
 603         transaction(&commit);
 604         if (!commit) {
 605             secinfo("#SecDB", "#SecDB nested transaction asked to not be committed");
 606         }
 607     } else {
 608         ok = SecDbBeginTransaction(dbconn, type, error);
 609         if (ok) {
 610             transaction(&commit);
 611             ok = SecDbEndTransaction(dbconn, commit, error);
 612         }
 613     }
 614
 615 done:
 616     return ok && commit;
 617 }
 618
 619 sqlite3 *SecDbHandle(SecDbConnectionRef dbconn) {
 620     return dbconn->handle;
 621 }
 622
 623 bool SecDbStep(SecDbConnectionRef dbconn, sqlite3_stmt *stmt, CFErrorRef *error, void (^row)(bool *stop)) {
 624     for (;;) {
 625         switch (_SecDbStep(dbconn, stmt, error)) {
 626             case kSecDbErrorStep:
 627                 secdebug("db", "kSecDbErrorStep %@", error?*error:NULL);
 628                 return false;
 629             case kSecDbRowStep:
 630                 secdebug("db", "kSecDbRowStep %@", error?*error:NULL);
 631                 if (row) {
 632                     bool stop = false;
 633                     row(&stop);
 634                     if (stop)
 635                         return true;
 636                     break;
 637                 }
 638                 SecDbError(SQLITE_ERROR, error, CFSTR("SecDbStep SQLITE_ROW returned without a row handler"));
 639                 return false;
 640             case kSecDbDoneStep:
 641                 secdebug("db", "kSecDbDoneStep %@", error?*error:NULL);
 642                 return true;
 643         }
 644     }
 645 }
 646
 647 bool SecDbCheckpoint(SecDbConnectionRef dbconn, CFErrorRef *error)
 648 {
 649     return SecDbConnectionCheckCode(dbconn, sqlite3_wal_checkpoint(dbconn->handle, NULL), error, CFSTR("wal_checkpoint"));
 650 }
 651
 652 static bool SecDbFileControl(SecDbConnectionRef dbconn, int op, void *arg, CFErrorRef *error) {
 653     return SecDbConnectionCheckCode(dbconn, sqlite3_file_control(dbconn->handle, NULL, op, arg), error, CFSTR("file_control"));
 654 }
 655
 656 static sqlite3 *_SecDbOpenV2(const char *path, int flags, CFErrorRef *error) {
 657 #if HAVE_UNLOCK_NOTIFY
 658     flags |= SQLITE_OPEN_SHAREDCACHE;
 659 #endif
 660     sqlite3 *handle = NULL;
 661     int s3e = sqlite3_open_v2(path, &handle, flags, NULL);
 662     if (s3e) {
 663         if (handle) {
 664             SecDbErrorWithDb(s3e, handle, error, CFSTR("open_v2 \"%s\" 0x%X"), path, flags);
 665             sqlite3_close(handle);
 666             handle = NULL;
 667         } else {
 668             SecDbError(s3e, error, CFSTR("open_v2 \"%s\" 0x%X"), path, flags);
 669         }
 670     }
 671     return handle;
 672 }
 673
 674 static bool SecDbOpenV2(SecDbConnectionRef dbconn, const char *path, int flags, CFErrorRef *error) {
 675     return (dbconn->handle = _SecDbOpenV2(path, flags, error)) != NULL;
 676 }
 677
 678 static bool SecDbTruncate(SecDbConnectionRef dbconn, CFErrorRef *error)
 679 {
 680     int flags = SQLITE_TRUNCATE_JOURNALMODE_WAL | SQLITE_TRUNCATE_AUTOVACUUM_FULL;
 681     __block bool ok = SecDbFileControl(dbconn, SQLITE_TRUNCATE_DATABASE, &flags, error);
 682     if (!ok) {
 683         sqlite3_close(dbconn->handle);
 684         dbconn->handle = NULL;
 685         CFStringPerformWithCString(dbconn->db->db_path, ^(const char *path) {
 686             if (error)
 687                 CFReleaseNull(*error);
 688             if (SecCheckErrno(unlink(path), error, CFSTR("unlink %s"), path)) {
 689                 ok = SecDbOpenHandle(dbconn, NULL, error);
 690             }
 691         });
 692         if (!ok) {
 693             secinfo("#SecDB", "#SecDB Failed to delete db handle: %{public}@", error ? *error : NULL);
 694             abort();
 695         }
 696     }
 697
 698     return ok;
 699 }
 700
 701 static bool SecDbHandleCorrupt(SecDbConnectionRef dbconn, int rc, CFErrorRef *error)
 702 {
 703     // Backup current db.
 704     __block bool didRename = false;
 705     CFStringPerformWithCString(dbconn->db->db_path, ^(const char *db_path) {
 706         sqlite3 *corrupt_db = NULL;
 707         char buf[PATH_MAX+1];
 708         snprintf(buf, sizeof(buf), "%s-corrupt", db_path);
 709         if (dbconn->handle && (corrupt_db = _SecDbOpenV2(buf, SQLITE_OPEN_READWRITE, error))) {
 710             int on = 1;
 711             didRename =  SecDbErrorWithDb(sqlite3_file_control(corrupt_db, NULL, SQLITE_FCNTL_PERSIST_WAL, &on), corrupt_db, error, CFSTR("persist wal"));
 712             didRename &=  SecDbErrorWithDb(sqlite3_file_control(corrupt_db, NULL, SQLITE_REPLACE_DATABASE, (void *)dbconn->handle), corrupt_db, error, CFSTR("replace database"));
 713             sqlite3_close(corrupt_db);
 714         }
 715         if (!didRename) {
 716             if (dbconn->handle)
 717                 secerror("Tried to rename corrupt database at path %@, but we failed: %@, trying explicit rename", dbconn->db->db_path, error ? *error : NULL);
 718             if (error)
 719                 CFReleaseNull(*error);
 720
 721             didRename = (!dbconn->handle || SecDbError(sqlite3_close(dbconn->handle), error, CFSTR("close"))) &&
 722                 SecCheckErrno(rename(db_path, buf), error, CFSTR("rename %s %s"), db_path, buf) &&
 723                 SecDbOpenHandle(dbconn, NULL, error);
 724         }
 725         if (didRename) {
 726             secerror("Database at path %@ is corrupt. Copied it to %s for further investigation.", dbconn->db->db_path, buf);
 727         } else {
 728             seccritical("Tried to copy corrupt database at path %@, but we failed: %@", dbconn->db->db_path, error ? *error : NULL);
 729         }
 730     });
 731
 732     bool ok = (didRename &&
 733                (dbconn->handle || SecDbOpenHandle(dbconn, NULL, error)) &&
 734                SecDbTruncate(dbconn, error));
 735
 736     // Mark the db as not corrupted, even if something failed.
 737     // Always note we are no longer in the corruption handler
 738     dbconn->isCorrupted = false;
 739
 740     // Invoke our callers opened callback, since we just created a new database
 741     if (ok && dbconn->db->opened) {
 742         dbconn->db->callOpenedHandlerForNextConnection = false;
 743         ok = dbconn->db->opened(dbconn, true, &dbconn->db->callOpenedHandlerForNextConnection, error);
 744     }
 745
 746     return ok;
 747 }
 748
 749 static bool SecDbProfileEnabled(void)
 750 {
 751     static dispatch_once_t onceToken;
 752     static bool profile_enabled = false;
 753
 754     // sudo defaults write /Library/Preferences/com.apple.security SQLProfile -bool true
 755     dispatch_once(&onceToken, ^{
 756         CFTypeRef profile = NULL;
 757
 758         if (csr_check(CSR_ALLOW_APPLE_INTERNAL) != 0)
 759             return;
 760
 761         profile = (CFNumberRef)CFPreferencesCopyValue(CFSTR("SQLProfile"), CFSTR("com.apple.security"), kCFPreferencesAnyUser, kCFPreferencesAnyHost);
 762
 763         if (profile == NULL)
 764             return;
 765
 766         if (CFGetTypeID(profile) == CFBooleanGetTypeID()) {
 767             profile_enabled = CFBooleanGetValue((CFBooleanRef)profile);
 768         } else if (CFGetTypeID(profile) == CFNumberGetTypeID()) {
 769             int32_t num = 0;
 770             CFNumberGetValue(profile, kCFNumberSInt32Type, &num);
 771             profile_enabled = !!num;
 772         }
 773
 774         secinfo("#SecDB", "sqlDb: sql profile: %{public}s", profile_enabled ? "enabled" : "disabled");
 775
 776         CFReleaseSafe(profile);
 777     });
 778
 779     return profile_enabled;
 780 }
 781
 782 static void SecDbProfile(void *context, const char *sql, sqlite3_uint64 ns) {
 783     sqlite3 *s3h = context;
 784     int code = sqlite3_extended_errcode(s3h);
 785     if (code == SQLITE_OK || code == SQLITE_DONE) {
 786         secinfo("#SecDB", "#SecDB sql: %{public}s\nTime: %llu ms", sql, ns >> 20);
 787     } else {
 788         secinfo("#SecDB", "#SecDB error[%d]: %{public}s lDb: %{public}s time: %llu ms", code, sqlite3_errmsg(s3h), sql, ns >> 20);
 789     }
 790 }
 791
 792 static bool SecDbTraceEnabled(void)
 793 {
 794 #if DEBUG
 795     return true;
 796 #else
 797     static dispatch_once_t onceToken;
 798     static bool trace_enabled = false;
 799
 800     // sudo defaults write /Library/Preferences/com.apple.security SQLTrace -bool true
 801     dispatch_once(&onceToken, ^{
 802         CFTypeRef trace = NULL;
 803
 804         if (csr_check(CSR_ALLOW_APPLE_INTERNAL) != 0)
 805             return;
 806
 807         trace = (CFNumberRef)CFPreferencesCopyValue(CFSTR("SQLTrace"), CFSTR("com.apple.security"), kCFPreferencesAnyUser, kCFPreferencesCurrentHost);
 808
 809         if (trace == NULL)
 810             return;
 811
 812         if (CFGetTypeID(trace) == CFBooleanGetTypeID()) {
 813             trace_enabled = CFBooleanGetValue((CFBooleanRef)trace);
 814         } else if (CFGetTypeID(trace) == CFNumberGetTypeID()) {
 815             int32_t num = 0;
 816             CFNumberGetValue(trace, kCFNumberSInt32Type, &num);
 817             trace_enabled = !!num;
 818         }
 819
 820         secinfo("#SecDB", "#SecDB sql trace: %{public}s", trace_enabled ? "enabled" : "disabled");
 821
 822         CFReleaseSafe(trace);
 823     });
 824
 825     return trace_enabled;
 826 #endif
 827 }
 828
 829 static void SecDbTrace(void *ctx, const char *trace) {
 830     SecDbConnectionRef dbconn __unused = ctx;
 831     static dispatch_queue_t queue;
 832     static dispatch_once_t once;
 833     dispatch_once(&once, ^{
 834         queue = dispatch_queue_create("trace_queue", DISPATCH_QUEUE_SERIAL);
 835     });
 836     dispatch_sync(queue, ^{
 837         secinfo("#SecDB", "#SecDB %{public}s", trace);
 838     });
 839 }
 840
 841 static bool SecDbOpenHandle(SecDbConnectionRef dbconn, bool *created, CFErrorRef *error)
 842 {
 843     __block bool ok = true;
 844     CFStringPerformWithCString(dbconn->db->db_path, ^(const char *db_path) {
 845         ok = created && SecDbOpenV2(dbconn, db_path, SQLITE_OPEN_READWRITE, NULL);
 846         if (!ok) {
 847             ok = true;
 848             if (created) {
 849                 char *tmp = dirname((char *)db_path);
 850                 if (tmp) {
 851                     int errnum = mkpath_np(tmp, 0700);
 852                     if (errnum != 0 && errnum != EEXIST) {
 853                         SecCFCreateErrorWithFormat(errnum, kSecErrnoDomain, NULL, error, NULL,
 854                                                    CFSTR("mkpath_np %s: [%d] %s"), tmp, errnum, strerror(errnum));
 855                         ok = false;
 856                     }
 857                 }
 858             }
 859             ok = ok && SecDbOpenV2(dbconn, db_path, SQLITE_OPEN_READWRITE | SQLITE_OPEN_CREATE, error);
 860             if (ok) {
 861                 chmod(db_path, S_IRUSR | S_IWUSR);
 862                 if (created)
 863                     *created = true;
 864             }
 865         }
 866
 867         if (ok && SecDbProfileEnabled()) {
 868             sqlite3_profile(dbconn->handle, SecDbProfile, dbconn->handle);
 869         }
 870         if (ok && SecDbTraceEnabled()) {
 871             sqlite3_trace(dbconn->handle, SecDbTrace, dbconn);
 872         }
 873         ok = ok && SecDbBusyHandler(dbconn, error);
 874     });
 875
 876 done:
 877     return ok;
 878 }
 879
 880 static SecDbConnectionRef
 881 SecDbConnectionCreate(SecDbRef db, bool readOnly, CFErrorRef *error)
 882 {
 883     SecDbConnectionRef dbconn = NULL;
 884
 885     dbconn = CFTypeAllocate(SecDbConnection, struct __OpaqueSecDbConnection, kCFAllocatorDefault);
 886     require(dbconn != NULL, done);
 887
 888     dbconn->db = db;
 889     dbconn->readOnly = readOnly;
 890     dbconn->inTransaction = false;
 891     dbconn->source = NULL;
 892     dbconn->isCorrupted = false;
 893     dbconn->maybeCorruptedCode = 0;
 894     dbconn->corruptionError = NULL;
 895     dbconn->handle = NULL;
 896     dbconn->changes = CFArrayCreateMutableForCFTypes(kCFAllocatorDefault);
 897
 898 done:
 899     return dbconn;
 900 }
 901
 902 static bool SecDbConnectionIsReadOnly(SecDbConnectionRef dbconn) {
 903     return dbconn->readOnly;
 904 }
 905
 906 static void SecDbConectionSetReadOnly(SecDbConnectionRef dbconn, bool readOnly) {
 907     dbconn->readOnly = readOnly;
 908 }
 909
 910 /* Read only connections go to the end of the queue, writeable connections
 911  go to the start of the queue. */
 912 SecDbConnectionRef SecDbConnectionAquire(SecDbRef db, bool readOnly, CFErrorRef *error) {
 913     CFRetain(db);
 914     secinfo("dbconn", "acquire %s connection", readOnly ? "ro" : "rw");
 915     dispatch_semaphore_wait(readOnly ? db->read_semaphore : db->write_semaphore, DISPATCH_TIME_FOREVER);
 916     __block SecDbConnectionRef dbconn = NULL;
 917     __block bool ok = true;
 918     __block bool ranOpenedHandler = false;
 919     dispatch_sync(db->queue, ^{
 920         if (!db->didFirstOpen) {
 921             bool didCreate = false;
 922             ok = dbconn = SecDbConnectionCreate(db, false, error);
 923             CFErrorRef localError = NULL;
 924             if (ok && !SecDbOpenHandle(dbconn, &didCreate, &localError)) {
 925                 secerror("Unable to create database: %@", localError);
 926                 if (localError && CFEqual(CFErrorGetDomain(localError), kSecDbErrorDomain)) {
 927                     int code = (int)CFErrorGetCode(localError);
 928                     dbconn->isCorrupted = (SQLITE_CORRUPT == code) || (SQLITE_NOTADB == code) || (SQLITE_IOERR == code) || (SQLITE_CANTOPEN == code);
 929                 }
 930                 // If the open failure isn't due to corruption, propagte the error.
 931                 ok = dbconn->isCorrupted;
 932                 if (!ok && error && *error == NULL) {
 933                     *error = localError;
 934                     localError = NULL;
 935                 }
 936             }
 937             CFReleaseNull(localError);
 938
 939             if (ok) {
 940                 db->didFirstOpen = ok = SecDbDidCreateFirstConnection(dbconn, didCreate, error);
 941                 ranOpenedHandler = true;
 942             }
 943             if (!ok)
 944                 CFReleaseNull(dbconn);
 945         } else {
 946             /* Try to get one from the cache */
 947             CFIndex count = CFArrayGetCount(db->connections);
 948             while (count && !dbconn) {
 949                 CFIndex ix = readOnly ? count - 1 : 0;
 950                 dbconn = (SecDbConnectionRef)CFArrayGetValueAtIndex(db->connections, ix);
 951                 if (dbconn)
 952                     CFRetain(dbconn);
 953                 else
 954                     secerror("got NULL dbconn at index: %" PRIdCFIndex " skipping", ix);
 955                 CFArrayRemoveValueAtIndex(db->connections, ix);
 956             }
 957         }
 958     });
 959
 960     if (dbconn) {
 961         /* Make sure the connection we found has the right access */
 962         if (SecDbConnectionIsReadOnly(dbconn) != readOnly) {
 963             SecDbConectionSetReadOnly(dbconn, readOnly);
 964         }
 965     } else if (ok) {
 966         /* Nothing found in cache, create a new connection */
 967         bool created = false;
 968         dbconn = SecDbConnectionCreate(db, readOnly, error);
 969         if (dbconn && !SecDbOpenHandle(dbconn, &created, error)) {
 970             CFReleaseNull(dbconn);
 971         }
 972     }
 973
 974     if (dbconn && !ranOpenedHandler && dbconn->db->opened) {
 975         dispatch_sync(db->queue, ^{
 976             if (dbconn->db->callOpenedHandlerForNextConnection) {
 977                 dbconn->db->callOpenedHandlerForNextConnection = false;
 978                 if (!dbconn->db->opened(dbconn, false, &dbconn->db->callOpenedHandlerForNextConnection, error)) {
 979                     if (!dbconn->isCorrupted || !SecDbHandleCorrupt(dbconn, 0, error)) {
 980                         CFReleaseNull(dbconn);
 981                     }
 982                 }
 983             }
 984         });
 985     }
 986
 987     if (!dbconn) {
 988         // If aquire fails we need to signal the semaphore again.
 989         dispatch_semaphore_signal(readOnly ? db->read_semaphore : db->write_semaphore);
 990         CFRelease(db);
 991     }
 992
 993     return dbconn;
 994 }
 995
 996 void SecDbConnectionRelease(SecDbConnectionRef dbconn) {
 997     if (!dbconn) {
 998         secerror("called with NULL dbconn");
 999         return;
1000     }
1001     SecDbRef db = dbconn->db;
1002     secinfo("dbconn", "release %@", dbconn);
1003     dispatch_sync(db->queue, ^{
1004         CFIndex count = CFArrayGetCount(db->connections);
1005         // Add back possible writable dbconn to the pool.
1006         bool readOnly = SecDbConnectionIsReadOnly(dbconn);
1007         CFArrayInsertValueAtIndex(db->connections, readOnly ? count : 0, dbconn);
1008         // Remove the last (probably read-only) dbconn from the pool.
1009         if (count >= kSecDbMaxIdleHandles) {
1010             CFArrayRemoveValueAtIndex(db->connections, count);
1011         }
1012         // Signal after we have put the connection back in the pool of connections
1013         dispatch_semaphore_signal(readOnly ? db->read_semaphore : db->write_semaphore);
1014         CFRelease(dbconn);
1015         CFRelease(db);
1016     });
1017 }
1018
1019 bool SecDbPerformRead(SecDbRef db, CFErrorRef *error, void (^perform)(SecDbConnectionRef dbconn)) {
1020     SecDbConnectionRef dbconn = SecDbConnectionAquire(db, true, error);
1021     bool success = false;
1022     if (dbconn) {
1023         perform(dbconn);
1024         success = true;
1025         SecDbConnectionRelease(dbconn);
1026     }
1027     return success;
1028 }
1029
1030 bool SecDbPerformWrite(SecDbRef db, CFErrorRef *error, void (^perform)(SecDbConnectionRef dbconn)) {
1031     SecDbConnectionRef dbconn = SecDbConnectionAquire(db, false, error);
1032     bool success = false;
1033     if (dbconn) {
1034         perform(dbconn);
1035         success = true;
1036         SecDbConnectionRelease(dbconn);
1037     }
1038     return success;
1039 }
1040
1041 static CFStringRef
1042 SecDbConnectionCopyFormatDescription(CFTypeRef value, CFDictionaryRef formatOptions)
1043 {
1044     SecDbConnectionRef dbconn = (SecDbConnectionRef)value;
1045     return CFStringCreateWithFormat(kCFAllocatorDefault, NULL, CFSTR("<SecDbConnection %s %s>"),
1046                                     dbconn->readOnly ? "ro" : "rw", dbconn->handle ? "open" : "closed");
1047 }
1048
1049 static void
1050 SecDbConnectionDestroy(CFTypeRef value)
1051 {
1052     SecDbConnectionRef dbconn = (SecDbConnectionRef)value;
1053     if (dbconn->handle) {
1054         sqlite3_close(dbconn->handle);
1055     }
1056     dbconn->db = NULL;
1057     CFReleaseNull(dbconn->changes);
1058     CFReleaseNull(dbconn->corruptionError);
1059
1060 }
1061
1062 void SecDbPerformOnCommitQueue(SecDbConnectionRef dbconn, bool barrier, dispatch_block_t perform) {
1063     if (barrier) {
1064         dispatch_barrier_sync(dbconn->db->commitQueue, ^{
1065             perform();
1066         });
1067     } else {
1068         dispatch_sync(dbconn->db->commitQueue, ^{
1069             perform();
1070         });
1071     }
1072 }
1073
1074 // MARK: -
1075 // MARK: Bind helpers
1076
1077 #if 0
1078 bool SecDbBindNull(sqlite3_stmt *stmt, int param, CFErrorRef *error) {
1079     bool ok = SecDbErrorWithStmt(sqlite3_bind_null(stmt, param),
1080                                  stmt, error, CFSTR("bind_null[%d]"), param);
1081     secinfo("bind", "bind_null[%d]: %@", param, error ? *error : NULL);
1082     return ok;
1083 }
1084 #endif
1085
1086 bool SecDbBindBlob(sqlite3_stmt *stmt, int param, const void *zData, size_t n, void(*xDel)(void*), CFErrorRef *error) {
1087     if (n > INT_MAX) {
1088         return SecDbErrorWithStmt(SQLITE_TOOBIG, stmt, error,
1089                                   CFSTR("bind_blob[%d]: blob bigger than INT_MAX"), param);
1090     }
1091     bool ok = SecDbErrorWithStmt(sqlite3_bind_blob(stmt, param, zData, (int)n, xDel),
1092                                  stmt, error, CFSTR("bind_blob[%d]"), param);
1093     secinfo("bind", "bind_blob[%d]: %.*s: %@", param, (int)n, zData, error ? *error : NULL);
1094     return ok;
1095 }
1096
1097 bool SecDbBindText(sqlite3_stmt *stmt, int param, const char *zData, size_t n, void(*xDel)(void*), CFErrorRef *error) {
1098     if (n > INT_MAX) {
1099         return SecDbErrorWithStmt(SQLITE_TOOBIG, stmt, error,
1100                                   CFSTR("bind_text[%d]: text bigger than INT_MAX"), param);
1101     }
1102     bool ok = SecDbErrorWithStmt(sqlite3_bind_text(stmt, param, zData, (int)n, xDel), stmt, error,
1103                                  CFSTR("bind_text[%d]"), param);
1104     secinfo("bind", "bind_text[%d]: \"%s\": %@", param, zData, error ? *error : NULL);
1105     return ok;
1106 }
1107
1108 bool SecDbBindDouble(sqlite3_stmt *stmt, int param, double value, CFErrorRef *error) {
1109     bool ok = SecDbErrorWithStmt(sqlite3_bind_double(stmt, param, value), stmt, error,
1110                                  CFSTR("bind_double[%d]"), param);
1111     secinfo("bind", "bind_double[%d]: %f: %@", param, value, error ? *error : NULL);
1112     return ok;
1113 }
1114
1115 bool SecDbBindInt(sqlite3_stmt *stmt, int param, int value, CFErrorRef *error) {
1116     bool ok = SecDbErrorWithStmt(sqlite3_bind_int(stmt, param, value), stmt, error,
1117                                  CFSTR("bind_int[%d]"), param);
1118     secinfo("bind", "bind_int[%d]: %d: %@", param, value, error ? *error : NULL);
1119     return ok;
1120 }
1121
1122 bool SecDbBindInt64(sqlite3_stmt *stmt, int param, sqlite3_int64 value, CFErrorRef *error) {
1123     bool ok = SecDbErrorWithStmt(sqlite3_bind_int64(stmt, param, value), stmt, error,
1124                                  CFSTR("bind_int64[%d]"), param);
1125     secinfo("bind", "bind_int64[%d]: %lld: %@", param, value, error ? *error : NULL);
1126     return ok;
1127 }
1128
1129
1130 /* AUDIT[securityd](done):
1131  value (ok) is a caller provided, non NULL CFTypeRef.
1132  */
1133 bool SecDbBindObject(sqlite3_stmt *stmt, int param, CFTypeRef value, CFErrorRef *error) {
1134     CFTypeID valueId;
1135     __block bool result = false;
1136
1137         /* TODO: Can we use SQLITE_STATIC below everwhere we currently use
1138      SQLITE_TRANSIENT since we finalize the statement before the value
1139      goes out of scope? */
1140     if (!value || (valueId = CFGetTypeID(value)) == CFNullGetTypeID()) {
1141         /* Skip bindings for NULL values.  sqlite3 will interpret unbound
1142          params as NULL which is exactly what we want. */
1143 #if 1
1144         result = true;
1145 #else
1146         result = SecDbBindNull(stmt, param, error);
1147 #endif
1148     } else if (valueId == CFStringGetTypeID()) {
1149         CFStringPerformWithCStringAndLength(value, ^(const char *cstr, size_t clen) {
1150             result = SecDbBindText(stmt, param, cstr, clen, SQLITE_TRANSIENT, error);
1151         });
1152     } else if (valueId == CFDataGetTypeID()) {
1153         CFIndex len = CFDataGetLength(value);
1154         if (len) {
1155             result = SecDbBindBlob(stmt, param, CFDataGetBytePtr(value),
1156                                    len, SQLITE_TRANSIENT, error);
1157         } else {
1158             result = SecDbBindText(stmt, param, "", 0, SQLITE_TRANSIENT, error);
1159         }
1160     } else if (valueId == CFDateGetTypeID()) {
1161         CFAbsoluteTime abs_time = CFDateGetAbsoluteTime(value);
1162         result = SecDbBindDouble(stmt, param, abs_time, error);
1163     } else if (valueId == CFBooleanGetTypeID()) {
1164         int bval = CFBooleanGetValue(value);
1165         result = SecDbBindInt(stmt, param, bval, error);
1166     } else if (valueId == CFNumberGetTypeID()) {
1167         Boolean convertOk;
1168         if (CFNumberIsFloatType(value)) {
1169             double nval;
1170             convertOk = CFNumberGetValue(value, kCFNumberDoubleType, &nval);
1171             result = SecDbBindDouble(stmt, param, nval, error);
1172         } else {
1173             SInt32 nval;
1174             convertOk = CFNumberGetValue(value, kCFNumberSInt32Type, &nval);
1175             if (convertOk) {
1176                 result = SecDbBindInt(stmt, param, nval, error);
1177             } else {
1178                 sqlite_int64 nval64;
1179                 convertOk = CFNumberGetValue(value, kCFNumberSInt64Type, &nval64);
1180                 if (convertOk)
1181                     result = SecDbBindInt64(stmt, param, nval64, error);
1182             }
1183         }
1184         if (!convertOk) {
1185             result = SecDbError(SQLITE_INTERNAL, error, CFSTR("bind CFNumberGetValue failed for %@"), value);
1186         }
1187     } else {
1188         if (error) {
1189             CFStringRef valueDesc = CFCopyTypeIDDescription(valueId);
1190             SecDbError(SQLITE_MISMATCH, error, CFSTR("bind unsupported type %@"), valueDesc);
1191             CFReleaseSafe(valueDesc);
1192         }
1193     }
1194
1195         return result;
1196 }
1197
1198 // MARK: -
1199 // MARK: SecDbStatementRef
1200
1201 bool SecDbReset(sqlite3_stmt *stmt, CFErrorRef *error) {
1202     return SecDbErrorWithStmt(sqlite3_reset(stmt), stmt, error, CFSTR("reset"));
1203 }
1204
1205 bool SecDbClearBindings(sqlite3_stmt *stmt, CFErrorRef *error) {
1206     return SecDbErrorWithStmt(sqlite3_clear_bindings(stmt), stmt, error, CFSTR("clear bindings"));
1207 }
1208
1209 bool SecDbFinalize(sqlite3_stmt *stmt, CFErrorRef *error) {
1210     sqlite3 *handle = sqlite3_db_handle(stmt);
1211     int s3e = sqlite3_finalize(stmt);
1212     return s3e == SQLITE_OK ? true : SecDbErrorWithDb(s3e, handle, error, CFSTR("finalize: %p"), stmt);
1213 }
1214
1215 sqlite3_stmt *SecDbPrepareV2(SecDbConnectionRef dbconn, const char *sql, size_t sqlLen, const char **sqlTail, CFErrorRef *error) {
1216     sqlite3 *db = SecDbHandle(dbconn);
1217     if (sqlLen > INT_MAX) {
1218         SecDbErrorWithDb(SQLITE_TOOBIG, db, error, CFSTR("prepare_v2: sql bigger than INT_MAX"));
1219         return NULL;
1220     }
1221     int ntries = 0;
1222     for (;;) {
1223         sqlite3_stmt *stmt = NULL;
1224         int s3e = sqlite3_prepare_v2(db, sql, (int)sqlLen, &stmt, sqlTail);
1225         if (s3e == SQLITE_OK)
1226             return stmt;
1227         else if (!SecDbWaitIfNeeded(dbconn, s3e, NULL, CFSTR("preparev2"), ntries, error))
1228             return NULL;
1229         ntries++;
1230     }
1231 }
1232
1233 static sqlite3_stmt *SecDbCopyStatementWithTailRange(SecDbConnectionRef dbconn, CFStringRef sql, CFRange *sqlTail, CFErrorRef *error) {
1234     __block sqlite3_stmt *stmt = NULL;
1235     if (sql) CFStringPerformWithCStringAndLength(sql, ^(const char *sqlStr, size_t sqlLen) {
1236         const char *tail = NULL;
1237         stmt = SecDbPrepareV2(dbconn, sqlStr, sqlLen, &tail, error);
1238         if (sqlTail && sqlStr < tail && tail < sqlStr + sqlLen) {
1239             sqlTail->location = tail - sqlStr;
1240             sqlTail->length = sqlLen - sqlTail->location;
1241         }
1242     });
1243
1244     return stmt;
1245 }
1246
1247 sqlite3_stmt *SecDbCopyStmt(SecDbConnectionRef dbconn, CFStringRef sql, CFStringRef *tail, CFErrorRef *error) {
1248     // TODO: Add caching and cache lookup of statements
1249     CFRange sqlTail = {};
1250     sqlite3_stmt *stmt = SecDbCopyStatementWithTailRange(dbconn, sql, &sqlTail, error);
1251     if (sqlTail.length > 0) {
1252         CFStringRef excess = CFStringCreateWithSubstring(CFGetAllocator(sql), sql, sqlTail);
1253         if (tail) {
1254             *tail = excess;
1255         } else {
1256             SecDbError(SQLITE_INTERNAL, error,
1257                        CFSTR("prepare_v2: %@ unused sql: %@"),
1258                        sql, excess);
1259             CFReleaseSafe(excess);
1260             SecDbFinalize(stmt, error);
1261             stmt = NULL;
1262         }
1263     }
1264     return stmt;
1265 }
1266
1267 /*
1268  TODO: Could do a hack here with a custom kCFAllocatorNULL allocator for a second CFRuntimeBase inside a SecDbStatement,
1269  TODO: Better yet make a full blow SecDbStatement instance whenever SecDbCopyStmt is called.  Then, when the statement is released, in the Dispose method, we Reset and ClearBindings the sqlite3_stmt * and hand it back to the SecDb with the original CFStringRef for the sql (or hash thereof) as an argument. */
1270 bool SecDbReleaseCachedStmt(SecDbConnectionRef dbconn, CFStringRef sql, sqlite3_stmt *stmt, CFErrorRef *error) {
1271     if (stmt) {
1272         return SecDbFinalize(stmt, error);
1273     }
1274     return true;
1275 }
1276
1277 bool SecDbPrepare(SecDbConnectionRef dbconn, CFStringRef sql, CFErrorRef *error, void(^exec)(sqlite3_stmt *stmt)) {
1278     assert(sql != NULL);
1279     sqlite3_stmt *stmt = SecDbCopyStmt(dbconn, sql, NULL, error);
1280     if (!stmt)
1281         return false;
1282
1283     exec(stmt);
1284     return SecDbReleaseCachedStmt(dbconn, sql, stmt, error);
1285 }
1286
1287 bool SecDbWithSQL(SecDbConnectionRef dbconn, CFStringRef sql, CFErrorRef *error, bool(^perform)(sqlite3_stmt *stmt)) {
1288     bool ok = true;
1289     CFRetain(sql);
1290     while (sql) {
1291         CFStringRef tail = NULL;
1292         if (ok) {
1293             sqlite3_stmt *stmt = SecDbCopyStmt(dbconn, sql, &tail, error);
1294             ok = stmt != NULL;
1295             if (stmt) {
1296                 if (perform) {
1297                     ok = perform(stmt);
1298                 } else {
1299                     // TODO: Use a different error scope here.
1300                     ok = SecError(-50 /* errSecParam */, error, CFSTR("SecDbWithSQL perform block missing"));
1301                 }
1302                 ok &= SecDbReleaseCachedStmt(dbconn, sql, stmt, error);
1303             }
1304         } else {
1305             // TODO We already have an error here we really just want the left over sql in it's userData
1306             ok = SecDbError(SQLITE_ERROR, error, CFSTR("Error with unexecuted sql remaining %@"), sql);
1307         }
1308         CFRelease(sql);
1309         sql = tail;
1310     }
1311     return ok;
1312 }
1313
1314 #if 1
1315 /* SecDbForEach returns true if all SQLITE_ROW returns of sqlite3_step() return true from the row block.
1316  If the row block returns false and doesn't set an error (to indicate it has reached a limit),
1317  this entire function returns false. In that case no error will be set. */
1318 bool SecDbForEach(sqlite3_stmt *stmt, CFErrorRef *error, bool(^row)(int row_index)) {
1319     bool result = false;
1320     for (int row_ix = 0;;++row_ix) {
1321         int s3e = sqlite3_step(stmt);
1322         if (s3e == SQLITE_ROW) {
1323             if (row) {
1324                 if (!row(row_ix)) {
1325                     break;
1326                 }
1327             } else {
1328                 // If we have no row block then getting SQLITE_ROW is an error
1329                 SecDbError(s3e, error,
1330                            CFSTR("step[%d]: %s returned SQLITE_ROW with NULL row block"),
1331                            row_ix, sqlite3_sql(stmt));
1332             }
1333         } else {
1334             if (s3e == SQLITE_DONE) {
1335                 result = true;
1336             } else {
1337                 SecDbErrorWithStmt(s3e, stmt, error, CFSTR("step[%d]"), row_ix);
1338             }
1339             break;
1340         }
1341     }
1342     return result;
1343 }
1344 #else
1345 bool SecDbForEach(sqlite3_stmt *stmt, CFErrorRef *error, bool(^row)(int row_index)) {
1346     int row_ix = 0;
1347     for (;;) {
1348         switch (_SecDbStep(dbconn, stmt, error)) {
1349             case kSecDbErrorStep:
1350                 return false;
1351             case kSecDbRowStep:
1352                 if (row) {
1353                     if (row(row_ix++))
1354                         break;
1355                 } else {
1356                     SecDbError(SQLITE_ERROR, error, CFSTR("SecDbStep SQLITE_ROW returned without a row handler"));
1357                 }
1358                 return false;
1359             case kSecDbDoneStep:
1360                 return true;
1361         }
1362     }
1363 }
1364 #endif
1365
1366 void SecDbRecordChange(SecDbConnectionRef dbconn, CFTypeRef deleted, CFTypeRef inserted) {
1367     if (!dbconn->db->notifyPhase) return;
1368     CFTypeRef entry = SecDbEventCreateWithComponents(deleted, inserted);
1369     if (entry) {
1370         CFArrayAppendValue(dbconn->changes, entry);
1371         CFRelease(entry);
1372
1373         if (!dbconn->inTransaction) {
1374             secerror("db %@ changed outside txn", dbconn);
1375             // Only notify of DidCommit, since WillCommit code assumes
1376             // we are in a txn.
1377             SecDbOnNotify(dbconn, ^{
1378                 SecDbNotifyPhase(dbconn, kSecDbTransactionDidCommit);
1379             });
1380         }
1381     }
1382 }
1383
1384
1385 CFGiblisFor(SecDbConnection)
1386
1387 //
1388 // SecDbEvent Creation and consumption
1389 //
1390
1391 static SecDbEventRef SecDbEventCreateInsert(CFTypeRef inserted) {
1392     return CFRetainSafe(inserted);
1393 }
1394
1395 static SecDbEventRef SecDbEventCreateDelete(CFTypeRef deleted) {
1396     return CFArrayCreate(kCFAllocatorDefault, &deleted, 1, &kCFTypeArrayCallBacks);
1397 }
1398
1399 static SecDbEventRef SecDbEventCreateUpdate(CFTypeRef deleted, CFTypeRef inserted) {
1400     const void *values[2] = { deleted, inserted };
1401     return CFArrayCreate(kCFAllocatorDefault, values, 2, &kCFTypeArrayCallBacks);
1402 }
1403
1404 SecDbEventRef SecDbEventCreateWithComponents(CFTypeRef deleted, CFTypeRef inserted) {
1405     if (deleted && inserted)
1406         return SecDbEventCreateUpdate(deleted, inserted);
1407     else if (deleted)
1408         return SecDbEventCreateDelete(deleted);
1409     else if (inserted)
1410         return SecDbEventCreateInsert(inserted);
1411     else
1412         return NULL;
1413 }
1414
1415 bool SecDbEventGetComponents(SecDbEventRef event, CFTypeRef *deleted, CFTypeRef *inserted, CFErrorRef *error) {
1416     if (isArray(event)) {
1417         CFArrayRef array = event;
1418         switch (CFArrayGetCount(array)) {
1419             case 2:
1420                 *deleted = CFArrayGetValueAtIndex(array, 0);
1421                 *inserted = CFArrayGetValueAtIndex(array, 1);
1422                 break;
1423             case 1:
1424                 *deleted = CFArrayGetValueAtIndex(array, 0);
1425                 *inserted = NULL;
1426                 break;
1427             default:
1428                 SecError(errSecParam, error, NULL, CFSTR("invalid entry in changes array: %@"), array);
1429                 break;
1430         }
1431     } else {
1432         *deleted = NULL;
1433         *inserted = event;
1434     }
1435     return true;
1436 }