OSX/utilities/src/SecDb.c

   1 /*
   2  * Copyright (c) 2012-2017 Apple Inc. All Rights Reserved.
   3  *
   4  * @APPLE_LICENSE_HEADER_START@
   5  *
   6  * This file contains Original Code and/or Modifications of Original Code
   7  * as defined in and that are subject to the Apple Public Source License
   8  * Version 2.0 (the 'License'). You may not use this file except in
   9  * compliance with the License. Please obtain a copy of the License at
  10  * http://www.opensource.apple.com/apsl/ and read it before using this
  11  * file.
  12  *
  13  * The Original Code and all software distributed under the License are
  14  * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  15  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  16  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
  17  * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
  18  * Please see the License for the specific language governing rights and
  19  * limitations under the License.
  20  *
  21  * @APPLE_LICENSE_HEADER_END@
  22  */
  23
  24
  25 #include "SecDb.h"
  26 #include "debugging.h"
  27
  28 #include <sqlite3.h>
  29 #include <sqlite3_private.h>
  30 #include <CoreFoundation/CoreFoundation.h>
  31 #include <libgen.h>
  32 #include <sys/csr.h>
  33 #include <sys/stat.h>
  34 #include <AssertMacros.h>
  35 #include "SecCFWrappers.h"
  36 #include "SecCFError.h"
  37 #include "SecIOFormat.h"
  38 #include <stdio.h>
  39 #include "Security/SecBase.h"
  40
  41
  42 //
  43 // Architecturally inverted files
  44 // These are in SecureObjectSync but utilities depends on them
  45 // <rdar://problem/20802079> Fix layer violation (SOSDigestVector, SOSManifest, SecDB.c)
  46 //
  47 #include <Security/SecureObjectSync/SOSDigestVector.h>
  48 #include <Security/SecureObjectSync/SOSManifest.h>
  49
  50 #define HAVE_UNLOCK_NOTIFY  0
  51
  52 struct __OpaqueSecDbStatement {
  53     CFRuntimeBase _base;
  54
  55     SecDbConnectionRef dbconn;
  56     sqlite3_stmt *stmt;
  57 };
  58
  59 struct __OpaqueSecDbConnection {
  60     CFRuntimeBase _base;
  61
  62     //CFMutableDictionaryRef statements;
  63
  64     SecDbRef db;     // NONRETAINED, since db or block retains us
  65     bool readOnly;
  66     bool inTransaction;
  67     SecDbTransactionSource source;
  68     bool isCorrupted;
  69     int maybeCorruptedCode;
  70     bool hasIOFailure;
  71     CFErrorRef corruptionError;
  72     sqlite3 *handle;
  73     // Pending deletions and additions for the current transaction
  74     // Entires are either:
  75     // 1) a CFArrayRef of 1 element representing a deletion,
  76     // 2) a CFArrayRef of 2 elements representing the element 0 having been replaced with element 1
  77     // 3) a CFTypeRef that is not a CFArrayRef, representing an add of the element in question.
  78     CFMutableArrayRef changes;
  79 };
  80
  81 struct __OpaqueSecDb {
  82     CFRuntimeBase _base;
  83
  84     CFStringRef db_path;
  85     dispatch_queue_t queue;
  86     dispatch_queue_t commitQueue;
  87     CFMutableArrayRef connections;
  88     dispatch_semaphore_t write_semaphore;
  89     dispatch_semaphore_t read_semaphore;
  90     bool didFirstOpen;
  91     bool (^opened)(SecDbConnectionRef dbconn, bool didCreate, bool *callMeAgainForNextConnection, CFErrorRef *error);
  92     bool callOpenedHandlerForNextConnection;
  93     CFMutableArrayRef notifyPhase; /* array of SecDBNotifyBlock */
  94     mode_t mode; /* database file permissions, default 0600 */
  95     bool readWrite; /* open database read-write, default true */
  96     bool allowRepair; /* allow database repair, default true */
  97     bool useWAL; /* use WAL mode, default true */
  98 };
  99
 100 // MARK: Error domains and error helper functions
 101
 102 CFStringRef kSecDbErrorDomain = CFSTR("com.apple.utilities.sqlite3");
 103
 104 bool SecDbError(int sql_code, CFErrorRef *error, CFStringRef format, ...) {
 105     if (sql_code == SQLITE_OK) return true;
 106
 107     if (error) {
 108         va_list args;
 109         CFIndex code = sql_code;
 110         CFErrorRef previousError = *error;
 111
 112         *error = NULL;
 113         va_start(args, format);
 114         SecCFCreateErrorWithFormatAndArguments(code, kSecDbErrorDomain, previousError, error, NULL, format, args);
 115         va_end(args);
 116     }
 117     return false;
 118 }
 119
 120 bool SecDbErrorWithDb(int sql_code, sqlite3 *db, CFErrorRef *error, CFStringRef format, ...) {
 121     if (sql_code == SQLITE_OK) return true;
 122     if (error) {
 123         va_list args;
 124         va_start(args, format);
 125         CFStringRef message = CFStringCreateWithFormatAndArguments(kCFAllocatorDefault, NULL, format, args);
 126         va_end(args);
 127
 128         int extended_code = sqlite3_extended_errcode(db);
 129         if (sql_code == extended_code)
 130             SecDbError(sql_code, error, CFSTR("%@: [%d] %s"), message, sql_code, sqlite3_errmsg(db));
 131         else
 132             SecDbError(sql_code, error, CFSTR("%@: [%d->%d] %s"), message, sql_code, extended_code, sqlite3_errmsg(db));
 133         CFReleaseSafe(message);
 134     }
 135     return false;
 136 }
 137
 138 bool SecDbErrorWithStmt(int sql_code, sqlite3_stmt *stmt, CFErrorRef *error, CFStringRef format, ...) {
 139     if (sql_code == SQLITE_OK) return true;
 140     if (error) {
 141         va_list args;
 142         va_start(args, format);
 143         CFStringRef message = CFStringCreateWithFormatAndArguments(kCFAllocatorDefault, NULL, format, args);
 144         va_end(args);
 145
 146         sqlite3 *db = sqlite3_db_handle(stmt);
 147         const char *sql = sqlite3_sql(stmt);
 148         int extended_code = sqlite3_extended_errcode(db);
 149         if (sql_code == extended_code)
 150             SecDbError(sql_code, error, CFSTR("%@: [%d] %s sql: %s"), message, sql_code, sqlite3_errmsg(db), sql);
 151         else
 152             SecDbError(sql_code, error, CFSTR("%@: [%d->%d] %s sql: %s"), message, sql_code, extended_code, sqlite3_errmsg(db), sql);
 153         CFReleaseSafe(message);
 154     }
 155     return false;
 156 }
 157
 158
 159 // MARK: -
 160 // MARK: Static helper functions
 161
 162 static bool SecDbOpenHandle(SecDbConnectionRef dbconn, bool *created, CFErrorRef *error);
 163 static bool SecDbHandleCorrupt(SecDbConnectionRef dbconn, int rc, CFErrorRef *error);
 164
 165 #pragma mark -
 166 #pragma mark SecDbRef
 167
 168 static CFStringRef
 169 SecDbCopyFormatDescription(CFTypeRef value, CFDictionaryRef formatOptions)
 170 {
 171     SecDbRef db = (SecDbRef)value;
 172     return CFStringCreateWithFormat(kCFAllocatorDefault, NULL, CFSTR("<SecDb path:%@ connections: %@>"), db->db_path, db->connections);
 173 }
 174
 175
 176 static void
 177 SecDbDestroy(CFTypeRef value)
 178 {
 179     SecDbRef db = (SecDbRef)value;
 180     CFReleaseNull(db->connections);
 181     CFReleaseNull(db->db_path);
 182     if (db->queue) {
 183         dispatch_release(db->queue);
 184         db->queue = NULL;
 185     }
 186     if (db->commitQueue) {
 187         dispatch_release(db->commitQueue);
 188         db->commitQueue = NULL;
 189     }
 190     if (db->read_semaphore) {
 191         dispatch_release(db->read_semaphore);
 192         db->read_semaphore = NULL;
 193     }
 194     if (db->write_semaphore) {
 195         dispatch_release(db->write_semaphore);
 196         db->write_semaphore = NULL;
 197     }
 198     if (db->opened) {
 199         Block_release(db->opened);
 200         db->opened = NULL;
 201     }
 202     CFReleaseNull(db->notifyPhase);
 203 }
 204
 205 CFGiblisFor(SecDb)
 206
 207 SecDbRef
 208 SecDbCreateWithOptions(CFStringRef dbName, mode_t mode, bool readWrite, bool allowRepair, bool useWAL,
 209                        bool (^opened)(SecDbConnectionRef dbconn, bool didCreate, bool *callMeAgainForNextConnection, CFErrorRef *error))
 210 {
 211     SecDbRef db = NULL;
 212
 213     db = CFTypeAllocate(SecDb, struct __OpaqueSecDb, kCFAllocatorDefault);
 214     require(db != NULL, done);
 215
 216     CFStringPerformWithCString(dbName, ^(const char *dbNameStr) {
 217         db->queue = dispatch_queue_create(dbNameStr, DISPATCH_QUEUE_SERIAL);
 218     });
 219     CFStringRef commitQueueStr = CFStringCreateWithFormat(kCFAllocatorDefault, NULL, CFSTR("%@-commit"), dbName);
 220     CFStringPerformWithCString(commitQueueStr, ^(const char *cqNameStr) {
 221         db->commitQueue = dispatch_queue_create(cqNameStr, DISPATCH_QUEUE_CONCURRENT);
 222     });
 223     CFReleaseNull(commitQueueStr);
 224     db->read_semaphore = dispatch_semaphore_create(kSecDbMaxReaders);
 225     db->write_semaphore = dispatch_semaphore_create(kSecDbMaxWriters);
 226     db->connections = CFArrayCreateMutableForCFTypes(kCFAllocatorDefault);
 227     db->opened = opened ? Block_copy(opened) : NULL;
 228     if (getenv("__OSINSTALL_ENVIRONMENT") != NULL) {
 229         // TODO: Move this code out of this layer
 230         secinfo("#SecDB", "SecDB: running from installer");
 231
 232         db->db_path = CFSTR("file::memory:?cache=shared");
 233     } else {
 234         db->db_path = CFStringCreateCopy(kCFAllocatorDefault, dbName);
 235     }
 236     db->mode = mode;
 237     db->readWrite = readWrite;
 238     db->allowRepair = allowRepair;
 239     db->useWAL = useWAL;
 240
 241 done:
 242     return db;
 243 }
 244
 245 SecDbRef
 246 SecDbCreate(CFStringRef dbName,
 247             bool (^opened)(SecDbConnectionRef dbconn, bool didCreate, bool *callMeAgainForNextConnection, CFErrorRef *error))
 248 {
 249     return SecDbCreateWithOptions(dbName, 0600, true, true, true, opened);
 250 }
 251
 252 CFIndex
 253 SecDbIdleConnectionCount(SecDbRef db) {
 254     __block CFIndex count = 0;
 255     dispatch_sync(db->queue, ^{
 256         count = CFArrayGetCount(db->connections);
 257     });
 258     return count;
 259 }
 260
 261 void SecDbAddNotifyPhaseBlock(SecDbRef db, SecDBNotifyBlock notifyPhase)
 262 {
 263     SecDBNotifyBlock block = Block_copy(notifyPhase); /* Force the block off the stack */
 264     if (db->notifyPhase == NULL) {
 265         db->notifyPhase = CFArrayCreateMutableForCFTypes(NULL);
 266     }
 267     CFArrayAppendValue(db->notifyPhase, block);
 268     Block_release(block);
 269 }
 270
 271 static void SecDbNotifyPhase(SecDbConnectionRef dbconn, SecDbTransactionPhase phase) {
 272     if (CFArrayGetCount(dbconn->changes)) {
 273         CFArrayRef changes = dbconn->changes;
 274         dbconn->changes = CFArrayCreateMutableForCFTypes(kCFAllocatorDefault);
 275         if (dbconn->db->notifyPhase) {
 276             CFArrayForEach(dbconn->db->notifyPhase, ^(const void *value) {
 277                 SecDBNotifyBlock notifyBlock = (SecDBNotifyBlock)value;
 278                 notifyBlock(dbconn, phase, dbconn->source, changes);
 279             });
 280         }
 281         CFReleaseSafe(changes);
 282     }
 283 }
 284
 285 static void SecDbOnNotify(SecDbConnectionRef dbconn, void (^perform)()) {
 286     perform();
 287 }
 288
 289 CFStringRef SecDbGetPath(SecDbRef db) {
 290     if(!db) {
 291         return NULL;
 292     }
 293     return db->db_path;
 294 }
 295
 296
 297 #pragma mark -
 298 #pragma mark SecDbConnectionRef
 299
 300 static bool SecDbCheckCorrupted(SecDbConnectionRef dbconn)
 301 {
 302     __block bool isCorrupted = true;
 303     __block CFErrorRef error = NULL;
 304     SecDbPrepare(dbconn, CFSTR("PRAGMA integrity_check"), &error, ^(sqlite3_stmt *stmt) {
 305         SecDbStep(dbconn, stmt, &error, ^(bool *stop) {
 306             const char * result = (const char*)sqlite3_column_text(stmt, 0);
 307             if (result && strncasecmp(result, "ok", 3) == 0) {
 308                 isCorrupted = false;
 309             }
 310         });
 311     });
 312     if (error) {
 313         secinfo("#SecDB", "#SecDB warning error %{public}@ when running integrity check", error);
 314         CFRelease(error);
 315     }
 316     return isCorrupted;
 317 }
 318
 319 static bool SecDbDidCreateFirstConnection(SecDbConnectionRef dbconn, bool didCreate, CFErrorRef *error)
 320 {
 321     secinfo("#SecDB", "#SecDB starting maintenance");
 322     bool ok = true;
 323
 324     if (!didCreate && !dbconn->isCorrupted) {
 325         dbconn->isCorrupted = SecDbCheckCorrupted(dbconn);
 326         if (dbconn->isCorrupted) {
 327             secinfo("#SecDB", "#SecDB integrity check=fail");
 328         } else {
 329             secinfo("#SecDB", "#SecDB starting maintenance");
 330         }
 331     }
 332
 333     if (!dbconn->isCorrupted && dbconn->db->opened) {
 334         CFErrorRef localError = NULL;
 335
 336         dbconn->db->callOpenedHandlerForNextConnection = false;
 337         ok = dbconn->db->opened(dbconn, didCreate, &dbconn->db->callOpenedHandlerForNextConnection, &localError);
 338
 339         if (!ok)
 340             secerror("opened block failed: %@", localError);
 341
 342         if (!dbconn->isCorrupted && error && *error == NULL) {
 343             *error = localError;
 344             localError = NULL;
 345         } else {
 346             if (localError)
 347                 secerror("opened block failed: error is released and lost");
 348             CFReleaseNull(localError);
 349         }
 350     }
 351
 352     if (dbconn->isCorrupted) {
 353         ok = SecDbHandleCorrupt(dbconn, 0, error);
 354     }
 355
 356     secinfo("#SecDB", "#SecDB starting maintenance");
 357     return ok;
 358 }
 359
 360 void SecDbCorrupt(SecDbConnectionRef dbconn, CFErrorRef error)
 361 {
 362     CFStringRef str = CFStringCreateWithFormat(NULL, NULL, CFSTR("SecDBCorrupt: %@"), error);
 363     if (str) {
 364         char buffer[1000] = "?";
 365         uint32_t errorCode = 0;
 366         CFStringGetCString(str, buffer, sizeof(buffer), kCFStringEncodingUTF8);
 367         os_log_fault(secLogObjForScope("SecEmergency"), "%s", buffer);
 368         if (error)
 369             errorCode = (uint32_t)CFErrorGetCode(error);
 370         __security_simulatecrash(str, __sec_exception_code_CorruptDb(errorCode));
 371         CFRelease(str);
 372     }
 373     dbconn->isCorrupted = true;
 374     CFRetainAssign(dbconn->corruptionError, error);
 375 }
 376
 377
 378 static uint8_t knownDbPathIndex(SecDbConnectionRef dbconn)
 379 {
 380
 381     if(CFEqual(dbconn->db->db_path, CFSTR("/Library/Keychains/keychain-2.db")))
 382         return 1;
 383     if(CFEqual(dbconn->db->db_path, CFSTR("/Library/Keychains/ocspcache.sqlite3")))
 384         return 2;
 385     if(CFEqual(dbconn->db->db_path, CFSTR("/Library/Keychains/TrustStore.sqlite3")))
 386         return 3;
 387     if(CFEqual(dbconn->db->db_path, CFSTR("/Library/Keychains/caissuercache.sqlite3")))
 388         return 4;
 389
 390     /* Unknown DB path */
 391     return 0;
 392 }
 393
 394
 395 // Return true if there was no error, returns false otherwise and set *error to an appropriate CFErrorRef.
 396 static bool SecDbConnectionCheckCode(SecDbConnectionRef dbconn, int code, CFErrorRef *error, CFStringRef desc, ...) {
 397     if (code == SQLITE_OK || code == SQLITE_DONE)
 398         return true;
 399
 400     if (error) {
 401         va_list args;
 402         va_start(args, desc);
 403         CFStringRef msg = CFStringCreateWithFormatAndArguments(kCFAllocatorDefault, NULL, desc, args);
 404         va_end(args);
 405         SecDbErrorWithDb(code, dbconn->handle, error, msg);
 406         CFRelease(msg);
 407     }
 408
 409     dbconn->hasIOFailure |= (SQLITE_IOERR == code);
 410
 411     /* If it's already corrupted, don't try to recover */
 412     if (dbconn->isCorrupted) {
 413         CFStringRef reason = CFStringCreateWithFormat(kCFAllocatorDefault, NULL,
 414                                                       CFSTR("SQL DB %@ is corrupted already. Not trying to recover, corruption error was: %d (previously %d)"),
 415                                                       dbconn->db->db_path, code, dbconn->maybeCorruptedCode);
 416         secerror("%@",reason);
 417         __security_simulatecrash(reason, __sec_exception_code_TwiceCorruptDb(knownDbPathIndex(dbconn)));
 418         CFReleaseSafe(reason);
 419         return false;
 420     }
 421
 422     // NOTADB means file is garbage, so it's functionally equivalent to corruption
 423     dbconn->isCorrupted = (SQLITE_CORRUPT == code) || (SQLITE_NOTADB == code);
 424     if (dbconn->isCorrupted) {
 425         /* Run integrity check and only make dbconn->isCorrupted true and
 426            run the corruption handler if the integrity check conclusively fails. */
 427         dbconn->maybeCorruptedCode = code;
 428         dbconn->isCorrupted = SecDbCheckCorrupted(dbconn);
 429         if (dbconn->isCorrupted) {
 430             secerror("operation returned code: %d integrity check=fail", code);
 431             SecDbHandleCorrupt(dbconn, code, error);
 432         } else {
 433             secerror("operation returned code: %d: integrity check=pass", code);
 434         }
 435     }
 436
 437     return false;
 438 }
 439
 440 #if HAVE_UNLOCK_NOTIFY
 441
 442 static void SecDbUnlockNotify(void **apArg, int nArg) {
 443     int i;
 444     for(i=0; i<nArg; i++) {
 445         dispatch_semaphore_t dsema = (dispatch_semaphore_t)apArg[i];
 446         dispatch_semaphore_signal(dsema);
 447     }
 448 }
 449
 450 static bool SecDbWaitForUnlockNotify(SecDbConnectionRef dbconn, sqlite3_stmt *stmt, CFErrorRef *error) {
 451     int rc;
 452     dispatch_semaphore_t dsema = dispatch_semaphore_create(0);
 453     rc = sqlite3_unlock_notify(dbconn->handle, SecDbUnlockNotify, dsema);
 454     assert(rc == SQLITE_LOCKED || rc == SQLITE_OK);
 455     if (rc == SQLITE_OK) {
 456         dispatch_semaphore_wait(dsema, DISPATCH_TIME_FOREVER);
 457     }
 458     dispatch_release(dsema);
 459     return (rc == SQLITE_OK
 460             ? true
 461             : (stmt
 462                ? SecDbErrorWithStmt(rc, stmt, error, CFSTR("sqlite3_unlock_notify"))
 463                : SecDbErrorWithDb(rc, dbconn->handle, error, CFSTR("sqlite3_unlock_notify"))));
 464 }
 465
 466 #endif
 467
 468 #define BUSY_TIMEOUT_MS (5 * 60 * 1000)  /* 5 minutes */
 469
 470 static bool SecDbBusyHandler(SecDbConnectionRef dbconn, CFErrorRef *error) {
 471     return SecDbErrorWithDb(sqlite3_busy_timeout(dbconn->handle, BUSY_TIMEOUT_MS), dbconn->handle, error, CFSTR("busy_handler"));
 472 }
 473
 474 static int sleepBackoff[] = { 10, 20, 50, 100, 250 };
 475 static int sumBackoff[]   = { 10, 30, 80, 180, 430 };
 476 static int NumberOfSleepBackoff = sizeof(sleepBackoff)/sizeof(sleepBackoff[0]);
 477
 478 // Return true causes the operation to be tried again.
 479 static bool SecDbWaitIfNeeded(SecDbConnectionRef dbconn, int s3e, sqlite3_stmt *stmt, CFStringRef desc, int nTries, CFErrorRef *error) {
 480 #if HAVE_UNLOCK_NOTIFY
 481     if (s3e == SQLITE_LOCKED) { // Optionally check for extended code being SQLITE_LOCKED_SHAREDCACHE
 482         return SecDbWaitForUnlockNotify(dbconn, stmt, error))
 483     }
 484 #endif
 485     if (((0xFF & s3e) == SQLITE_BUSY) || ((0xFF & s3e) == SQLITE_LOCKED)) {
 486         int totaltimeout, timeout;
 487
 488         _Static_assert(sizeof(sumBackoff) == sizeof(sleepBackoff), "matching arrays not matching");
 489         _Static_assert(sizeof(sumBackoff[0]) == sizeof(sleepBackoff[0]), "matching arrays not matching");
 490
 491         if (nTries < NumberOfSleepBackoff) {
 492             timeout = sleepBackoff[nTries];
 493             totaltimeout = sumBackoff[nTries];
 494         } else {
 495             timeout = sleepBackoff[NumberOfSleepBackoff - 1];
 496             totaltimeout = sumBackoff[NumberOfSleepBackoff - 1] + (timeout * (nTries - NumberOfSleepBackoff));
 497         }
 498         if (totaltimeout < BUSY_TIMEOUT_MS) {
 499             secinfo("#SecDB", "sqlite busy/locked: %d ntries: %d totaltimeout: %d", s3e, nTries, totaltimeout);
 500             sqlite3_sleep(timeout);
 501             return true;
 502         } else {
 503             secinfo("#SecDB", "sqlite busy/locked: too long: %d ms, giving up", totaltimeout);
 504         }
 505     }
 506
 507     return SecDbConnectionCheckCode(dbconn, s3e, error, desc);
 508 }
 509
 510 enum SecDbStepResult {
 511     kSecDbErrorStep = 0,
 512     kSecDbRowStep = 1,
 513     kSecDbDoneStep = 2,
 514 };
 515 typedef enum SecDbStepResult SecDbStepResult;
 516
 517 static SecDbStepResult _SecDbStep(SecDbConnectionRef dbconn, sqlite3_stmt *stmt, CFErrorRef *error) {
 518     assert(stmt != NULL);
 519     int s3e;
 520     int ntries = 0;
 521     for (;;) {
 522         s3e = sqlite3_step(stmt);
 523         if (s3e == SQLITE_ROW) {
 524             return kSecDbRowStep;
 525         } else if (s3e == SQLITE_DONE) {
 526             /*
 527              ** ^[SQLITE_DONE] means that the statement has finished executing
 528              ** successfully.  sqlite3_step() should not be called again on this virtual
 529              ** machine without first calling [] to reset the virtual
 530              ** machine back to its initial state.
 531              */
 532             sqlite3_reset(stmt);
 533             return kSecDbDoneStep;
 534         } else if (!SecDbWaitIfNeeded(dbconn, s3e, stmt, CFSTR("step"), ntries, error)) {
 535             return kSecDbErrorStep;
 536         }
 537         ntries++;
 538     };
 539 }
 540
 541 bool
 542 SecDbExec(SecDbConnectionRef dbconn, CFStringRef sql, CFErrorRef *error)
 543 {
 544     bool ok = true;
 545     CFRetain(sql);
 546     while (sql) {
 547         CFStringRef tail = NULL;
 548         if (ok) {
 549             sqlite3_stmt *stmt = SecDbCopyStmt(dbconn, sql, &tail, error);
 550             ok = stmt != NULL;
 551             if (stmt) {
 552                 SecDbStepResult sr;
 553                 while ((sr = _SecDbStep(dbconn, stmt, error)) == kSecDbRowStep);
 554                 if (sr == kSecDbErrorStep)
 555                     ok = false;
 556                 ok &= SecDbReleaseCachedStmt(dbconn, sql, stmt, error);
 557             }
 558         } else {
 559             // TODO We already have an error here we really just want the left over sql in it's userData
 560             ok = SecDbError(SQLITE_ERROR, error, CFSTR("Error with unexecuted sql remaining %@"), sql);
 561         }
 562         CFRelease(sql);
 563         sql = tail;
 564     }
 565     return ok;
 566 }
 567
 568 static bool SecDbBeginTransaction(SecDbConnectionRef dbconn, SecDbTransactionType type, CFErrorRef *error)
 569 {
 570     bool ok = true;
 571     CFStringRef query;
 572     switch (type) {
 573         case kSecDbImmediateTransactionType:
 574             secnoticeq("db", "SecDbBeginTransaction SecDbBeginTransaction %p", dbconn);
 575             query = CFSTR("BEGIN IMMEDATE");
 576             break;
 577         case kSecDbExclusiveRemoteTransactionType:
 578             secnoticeq("db", "SecDbBeginTransaction kSecDbExclusiveRemoteTransactionType %p", dbconn);
 579             dbconn->source = kSecDbSOSTransaction;
 580             // FALL THROUGH
 581         case kSecDbExclusiveTransactionType:
 582             if (type==kSecDbExclusiveTransactionType)
 583                 secnoticeq("db", "SecDbBeginTransaction kSecDbExclusiveTransactionType %p", dbconn);
 584             query = CFSTR("BEGIN EXCLUSIVE");
 585             break;
 586         case kSecDbNormalTransactionType:
 587             secnoticeq("db", "SecDbBeginTransaction kSecDbNormalTransactionType %p", dbconn);
 588             query = CFSTR("BEGIN");
 589             break;
 590         default:
 591             secnoticeq("db", "SecDbBeginTransaction invalid transaction type %lu", type);
 592             ok = SecDbError(SQLITE_ERROR, error, CFSTR("invalid transaction type %" PRIu32), type);
 593             query = NULL;
 594             break;
 595     }
 596
 597     if (query != NULL && sqlite3_get_autocommit(dbconn->handle) != 0) {
 598         ok = SecDbExec(dbconn, query, error);
 599     }
 600     if (ok)
 601         dbconn->inTransaction = true;
 602
 603     return ok;
 604 }
 605
 606 static bool SecDbEndTransaction(SecDbConnectionRef dbconn, bool commit, CFErrorRef *error)
 607 {
 608     __block bool ok = true;
 609     __block bool commited = false;
 610
 611     dispatch_block_t notifyAndExec = ^{
 612         if (commit) {
 613             secnoticeq("db", "SecDbEndTransaction kSecDbTransactionWillCommit %p", dbconn);
 614             SecDbNotifyPhase(dbconn, kSecDbTransactionWillCommit);
 615             commited = ok = SecDbExec(dbconn, CFSTR("END"), error);
 616             secnoticeq("db", "SecDbEndTransaction kSecDbTransactionWillCommit %p (after notify)", dbconn);
 617         } else {
 618             ok = SecDbExec(dbconn, CFSTR("ROLLBACK"), error);
 619             commited = false;
 620         }
 621         dbconn->inTransaction = false;
 622         SecDbNotifyPhase(dbconn, commited ? kSecDbTransactionDidCommit : kSecDbTransactionDidRollback);
 623         secnoticeq("db", "SecDbEndTransaction %s %p", commited ? "kSecDbTransactionDidCommit" : "kSecDbTransactionDidRollback", dbconn);
 624         dbconn->source = kSecDbAPITransaction;
 625     };
 626
 627     SecDbPerformOnCommitQueue(dbconn, true, notifyAndExec);
 628
 629     return ok;
 630 }
 631
 632 bool SecDbTransaction(SecDbConnectionRef dbconn, SecDbTransactionType type,
 633                       CFErrorRef *error, void (^transaction)(bool *commit))
 634 {
 635     bool ok = true;
 636     bool commit = true;
 637
 638     if (dbconn->inTransaction) {
 639         transaction(&commit);
 640         if (!commit) {
 641             secinfo("#SecDB", "#SecDB nested transaction asked to not be committed");
 642         }
 643     } else {
 644         ok = SecDbBeginTransaction(dbconn, type, error);
 645         if (ok) {
 646             transaction(&commit);
 647             ok = SecDbEndTransaction(dbconn, commit, error);
 648         }
 649     }
 650
 651     return ok && commit;
 652 }
 653
 654 sqlite3 *SecDbHandle(SecDbConnectionRef dbconn) {
 655     return dbconn->handle;
 656 }
 657
 658 bool SecDbStep(SecDbConnectionRef dbconn, sqlite3_stmt *stmt, CFErrorRef *error, void (^row)(bool *stop)) {
 659     for (;;) {
 660         switch (_SecDbStep(dbconn, stmt, error)) {
 661             case kSecDbErrorStep:
 662                 secdebug("db", "kSecDbErrorStep %@", error?*error:NULL);
 663                 return false;
 664             case kSecDbRowStep:
 665                 secdebug("db", "kSecDbRowStep %@", error?*error:NULL);
 666                 if (row) {
 667                     bool stop = false;
 668                     row(&stop);
 669                     if (stop)
 670                         return true;
 671                     break;
 672                 }
 673                 SecDbError(SQLITE_ERROR, error, CFSTR("SecDbStep SQLITE_ROW returned without a row handler"));
 674                 return false;
 675             case kSecDbDoneStep:
 676                 secdebug("db", "kSecDbDoneStep %@", error?*error:NULL);
 677                 return true;
 678         }
 679     }
 680 }
 681
 682 bool SecDbCheckpoint(SecDbConnectionRef dbconn, CFErrorRef *error)
 683 {
 684     return SecDbConnectionCheckCode(dbconn, sqlite3_wal_checkpoint(dbconn->handle, NULL), error, CFSTR("wal_checkpoint"));
 685 }
 686
 687 static bool SecDbFileControl(SecDbConnectionRef dbconn, int op, void *arg, CFErrorRef *error) {
 688     return SecDbConnectionCheckCode(dbconn, sqlite3_file_control(dbconn->handle, NULL, op, arg), error, CFSTR("file_control"));
 689 }
 690
 691 static sqlite3 *_SecDbOpenV2(const char *path, int flags, CFErrorRef *error) {
 692 #if HAVE_UNLOCK_NOTIFY
 693     flags |= SQLITE_OPEN_SHAREDCACHE;
 694 #endif
 695     sqlite3 *handle = NULL;
 696     int s3e = sqlite3_open_v2(path, &handle, flags, NULL);
 697     if (s3e) {
 698         if (handle) {
 699             SecDbErrorWithDb(s3e, handle, error, CFSTR("open_v2 \"%s\" 0x%X"), path, flags);
 700             sqlite3_close(handle);
 701             handle = NULL;
 702         } else {
 703             SecDbError(s3e, error, CFSTR("open_v2 \"%s\" 0x%X"), path, flags);
 704         }
 705     }
 706     return handle;
 707 }
 708
 709 static bool SecDbOpenV2(SecDbConnectionRef dbconn, const char *path, int flags, CFErrorRef *error) {
 710     return (dbconn->handle = _SecDbOpenV2(path, flags, error)) != NULL;
 711 }
 712
 713 static bool SecDbTruncate(SecDbConnectionRef dbconn, CFErrorRef *error)
 714 {
 715     int flags = SQLITE_TRUNCATE_AUTOVACUUM_FULL;
 716     if (dbconn->db->useWAL) {
 717         flags |= SQLITE_TRUNCATE_JOURNALMODE_WAL;
 718     }
 719     __block bool ok = SecDbFileControl(dbconn, SQLITE_TRUNCATE_DATABASE, &flags, error);
 720     if (!ok) {
 721         sqlite3_close(dbconn->handle);
 722         dbconn->handle = NULL;
 723         CFStringPerformWithCString(dbconn->db->db_path, ^(const char *path) {
 724             if (error)
 725                 CFReleaseNull(*error);
 726             if (SecCheckErrno(unlink(path), error, CFSTR("unlink %s"), path)) {
 727                 ok = SecDbOpenHandle(dbconn, NULL, error);
 728             }
 729         });
 730         if (!ok) {
 731             secinfo("#SecDB", "#SecDB Failed to delete db handle: %{public}@", error ? *error : NULL);
 732             abort();
 733         }
 734     }
 735
 736     return ok;
 737 }
 738
 739 static bool SecDbHandleCorrupt(SecDbConnectionRef dbconn, int rc, CFErrorRef *error)
 740 {
 741     if (!dbconn->db->allowRepair) {
 742         SecCFCreateErrorWithFormat(rc, kSecErrnoDomain, NULL, error, NULL,
 743                                    CFSTR("SecDbHandleCorrupt repair not allowed: [%d] %s"), rc, strerror(rc));
 744         return false;
 745     }
 746
 747     // Backup current db.
 748     __block bool didRename = false;
 749     CFStringPerformWithCString(dbconn->db->db_path, ^(const char *db_path) {
 750         sqlite3 *corrupt_db = NULL;
 751         char buf[PATH_MAX+1];
 752         snprintf(buf, sizeof(buf), "%s-corrupt", db_path);
 753         if (dbconn->handle && (corrupt_db = _SecDbOpenV2(buf, SQLITE_OPEN_READWRITE, error))) {
 754             int on = 1;
 755             didRename =  SecDbErrorWithDb(sqlite3_file_control(corrupt_db, NULL, SQLITE_FCNTL_PERSIST_WAL, &on), corrupt_db, error, CFSTR("persist wal"));
 756             didRename &=  SecDbErrorWithDb(sqlite3_file_control(corrupt_db, NULL, SQLITE_REPLACE_DATABASE, (void *)dbconn->handle), corrupt_db, error, CFSTR("replace database"));
 757             sqlite3_close(corrupt_db);
 758         }
 759         if (!didRename) {
 760             if (dbconn->handle)
 761                 secerror("Tried to rename corrupt database at path %@, but we failed: %@, trying explicit rename", dbconn->db->db_path, error ? *error : NULL);
 762             if (error)
 763                 CFReleaseNull(*error);
 764
 765             didRename = (!dbconn->handle || SecDbError(sqlite3_close(dbconn->handle), error, CFSTR("close"))) &&
 766                 SecCheckErrno(rename(db_path, buf), error, CFSTR("rename %s %s"), db_path, buf) &&
 767                 SecDbOpenHandle(dbconn, NULL, error);
 768         }
 769         if (didRename) {
 770             secerror("Database at path %@ is corrupt. Copied it to %s for further investigation.", dbconn->db->db_path, buf);
 771         } else {
 772             seccritical("Tried to copy corrupt database at path %@, but we failed: %@", dbconn->db->db_path, error ? *error : NULL);
 773         }
 774     });
 775
 776     bool ok = (didRename &&
 777                (dbconn->handle || SecDbOpenHandle(dbconn, NULL, error)) &&
 778                SecDbTruncate(dbconn, error));
 779
 780     // Mark the db as not corrupted, even if something failed.
 781     // Always note we are no longer in the corruption handler
 782     dbconn->isCorrupted = false;
 783
 784     // Invoke our callers opened callback, since we just created a new database
 785     if (ok && dbconn->db->opened) {
 786         dbconn->db->callOpenedHandlerForNextConnection = false;
 787         ok = dbconn->db->opened(dbconn, true, &dbconn->db->callOpenedHandlerForNextConnection, error);
 788     }
 789
 790     return ok;
 791 }
 792
 793 static bool SecDbLoggingEnabled(CFStringRef type)
 794 {
 795     CFTypeRef profile = NULL;
 796     bool enabled = false;
 797
 798     if (csr_check(CSR_ALLOW_APPLE_INTERNAL) != 0)
 799         return false;
 800
 801     profile = (CFNumberRef)CFPreferencesCopyValue(CFSTR("SQLProfile"), CFSTR("com.apple.security"), kCFPreferencesAnyUser, kCFPreferencesAnyHost);
 802
 803     if (profile == NULL)
 804         return false;
 805
 806     if (CFGetTypeID(profile) == CFBooleanGetTypeID()) {
 807         enabled = CFBooleanGetValue((CFBooleanRef)profile);
 808     } else if (CFGetTypeID(profile) == CFNumberGetTypeID()) {
 809         int32_t num = 0;
 810         CFNumberGetValue(profile, kCFNumberSInt32Type, &num);
 811         enabled = !!num;
 812     }
 813
 814     CFReleaseSafe(profile);
 815
 816     return enabled;
 817 }
 818
 819 static unsigned
 820 SecDbProfileMask(void)
 821 {
 822     static dispatch_once_t onceToken;
 823     static unsigned profile_mask = 0;
 824
 825     // sudo defaults write /Library/Preferences/com.apple.security SQLProfile -bool true
 826     dispatch_once(&onceToken, ^{
 827         if (SecDbLoggingEnabled(CFSTR("SQLProfile")))
 828             profile_mask = SQLITE_TRACE_PROFILE;
 829 #if DEBUG
 830         profile_mask |= SQLITE_TRACE_STMT;
 831 #else
 832         if (SecDbLoggingEnabled(CFSTR("SQLTrace")))
 833             profile_mask = SQLITE_TRACE_STMT;
 834 #endif
 835         if (SecDbLoggingEnabled(CFSTR("SQLRow")))
 836             profile_mask = SQLITE_TRACE_ROW;
 837         secinfo("#SecDB", "sqlDb: sql trace mask: 0x%08x", profile_mask);
 838     });
 839     return profile_mask;
 840 }
 841
 842 static int
 843 SecDbTraceV2(unsigned mask, void *ctx, void *p, void *x) {
 844     SecDbConnectionRef dbconn __unused = ctx;
 845     const char *trace = "unknown";
 846
 847     if (mask == SQLITE_TRACE_PROFILE)
 848         trace = "profile";
 849     else if (mask == SQLITE_TRACE_STMT)
 850         trace = "stmt";
 851     else if (mask == SQLITE_TRACE_ROW)
 852         trace = "row";
 853
 854     secinfo("#SecDB", "#SecDB %{public}s", trace);
 855     return 0;
 856 }
 857
 858
 859 static bool SecDbOpenHandle(SecDbConnectionRef dbconn, bool *created, CFErrorRef *error)
 860 {
 861     __block bool ok = true;
 862
 863     CFStringPerformWithCString(dbconn->db->db_path, ^(const char *db_path) {
 864         int flags = (dbconn->db->readWrite) ? SQLITE_OPEN_READWRITE : SQLITE_OPEN_READONLY;
 865         ok = created && SecDbOpenV2(dbconn, db_path, flags, NULL);
 866         if (!ok) {
 867             ok = true;
 868             if (created) {
 869                 char *tmp = dirname((char *)db_path);
 870                 if (tmp) {
 871                     mode_t omode = dbconn->db->mode;
 872                     if (omode & S_IRUSR) { omode |= S_IXUSR; } // owner can read
 873                     if (omode & S_IRGRP) { omode |= S_IXGRP; } // group can read
 874                     if (omode & S_IROTH) { omode |= S_IXOTH; } // other can read
 875                     int errnum = mkpath_np(tmp, omode);
 876                     if (errnum != 0 && errnum != EEXIST) {
 877                         SecCFCreateErrorWithFormat(errnum, kSecErrnoDomain, NULL, error, NULL,
 878                                                    CFSTR("mkpath_np %s: [%d] %s"), tmp, errnum, strerror(errnum));
 879                         ok = false;
 880                     }
 881                 }
 882             }
 883             // if the enclosing directory is ok, try to create the database.
 884             // this forces us to open it read-write, so we'll need to be the owner here.
 885             ok = ok && SecDbOpenV2(dbconn, db_path, SQLITE_OPEN_READWRITE | SQLITE_OPEN_CREATE, error);
 886             if (ok) {
 887                 chmod(db_path, dbconn->db->mode); // default: 0600 (S_IRUSR | S_IWUSR)
 888                 if (created)
 889                     *created = true;
 890             }
 891         }
 892
 893         if (ok) {
 894             unsigned mask = SecDbProfileMask();
 895             if (mask) {
 896                 (void)sqlite3_trace_v2(dbconn->handle,
 897                                        mask,
 898                                        SecDbTraceV2,
 899                                        dbconn);
 900             }
 901         }
 902
 903         ok = ok && SecDbBusyHandler(dbconn, error);
 904     });
 905
 906     return ok;
 907 }
 908
 909 static SecDbConnectionRef
 910 SecDbConnectionCreate(SecDbRef db, bool readOnly, CFErrorRef *error)
 911 {
 912     SecDbConnectionRef dbconn = NULL;
 913
 914     dbconn = CFTypeAllocate(SecDbConnection, struct __OpaqueSecDbConnection, kCFAllocatorDefault);
 915     require(dbconn != NULL, done);
 916
 917     dbconn->db = db;
 918     dbconn->readOnly = readOnly;
 919     dbconn->inTransaction = false;
 920     dbconn->source = kSecDbInvalidTransaction;
 921     dbconn->isCorrupted = false;
 922     dbconn->maybeCorruptedCode = 0;
 923     dbconn->hasIOFailure = false;
 924     dbconn->corruptionError = NULL;
 925     dbconn->handle = NULL;
 926     dbconn->changes = CFArrayCreateMutableForCFTypes(kCFAllocatorDefault);
 927
 928 done:
 929     return dbconn;
 930 }
 931
 932 static bool SecDbConnectionIsReadOnly(SecDbConnectionRef dbconn) {
 933     return dbconn->readOnly;
 934 }
 935
 936 static void SecDbConectionSetReadOnly(SecDbConnectionRef dbconn, bool readOnly) {
 937     dbconn->readOnly = readOnly;
 938 }
 939
 940 /* Read only connections go to the end of the queue, writeable connections
 941  go to the start of the queue. */
 942 SecDbConnectionRef SecDbConnectionAcquire(SecDbRef db, bool readOnly, CFErrorRef *error) {
 943     CFRetain(db);
 944     secinfo("dbconn", "acquire %s connection", readOnly ? "ro" : "rw");
 945     dispatch_semaphore_wait(readOnly ? db->read_semaphore : db->write_semaphore, DISPATCH_TIME_FOREVER);
 946     __block SecDbConnectionRef dbconn = NULL;
 947     __block bool ok = true;
 948     __block bool ranOpenedHandler = false;
 949     dispatch_sync(db->queue, ^{
 950         if (!db->didFirstOpen) {
 951             bool didCreate = false;
 952             ok = dbconn = SecDbConnectionCreate(db, false, error);
 953             CFErrorRef localError = NULL;
 954             if (ok && !SecDbOpenHandle(dbconn, &didCreate, &localError)) {
 955                 secerror("Unable to create database: %@", localError);
 956                 if (localError && CFEqual(CFErrorGetDomain(localError), kSecDbErrorDomain)) {
 957                     int code = (int)CFErrorGetCode(localError);
 958                     dbconn->isCorrupted = (SQLITE_CORRUPT == code) || (SQLITE_NOTADB == code);
 959                 }
 960                 // If the open failure isn't due to corruption, propagate the error.
 961                 ok = dbconn->isCorrupted;
 962                 if (!ok && error && *error == NULL) {
 963                     *error = localError;
 964                     localError = NULL;
 965                 }
 966             }
 967             CFReleaseNull(localError);
 968
 969             if (ok) {
 970                 db->didFirstOpen = ok = SecDbDidCreateFirstConnection(dbconn, didCreate, error);
 971                 ranOpenedHandler = true;
 972             }
 973             if (!ok)
 974                 CFReleaseNull(dbconn);
 975         } else {
 976             /* Try to get one from the cache */
 977             CFIndex count = CFArrayGetCount(db->connections);
 978             while (count && !dbconn) {
 979                 CFIndex ix = readOnly ? count - 1 : 0;
 980                 dbconn = (SecDbConnectionRef)CFArrayGetValueAtIndex(db->connections, ix);
 981                 if (dbconn)
 982                     CFRetain(dbconn);
 983                 else
 984                     secerror("got NULL dbconn at index: %" PRIdCFIndex " skipping", ix);
 985                 CFArrayRemoveValueAtIndex(db->connections, ix);
 986             }
 987         }
 988     });
 989
 990     if (dbconn) {
 991         /* Make sure the connection we found has the right access */
 992         if (SecDbConnectionIsReadOnly(dbconn) != readOnly) {
 993             SecDbConectionSetReadOnly(dbconn, readOnly);
 994         }
 995     } else if (ok) {
 996         /* Nothing found in cache, create a new connection */
 997         bool created = false;
 998         dbconn = SecDbConnectionCreate(db, readOnly, error);
 999         if (dbconn && !SecDbOpenHandle(dbconn, &created, error)) {
1000             CFReleaseNull(dbconn);
1001         }
1002     }
1003
1004     if (dbconn && !ranOpenedHandler && dbconn->db->opened) {
1005         dispatch_sync(db->queue, ^{
1006             if (dbconn->db->callOpenedHandlerForNextConnection) {
1007                 dbconn->db->callOpenedHandlerForNextConnection = false;
1008                 if (!dbconn->db->opened(dbconn, false, &dbconn->db->callOpenedHandlerForNextConnection, error)) {
1009                     if (!dbconn->isCorrupted || !SecDbHandleCorrupt(dbconn, 0, error)) {
1010                         CFReleaseNull(dbconn);
1011                     }
1012                 }
1013             }
1014         });
1015     }
1016
1017     if (!dbconn) {
1018         // If acquire fails we need to signal the semaphore again.
1019         dispatch_semaphore_signal(readOnly ? db->read_semaphore : db->write_semaphore);
1020         CFRelease(db);
1021     }
1022
1023     return dbconn;
1024 }
1025
1026 void SecDbConnectionRelease(SecDbConnectionRef dbconn) {
1027     if (!dbconn) {
1028         secerror("called with NULL dbconn");
1029         return;
1030     }
1031     SecDbRef db = dbconn->db;
1032     secinfo("dbconn", "release %@", dbconn);
1033     dispatch_sync(db->queue, ^{
1034         bool readOnly = SecDbConnectionIsReadOnly(dbconn);
1035         if (dbconn->hasIOFailure) {
1036             // Something wrong on the file layer (e.g. revoked file descriptor for networked home)
1037             // so we don't trust our existing connections anymore.
1038             CFArrayRemoveAllValues(db->connections);
1039         } else {
1040             CFIndex count = CFArrayGetCount(db->connections);
1041             // Add back possible writable dbconn to the pool.
1042             CFArrayInsertValueAtIndex(db->connections, readOnly ? count : 0, dbconn);
1043             // Remove the last (probably read-only) dbconn from the pool.
1044             if (count >= kSecDbMaxIdleHandles) {
1045                 CFArrayRemoveValueAtIndex(db->connections, count);
1046             }
1047         }
1048         // Signal after we have put the connection back in the pool of connections
1049         dispatch_semaphore_signal(readOnly ? db->read_semaphore : db->write_semaphore);
1050         CFRelease(dbconn);
1051         CFRelease(db);
1052     });
1053 }
1054
1055 bool SecDbPerformRead(SecDbRef db, CFErrorRef *error, void (^perform)(SecDbConnectionRef dbconn)) {
1056     SecDbConnectionRef dbconn = SecDbConnectionAcquire(db, true, error);
1057     bool success = false;
1058     if (dbconn) {
1059         perform(dbconn);
1060         success = true;
1061         SecDbConnectionRelease(dbconn);
1062     }
1063     return success;
1064 }
1065
1066 bool SecDbPerformWrite(SecDbRef db, CFErrorRef *error, void (^perform)(SecDbConnectionRef dbconn)) {
1067     if(!db) {
1068         SecError(errSecNotAvailable, error, CFSTR("failed to get a db handle"));
1069         return false;
1070     }
1071     SecDbConnectionRef dbconn = SecDbConnectionAcquire(db, false, error);
1072     bool success = false;
1073     if (dbconn) {
1074         perform(dbconn);
1075         success = true;
1076         SecDbConnectionRelease(dbconn);
1077     }
1078     return success;
1079 }
1080
1081 static CFStringRef
1082 SecDbConnectionCopyFormatDescription(CFTypeRef value, CFDictionaryRef formatOptions)
1083 {
1084     SecDbConnectionRef dbconn = (SecDbConnectionRef)value;
1085     return CFStringCreateWithFormat(kCFAllocatorDefault, NULL, CFSTR("<SecDbConnection %s %s>"),
1086                                     dbconn->readOnly ? "ro" : "rw", dbconn->handle ? "open" : "closed");
1087 }
1088
1089 static void
1090 SecDbConnectionDestroy(CFTypeRef value)
1091 {
1092     SecDbConnectionRef dbconn = (SecDbConnectionRef)value;
1093     if (dbconn->handle) {
1094         sqlite3_close(dbconn->handle);
1095     }
1096     dbconn->db = NULL;
1097     CFReleaseNull(dbconn->changes);
1098     CFReleaseNull(dbconn->corruptionError);
1099
1100 }
1101
1102 void SecDbPerformOnCommitQueue(SecDbConnectionRef dbconn, bool barrier, dispatch_block_t perform) {
1103     if (barrier) {
1104         dispatch_barrier_sync(dbconn->db->commitQueue, ^{
1105             perform();
1106         });
1107     } else {
1108         dispatch_sync(dbconn->db->commitQueue, ^{
1109             perform();
1110         });
1111     }
1112 }
1113
1114 // MARK: -
1115 // MARK: Bind helpers
1116
1117 #if 0
1118 bool SecDbBindNull(sqlite3_stmt *stmt, int param, CFErrorRef *error) {
1119     bool ok = SecDbErrorWithStmt(sqlite3_bind_null(stmt, param),
1120                                  stmt, error, CFSTR("bind_null[%d]"), param);
1121     secinfo("bind", "bind_null[%d]: %@", param, error ? *error : NULL);
1122     return ok;
1123 }
1124 #endif
1125
1126 bool SecDbBindBlob(sqlite3_stmt *stmt, int param, const void *zData, size_t n, void(*xDel)(void*), CFErrorRef *error) {
1127     if (n > INT_MAX) {
1128         return SecDbErrorWithStmt(SQLITE_TOOBIG, stmt, error,
1129                                   CFSTR("bind_blob[%d]: blob bigger than INT_MAX"), param);
1130     }
1131     bool ok = SecDbErrorWithStmt(sqlite3_bind_blob(stmt, param, zData, (int)n, xDel),
1132                                  stmt, error, CFSTR("bind_blob[%d]"), param);
1133     secinfo("bind", "bind_blob[%d]: %.*s: %@", param, (int)n, zData, error ? *error : NULL);
1134     return ok;
1135 }
1136
1137 bool SecDbBindText(sqlite3_stmt *stmt, int param, const char *zData, size_t n, void(*xDel)(void*), CFErrorRef *error) {
1138     if (n > INT_MAX) {
1139         return SecDbErrorWithStmt(SQLITE_TOOBIG, stmt, error,
1140                                   CFSTR("bind_text[%d]: text bigger than INT_MAX"), param);
1141     }
1142     bool ok = SecDbErrorWithStmt(sqlite3_bind_text(stmt, param, zData, (int)n, xDel), stmt, error,
1143                                  CFSTR("bind_text[%d]"), param);
1144     secinfo("bind", "bind_text[%d]: \"%s\": %@", param, zData, error ? *error : NULL);
1145     return ok;
1146 }
1147
1148 bool SecDbBindDouble(sqlite3_stmt *stmt, int param, double value, CFErrorRef *error) {
1149     bool ok = SecDbErrorWithStmt(sqlite3_bind_double(stmt, param, value), stmt, error,
1150                                  CFSTR("bind_double[%d]"), param);
1151     secinfo("bind", "bind_double[%d]: %f: %@", param, value, error ? *error : NULL);
1152     return ok;
1153 }
1154
1155 bool SecDbBindInt(sqlite3_stmt *stmt, int param, int value, CFErrorRef *error) {
1156     bool ok = SecDbErrorWithStmt(sqlite3_bind_int(stmt, param, value), stmt, error,
1157                                  CFSTR("bind_int[%d]"), param);
1158     secinfo("bind", "bind_int[%d]: %d: %@", param, value, error ? *error : NULL);
1159     return ok;
1160 }
1161
1162 bool SecDbBindInt64(sqlite3_stmt *stmt, int param, sqlite3_int64 value, CFErrorRef *error) {
1163     bool ok = SecDbErrorWithStmt(sqlite3_bind_int64(stmt, param, value), stmt, error,
1164                                  CFSTR("bind_int64[%d]"), param);
1165     secinfo("bind", "bind_int64[%d]: %lld: %@", param, value, error ? *error : NULL);
1166     return ok;
1167 }
1168
1169
1170 /* AUDIT[securityd](done):
1171  value (ok) is a caller provided, non NULL CFTypeRef.
1172  */
1173 bool SecDbBindObject(sqlite3_stmt *stmt, int param, CFTypeRef value, CFErrorRef *error) {
1174     CFTypeID valueId;
1175     __block bool result = false;
1176
1177         /* TODO: Can we use SQLITE_STATIC below everwhere we currently use
1178      SQLITE_TRANSIENT since we finalize the statement before the value
1179      goes out of scope? */
1180     if (!value || (valueId = CFGetTypeID(value)) == CFNullGetTypeID()) {
1181         /* Skip bindings for NULL values.  sqlite3 will interpret unbound
1182          params as NULL which is exactly what we want. */
1183 #if 1
1184         result = true;
1185 #else
1186         result = SecDbBindNull(stmt, param, error);
1187 #endif
1188     } else if (valueId == CFStringGetTypeID()) {
1189         CFStringPerformWithCStringAndLength(value, ^(const char *cstr, size_t clen) {
1190             result = SecDbBindText(stmt, param, cstr, clen, SQLITE_TRANSIENT, error);
1191         });
1192     } else if (valueId == CFDataGetTypeID()) {
1193         CFIndex len = CFDataGetLength(value);
1194         if (len) {
1195             result = SecDbBindBlob(stmt, param, CFDataGetBytePtr(value),
1196                                    len, SQLITE_TRANSIENT, error);
1197         } else {
1198             result = SecDbBindText(stmt, param, "", 0, SQLITE_TRANSIENT, error);
1199         }
1200     } else if (valueId == CFDateGetTypeID()) {
1201         CFAbsoluteTime abs_time = CFDateGetAbsoluteTime(value);
1202         result = SecDbBindDouble(stmt, param, abs_time, error);
1203     } else if (valueId == CFBooleanGetTypeID()) {
1204         int bval = CFBooleanGetValue(value);
1205         result = SecDbBindInt(stmt, param, bval, error);
1206     } else if (valueId == CFNumberGetTypeID()) {
1207         Boolean convertOk;
1208         if (CFNumberIsFloatType(value)) {
1209             double nval;
1210             convertOk = CFNumberGetValue(value, kCFNumberDoubleType, &nval);
1211             result = SecDbBindDouble(stmt, param, nval, error);
1212         } else {
1213             SInt32 nval;
1214             convertOk = CFNumberGetValue(value, kCFNumberSInt32Type, &nval);
1215             if (convertOk) {
1216                 result = SecDbBindInt(stmt, param, nval, error);
1217             } else {
1218                 sqlite_int64 nval64;
1219                 convertOk = CFNumberGetValue(value, kCFNumberSInt64Type, &nval64);
1220                 if (convertOk)
1221                     result = SecDbBindInt64(stmt, param, nval64, error);
1222             }
1223         }
1224         if (!convertOk) {
1225             result = SecDbError(SQLITE_INTERNAL, error, CFSTR("bind CFNumberGetValue failed for %@"), value);
1226         }
1227     } else {
1228         if (error) {
1229             CFStringRef valueDesc = CFCopyTypeIDDescription(valueId);
1230             SecDbError(SQLITE_MISMATCH, error, CFSTR("bind unsupported type %@"), valueDesc);
1231             CFReleaseSafe(valueDesc);
1232         }
1233     }
1234
1235         return result;
1236 }
1237
1238 // MARK: -
1239 // MARK: SecDbStatementRef
1240
1241 bool SecDbReset(sqlite3_stmt *stmt, CFErrorRef *error) {
1242     return SecDbErrorWithStmt(sqlite3_reset(stmt), stmt, error, CFSTR("reset"));
1243 }
1244
1245 bool SecDbClearBindings(sqlite3_stmt *stmt, CFErrorRef *error) {
1246     return SecDbErrorWithStmt(sqlite3_clear_bindings(stmt), stmt, error, CFSTR("clear bindings"));
1247 }
1248
1249 bool SecDbFinalize(sqlite3_stmt *stmt, CFErrorRef *error) {
1250     sqlite3 *handle = sqlite3_db_handle(stmt);
1251     int s3e = sqlite3_finalize(stmt);
1252     return s3e == SQLITE_OK ? true : SecDbErrorWithDb(s3e, handle, error, CFSTR("finalize: %p"), stmt);
1253 }
1254
1255 sqlite3_stmt *SecDbPrepareV2(SecDbConnectionRef dbconn, const char *sql, size_t sqlLen, const char **sqlTail, CFErrorRef *error) {
1256     sqlite3 *db = SecDbHandle(dbconn);
1257     if (sqlLen > INT_MAX) {
1258         SecDbErrorWithDb(SQLITE_TOOBIG, db, error, CFSTR("prepare_v2: sql bigger than INT_MAX"));
1259         return NULL;
1260     }
1261     int ntries = 0;
1262     for (;;) {
1263         sqlite3_stmt *stmt = NULL;
1264         int s3e = sqlite3_prepare_v2(db, sql, (int)sqlLen, &stmt, sqlTail);
1265         if (s3e == SQLITE_OK)
1266             return stmt;
1267         else if (!SecDbWaitIfNeeded(dbconn, s3e, NULL, CFSTR("preparev2"), ntries, error))
1268             return NULL;
1269         ntries++;
1270     }
1271 }
1272
1273 static sqlite3_stmt *SecDbCopyStatementWithTailRange(SecDbConnectionRef dbconn, CFStringRef sql, CFRange *sqlTail, CFErrorRef *error) {
1274     __block sqlite3_stmt *stmt = NULL;
1275     if (sql) CFStringPerformWithCStringAndLength(sql, ^(const char *sqlStr, size_t sqlLen) {
1276         const char *tail = NULL;
1277         stmt = SecDbPrepareV2(dbconn, sqlStr, sqlLen, &tail, error);
1278         if (sqlTail && sqlStr < tail && tail < sqlStr + sqlLen) {
1279             sqlTail->location = tail - sqlStr;
1280             sqlTail->length = sqlLen - sqlTail->location;
1281         }
1282     });
1283
1284     return stmt;
1285 }
1286
1287 sqlite3_stmt *SecDbCopyStmt(SecDbConnectionRef dbconn, CFStringRef sql, CFStringRef *tail, CFErrorRef *error) {
1288     // TODO: Add caching and cache lookup of statements
1289     CFRange sqlTail = {};
1290     sqlite3_stmt *stmt = SecDbCopyStatementWithTailRange(dbconn, sql, &sqlTail, error);
1291     if (sqlTail.length > 0) {
1292         CFStringRef excess = CFStringCreateWithSubstring(CFGetAllocator(sql), sql, sqlTail);
1293         if (tail) {
1294             *tail = excess;
1295         } else {
1296             SecDbError(SQLITE_INTERNAL, error,
1297                        CFSTR("prepare_v2: %@ unused sql: %@"),
1298                        sql, excess);
1299             CFReleaseSafe(excess);
1300             SecDbFinalize(stmt, error);
1301             stmt = NULL;
1302         }
1303     }
1304     return stmt;
1305 }
1306
1307 /*
1308  TODO: Could do a hack here with a custom kCFAllocatorNULL allocator for a second CFRuntimeBase inside a SecDbStatement,
1309  TODO: Better yet make a full blow SecDbStatement instance whenever SecDbCopyStmt is called.  Then, when the statement is released, in the Dispose method, we Reset and ClearBindings the sqlite3_stmt * and hand it back to the SecDb with the original CFStringRef for the sql (or hash thereof) as an argument. */
1310 bool SecDbReleaseCachedStmt(SecDbConnectionRef dbconn, CFStringRef sql, sqlite3_stmt *stmt, CFErrorRef *error) {
1311     if (stmt) {
1312         return SecDbFinalize(stmt, error);
1313     }
1314     return true;
1315 }
1316
1317 bool SecDbPrepare(SecDbConnectionRef dbconn, CFStringRef sql, CFErrorRef *error, void(^exec)(sqlite3_stmt *stmt)) {
1318     assert(sql != NULL);
1319     sqlite3_stmt *stmt = SecDbCopyStmt(dbconn, sql, NULL, error);
1320     if (!stmt)
1321         return false;
1322
1323     exec(stmt);
1324     return SecDbReleaseCachedStmt(dbconn, sql, stmt, error);
1325 }
1326
1327 bool SecDbWithSQL(SecDbConnectionRef dbconn, CFStringRef sql, CFErrorRef *error, bool(^perform)(sqlite3_stmt *stmt)) {
1328     bool ok = true;
1329     CFRetain(sql);
1330     while (sql) {
1331         CFStringRef tail = NULL;
1332         if (ok) {
1333             sqlite3_stmt *stmt = SecDbCopyStmt(dbconn, sql, &tail, error);
1334             ok = stmt != NULL;
1335             if (stmt) {
1336                 if (perform) {
1337                     ok = perform(stmt);
1338                 } else {
1339                     // TODO: Use a different error scope here.
1340                     ok = SecError(-50 /* errSecParam */, error, CFSTR("SecDbWithSQL perform block missing"));
1341                 }
1342                 ok &= SecDbReleaseCachedStmt(dbconn, sql, stmt, error);
1343             }
1344         } else {
1345             // TODO We already have an error here we really just want the left over sql in it's userData
1346             ok = SecDbError(SQLITE_ERROR, error, CFSTR("Error with unexecuted sql remaining %@"), sql);
1347         }
1348         CFRelease(sql);
1349         sql = tail;
1350     }
1351     return ok;
1352 }
1353
1354 #if 1
1355 /* SecDbForEach returns true if all SQLITE_ROW returns of sqlite3_step() return true from the row block.
1356  If the row block returns false and doesn't set an error (to indicate it has reached a limit),
1357  this entire function returns false. In that case no error will be set. */
1358 bool SecDbForEach(SecDbConnectionRef dbconn, sqlite3_stmt *stmt, CFErrorRef *error, bool(^row)(int row_index)) {
1359     bool result = false;
1360     for (int row_ix = 0;;++row_ix) {
1361         int s3e = sqlite3_step(stmt);
1362         if (s3e == SQLITE_ROW) {
1363             if (row) {
1364                 if (!row(row_ix)) {
1365                     break;
1366                 }
1367             } else {
1368                 // If we have no row block then getting SQLITE_ROW is an error
1369                 SecDbError(s3e, error,
1370                            CFSTR("step[%d]: %s returned SQLITE_ROW with NULL row block"),
1371                            row_ix, sqlite3_sql(stmt));
1372             }
1373         } else {
1374             if (s3e == SQLITE_DONE) {
1375                 result = true;
1376             } else {
1377                 dbconn->hasIOFailure |= (s3e == SQLITE_IOERR);
1378                 SecDbErrorWithStmt(s3e, stmt, error, CFSTR("step[%d]"), row_ix);
1379             }
1380             break;
1381         }
1382     }
1383     return result;
1384 }
1385 #else
1386 bool SecDbForEach(sqlite3_stmt *stmt, CFErrorRef *error, bool(^row)(int row_index)) {
1387     int row_ix = 0;
1388     for (;;) {
1389         switch (_SecDbStep(dbconn, stmt, error)) {
1390             case kSecDbErrorStep:
1391                 return false;
1392             case kSecDbRowStep:
1393                 if (row) {
1394                     if (row(row_ix++))
1395                         break;
1396                 } else {
1397                     SecDbError(SQLITE_ERROR, error, CFSTR("SecDbStep SQLITE_ROW returned without a row handler"));
1398                 }
1399                 return false;
1400             case kSecDbDoneStep:
1401                 return true;
1402         }
1403     }
1404 }
1405 #endif
1406
1407 void SecDbRecordChange(SecDbConnectionRef dbconn, CFTypeRef deleted, CFTypeRef inserted) {
1408     if (!dbconn->db->notifyPhase) return;
1409     CFTypeRef entry = SecDbEventCreateWithComponents(deleted, inserted);
1410     if (entry) {
1411         CFArrayAppendValue(dbconn->changes, entry);
1412         CFRelease(entry);
1413
1414         if (!dbconn->inTransaction) {
1415             secerror("db %@ changed outside txn", dbconn);
1416             // Only notify of DidCommit, since WillCommit code assumes
1417             // we are in a txn.
1418             SecDbOnNotify(dbconn, ^{
1419                 SecDbNotifyPhase(dbconn, kSecDbTransactionDidCommit);
1420             });
1421         }
1422     }
1423 }
1424
1425
1426 CFGiblisFor(SecDbConnection)
1427
1428 //
1429 // SecDbEvent Creation and consumption
1430 //
1431
1432 static SecDbEventRef SecDbEventCreateInsert(CFTypeRef inserted) {
1433     return CFRetainSafe(inserted);
1434 }
1435
1436 static SecDbEventRef SecDbEventCreateDelete(CFTypeRef deleted) {
1437     return CFArrayCreate(kCFAllocatorDefault, &deleted, 1, &kCFTypeArrayCallBacks);
1438 }
1439
1440 static SecDbEventRef SecDbEventCreateUpdate(CFTypeRef deleted, CFTypeRef inserted) {
1441     const void *values[2] = { deleted, inserted };
1442     return CFArrayCreate(kCFAllocatorDefault, values, 2, &kCFTypeArrayCallBacks);
1443 }
1444
1445 SecDbEventRef SecDbEventCreateWithComponents(CFTypeRef deleted, CFTypeRef inserted) {
1446     if (deleted && inserted)
1447         return SecDbEventCreateUpdate(deleted, inserted);
1448     else if (deleted)
1449         return SecDbEventCreateDelete(deleted);
1450     else if (inserted)
1451         return SecDbEventCreateInsert(inserted);
1452     else
1453         return NULL;
1454 }
1455
1456 bool SecDbEventGetComponents(SecDbEventRef event, CFTypeRef *deleted, CFTypeRef *inserted, CFErrorRef *error) {
1457     if (isArray(event)) {
1458         CFArrayRef array = event;
1459         switch (CFArrayGetCount(array)) {
1460             case 2:
1461                 *deleted = CFArrayGetValueAtIndex(array, 0);
1462                 *inserted = CFArrayGetValueAtIndex(array, 1);
1463                 break;
1464             case 1:
1465                 *deleted = CFArrayGetValueAtIndex(array, 0);
1466                 *inserted = NULL;
1467                 break;
1468             default:
1469                 SecError(errSecParam, error, NULL, CFSTR("invalid entry in changes array: %@"), array);
1470                 break;
1471         }
1472     } else {
1473         *deleted = NULL;
1474         *inserted = event;
1475     }
1476     return true;
1477 }