OSX/utilities/src/SecDb.c

   1 /*
   2  * Copyright (c) 2012-2014 Apple Inc. All Rights Reserved.
   3  *
   4  * @APPLE_LICENSE_HEADER_START@
   5  *
   6  * This file contains Original Code and/or Modifications of Original Code
   7  * as defined in and that are subject to the Apple Public Source License
   8  * Version 2.0 (the 'License'). You may not use this file except in
   9  * compliance with the License. Please obtain a copy of the License at
  10  * http://www.opensource.apple.com/apsl/ and read it before using this
  11  * file.
  12  *
  13  * The Original Code and all software distributed under the License are
  14  * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  15  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  16  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
  17  * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
  18  * Please see the License for the specific language governing rights and
  19  * limitations under the License.
  20  *
  21  * @APPLE_LICENSE_HEADER_END@
  22  */
  23
  24
  25 #include "SecDb.h"
  26 #include "debugging.h"
  27
  28 #include <sqlite3.h>
  29 #include <sqlite3_private.h>
  30 #include <CoreFoundation/CoreFoundation.h>
  31 #include <libgen.h>
  32 #include <sys/stat.h>
  33 #include <AssertMacros.h>
  34 #include "SecCFWrappers.h"
  35 #include "SecCFError.h"
  36 #include "SecIOFormat.h"
  37 #include <stdio.h>
  38 #include "Security/SecBase.h"
  39
  40
  41 //
  42 // Architecturally inverted files
  43 // These are in SecureObjectSync but utilities depends on them
  44 // <rdar://problem/20802079> Fix layer violation (SOSDigestVector, SOSManifest, SecDB.c)
  45 //
  46 #include <Security/SecureObjectSync/SOSDigestVector.h>
  47 #include <Security/SecureObjectSync/SOSManifest.h>
  48
  49 #define LOGE(ARG,...) secerror(ARG, ## __VA_ARGS__)
  50 #define LOGV(ARG,...) secdebug("secdb", ARG, ## __VA_ARGS__)
  51 #define LOGD(ARG,...) secdebug("secdb", ARG, ## __VA_ARGS__)
  52
  53 #define HAVE_UNLOCK_NOTIFY  0
  54 #define USE_BUSY_HANDLER  1
  55
  56 struct __OpaqueSecDbStatement {
  57     CFRuntimeBase _base;
  58
  59     SecDbConnectionRef dbconn;
  60     sqlite3_stmt *stmt;
  61 };
  62
  63 struct __OpaqueSecDbConnection {
  64     CFRuntimeBase _base;
  65
  66     //CFMutableDictionaryRef statements;
  67
  68     SecDbRef db;     // NONRETAINED, since db or block retains us
  69     bool readOnly;
  70     bool inTransaction;
  71     SecDbTransactionSource source;
  72     bool isCorrupted;
  73     CFErrorRef corruptionError;
  74     sqlite3 *handle;
  75     // Pending deletions and additions for the current transaction
  76     // Entires are either:
  77     // 1) a CFArrayRef of 1 element representing a deletion,
  78     // 2) a CFArrayRef of 2 elements representing the element 0 having been replaced with element 1
  79     // 3) a CFTypeRef that is not a CFArrayRef, representing an add of the element in question.
  80     CFMutableArrayRef changes;
  81 };
  82
  83 struct __OpaqueSecDb {
  84     CFRuntimeBase _base;
  85
  86     CFStringRef db_path;
  87     dispatch_queue_t queue;
  88     CFMutableArrayRef connections;
  89     dispatch_semaphore_t write_semaphore;
  90     dispatch_semaphore_t read_semaphore;
  91     bool didFirstOpen;
  92     bool (^opened)(SecDbConnectionRef dbconn, bool didCreate, bool *callMeAgainForNextConnection, CFErrorRef *error);
  93     bool callOpenedHandlerForNextConnection;
  94     dispatch_queue_t notifyQueue;
  95     SecDBNotifyBlock notifyPhase;
  96 };
  97
  98 // MARK: Error domains and error helper functions
  99
 100 CFStringRef kSecDbErrorDomain = CFSTR("com.apple.utilities.sqlite3");
 101
 102 bool SecDbError(int sql_code, CFErrorRef *error, CFStringRef format, ...) {
 103     if (sql_code == SQLITE_OK) return true;
 104     if (error) {
 105         va_list args;
 106         CFIndex code = sql_code;
 107         CFErrorRef previousError = *error;
 108
 109         *error = NULL;
 110         va_start(args, format);
 111         SecCFCreateErrorWithFormatAndArguments(code, kSecDbErrorDomain, previousError, error, NULL, format, args);
 112         CFReleaseNull(previousError);
 113         va_end(args);
 114     }
 115     return false;
 116 }
 117
 118 bool SecDbErrorWithDb(int sql_code, sqlite3 *db, CFErrorRef *error, CFStringRef format, ...) {
 119     if (sql_code == SQLITE_OK) return true;
 120     if (error) {
 121         va_list args;
 122         va_start(args, format);
 123         CFStringRef message = CFStringCreateWithFormatAndArguments(kCFAllocatorDefault, NULL, format, args);
 124         va_end(args);
 125
 126         int extended_code = sqlite3_extended_errcode(db);
 127         if (sql_code == extended_code)
 128             SecDbError(sql_code, error, CFSTR("%@: [%d] %s"), message, sql_code, sqlite3_errmsg(db));
 129         else
 130             SecDbError(sql_code, error, CFSTR("%@: [%d->%d] %s"), message, sql_code, extended_code, sqlite3_errmsg(db));
 131         CFReleaseSafe(message);
 132     }
 133     return false;
 134 }
 135
 136 bool SecDbErrorWithStmt(int sql_code, sqlite3_stmt *stmt, CFErrorRef *error, CFStringRef format, ...) {
 137     if (sql_code == SQLITE_OK) return true;
 138     if (error) {
 139         va_list args;
 140         va_start(args, format);
 141         CFStringRef message = CFStringCreateWithFormatAndArguments(kCFAllocatorDefault, NULL, format, args);
 142         va_end(args);
 143
 144         sqlite3 *db = sqlite3_db_handle(stmt);
 145         const char *sql = sqlite3_sql(stmt);
 146         int extended_code = sqlite3_extended_errcode(db);
 147         if (sql_code == extended_code)
 148             SecDbError(sql_code, error, CFSTR("%@: [%d] %s sql: %s"), message, sql_code, sqlite3_errmsg(db), sql);
 149         else
 150             SecDbError(sql_code, error, CFSTR("%@: [%d->%d] %s sql: %s"), message, sql_code, extended_code, sqlite3_errmsg(db), sql);
 151         CFReleaseSafe(message);
 152     }
 153     return false;
 154 }
 155
 156
 157 // MARK: -
 158 // MARK: Static helper functions
 159
 160 static bool SecDbOpenHandle(SecDbConnectionRef dbconn, bool *created, CFErrorRef *error);
 161 static bool SecDbHandleCorrupt(SecDbConnectionRef dbconn, int rc, CFErrorRef *error);
 162
 163 #pragma mark -
 164 #pragma mark SecDbRef
 165
 166 static CFStringRef
 167 SecDbCopyFormatDescription(CFTypeRef value, CFDictionaryRef formatOptions)
 168 {
 169     SecDbRef db = (SecDbRef)value;
 170     return CFStringCreateWithFormat(kCFAllocatorDefault, NULL, CFSTR("<SecDb path:%@ connections: %@>"), db->db_path, db->connections);
 171 }
 172
 173
 174 static void
 175 SecDbDestroy(CFTypeRef value)
 176 {
 177     SecDbRef db = (SecDbRef)value;
 178     CFReleaseSafe(db->connections);
 179     CFReleaseSafe(db->db_path);
 180     dispatch_release(db->queue);
 181     dispatch_release(db->read_semaphore);
 182     dispatch_release(db->write_semaphore);
 183     if (db->opened)
 184         Block_release(db->opened);
 185 }
 186
 187 CFGiblisFor(SecDb)
 188
 189 SecDbRef
 190 SecDbCreate(CFStringRef dbName,
 191             bool (^opened)(SecDbConnectionRef dbconn, bool didCreate, bool *callMeAgainForNextConnection, CFErrorRef *error))
 192 {
 193     SecDbRef db = NULL;
 194
 195     db = CFTypeAllocate(SecDb, struct __OpaqueSecDb, kCFAllocatorDefault);
 196     require(db != NULL, done);
 197
 198     CFStringPerformWithCString(dbName, ^(const char *dbNameStr) {
 199         db->queue = dispatch_queue_create(dbNameStr, DISPATCH_QUEUE_SERIAL);
 200     });
 201     db->read_semaphore = dispatch_semaphore_create(kSecDbMaxReaders);
 202     db->write_semaphore = dispatch_semaphore_create(kSecDbMaxWriters);
 203     db->connections = CFArrayCreateMutableForCFTypes(kCFAllocatorDefault);
 204     db->opened = opened ? Block_copy(opened) : NULL;
 205     if (getenv("__OSINSTALL_ENVIRONMENT") != NULL) {
 206         // TODO: Move this code out of this layer
 207         LOGV("sqlDb: running from installer");
 208         db->db_path = CFSTR("file::memory:?cache=shared");
 209     } else {
 210         db->db_path = CFStringCreateCopy(kCFAllocatorDefault, dbName);
 211     }
 212 done:
 213     return db;
 214 }
 215
 216 CFIndex
 217 SecDbIdleConnectionCount(SecDbRef db) {
 218     __block CFIndex count = 0;
 219     dispatch_sync(db->queue, ^{
 220         count = CFArrayGetCount(db->connections);
 221     });
 222     return count;
 223 }
 224
 225 void SecDbSetNotifyPhaseBlock(SecDbRef db, dispatch_queue_t queue, SecDBNotifyBlock notifyPhase) {
 226     if (db->notifyQueue)
 227         dispatch_release(db->notifyQueue);
 228     if (db->notifyPhase)
 229         Block_release(db->notifyPhase);
 230
 231     if (queue) {
 232         db->notifyQueue = queue;
 233         dispatch_retain(db->notifyQueue);
 234     } else {
 235         db->notifyQueue = NULL;
 236     }
 237     if (notifyPhase)
 238         db->notifyPhase = Block_copy(notifyPhase);
 239     else
 240         db->notifyPhase = NULL;
 241 }
 242
 243 static void SecDbNotifyPhase(SecDbConnectionRef dbconn, SecDbTransactionPhase phase) {
 244     if (CFArrayGetCount(dbconn->changes)) {
 245         CFArrayRef changes = dbconn->changes;
 246         dbconn->changes = CFArrayCreateMutableForCFTypes(kCFAllocatorDefault);
 247         if (dbconn->db->notifyPhase)
 248             dbconn->db->notifyPhase(dbconn, phase, dbconn->source, changes);
 249         CFReleaseSafe(changes);
 250     }
 251 }
 252
 253 static void SecDbOnNotifyQueue(SecDbConnectionRef dbconn, void (^perform)()) {
 254     if (dbconn->db->notifyQueue) {
 255         dispatch_sync(dbconn->db->notifyQueue, ^{
 256             perform();
 257         });
 258     } else {
 259         perform();
 260     }
 261 }
 262
 263 CFStringRef SecDbGetPath(SecDbRef db) {
 264     return db->db_path;
 265 }
 266
 267
 268 #pragma mark -
 269 #pragma mark SecDbConnectionRef
 270
 271 static bool SecDbCheckCorrupted(SecDbConnectionRef dbconn)
 272 {
 273     __block bool isCorrupted = true;
 274     __block CFErrorRef error = NULL;
 275     SecDbPrepare(dbconn, CFSTR("PRAGMA integrity_check"), &error, ^(sqlite3_stmt *stmt) {
 276         SecDbStep(dbconn, stmt, &error, ^(bool *stop) {
 277             const char * result = (const char*)sqlite3_column_text(stmt, 0);
 278             if (result && strncasecmp(result, "ok", 3) == 0) {
 279                 isCorrupted = false;
 280             }
 281         });
 282     });
 283     if (error) {
 284         LOGV("sqlDb: warning error %@ when running integrity check", error);
 285         CFRelease(error);
 286     }
 287     return isCorrupted;
 288 }
 289
 290 static bool SecDbDidCreateFirstConnection(SecDbConnectionRef dbconn, bool didCreate, CFErrorRef *error)
 291 {
 292     LOGD("sqlDb: starting maintenance");
 293     bool ok = true;
 294
 295     if (!didCreate && !dbconn->isCorrupted) {
 296         dbconn->isCorrupted = SecDbCheckCorrupted(dbconn);
 297         if (dbconn->isCorrupted)
 298             secerror("integrity check=fail");
 299         else
 300             LOGD("sqlDb: integrity check=pass");
 301     }
 302
 303     if (!dbconn->isCorrupted && dbconn->db->opened) {
 304         CFErrorRef localError = NULL;
 305
 306         dbconn->db->callOpenedHandlerForNextConnection = false;
 307         ok = dbconn->db->opened(dbconn, didCreate, &dbconn->db->callOpenedHandlerForNextConnection, &localError);
 308
 309         if (!ok)
 310             secerror("opened block failed: %@", localError);
 311
 312         if (!dbconn->isCorrupted && error && *error == NULL) {
 313             *error = localError;
 314             localError = NULL;
 315         } else {
 316             secerror("opened block failed: error is released and lost");
 317             CFReleaseNull(localError);
 318         }
 319     }
 320
 321     if (dbconn->isCorrupted) {
 322         ok = SecDbHandleCorrupt(dbconn, 0, error);
 323     }
 324
 325     LOGD("sqlDb: finished maintenance");
 326     return ok;
 327 }
 328
 329 void SecDbCorrupt(SecDbConnectionRef dbconn, CFErrorRef error)
 330 {
 331     dbconn->isCorrupted = true;
 332     CFAssignRetained(dbconn->corruptionError, error);
 333 }
 334
 335
 336 static uint8_t knownDbPathIndex(SecDbConnectionRef dbconn)
 337 {
 338
 339     if(CFEqual(dbconn->db->db_path, CFSTR("/Library/Keychains/keychain-2.db")))
 340         return 1;
 341     if(CFEqual(dbconn->db->db_path, CFSTR("/Library/Keychains/ocspcache.sqlite3")))
 342         return 2;
 343     if(CFEqual(dbconn->db->db_path, CFSTR("/Library/Keychains/TrustStore.sqlite3")))
 344         return 3;
 345     if(CFEqual(dbconn->db->db_path, CFSTR("/Library/Keychains/caissuercache.sqlite3")))
 346         return 4;
 347
 348     /* Unknown DB path */
 349     return 0;
 350 }
 351
 352
 353 // Return true if there was no error, returns false otherwise and set *error to an appropriate CFErrorRef.
 354 static bool SecDbConnectionCheckCode(SecDbConnectionRef dbconn, int code, CFErrorRef *error, CFStringRef desc, ...) {
 355     if (code == SQLITE_OK || code == SQLITE_DONE)
 356         return true;
 357
 358     if (error) {
 359         va_list args;
 360         va_start(args, desc);
 361         CFStringRef msg = CFStringCreateWithFormatAndArguments(kCFAllocatorDefault, NULL, desc, args);
 362         va_end(args);
 363         SecDbErrorWithDb(code, dbconn->handle, error, msg);
 364         CFRelease(msg);
 365     }
 366
 367     /* If it's already corrupted, don't try to recover */
 368     if (dbconn->isCorrupted) {
 369         CFStringRef reason = CFStringCreateWithFormat(kCFAllocatorDefault, NULL, CFSTR("SQL DB %@ is corrupted already. Not trying to recover"), dbconn->db->db_path);
 370         secerror("%@",reason);
 371         __security_simulatecrash(reason, __sec_exception_code_TwiceCorruptDb(knownDbPathIndex(dbconn)));
 372         CFReleaseSafe(reason);
 373         return false;
 374     }
 375
 376     dbconn->isCorrupted = (SQLITE_CORRUPT == code) || (SQLITE_NOTADB == code) || (SQLITE_IOERR == code) || (SQLITE_CANTOPEN == code);
 377     if (dbconn->isCorrupted) {
 378         /* Run integrity check and only make dbconn->isCorrupted true and
 379            run the corruption handler if the integrity check conclusively fails. */
 380         dbconn->isCorrupted = SecDbCheckCorrupted(dbconn);
 381         if (dbconn->isCorrupted) {
 382             secerror("operation returned code: %d integrity check=fail", code);
 383             SecDbHandleCorrupt(dbconn, code, error);
 384         } else {
 385             secerror("operation returned code: %d: integrity check=pass", code);
 386         }
 387     }
 388
 389     return false;
 390 }
 391
 392 #if HAVE_UNLOCK_NOTIFY
 393
 394 static void SecDbUnlockNotify(void **apArg, int nArg) {
 395     int i;
 396     for(i=0; i<nArg; i++) {
 397         dispatch_semaphore_t dsema = (dispatch_semaphore_t)apArg[i];
 398         dispatch_semaphore_signal(dsema);
 399     }
 400 }
 401
 402 static bool SecDbWaitForUnlockNotify(SecDbConnectionRef dbconn, sqlite3_stmt *stmt, CFErrorRef *error) {
 403     int rc;
 404     dispatch_semaphore_t dsema = dispatch_semaphore_create(0);
 405     rc = sqlite3_unlock_notify(dbconn->handle, SecDbUnlockNotify, dsema);
 406     assert(rc == SQLITE_LOCKED || rc == SQLITE_OK);
 407     if (rc == SQLITE_OK) {
 408         dispatch_semaphore_wait(dsema, DISPATCH_TIME_FOREVER);
 409     }
 410     dispatch_release(dsema);
 411     return (rc == SQLITE_OK
 412             ? true
 413             : (stmt
 414                ? SecDbErrorWithStmt(rc, stmt, error, CFSTR("sqlite3_unlock_notify"))
 415                : SecDbErrorWithDb(rc, dbconn->handle, error, CFSTR("sqlite3_unlock_notify"))));
 416 }
 417
 418 #endif
 419
 420 #if USE_BUSY_HANDLER
 421
 422 // Return 0 to stop retrying.
 423 static int SecDbHandleBusy(void *ctx, int retryCount) {
 424     SecDbConnectionRef dbconn __unused = ctx;
 425     struct timespec sleeptime = { .tv_sec = 0, .tv_nsec = 10000 };
 426     while (retryCount--) {
 427         // Double sleeptime until we hit one second then add one
 428         // second more every time we sleep.
 429         if (sleeptime.tv_sec) {
 430             sleeptime.tv_sec++;
 431         } else {
 432             sleeptime.tv_nsec *= 2;
 433             if (sleeptime.tv_nsec > NSEC_PER_SEC) {
 434                 sleeptime.tv_nsec = 0;
 435                 sleeptime.tv_sec++;
 436             }
 437         }
 438     }
 439     struct timespec unslept = {};
 440     nanosleep(&sleeptime, &unslept);
 441
 442     return 1;
 443 }
 444
 445 static bool SecDbBusyHandler(SecDbConnectionRef dbconn, CFErrorRef *error) {
 446     return SecDbErrorWithDb(sqlite3_busy_handler(dbconn->handle, SecDbHandleBusy, dbconn), dbconn->handle, error, CFSTR("busy_handler"));
 447 }
 448
 449 #endif // USE_BUSY_HANDLER
 450
 451 // Return true causes the operation to be tried again.
 452 static bool SecDbWaitIfNeeded(SecDbConnectionRef dbconn, int s3e, sqlite3_stmt *stmt, CFStringRef desc, struct timespec *sleeptime, CFErrorRef *error) {
 453 #if HAVE_UNLOCK_NOTIFY
 454     if (s3e == SQLITE_LOCKED) { // Optionally check for extended code being SQLITE_LOCKED_SHAREDCACHE
 455         return SecDbWaitForUnlockNotify(dbconn, stmt, error))
 456     }
 457 #endif
 458
 459 #if !USE_BUSY_HANDLER
 460     if (s3e == SQLITE_LOCKED || s3e == SQLITE_BUSY) {
 461         LOGV("sqlDb: %s", sqlite3_errmsg(dbconn->handle));
 462         while (s3e == SQLITE_LOCKED || s3e == SQLITE_BUSY) {
 463             struct timespec unslept = {};
 464             nanosleep(sleeptime, &unslept);
 465             s3e = SQLITE_OK;
 466             if (stmt)
 467                 s3e = sqlite3_reset(stmt);
 468
 469             // Double sleeptime until we hit one second the add one
 470             // second more every time we sleep.
 471             if (sleeptime->tv_sec) {
 472                 sleeptime->tv_sec++;
 473             } else {
 474                 sleeptime->tv_nsec *= 2;
 475                 if (sleeptime->tv_nsec > NSEC_PER_SEC) {
 476                     sleeptime->tv_nsec = 0;
 477                     sleeptime->tv_sec++;
 478                 }
 479             }
 480         }
 481         if (s3e)
 482             return SecDbErrorWithStmt(s3e, stmt, error, CFSTR("reset"));
 483     } else
 484 #endif // !USE_BUSY_HANDLER
 485     {
 486         return SecDbConnectionCheckCode(dbconn, s3e, error, desc);
 487     }
 488     return true;
 489 }
 490
 491 enum SecDbStepResult {
 492     kSecDbErrorStep = 0,
 493     kSecDbRowStep = 1,
 494     kSecDbDoneStep = 2,
 495 };
 496 typedef enum SecDbStepResult SecDbStepResult;
 497
 498 static SecDbStepResult _SecDbStep(SecDbConnectionRef dbconn, sqlite3_stmt *stmt, CFErrorRef *error) {
 499     assert(stmt != NULL);
 500     int s3e;
 501     struct timespec sleeptime = { .tv_sec = 0, .tv_nsec = 10000 };
 502     for (;;) {
 503         s3e = sqlite3_step(stmt);
 504         if (s3e == SQLITE_ROW)
 505             return kSecDbRowStep;
 506         else if (s3e == SQLITE_DONE)
 507             return kSecDbDoneStep;
 508         else if (!SecDbWaitIfNeeded(dbconn, s3e, stmt, CFSTR("step"), &sleeptime, error))
 509             return kSecDbErrorStep;
 510     };
 511 }
 512
 513 bool
 514 SecDbExec(SecDbConnectionRef dbconn, CFStringRef sql, CFErrorRef *error)
 515 {
 516     bool ok = true;
 517     CFRetain(sql);
 518     while (sql) {
 519         CFStringRef tail = NULL;
 520         if (ok) {
 521             sqlite3_stmt *stmt = SecDbCopyStmt(dbconn, sql, &tail, error);
 522             ok = stmt != NULL;
 523             if (stmt) {
 524                 SecDbStepResult sr;
 525                 while ((sr = _SecDbStep(dbconn, stmt, error)) == kSecDbRowStep);
 526                 if (sr == kSecDbErrorStep)
 527                     ok = false;
 528                 ok &= SecDbReleaseCachedStmt(dbconn, sql, stmt, error);
 529             }
 530         } else {
 531             // TODO We already have an error here we really just want the left over sql in it's userData
 532             ok = SecDbError(SQLITE_ERROR, error, CFSTR("Error with unexecuted sql remaining %@"), sql);
 533         }
 534         CFRelease(sql);
 535         sql = tail;
 536     }
 537     return ok;
 538 }
 539
 540 static bool SecDbBeginTransaction(SecDbConnectionRef dbconn, SecDbTransactionType type, CFErrorRef *error)
 541 {
 542     bool ok = true;
 543     CFStringRef query;
 544     switch (type) {
 545         case kSecDbImmediateTransactionType:
 546             query = CFSTR("BEGIN IMMEDATE");
 547             break;
 548         case kSecDbExclusiveRemoteTransactionType:
 549             dbconn->source = kSecDbSOSTransaction;
 550         case kSecDbExclusiveTransactionType:
 551             query = CFSTR("BEGIN EXCLUSIVE");
 552             break;
 553         case kSecDbNormalTransactionType:
 554             query = CFSTR("BEGIN");
 555             break;
 556         default:
 557             ok = SecDbError(SQLITE_ERROR, error, CFSTR("invalid transaction type %" PRIu32), type);
 558             query = NULL;
 559             break;
 560     }
 561
 562     if (query != NULL && sqlite3_get_autocommit(dbconn->handle) != 0) {
 563         ok = SecDbExec(dbconn, query, error);
 564     }
 565     if (ok)
 566         dbconn->inTransaction = true;
 567
 568     return ok;
 569 }
 570
 571 static bool SecDbEndTransaction(SecDbConnectionRef dbconn, bool commit, CFErrorRef *error)
 572 {
 573     __block bool ok = true;
 574     SecDbOnNotifyQueue(dbconn, ^{
 575         bool commited = false;
 576         if (commit) {
 577             SecDbNotifyPhase(dbconn, kSecDbTransactionWillCommit);
 578             commited = ok = SecDbExec(dbconn, CFSTR("END"), error);
 579         } else {
 580             ok = SecDbExec(dbconn, CFSTR("ROLLBACK"), error);
 581             commited = false;
 582         }
 583         dbconn->inTransaction = false;
 584         SecDbNotifyPhase(dbconn, commited ? kSecDbTransactionDidCommit : kSecDbTransactionDidRollback);
 585         dbconn->source = kSecDbAPITransaction;
 586     });
 587     return ok;
 588 }
 589
 590 bool SecDbTransaction(SecDbConnectionRef dbconn, SecDbTransactionType type,
 591                       CFErrorRef *error, void (^transaction)(bool *commit))
 592 {
 593     bool ok = true;
 594     bool commit = true;
 595
 596     if (dbconn->inTransaction) {
 597         transaction(&commit);
 598         if (!commit) {
 599             LOGV("sqlDb: nested transaction asked to not be committed");
 600         }
 601     } else {
 602         ok = SecDbBeginTransaction(dbconn, type, error);
 603         if (ok) {
 604             transaction(&commit);
 605             ok = SecDbEndTransaction(dbconn, commit, error);
 606         }
 607     }
 608
 609 done:
 610     return ok && commit;
 611 }
 612
 613 sqlite3 *SecDbHandle(SecDbConnectionRef dbconn) {
 614     return dbconn->handle;
 615 }
 616
 617 bool SecDbStep(SecDbConnectionRef dbconn, sqlite3_stmt *stmt, CFErrorRef *error, void (^row)(bool *stop)) {
 618     for (;;) {
 619         switch (_SecDbStep(dbconn, stmt, error)) {
 620             case kSecDbErrorStep:
 621                 return false;
 622             case kSecDbRowStep:
 623                 if (row) {
 624                     bool stop = false;
 625                     row(&stop);
 626                     if (stop)
 627                         return true;
 628                     break;
 629                 }
 630                 SecDbError(SQLITE_ERROR, error, CFSTR("SecDbStep SQLITE_ROW returned without a row handler"));
 631                 return false;
 632             case kSecDbDoneStep:
 633                 return true;
 634         }
 635     }
 636 }
 637
 638 bool SecDbCheckpoint(SecDbConnectionRef dbconn, CFErrorRef *error)
 639 {
 640     return SecDbConnectionCheckCode(dbconn, sqlite3_wal_checkpoint(dbconn->handle, NULL), error, CFSTR("wal_checkpoint"));
 641 }
 642
 643 static bool SecDbFileControl(SecDbConnectionRef dbconn, int op, void *arg, CFErrorRef *error) {
 644     return SecDbConnectionCheckCode(dbconn, sqlite3_file_control(dbconn->handle, NULL, op, arg), error, CFSTR("file_control"));
 645 }
 646
 647 static sqlite3 *_SecDbOpenV2(const char *path, int flags, CFErrorRef *error) {
 648 #if HAVE_UNLOCK_NOTIFY
 649     flags |= SQLITE_OPEN_SHAREDCACHE;
 650 #endif
 651     sqlite3 *handle = NULL;
 652     int s3e = sqlite3_open_v2(path, &handle, flags, NULL);
 653     if (s3e) {
 654         if (handle) {
 655             SecDbErrorWithDb(s3e, handle, error, CFSTR("open_v2 \"%s\" 0x%X"), path, flags);
 656             sqlite3_close(handle);
 657             handle = NULL;
 658         } else {
 659             SecDbError(s3e, error, CFSTR("open_v2 \"%s\" 0x%X"), path, flags);
 660         }
 661     }
 662     return handle;
 663 }
 664
 665 static bool SecDbOpenV2(SecDbConnectionRef dbconn, const char *path, int flags, CFErrorRef *error) {
 666     return (dbconn->handle = _SecDbOpenV2(path, flags, error)) != NULL;
 667 }
 668
 669 static bool SecDbTruncate(SecDbConnectionRef dbconn, CFErrorRef *error)
 670 {
 671     int flags = SQLITE_TRUNCATE_JOURNALMODE_WAL | SQLITE_TRUNCATE_AUTOVACUUM_FULL;
 672     __block bool ok = SecDbFileControl(dbconn, SQLITE_TRUNCATE_DATABASE, &flags, error);
 673     if (!ok) {
 674         sqlite3_close(dbconn->handle);
 675         dbconn->handle = NULL;
 676         CFStringPerformWithCString(dbconn->db->db_path, ^(const char *path) {
 677             if (error)
 678                 CFReleaseNull(*error);
 679             if (SecCheckErrno(unlink(path), error, CFSTR("unlink %s"), path)) {
 680                 ok = SecDbOpenHandle(dbconn, NULL, error);
 681             }
 682         });
 683         if (!ok) {
 684             secerror("Failed to delete db handle: %@", error ? *error : NULL);
 685             abort();
 686         }
 687     }
 688
 689     return ok;
 690 }
 691
 692 static bool SecDbHandleCorrupt(SecDbConnectionRef dbconn, int rc, CFErrorRef *error)
 693 {
 694     CFStringRef reason = CFStringCreateWithFormat(kCFAllocatorDefault, NULL, CFSTR("SQL DB %@ is corrupted, trying to recover (rc=%d) %@"), dbconn->db->db_path, rc, dbconn->corruptionError);
 695     __security_simulatecrash(reason, __sec_exception_code_CorruptDb(knownDbPathIndex(dbconn), rc));
 696     CFReleaseSafe(reason);
 697
 698     // Backup current db.
 699     __block bool didRename = false;
 700     CFStringPerformWithCString(dbconn->db->db_path, ^(const char *db_path) {
 701         sqlite3 *corrupt_db = NULL;
 702         char buf[PATH_MAX+1];
 703         snprintf(buf, sizeof(buf), "%s-corrupt", db_path);
 704         if (dbconn->handle && (corrupt_db = _SecDbOpenV2(buf, SQLITE_OPEN_READWRITE, error))) {
 705             int on = 1;
 706             didRename =  SecDbErrorWithDb(sqlite3_file_control(corrupt_db, NULL, SQLITE_FCNTL_PERSIST_WAL, &on), corrupt_db, error, CFSTR("persist wal"));
 707             didRename &=  SecDbErrorWithDb(sqlite3_file_control(corrupt_db, NULL, SQLITE_REPLACE_DATABASE, (void *)dbconn->handle), corrupt_db, error, CFSTR("replace database"));
 708             sqlite3_close(corrupt_db);
 709         }
 710         if (!didRename) {
 711             if (dbconn->handle)
 712                 secerror("Tried to rename corrupt database at path %@, but we failed: %@, trying explicit rename", dbconn->db->db_path, error ? *error : NULL);
 713             if (error)
 714                 CFReleaseNull(*error);
 715
 716             didRename = SecCheckErrno(rename(db_path, buf), error, CFSTR("rename %s %s"), db_path, buf) &&
 717                 (!dbconn->handle || SecDbError(sqlite3_close(dbconn->handle), error, CFSTR("close"))) &&
 718                 SecDbOpenHandle(dbconn, NULL, error);
 719         }
 720         if (didRename) {
 721             secerror("Database at path %@ is corrupt. Copied it to %s for further investigation.", dbconn->db->db_path, buf);
 722         } else {
 723             seccritical("Tried to copy corrupt database at path %@, but we failed: %@", dbconn->db->db_path, error ? *error : NULL);
 724         }
 725     });
 726
 727     bool ok = (didRename &&
 728                (dbconn->handle || SecDbOpenHandle(dbconn, NULL, error)) &&
 729                SecDbTruncate(dbconn, error));
 730
 731     // Mark the db as not corrupted, even if something failed.
 732     // Always note we are no longer in the corruption handler
 733     dbconn->isCorrupted = false;
 734
 735     // Invoke our callers opened callback, since we just created a new database
 736     if (ok && dbconn->db->opened) {
 737         dbconn->db->callOpenedHandlerForNextConnection = false;
 738         ok = dbconn->db->opened(dbconn, true, &dbconn->db->callOpenedHandlerForNextConnection, error);
 739     }
 740
 741     return ok;
 742 }
 743
 744 static bool SecDbProfileEnabled(void)
 745 {
 746 #if 0
 747     static dispatch_once_t onceToken;
 748     static bool profile_enabled = false;
 749
 750 #if DEBUG
 751     //sudo defaults write /Library/Preferences/com.apple.security.auth profile -bool true
 752     dispatch_once(&onceToken, ^{
 753                 CFTypeRef profile = (CFNumberRef)CFPreferencesCopyValue(CFSTR("profile"), CFSTR(SECURITY_AUTH_NAME), kCFPreferencesAnyUser, kCFPreferencesCurrentHost);
 754
 755         if (profile && CFGetTypeID(profile) == CFBooleanGetTypeID()) {
 756             profile_enabled = CFBooleanGetValue((CFBooleanRef)profile);
 757         }
 758
 759         LOGV("sqlDb: sql profile: %s", profile_enabled ? "enabled" : "disabled");
 760
 761         CFReleaseSafe(profile);
 762     });
 763 #endif
 764
 765     return profile_enabled;
 766 #else
 767 #if DEBUG
 768     return true;
 769 #else
 770     return false;
 771 #endif
 772 #endif
 773 }
 774
 775 #if 0
 776 static void SecDbProfile(void *context __unused, const char *sql, sqlite3_uint64 ns) {
 777     LOGV("==\nsqlDb: %s\nTime: %llu ms\n", sql, ns >> 20);
 778 }
 779 #else
 780 static void SecDbProfile(void *context, const char *sql, sqlite3_uint64 ns) {
 781     sqlite3 *s3h = context;
 782     int code = sqlite3_extended_errcode(s3h);
 783     if (code == SQLITE_OK || code == SQLITE_DONE) {
 784         secdebug("profile", "==\nsqlDb: %s\nTime: %llu ms\n", sql, ns >> 20);
 785     } else {
 786         secdebug("profile", "==error[%d]: %s==\nsqlDb: %s\nTime: %llu ms \n", code, sqlite3_errmsg(s3h), sql, ns >> 20);
 787     }
 788 }
 789 #endif
 790
 791 static bool SecDbTraceEnabled(void)
 792 {
 793 #if DEBUG
 794     return true;
 795 #else
 796     return false;
 797 #endif
 798 }
 799
 800 static void SecDbTrace(void *ctx, const char *trace) {
 801     SecDbConnectionRef dbconn __unused = ctx;
 802     static dispatch_queue_t queue;
 803     static dispatch_once_t once;
 804     dispatch_once(&once, ^{
 805         queue = dispatch_queue_create("trace_queue", DISPATCH_QUEUE_SERIAL);
 806     });
 807     dispatch_sync(queue, ^{
 808         __security_debug(CFSTR("trace"), "", "", 0, CFSTR("%s"), trace);
 809     });
 810 }
 811
 812 static bool SecDbOpenHandle(SecDbConnectionRef dbconn, bool *created, CFErrorRef *error)
 813 {
 814     __block bool ok = true;
 815     CFStringPerformWithCString(dbconn->db->db_path, ^(const char *db_path) {
 816         ok = created && SecDbOpenV2(dbconn, db_path, SQLITE_OPEN_READWRITE, NULL);
 817         if (!ok) {
 818             ok = true;
 819             if (created) {
 820                 char *tmp = dirname((char *)db_path);
 821                 if (tmp) {
 822                     int errnum = mkpath_np(tmp, 0700);
 823                     if (errnum != 0 && errnum != EEXIST) {
 824                         SecCFCreateErrorWithFormat(errnum, kSecErrnoDomain, NULL, error, NULL,
 825                                                    CFSTR("mkpath_np %s: [%d] %s"), tmp, errnum, strerror(errnum));
 826                         ok = false;
 827                     }
 828                 }
 829             }
 830             ok = ok && SecDbOpenV2(dbconn, db_path, SQLITE_OPEN_READWRITE | SQLITE_OPEN_CREATE, error);
 831             if (ok) {
 832                 chmod(db_path, S_IRUSR | S_IWUSR);
 833                 if (created)
 834                     *created = true;
 835             }
 836         }
 837
 838         if (ok && SecDbProfileEnabled()) {
 839             sqlite3_profile(dbconn->handle, SecDbProfile, dbconn->handle);
 840         }
 841         if (ok && SecDbTraceEnabled()) {
 842             sqlite3_trace(dbconn->handle, SecDbTrace, dbconn);
 843         }
 844 #if USE_BUSY_HANDLER
 845         ok = ok && SecDbBusyHandler(dbconn, error);
 846 #endif
 847     });
 848
 849 done:
 850     return ok;
 851 }
 852
 853 static SecDbConnectionRef
 854 SecDbConnectionCreate(SecDbRef db, bool readOnly, CFErrorRef *error)
 855 {
 856     SecDbConnectionRef dbconn = NULL;
 857
 858     dbconn = CFTypeAllocate(SecDbConnection, struct __OpaqueSecDbConnection, kCFAllocatorDefault);
 859     require(dbconn != NULL, done);
 860
 861     dbconn->db = db;
 862     dbconn->readOnly = readOnly;
 863     dbconn->inTransaction = false;
 864     dbconn->source = NULL;
 865     dbconn->isCorrupted = false;
 866     dbconn->corruptionError = NULL;
 867     dbconn->handle = NULL;
 868     dbconn->changes = CFArrayCreateMutableForCFTypes(kCFAllocatorDefault);
 869
 870 done:
 871     return dbconn;
 872 }
 873
 874 static bool SecDbConnectionIsReadOnly(SecDbConnectionRef dbconn) {
 875     return dbconn->readOnly;
 876 }
 877
 878 static void SecDbConectionSetReadOnly(SecDbConnectionRef dbconn, bool readOnly) {
 879     dbconn->readOnly = readOnly;
 880 }
 881
 882 /* Read only connections go to the end of the queue, writeable connections
 883  go to the start of the queue. */
 884 SecDbConnectionRef SecDbConnectionAquire(SecDbRef db, bool readOnly, CFErrorRef *error) {
 885     CFRetain(db);
 886     secdebug("dbconn", "acquire %s connection", readOnly ? "ro" : "rw");
 887     dispatch_semaphore_wait(readOnly ? db->read_semaphore : db->write_semaphore, DISPATCH_TIME_FOREVER);
 888     __block SecDbConnectionRef dbconn = NULL;
 889     __block bool ok = true;
 890     __block bool ranOpenedHandler = false;
 891     dispatch_sync(db->queue, ^{
 892         if (!db->didFirstOpen) {
 893             bool didCreate = false;
 894             ok = dbconn = SecDbConnectionCreate(db, false, error);
 895             CFErrorRef localError = NULL;
 896             if (ok && !SecDbOpenHandle(dbconn, &didCreate, &localError)) {
 897                 secerror("Unable to create database: %@", localError);
 898                 if (localError && CFEqual(CFErrorGetDomain(localError), kSecDbErrorDomain)) {
 899                     int code = (int)CFErrorGetCode(localError);
 900                     dbconn->isCorrupted = (SQLITE_CORRUPT == code) || (SQLITE_NOTADB == code) || (SQLITE_IOERR == code) || (SQLITE_CANTOPEN == code);
 901                 }
 902                 // If the open failure isn't due to corruption, propagte the error.
 903                 ok = dbconn->isCorrupted;
 904                 if (!ok && error && *error == NULL) {
 905                     *error = localError;
 906                     localError = NULL;
 907                 }
 908             }
 909             CFReleaseNull(localError);
 910
 911             if (ok) {
 912                 db->didFirstOpen = ok = SecDbDidCreateFirstConnection(dbconn, didCreate, error);
 913                 ranOpenedHandler = true;
 914             }
 915             if (!ok)
 916                 CFReleaseNull(dbconn);
 917         } else {
 918             /* Try to get one from the cache */
 919             CFIndex count = CFArrayGetCount(db->connections);
 920             while (count && !dbconn) {
 921                 CFIndex ix = readOnly ? count - 1 : 0;
 922                 dbconn = (SecDbConnectionRef)CFArrayGetValueAtIndex(db->connections, ix);
 923                 if (dbconn)
 924                     CFRetain(dbconn);
 925                 else
 926                     secerror("got NULL dbconn at index: %" PRIdCFIndex " skipping", ix);
 927                 CFArrayRemoveValueAtIndex(db->connections, ix);
 928             }
 929         }
 930     });
 931
 932     if (dbconn) {
 933         /* Make sure the connection we found has the right access */
 934         if (SecDbConnectionIsReadOnly(dbconn) != readOnly) {
 935             SecDbConectionSetReadOnly(dbconn, readOnly);
 936         }
 937     } else if (ok) {
 938         /* Nothing found in cache, create a new connection */
 939         bool created = false;
 940         dbconn = SecDbConnectionCreate(db, readOnly, error);
 941         if (dbconn && !SecDbOpenHandle(dbconn, &created, error)) {
 942             CFReleaseNull(dbconn);
 943         }
 944     }
 945
 946     if (dbconn && !ranOpenedHandler && dbconn->db->opened) {
 947         dispatch_sync(db->queue, ^{
 948             if (dbconn->db->callOpenedHandlerForNextConnection) {
 949                 dbconn->db->callOpenedHandlerForNextConnection = false;
 950                 if (!dbconn->db->opened(dbconn, false, &dbconn->db->callOpenedHandlerForNextConnection, error)) {
 951                     if (!dbconn->isCorrupted || !SecDbHandleCorrupt(dbconn, 0, error)) {
 952                         CFReleaseNull(dbconn);
 953                     }
 954                 }
 955             }
 956         });
 957     }
 958
 959     if (!dbconn) {
 960         // If aquire fails we need to signal the semaphore again.
 961         dispatch_semaphore_signal(readOnly ? db->read_semaphore : db->write_semaphore);
 962         CFRelease(db);
 963     }
 964
 965     return dbconn;
 966 }
 967
 968 void SecDbConnectionRelease(SecDbConnectionRef dbconn) {
 969     if (!dbconn) {
 970         secerror("called with NULL dbconn");
 971         return;
 972     }
 973     SecDbRef db = dbconn->db;
 974     secdebug("dbconn", "release %@", dbconn);
 975     dispatch_sync(db->queue, ^{
 976         CFIndex count = CFArrayGetCount(db->connections);
 977         // Add back possible writable dbconn to the pool.
 978         bool readOnly = SecDbConnectionIsReadOnly(dbconn);
 979         CFArrayInsertValueAtIndex(db->connections, readOnly ? count : 0, dbconn);
 980         // Remove the last (probably read-only) dbconn from the pool.
 981         if (count >= kSecDbMaxIdleHandles) {
 982             CFArrayRemoveValueAtIndex(db->connections, count);
 983         }
 984         // Signal after we have put the connection back in the pool of connections
 985         dispatch_semaphore_signal(readOnly ? db->read_semaphore : db->write_semaphore);
 986         CFRelease(dbconn);
 987         CFRelease(db);
 988     });
 989 }
 990
 991 bool SecDbPerformRead(SecDbRef db, CFErrorRef *error, void (^perform)(SecDbConnectionRef dbconn)) {
 992     SecDbConnectionRef dbconn = SecDbConnectionAquire(db, true, error);
 993     bool success = false;
 994     if (dbconn) {
 995         perform(dbconn);
 996         success = true;
 997         SecDbConnectionRelease(dbconn);
 998     }
 999     return success;
1000 }
1001
1002 bool SecDbPerformWrite(SecDbRef db, CFErrorRef *error, void (^perform)(SecDbConnectionRef dbconn)) {
1003     SecDbConnectionRef dbconn = SecDbConnectionAquire(db, false, error);
1004     bool success = false;
1005     if (dbconn) {
1006         perform(dbconn);
1007         success = true;
1008         SecDbConnectionRelease(dbconn);
1009     }
1010     return success;
1011 }
1012
1013 static CFStringRef
1014 SecDbConnectionCopyFormatDescription(CFTypeRef value, CFDictionaryRef formatOptions)
1015 {
1016     SecDbConnectionRef dbconn = (SecDbConnectionRef)value;
1017     return CFStringCreateWithFormat(kCFAllocatorDefault, NULL, CFSTR("<SecDbConnection %s %s>"),
1018                                     dbconn->readOnly ? "ro" : "rw", dbconn->handle ? "open" : "closed");
1019 }
1020
1021 static void
1022 SecDbConnectionDestroy(CFTypeRef value)
1023 {
1024     SecDbConnectionRef dbconn = (SecDbConnectionRef)value;
1025     if (dbconn->handle) {
1026         sqlite3_close(dbconn->handle);
1027     }
1028     dbconn->db = NULL;
1029     CFReleaseNull(dbconn->changes);
1030     CFReleaseNull(dbconn->corruptionError);
1031
1032 }
1033
1034
1035 // MARK: -
1036 // MARK: Bind helpers
1037
1038 #if 0
1039 bool SecDbBindNull(sqlite3_stmt *stmt, int param, CFErrorRef *error) {
1040     bool ok = SecDbErrorWithStmt(sqlite3_bind_null(stmt, param),
1041                                  stmt, error, CFSTR("bind_null[%d]"), param);
1042     secdebug("bind", "bind_null[%d]: %@", param, error ? *error : NULL);
1043     return ok;
1044 }
1045 #endif
1046
1047 bool SecDbBindBlob(sqlite3_stmt *stmt, int param, const void *zData, size_t n, void(*xDel)(void*), CFErrorRef *error) {
1048     if (n > INT_MAX) {
1049         return SecDbErrorWithStmt(SQLITE_TOOBIG, stmt, error,
1050                                   CFSTR("bind_blob[%d]: blob bigger than INT_MAX"), param);
1051     }
1052     bool ok = SecDbErrorWithStmt(sqlite3_bind_blob(stmt, param, zData, (int)n, xDel),
1053                                  stmt, error, CFSTR("bind_blob[%d]"), param);
1054     secdebug("bind", "bind_blob[%d]: %.*s: %@", param, (int)n, zData, error ? *error : NULL);
1055     return ok;
1056 }
1057
1058 bool SecDbBindText(sqlite3_stmt *stmt, int param, const char *zData, size_t n, void(*xDel)(void*), CFErrorRef *error) {
1059     if (n > INT_MAX) {
1060         return SecDbErrorWithStmt(SQLITE_TOOBIG, stmt, error,
1061                                   CFSTR("bind_text[%d]: text bigger than INT_MAX"), param);
1062     }
1063     bool ok = SecDbErrorWithStmt(sqlite3_bind_text(stmt, param, zData, (int)n, xDel), stmt, error,
1064                                  CFSTR("bind_text[%d]"), param);
1065     secdebug("bind", "bind_text[%d]: \"%s\": %@", param, zData, error ? *error : NULL);
1066     return ok;
1067 }
1068
1069 bool SecDbBindDouble(sqlite3_stmt *stmt, int param, double value, CFErrorRef *error) {
1070     bool ok = SecDbErrorWithStmt(sqlite3_bind_double(stmt, param, value), stmt, error,
1071                                  CFSTR("bind_double[%d]"), param);
1072     secdebug("bind", "bind_double[%d]: %f: %@", param, value, error ? *error : NULL);
1073     return ok;
1074 }
1075
1076 bool SecDbBindInt(sqlite3_stmt *stmt, int param, int value, CFErrorRef *error) {
1077     bool ok = SecDbErrorWithStmt(sqlite3_bind_int(stmt, param, value), stmt, error,
1078                                  CFSTR("bind_int[%d]"), param);
1079     secdebug("bind", "bind_int[%d]: %d: %@", param, value, error ? *error : NULL);
1080     return ok;
1081 }
1082
1083 bool SecDbBindInt64(sqlite3_stmt *stmt, int param, sqlite3_int64 value, CFErrorRef *error) {
1084     bool ok = SecDbErrorWithStmt(sqlite3_bind_int64(stmt, param, value), stmt, error,
1085                                  CFSTR("bind_int64[%d]"), param);
1086     secdebug("bind", "bind_int64[%d]: %lld: %@", param, value, error ? *error : NULL);
1087     return ok;
1088 }
1089
1090
1091 /* AUDIT[securityd](done):
1092  value (ok) is a caller provided, non NULL CFTypeRef.
1093  */
1094 bool SecDbBindObject(sqlite3_stmt *stmt, int param, CFTypeRef value, CFErrorRef *error) {
1095     CFTypeID valueId;
1096     __block bool result = false;
1097
1098         /* TODO: Can we use SQLITE_STATIC below everwhere we currently use
1099      SQLITE_TRANSIENT since we finalize the statement before the value
1100      goes out of scope? */
1101     if (!value || (valueId = CFGetTypeID(value)) == CFNullGetTypeID()) {
1102         /* Skip bindings for NULL values.  sqlite3 will interpret unbound
1103          params as NULL which is exactly what we want. */
1104 #if 1
1105         result = true;
1106 #else
1107         result = SecDbBindNull(stmt, param, error);
1108 #endif
1109     } else if (valueId == CFStringGetTypeID()) {
1110         CFStringPerformWithCStringAndLength(value, ^(const char *cstr, size_t clen) {
1111             result = SecDbBindText(stmt, param, cstr, clen, SQLITE_TRANSIENT, error);
1112         });
1113     } else if (valueId == CFDataGetTypeID()) {
1114         CFIndex len = CFDataGetLength(value);
1115         if (len) {
1116             result = SecDbBindBlob(stmt, param, CFDataGetBytePtr(value),
1117                                    len, SQLITE_TRANSIENT, error);
1118         } else {
1119             result = SecDbBindText(stmt, param, "", 0, SQLITE_TRANSIENT, error);
1120         }
1121     } else if (valueId == CFDateGetTypeID()) {
1122         CFAbsoluteTime abs_time = CFDateGetAbsoluteTime(value);
1123         result = SecDbBindDouble(stmt, param, abs_time, error);
1124     } else if (valueId == CFBooleanGetTypeID()) {
1125         int bval = CFBooleanGetValue(value);
1126         result = SecDbBindInt(stmt, param, bval, error);
1127     } else if (valueId == CFNumberGetTypeID()) {
1128         Boolean convertOk;
1129         if (CFNumberIsFloatType(value)) {
1130             double nval;
1131             convertOk = CFNumberGetValue(value, kCFNumberDoubleType, &nval);
1132             result = SecDbBindDouble(stmt, param, nval, error);
1133         } else {
1134             int nval;
1135             convertOk = CFNumberGetValue(value, kCFNumberSInt32Type, &nval);
1136             if (convertOk) {
1137                 result = SecDbBindInt(stmt, param, nval, error);
1138             } else {
1139                 sqlite_int64 nval64;
1140                 convertOk = CFNumberGetValue(value, kCFNumberSInt64Type, &nval64);
1141                 if (convertOk)
1142                     result = SecDbBindInt64(stmt, param, nval64, error);
1143             }
1144         }
1145         if (!convertOk) {
1146             result = SecDbError(SQLITE_INTERNAL, error, CFSTR("bind CFNumberGetValue failed for %@"), value);
1147         }
1148     } else {
1149         if (error) {
1150             CFStringRef valueDesc = CFCopyTypeIDDescription(valueId);
1151             SecDbError(SQLITE_MISMATCH, error, CFSTR("bind unsupported type %@"), valueDesc);
1152             CFReleaseSafe(valueDesc);
1153         }
1154     }
1155
1156         return result;
1157 }
1158
1159 // MARK: -
1160 // MARK: SecDbStatementRef
1161
1162 bool SecDbReset(sqlite3_stmt *stmt, CFErrorRef *error) {
1163     return SecDbErrorWithStmt(sqlite3_reset(stmt), stmt, error, CFSTR("reset"));
1164 }
1165
1166 bool SecDbClearBindings(sqlite3_stmt *stmt, CFErrorRef *error) {
1167     return SecDbErrorWithStmt(sqlite3_clear_bindings(stmt), stmt, error, CFSTR("clear bindings"));
1168 }
1169
1170 bool SecDbFinalize(sqlite3_stmt *stmt, CFErrorRef *error) {
1171     sqlite3 *handle = sqlite3_db_handle(stmt);
1172     int s3e = sqlite3_finalize(stmt);
1173     return s3e == SQLITE_OK ? true : SecDbErrorWithDb(s3e, handle, error, CFSTR("finalize: %p"), stmt);
1174 }
1175
1176 sqlite3_stmt *SecDbPrepareV2(SecDbConnectionRef dbconn, const char *sql, size_t sqlLen, const char **sqlTail, CFErrorRef *error) {
1177     sqlite3 *db = SecDbHandle(dbconn);
1178     if (sqlLen > INT_MAX) {
1179         SecDbErrorWithDb(SQLITE_TOOBIG, db, error, CFSTR("prepare_v2: sql bigger than INT_MAX"));
1180         return NULL;
1181     }
1182     struct timespec sleeptime = { .tv_sec = 0, .tv_nsec = 10000 };
1183     for (;;) {
1184         sqlite3_stmt *stmt = NULL;
1185         int s3e = sqlite3_prepare_v2(db, sql, (int)sqlLen, &stmt, sqlTail);
1186         if (s3e == SQLITE_OK)
1187             return stmt;
1188         else if (!SecDbWaitIfNeeded(dbconn, s3e, NULL, CFSTR("preparev2"), &sleeptime, error))
1189             return NULL;
1190     }
1191 }
1192
1193 static sqlite3_stmt *SecDbCopyStatementWithTailRange(SecDbConnectionRef dbconn, CFStringRef sql, CFRange *sqlTail, CFErrorRef *error) {
1194     __block sqlite3_stmt *stmt = NULL;
1195     if (sql) CFStringPerformWithCStringAndLength(sql, ^(const char *sqlStr, size_t sqlLen) {
1196         const char *tail = NULL;
1197         stmt = SecDbPrepareV2(dbconn, sqlStr, sqlLen, &tail, error);
1198         if (sqlTail && sqlStr < tail && tail < sqlStr + sqlLen) {
1199             sqlTail->location = tail - sqlStr;
1200             sqlTail->length = sqlLen - sqlTail->location;
1201         }
1202     });
1203
1204     return stmt;
1205 }
1206
1207 sqlite3_stmt *SecDbCopyStmt(SecDbConnectionRef dbconn, CFStringRef sql, CFStringRef *tail, CFErrorRef *error) {
1208     // TODO: Add caching and cache lookup of statements
1209     CFRange sqlTail = {};
1210     sqlite3_stmt *stmt = SecDbCopyStatementWithTailRange(dbconn, sql, &sqlTail, error);
1211     if (sqlTail.length > 0) {
1212         CFStringRef excess = CFStringCreateWithSubstring(CFGetAllocator(sql), sql, sqlTail);
1213         if (tail) {
1214             *tail = excess;
1215         } else {
1216             SecDbError(SQLITE_INTERNAL, error,
1217                        CFSTR("prepare_v2: %@ unused sql: %@"),
1218                        sql, excess);
1219             CFReleaseSafe(excess);
1220             SecDbFinalize(stmt, error);
1221             stmt = NULL;
1222         }
1223     }
1224     return stmt;
1225 }
1226
1227 /*
1228  TODO: Could do a hack here with a custom kCFAllocatorNULL allocator for a second CFRuntimeBase inside a SecDbStatement,
1229  TODO: Better yet make a full blow SecDbStatement instance whenever SecDbCopyStmt is called.  Then, when the statement is released, in the Dispose method, we Reset and ClearBindings the sqlite3_stmt * and hand it back to the SecDb with the original CFStringRef for the sql (or hash thereof) as an argument. */
1230 bool SecDbReleaseCachedStmt(SecDbConnectionRef dbconn, CFStringRef sql, sqlite3_stmt *stmt, CFErrorRef *error) {
1231     if (stmt) {
1232         return SecDbFinalize(stmt, error);
1233     }
1234     return true;
1235 }
1236
1237 bool SecDbPrepare(SecDbConnectionRef dbconn, CFStringRef sql, CFErrorRef *error, void(^exec)(sqlite3_stmt *stmt)) {
1238     assert(sql != NULL);
1239     sqlite3_stmt *stmt = SecDbCopyStmt(dbconn, sql, NULL, error);
1240     if (!stmt)
1241         return false;
1242
1243     exec(stmt);
1244     return SecDbReleaseCachedStmt(dbconn, sql, stmt, error);
1245 }
1246
1247 bool SecDbWithSQL(SecDbConnectionRef dbconn, CFStringRef sql, CFErrorRef *error, bool(^perform)(sqlite3_stmt *stmt)) {
1248     bool ok = true;
1249     CFRetain(sql);
1250     while (sql) {
1251         CFStringRef tail = NULL;
1252         if (ok) {
1253             sqlite3_stmt *stmt = SecDbCopyStmt(dbconn, sql, &tail, error);
1254             ok = stmt != NULL;
1255             if (stmt) {
1256                 if (perform) {
1257                     ok = perform(stmt);
1258                 } else {
1259                     // TODO: Use a different error scope here.
1260                     ok = SecError(-50 /* errSecParam */, error, CFSTR("SecDbWithSQL perform block missing"));
1261                 }
1262                 ok &= SecDbReleaseCachedStmt(dbconn, sql, stmt, error);
1263             }
1264         } else {
1265             // TODO We already have an error here we really just want the left over sql in it's userData
1266             ok = SecDbError(SQLITE_ERROR, error, CFSTR("Error with unexecuted sql remaining %@"), sql);
1267         }
1268         CFRelease(sql);
1269         sql = tail;
1270     }
1271     return ok;
1272 }
1273
1274 #if 1
1275 /* SecDbForEach returns true if all SQLITE_ROW returns of sqlite3_step() return true from the row block.
1276  If the row block returns false and doesn't set an error (to indicate it has reached a limit),
1277  this entire function returns false. In that case no error will be set. */
1278 bool SecDbForEach(sqlite3_stmt *stmt, CFErrorRef *error, bool(^row)(int row_index)) {
1279     bool result = false;
1280     for (int row_ix = 0;;++row_ix) {
1281         int s3e = sqlite3_step(stmt);
1282         if (s3e == SQLITE_ROW) {
1283             if (row) {
1284                 if (!row(row_ix)) {
1285                     break;
1286                 }
1287             } else {
1288                 // If we have no row block then getting SQLITE_ROW is an error
1289                 SecDbError(s3e, error,
1290                            CFSTR("step[%d]: %s returned SQLITE_ROW with NULL row block"),
1291                            row_ix, sqlite3_sql(stmt));
1292             }
1293         } else {
1294             if (s3e == SQLITE_DONE) {
1295                 result = true;
1296             } else {
1297                 SecDbErrorWithStmt(s3e, stmt, error, CFSTR("step[%d]"), row_ix);
1298             }
1299             break;
1300         }
1301     }
1302     return result;
1303 }
1304 #else
1305 bool SecDbForEach(sqlite3_stmt *stmt, CFErrorRef *error, bool(^row)(int row_index)) {
1306     int row_ix = 0;
1307     for (;;) {
1308         switch (_SecDbStep(dbconn, stmt, error)) {
1309             case kSecDbErrorStep:
1310                 return false;
1311             case kSecDbRowStep:
1312                 if (row) {
1313                     if (row(row_ix++))
1314                         break;
1315                 } else {
1316                     SecDbError(SQLITE_ERROR, error, CFSTR("SecDbStep SQLITE_ROW returned without a row handler"));
1317                 }
1318                 return false;
1319             case kSecDbDoneStep:
1320                 return true;
1321         }
1322     }
1323 }
1324 #endif
1325
1326 void SecDbRecordChange(SecDbConnectionRef dbconn, CFTypeRef deleted, CFTypeRef inserted) {
1327     if (!dbconn->db->notifyPhase) return;
1328     CFTypeRef entry = SecDbEventCreateWithComponents(deleted, inserted);
1329     if (entry) {
1330         CFArrayAppendValue(dbconn->changes, entry);
1331         CFRelease(entry);
1332
1333         if (!dbconn->inTransaction) {
1334             secerror("db %@ changed outside txn", dbconn);
1335             // Only notify of DidCommit, since WillCommit code assumes
1336             // we are in a txn.
1337             SecDbOnNotifyQueue(dbconn, ^{
1338                 SecDbNotifyPhase(dbconn, kSecDbTransactionDidCommit);
1339             });
1340         }
1341     }
1342 }
1343
1344
1345 CFGiblisFor(SecDbConnection)
1346
1347 //
1348 // SecDbEvent Creation and consumption
1349 //
1350
1351 static SecDbEventRef SecDbEventCreateInsert(CFTypeRef inserted) {
1352     return CFRetainSafe(inserted);
1353 }
1354
1355 static SecDbEventRef SecDbEventCreateDelete(CFTypeRef deleted) {
1356     return CFArrayCreate(kCFAllocatorDefault, &deleted, 1, &kCFTypeArrayCallBacks);
1357 }
1358
1359 static SecDbEventRef SecDbEventCreateUpdate(CFTypeRef deleted, CFTypeRef inserted) {
1360     const void *values[2] = { deleted, inserted };
1361     return CFArrayCreate(kCFAllocatorDefault, values, 2, &kCFTypeArrayCallBacks);
1362 }
1363
1364 SecDbEventRef SecDbEventCreateWithComponents(CFTypeRef deleted, CFTypeRef inserted) {
1365     if (deleted && inserted)
1366         return SecDbEventCreateUpdate(deleted, inserted);
1367     else if (deleted)
1368         return SecDbEventCreateDelete(deleted);
1369     else if (inserted)
1370         return SecDbEventCreateInsert(inserted);
1371     else
1372         return NULL;
1373 }
1374
1375 bool SecDbEventGetComponents(SecDbEventRef event, CFTypeRef *deleted, CFTypeRef *inserted, CFErrorRef *error) {
1376     if (isArray(event)) {
1377         CFArrayRef array = event;
1378         switch (CFArrayGetCount(array)) {
1379             case 2:
1380                 *deleted = CFArrayGetValueAtIndex(array, 0);
1381                 *inserted = CFArrayGetValueAtIndex(array, 1);
1382                 break;
1383             case 1:
1384                 *deleted = CFArrayGetValueAtIndex(array, 0);
1385                 *inserted = NULL;
1386                 break;
1387             default:
1388                 SecError(errSecParam, error, NULL, CFSTR("invalid entry in changes array: %@"), array);
1389                 break;
1390         }
1391     } else {
1392         *deleted = NULL;
1393         *inserted = event;
1394     }
1395     return true;
1396 }