OSX/utilities/SecDb.c

   1 /*
   2  * Copyright (c) 2012-2017 Apple Inc. All Rights Reserved.
   3  *
   4  * @APPLE_LICENSE_HEADER_START@
   5  *
   6  * This file contains Original Code and/or Modifications of Original Code
   7  * as defined in and that are subject to the Apple Public Source License
   8  * Version 2.0 (the 'License'). You may not use this file except in
   9  * compliance with the License. Please obtain a copy of the License at
  10  * http://www.opensource.apple.com/apsl/ and read it before using this
  11  * file.
  12  *
  13  * The Original Code and all software distributed under the License are
  14  * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  15  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  16  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
  17  * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
  18  * Please see the License for the specific language governing rights and
  19  * limitations under the License.
  20  *
  21  * @APPLE_LICENSE_HEADER_END@
  22  */
  23
  24
  25 #include "SecDb.h"
  26 #include "SecDbInternal.h"
  27 #include "debugging.h"
  28
  29 #include <sqlite3.h>
  30 #include <sqlite3_private.h>
  31 #include <CoreFoundation/CoreFoundation.h>
  32 #include <libgen.h>
  33 #include <sys/csr.h>
  34 #include <sys/stat.h>
  35 #include <AssertMacros.h>
  36 #include "SecCFWrappers.h"
  37 #include "SecCFError.h"
  38 #include "SecIOFormat.h"
  39 #include <stdio.h>
  40 #include "Security/SecBase.h"
  41 #include "SecAutorelease.h"
  42 #include <os/assumes.h>
  43 #include <xpc/private.h>    // xpc_transaction_exit_clean()
  44
  45
  46 //
  47 // Architecturally inverted files
  48 // These are in SecureObjectSync but utilities depends on them
  49 // <rdar://problem/20802079> Fix layer violation (SOSDigestVector, SOSManifest, SecDB.c)
  50 //
  51 #include "keychain/SecureObjectSync/SOSDigestVector.h"
  52 #include "keychain/SecureObjectSync/SOSManifest.h"
  53
  54 #define SECDB_DEBUGGING 0
  55
  56 struct __OpaqueSecDbStatement {
  57     CFRuntimeBase _base;
  58
  59     SecDbConnectionRef dbconn;
  60     sqlite3_stmt *stmt;
  61 };
  62
  63 struct __OpaqueSecDbConnection {
  64     CFRuntimeBase _base;
  65
  66     //CFMutableDictionaryRef statements;
  67
  68     SecDbRef db;     // NONRETAINED, since db or block retains us
  69     bool readOnly;
  70     bool inTransaction;
  71     SecDbTransactionSource source;
  72     bool isCorrupted;
  73     int maybeCorruptedCode;
  74     bool hasIOFailure;
  75     CFErrorRef corruptionError;
  76     sqlite3 *handle;
  77     // Pending deletions and additions for the current transaction
  78     // Entires are either:
  79     // 1) a CFArrayRef of 1 element representing a deletion,
  80     // 2) a CFArrayRef of 2 elements representing the element 0 having been replaced with element 1
  81     // 3) a CFTypeRef that is not a CFArrayRef, representing an add of the element in question.
  82     CFMutableArrayRef changes;
  83 };
  84
  85 struct __OpaqueSecDb {
  86     CFRuntimeBase _base;
  87
  88     CFStringRef db_path;
  89     dispatch_queue_t queue;
  90     dispatch_queue_t commitQueue;
  91
  92     CFMutableArrayRef idleWriteConnections;     // up to kSecDbMaxWriters of them (currently 1, requires locking change for >1)
  93     CFMutableArrayRef idleReadConnections;      // up to kSecDbMaxReaders of them
  94     pthread_mutex_t writeMutex;
  95     // TODO: Replace after we have rdar://problem/60961964
  96     dispatch_semaphore_t readSemaphore;
  97
  98     bool didFirstOpen;
  99     bool (^opened)(SecDbRef db, SecDbConnectionRef dbconn, bool didCreate, bool *callMeAgainForNextConnection, CFErrorRef *error);
 100     bool callOpenedHandlerForNextConnection;
 101     CFMutableArrayRef notifyPhase; /* array of SecDBNotifyBlock */
 102     mode_t mode; /* database file permissions */
 103     bool readWrite; /* open database read-write */
 104     bool allowRepair; /* allow database repair */
 105     bool useWAL; /* use WAL mode */
 106     bool useRobotVacuum; /* use if SecDB should manage vacuum behind your back */
 107     uint8_t maxIdleHandles;
 108     void (^corruptionReset)(void);
 109 };
 110
 111 // MARK: Error domains and error helper functions
 112
 113 CFStringRef kSecDbErrorDomain = CFSTR("com.apple.utilities.sqlite3");
 114
 115 bool SecDbError(int sql_code, CFErrorRef *error, CFStringRef format, ...) {
 116     if (sql_code == SQLITE_OK) return true;
 117
 118     if (error) {
 119         va_list args;
 120         CFIndex code = sql_code;
 121         CFErrorRef previousError = *error;
 122
 123         *error = NULL;
 124         va_start(args, format);
 125         SecCFCreateErrorWithFormatAndArguments(code, kSecDbErrorDomain, previousError, error, NULL, format, args);
 126         va_end(args);
 127     }
 128     return false;
 129 }
 130
 131 bool SecDbErrorWithDb(int sql_code, sqlite3 *db, CFErrorRef *error, CFStringRef format, ...) {
 132     if (sql_code == SQLITE_OK) return true;
 133     if (error) {
 134         va_list args;
 135         va_start(args, format);
 136         CFStringRef message = CFStringCreateWithFormatAndArguments(kCFAllocatorDefault, NULL, format, args);
 137         va_end(args);
 138         CFStringRef errno_code = NULL;
 139
 140         if (sql_code == SQLITE_CANTOPEN) {
 141             int errno_number = sqlite3_system_errno(db);
 142             errno_code = CFStringCreateWithFormat(NULL, NULL, CFSTR("%d"), errno_number);
 143         } else {
 144             errno_code = CFRetain(CFSTR(""));
 145         }
 146
 147         int extended_code = sqlite3_extended_errcode(db);
 148         if (sql_code == extended_code)
 149             SecDbError(sql_code, error, CFSTR("%@: [%d]%@ %s"), message, sql_code, errno_code, sqlite3_errmsg(db));
 150         else
 151             SecDbError(sql_code, error, CFSTR("%@: [%d->%d]%@ %s"), message, sql_code, extended_code, errno_code, sqlite3_errmsg(db));
 152         CFReleaseSafe(message);
 153         CFReleaseSafe(errno_code);
 154     }
 155     return false;
 156 }
 157
 158 bool SecDbErrorWithStmt(int sql_code, sqlite3_stmt *stmt, CFErrorRef *error, CFStringRef format, ...) {
 159     if (sql_code == SQLITE_OK) return true;
 160     if (error) {
 161         va_list args;
 162         va_start(args, format);
 163         CFStringRef message = CFStringCreateWithFormatAndArguments(kCFAllocatorDefault, NULL, format, args);
 164         va_end(args);
 165
 166         sqlite3 *db = sqlite3_db_handle(stmt);
 167         const char *sql = sqlite3_sql(stmt);
 168         int extended_code = sqlite3_extended_errcode(db);
 169         if (sql_code == extended_code)
 170             SecDbError(sql_code, error, CFSTR("%@: [%d] %s sql: %s"), message, sql_code, sqlite3_errmsg(db), sql);
 171         else
 172             SecDbError(sql_code, error, CFSTR("%@: [%d->%d] %s sql: %s"), message, sql_code, extended_code, sqlite3_errmsg(db), sql);
 173         CFReleaseSafe(message);
 174     }
 175     return false;
 176 }
 177
 178 // A callback for the sqlite3_log() interface.
 179 static void sqlite3Log(void *pArg, int iErrCode, const char *zMsg){
 180     secdebug("sqlite3", "(%d) %s", iErrCode, zMsg);
 181 }
 182
 183 void _SecDbServerSetup(void)
 184 {
 185     static dispatch_once_t onceToken;
 186     dispatch_once(&onceToken, ^{
 187         int rx = sqlite3_config(SQLITE_CONFIG_LOG, sqlite3Log, NULL);
 188         if (SQLITE_OK != rx) {
 189             secwarning("Could not set up sqlite global error logging to syslog: %d", rx);
 190         }
 191     });
 192 }
 193
 194
 195 // MARK: -
 196 // MARK: Static helper functions
 197
 198 static bool SecDbOpenHandle(SecDbConnectionRef dbconn, bool *created, CFErrorRef *error);
 199 static bool SecDbHandleCorrupt(SecDbConnectionRef dbconn, int rc, CFErrorRef *error);
 200
 201 #pragma mark -
 202 #pragma mark SecDbRef
 203
 204 static CFStringRef
 205 SecDbCopyFormatDescription(CFTypeRef value, CFDictionaryRef formatOptions)
 206 {
 207     SecDbRef db = (SecDbRef)value;
 208     return CFStringCreateWithFormat(kCFAllocatorDefault, NULL, CFSTR("<SecDb path:%@ connections: %@>"), db->db_path, db->idleReadConnections);
 209 }
 210
 211
 212 static void
 213 SecDbDestroy(CFTypeRef value)
 214 {
 215     SecDbRef db = (SecDbRef)value;
 216
 217     CFReleaseNull(db->db_path);
 218     dispatch_sync(db->queue, ^{
 219         CFReleaseNull(db->idleWriteConnections);
 220         CFReleaseNull(db->idleReadConnections);
 221     });
 222
 223     if (db->queue) {
 224         dispatch_release(db->queue);
 225         db->queue = NULL;
 226     }
 227     if (db->commitQueue) {
 228         dispatch_release(db->commitQueue);
 229         db->commitQueue = NULL;
 230     }
 231
 232     pthread_mutex_destroy(&(db->writeMutex));
 233
 234     if (db->readSemaphore) {
 235         dispatch_release(db->readSemaphore);
 236         db->readSemaphore = NULL;
 237     }
 238
 239     if (db->opened) {
 240         Block_release(db->opened);
 241         db->opened = NULL;
 242     }
 243     CFReleaseNull(db->notifyPhase);
 244 }
 245
 246 CFGiblisFor(SecDb)
 247
 248 SecDbRef
 249 SecDbCreate(CFStringRef dbName, mode_t mode, bool readWrite, bool allowRepair, bool useWAL, bool useRobotVacuum, uint8_t maxIdleHandles,
 250                        bool (^opened)(SecDbRef db, SecDbConnectionRef dbconn, bool didCreate, bool *callMeAgainForNextConnection, CFErrorRef *error))
 251 {
 252     SecDbRef db = NULL;
 253
 254     db = CFTypeAllocate(SecDb, struct __OpaqueSecDb, kCFAllocatorDefault);
 255     require(db != NULL, done);
 256
 257     CFStringPerformWithCString(dbName, ^(const char *dbNameStr) {
 258         db->queue = dispatch_queue_create(dbNameStr, DISPATCH_QUEUE_SERIAL);
 259     });
 260     CFStringRef commitQueueStr = CFStringCreateWithFormat(kCFAllocatorDefault, NULL, CFSTR("%@-commit"), dbName);
 261     CFStringPerformWithCString(commitQueueStr, ^(const char *cqNameStr) {
 262         db->commitQueue = dispatch_queue_create(cqNameStr, DISPATCH_QUEUE_CONCURRENT);
 263     });
 264     CFReleaseNull(commitQueueStr);
 265     db->readSemaphore = dispatch_semaphore_create(kSecDbMaxReaders);
 266     if (pthread_mutex_init(&(db->writeMutex), NULL) != 0) {
 267         seccritical("SecDb: SecDbCreate failed to init the write mutex, this will end badly");
 268     }
 269     db->idleWriteConnections = CFArrayCreateMutableForCFTypes(kCFAllocatorDefault);
 270     db->idleReadConnections = CFArrayCreateMutableForCFTypes(kCFAllocatorDefault);
 271     db->opened = opened ? Block_copy(opened) : NULL;
 272     if (getenv("__OSINSTALL_ENVIRONMENT") != NULL) {
 273         // TODO: Move this code out of this layer
 274         secinfo("#SecDB", "SecDB: running from installer");
 275
 276         db->db_path = CFSTR("file::memory:?cache=shared");
 277     } else {
 278         db->db_path = CFStringCreateCopy(kCFAllocatorDefault, dbName);
 279     }
 280     db->mode = mode;
 281     db->readWrite = readWrite;
 282     db->allowRepair = allowRepair;
 283     db->useWAL = useWAL;
 284     db->useRobotVacuum = useRobotVacuum;
 285     db->maxIdleHandles = maxIdleHandles;
 286     db->corruptionReset = NULL;
 287
 288 done:
 289     return db;
 290 }
 291
 292 CFIndex
 293 SecDbIdleConnectionCount(SecDbRef db) {
 294     __block CFIndex count = 0;
 295     dispatch_sync(db->queue, ^{
 296         count = CFArrayGetCount(db->idleReadConnections);
 297         count += CFArrayGetCount(db->idleWriteConnections);
 298     });
 299     return count;
 300 }
 301
 302 void SecDbAddNotifyPhaseBlock(SecDbRef db, SecDBNotifyBlock notifyPhase)
 303 {
 304 #if !TARGET_OS_BRIDGE
 305     // SecDbNotifyPhase seems to mostly be called on the db's commitQueue, and not the db's queue. Therefore, protect the array with that queue.
 306     dispatch_sync(db->commitQueue, ^{
 307         SecDBNotifyBlock block = Block_copy(notifyPhase); /* Force the block off the stack */
 308         if (db->notifyPhase == NULL) {
 309             db->notifyPhase = CFArrayCreateMutableForCFTypes(NULL);
 310         }
 311         CFArrayAppendValue(db->notifyPhase, block);
 312         Block_release(block);
 313     });
 314 #endif
 315 }
 316
 317 static void SecDbNotifyPhase(SecDbConnectionRef dbconn, SecDbTransactionPhase phase) {
 318 #if !TARGET_OS_BRIDGE
 319     if (CFArrayGetCount(dbconn->changes)) {
 320         CFArrayRef changes = dbconn->changes;
 321         dbconn->changes = CFArrayCreateMutableForCFTypes(kCFAllocatorDefault);
 322         if (dbconn->db->notifyPhase) {
 323             CFArrayForEach(dbconn->db->notifyPhase, ^(const void *value) {
 324                 SecDBNotifyBlock notifyBlock = (SecDBNotifyBlock)value;
 325                 notifyBlock(dbconn, phase, dbconn->source, changes);
 326             });
 327         }
 328         CFReleaseSafe(changes);
 329     }
 330 #endif
 331 }
 332
 333 static void SecDbOnNotify(SecDbConnectionRef dbconn, void (^perform)(void)) {
 334     perform();
 335 }
 336
 337 CFStringRef SecDbGetPath(SecDbRef db) {
 338     if(!db) {
 339         return NULL;
 340     }
 341     return db->db_path;
 342 }
 343
 344
 345 #pragma mark -
 346 #pragma mark SecDbConnectionRef
 347
 348 static bool SecDbCheckCorrupted(SecDbConnectionRef dbconn)
 349 {
 350     __block bool checkDidRun = false;
 351     __block bool isCorrupted = false;
 352     __block CFErrorRef error = NULL;
 353     SecDbPrepare(dbconn, CFSTR("PRAGMA integrity_check"), &error, ^(sqlite3_stmt *stmt) {
 354         SecDbStep(dbconn, stmt, &error, ^(bool *stop) {
 355             const char * result = (const char*)sqlite3_column_text(stmt, 0);
 356             if (!result || strncasecmp(result, "ok", 3) != 0) {
 357                 isCorrupted = true;
 358                 secerror("SecDBCheckCorrupted integrity_check returned %s", (result) ? result : "NULL");
 359             }
 360             checkDidRun = true;
 361         });
 362     });
 363     if (!checkDidRun) {
 364         // An error occurred in SecDbPrepare before we could run the block.
 365         if (error) {
 366             CFIndex code = CFErrorGetCode(error);
 367             if (SQLITE_CORRUPT == code || SQLITE_NOTADB == code) {
 368                 isCorrupted = true;
 369             }
 370             secinfo("#SecDB", "#SecDB warning error %{public}@ when running integrity check", error);
 371         } else {
 372             // We don't have an error ref if SecDbPrepare has called SecDbConnectionCheckCode,
 373             // which then called SecDbHandleCorrupt. That code path is only entered when the
 374             // original error was SQLITE_CORRUPT or SQLITE_NOTADB. On other errors, the
 375             // CFErrorRef is not cleared and we can just check the code above.
 376             isCorrupted = true;
 377             secinfo("#SecDB", "#SecDB warning: failed to run integrity check due to corruption");
 378         }
 379     }
 380     if (isCorrupted) {
 381         if (checkDidRun) {
 382             secerror("SecDBCheckCorrupted ran integrity_check, and that didn't return ok");
 383         } else {
 384             secerror("SecDBCheckCorrupted failed to run integrity check");
 385         }
 386     }
 387     CFReleaseNull(error);
 388
 389     return isCorrupted;
 390 }
 391
 392 static bool SecDbDidCreateFirstConnection(SecDbConnectionRef dbconn, bool didCreate, CFErrorRef *error)
 393 {
 394     secinfo("#SecDB", "#SecDB starting maintenance");
 395     bool ok = true;
 396
 397     // Historical note: this used to check for integrity but that became too slow and caused panics at boot.
 398     // Now, just react to SQLite errors when doing an operation. If file on disk is borked it'll tell us right away.
 399
 400     if (!dbconn->isCorrupted && dbconn->db->opened) {
 401         CFErrorRef localError = NULL;
 402
 403         dbconn->db->callOpenedHandlerForNextConnection = false;
 404         ok = dbconn->db->opened(dbconn->db, dbconn, didCreate, &dbconn->db->callOpenedHandlerForNextConnection, &localError);
 405
 406         if (!ok)
 407             secerror("opened block failed: %@", localError);
 408
 409         if (!dbconn->isCorrupted && error && *error == NULL) {
 410             *error = localError;
 411             localError = NULL;
 412         } else {
 413             if (localError)
 414                 secerror("opened block failed: error (%@) is being released and lost", localError);
 415             CFReleaseNull(localError);
 416         }
 417     }
 418
 419     if (dbconn->isCorrupted) {
 420         ok = SecDbHandleCorrupt(dbconn, 0, error);
 421     }
 422
 423     secinfo("#SecDB", "#SecDB starting maintenance");
 424     return ok;
 425 }
 426
 427 void SecDbCorrupt(SecDbConnectionRef dbconn, CFErrorRef error)
 428 {
 429     if (__security_simulatecrash_enabled()) {
 430         os_log_fault(secLogObjForScope("SecEmergency"), "SecDBCorrupt: %@", error);
 431     }
 432     dbconn->isCorrupted = true;
 433     CFRetainAssign(dbconn->corruptionError, error);
 434 }
 435
 436
 437 static uint8_t knownDbPathIndex(SecDbConnectionRef dbconn)
 438 {
 439
 440     if(CFEqual(dbconn->db->db_path, CFSTR("/Library/Keychains/keychain-2.db")))
 441         return 1;
 442     if(CFEqual(dbconn->db->db_path, CFSTR("/Library/Keychains/ocspcache.sqlite3")))
 443         return 2;
 444     if(CFEqual(dbconn->db->db_path, CFSTR("/Library/Keychains/TrustStore.sqlite3")))
 445         return 3;
 446     if(CFEqual(dbconn->db->db_path, CFSTR("/Library/Keychains/caissuercache.sqlite3")))
 447         return 4;
 448
 449     /* Unknown DB path */
 450     return 0;
 451 }
 452
 453 static bool SecDbConnectionCheckCode(SecDbConnectionRef dbconn, int code, CFErrorRef *error, CFStringRef desc, ...)
 454     CF_FORMAT_FUNCTION(4, 5);
 455
 456 // Return true if there was no error, returns false otherwise and set *error to an appropriate CFErrorRef.
 457 static bool SecDbConnectionCheckCode(SecDbConnectionRef dbconn, int code, CFErrorRef *error, CFStringRef desc, ...) {
 458     if (code == SQLITE_OK || code == SQLITE_DONE)
 459         return true;
 460
 461     if (error) {
 462         va_list args;
 463         va_start(args, desc);
 464         CFStringRef msg = CFStringCreateWithFormatAndArguments(kCFAllocatorDefault, NULL, desc, args);
 465         va_end(args);
 466         SecDbErrorWithDb(code, dbconn->handle, error, CFSTR("%@"), msg);
 467         CFRelease(msg);
 468     }
 469
 470     dbconn->hasIOFailure |= (SQLITE_IOERR == code);
 471
 472     /* If it's already corrupted, don't try to recover */
 473     if (dbconn->isCorrupted) {
 474         CFStringRef reason = CFStringCreateWithFormat(kCFAllocatorDefault, NULL,
 475                                                       CFSTR("SQL DB %@ is corrupted already. Corruption error was: %d (previously %d)"),
 476                                                       dbconn->db->db_path, code, dbconn->maybeCorruptedCode);
 477         secerror("%@",reason);
 478         __security_simulatecrash(reason, __sec_exception_code_TwiceCorruptDb(knownDbPathIndex(dbconn)));
 479         CFReleaseSafe(reason);
 480         // We can't fall through to the checking case because it eventually calls SecDbConnectionCheckCode again.
 481         // However, this is the second time we're seeing corruption so let's take the ultimate measure.
 482         if ((SQLITE_CORRUPT == code) || (SQLITE_NOTADB == code)) {
 483             secerror("SecDbConnectionCheckCode detected corruption twice: going to handle corrupt DB");
 484             (void)SecDbHandleCorrupt(dbconn, code, error);
 485         }
 486         return false;
 487     }
 488
 489     // NOTADB means file is garbage, so it's functionally equivalent to corruption
 490     dbconn->isCorrupted = (SQLITE_CORRUPT == code) || (SQLITE_NOTADB == code);
 491     if (dbconn->isCorrupted) {
 492         /* Run integrity check and only make dbconn->isCorrupted true and
 493            run the corruption handler if the integrity check conclusively fails. */
 494         dbconn->maybeCorruptedCode = code;
 495         dbconn->isCorrupted = SecDbCheckCorrupted(dbconn);
 496         if (dbconn->isCorrupted) {
 497             secerror("operation returned code: %d integrity check=fail", code);
 498             (void)SecDbHandleCorrupt(dbconn, code, error);
 499         } else {
 500             secerror("operation returned code: %d: integrity check=pass", code);
 501         }
 502     }
 503
 504     return false;
 505 }
 506
 507 #define BUSY_TIMEOUT_MS (5 * 60 * 1000)  /* 5 minutes */
 508
 509 static int sleepBackoff[] = { 10, 20, 50, 100, 250 };
 510 static int sumBackoff[]   = { 10, 30, 80, 180, 430 };
 511 static int NumberOfSleepBackoff = sizeof(sleepBackoff)/sizeof(sleepBackoff[0]);
 512
 513 // Use these as silly hacks to encode the SQLite return code in the backtrace, for hang debugging purposes
 514 static void __attribute__((noinline)) SecDbLockSleep(int ms) {
 515     sqlite3_sleep(ms);
 516 }
 517
 518 static void __attribute__((noinline)) SecDbBusySleep(int ms) {
 519     sqlite3_sleep(ms);
 520 }
 521
 522 // Return true causes the operation to be tried again.
 523 // Note that we set sqlite3_busy_timeout on the connection, so anytime you're in here, it's likely due to SQLITE_LOCKED.
 524 static bool SecDbWaitIfNeeded(SecDbConnectionRef dbconn, int s3e, sqlite3_stmt *stmt, CFStringRef desc, int nTries, CFErrorRef *error) {
 525     if (((0xFF & s3e) == SQLITE_BUSY) || ((0xFF & s3e) == SQLITE_LOCKED)) {
 526         int totaltimeout, timeout;
 527
 528         _Static_assert(sizeof(sumBackoff) == sizeof(sleepBackoff), "matching arrays not matching");
 529         _Static_assert(sizeof(sumBackoff[0]) == sizeof(sleepBackoff[0]), "matching arrays not matching");
 530
 531         if (nTries < NumberOfSleepBackoff) {
 532             timeout = sleepBackoff[nTries];
 533             totaltimeout = sumBackoff[nTries];
 534         } else {
 535             timeout = sleepBackoff[NumberOfSleepBackoff - 1];
 536             totaltimeout = sumBackoff[NumberOfSleepBackoff - 1] + (timeout * (nTries - NumberOfSleepBackoff));
 537         }
 538         if (totaltimeout < BUSY_TIMEOUT_MS) {
 539             secinfo("#SecDB", "sqlite busy/locked: %d ntries: %d totaltimeout: %d", s3e, nTries, totaltimeout);
 540             if(((0xFF & s3e) == SQLITE_LOCKED)) {
 541                 SecDbLockSleep(timeout);
 542             } else {
 543                 SecDbBusySleep(timeout);
 544             }
 545             return true;
 546         } else {
 547             secinfo("#SecDB", "sqlite busy/locked: too long: %d ms, giving up", totaltimeout);
 548         }
 549     }
 550
 551     return SecDbConnectionCheckCode(dbconn, s3e, error, CFSTR("%@"), desc);
 552 }
 553
 554 enum SecDbStepResult {
 555     kSecDbErrorStep = 0,
 556     kSecDbRowStep = 1,
 557     kSecDbDoneStep = 2,
 558 };
 559 typedef enum SecDbStepResult SecDbStepResult;
 560
 561 static SecDbStepResult _SecDbStep(SecDbConnectionRef dbconn, sqlite3_stmt *stmt, CFErrorRef *error) {
 562     assert(stmt != NULL);
 563     int s3e;
 564     int ntries = 0;
 565     for (;;) {
 566         if (SecDbConnectionIsReadOnly(dbconn) && !sqlite3_stmt_readonly(stmt)) {
 567             secerror("_SecDbStep: SecDbConnection is readonly but we're about to write: %s", sqlite3_sql(stmt));
 568         }
 569         s3e = sqlite3_step(stmt);
 570         if (s3e == SQLITE_ROW) {
 571             return kSecDbRowStep;
 572         } else if (s3e == SQLITE_DONE) {
 573             /*
 574              ** ^[SQLITE_DONE] means that the statement has finished executing
 575              ** successfully.  sqlite3_step() should not be called again on this virtual
 576              ** machine without first calling [] to reset the virtual
 577              ** machine back to its initial state.
 578              */
 579             sqlite3_reset(stmt);
 580             return kSecDbDoneStep;
 581         } else if (!SecDbWaitIfNeeded(dbconn, s3e, stmt, CFSTR("step"), ntries, error)) {
 582             return kSecDbErrorStep;
 583         }
 584         ntries++;
 585     };
 586 }
 587
 588 bool
 589 SecDbExec(SecDbConnectionRef dbconn, CFStringRef sql, CFErrorRef *error)
 590 {
 591     bool ok = true;
 592     CFRetain(sql);
 593     while (sql) {
 594         CFStringRef tail = NULL;
 595         if (ok) {
 596             sqlite3_stmt *stmt = SecDbCopyStmt(dbconn, sql, &tail, error);
 597             ok = stmt != NULL;
 598             if (stmt) {
 599                 SecDbStepResult sr;
 600                 while ((sr = _SecDbStep(dbconn, stmt, error)) == kSecDbRowStep);
 601                 if (sr == kSecDbErrorStep)
 602                     ok = false;
 603                 ok &= SecDbReleaseCachedStmt(dbconn, sql, stmt, error);
 604             }
 605         } else {
 606             // TODO We already have an error here we really just want the left over sql in it's userData
 607             ok = SecDbError(SQLITE_ERROR, error, CFSTR("Error with unexecuted sql remaining %@"), sql);
 608         }
 609         CFRelease(sql);
 610         sql = tail;
 611     }
 612     return ok;
 613 }
 614
 615 static int SecDBGetInteger(SecDbConnectionRef dbconn, CFStringRef sql)
 616 {
 617     __block int number = -1;
 618     __block CFErrorRef error = NULL;
 619
 620     (void)SecDbWithSQL(dbconn, sql, &error, ^bool(sqlite3_stmt *sqlStmt) {
 621         (void)SecDbStep(dbconn, sqlStmt, &error, ^(bool *stop) {
 622             number = sqlite3_column_int(sqlStmt, 0);
 623             *stop = true;
 624         });
 625         return true;
 626     });
 627     CFReleaseNull(error);
 628     return number;
 629 }
 630
 631
 632 void SecDBManagementTasks(SecDbConnectionRef dbconn)
 633 {
 634     int64_t page_count = SecDBGetInteger(dbconn, CFSTR("pragma page_count"));
 635     if (page_count <= 0) {
 636         return;
 637     }
 638     int64_t free_count = SecDBGetInteger(dbconn, CFSTR("pragma freelist_count"));
 639     if (free_count < 0) {
 640         return;
 641     }
 642
 643     int64_t max_free = 8192;
 644
 645     int64_t pages_in_use = page_count - free_count;
 646     double loadFactor = ((double)pages_in_use/(double)page_count);
 647     if (0.85 < loadFactor && free_count < max_free) {
 648         /* no work yet */
 649     } else {
 650         int64_t pages_to_free = (int64_t)(0.2 * free_count);
 651         if (0.4 > loadFactor) {
 652             pages_to_free = free_count;
 653         }
 654
 655         char *formatString = NULL;
 656         asprintf(&formatString, "pragma incremental_vacuum(%d)", (int)pages_to_free);
 657         if (formatString) {
 658             char *sqlerror = NULL;
 659             int rc = sqlite3_exec(dbconn->handle, formatString, NULL, NULL, &sqlerror);
 660             if (rc) {
 661                 secerror("incremental_vacuum failed with: (%d) %{public}s", rc, sqlerror);
 662             }
 663             sqlite3_free(sqlerror);
 664             free(formatString);
 665         }
 666     }
 667 }
 668
 669
 670 static bool SecDbBeginTransaction(SecDbConnectionRef dbconn, SecDbTransactionType type, CFErrorRef *error)
 671 {
 672     bool ok = true;
 673     CFStringRef query;
 674     switch (type) {
 675         case kSecDbImmediateTransactionType:
 676             secdebug("db", "SecDbBeginTransaction SecDbBeginTransaction %p", dbconn);
 677             query = CFSTR("BEGIN IMMEDIATE");
 678             break;
 679         case kSecDbExclusiveRemoteSOSTransactionType:
 680             secdebug("db", "SecDbBeginTransaction kSecDbExclusiveRemoteSOSTransactionType %p", dbconn);
 681             dbconn->source = kSecDbSOSTransaction;
 682             query = CFSTR("BEGIN EXCLUSIVE");
 683             break;
 684         case kSecDbExclusiveRemoteCKKSTransactionType:
 685             secdebug("db", "SecDbBeginTransaction kSecDbExclusiveRemoteCKKSTransactionType %p", dbconn);
 686             dbconn->source = kSecDbCKKSTransaction;
 687             query = CFSTR("BEGIN EXCLUSIVE");
 688             break;
 689         case kSecDbExclusiveTransactionType:
 690             if (type==kSecDbExclusiveTransactionType)
 691                 secdebug("db", "SecDbBeginTransaction kSecDbExclusiveTransactionType %p", dbconn);
 692             query = CFSTR("BEGIN EXCLUSIVE");
 693             break;
 694         case kSecDbNormalTransactionType:
 695             secdebug("db", "SecDbBeginTransaction kSecDbNormalTransactionType %p", dbconn);
 696             query = CFSTR("BEGIN");
 697             break;
 698         default:
 699             secdebug("db", "SecDbBeginTransaction invalid transaction type %lu", type);
 700             ok = SecDbError(SQLITE_ERROR, error, CFSTR("invalid transaction type %d"), (int)type);
 701             query = NULL;
 702             break;
 703     }
 704
 705     if (query != NULL && sqlite3_get_autocommit(dbconn->handle) != 0) {
 706         ok = SecDbExec(dbconn, query, error);
 707     }
 708     if (ok)
 709         dbconn->inTransaction = true;
 710
 711     return ok;
 712 }
 713
 714 static bool SecDbEndTransaction(SecDbConnectionRef dbconn, bool commit, CFErrorRef *error)
 715 {
 716     __block bool ok = true;
 717     __block bool commited = false;
 718
 719     dispatch_block_t notifyAndExec = ^{
 720         if (commit) {
 721             //secdebug("db", "SecDbEndTransaction kSecDbTransactionWillCommit %p", dbconn);
 722             SecDbNotifyPhase(dbconn, kSecDbTransactionWillCommit);
 723             commited = ok = SecDbExec(dbconn, CFSTR("END"), error);
 724             //secdebug("db", "SecDbEndTransaction kSecDbTransactionWillCommit %p (after notify)", dbconn);
 725         } else {
 726             ok = SecDbExec(dbconn, CFSTR("ROLLBACK"), error);
 727             commited = false;
 728         }
 729         dbconn->inTransaction = false;
 730         SecDbNotifyPhase(dbconn, commited ? kSecDbTransactionDidCommit : kSecDbTransactionDidRollback);
 731         secdebug("db", "SecDbEndTransaction %s %p", commited ? "kSecDbTransactionDidCommit" : "kSecDbTransactionDidRollback", dbconn);
 732         dbconn->source = kSecDbAPITransaction;
 733
 734         if (commit && dbconn->db->useRobotVacuum) {
 735             SecDBManagementTasks(dbconn);
 736         }
 737     };
 738
 739     SecDbPerformOnCommitQueue(dbconn, true, notifyAndExec);
 740
 741     return ok;
 742 }
 743
 744 bool SecDbTransaction(SecDbConnectionRef dbconn, SecDbTransactionType type,
 745                       CFErrorRef *error, void (^transaction)(bool *commit))
 746 {
 747     bool ok = true;
 748     bool commit = true;
 749
 750     if (dbconn->inTransaction) {
 751         transaction(&commit);
 752         if (!commit) {
 753             secinfo("#SecDB", "#SecDB nested transaction asked to not be committed");
 754         }
 755     } else {
 756         ok = SecDbBeginTransaction(dbconn, type, error);
 757         if (ok) {
 758             transaction(&commit);
 759             ok = SecDbEndTransaction(dbconn, commit, error);
 760         }
 761     }
 762
 763     return ok && commit;
 764 }
 765
 766 sqlite3 *SecDbHandle(SecDbConnectionRef dbconn) {
 767     return dbconn->handle;
 768 }
 769
 770 bool SecDbStep(SecDbConnectionRef dbconn, sqlite3_stmt *stmt, CFErrorRef *error, void (^row)(bool *stop)) {
 771     for (;;) {
 772         switch (_SecDbStep(dbconn, stmt, error)) {
 773             case kSecDbErrorStep:
 774                 secdebug("db", "kSecDbErrorStep %@", error ? *error : NULL);
 775                 return false;
 776             case kSecDbRowStep:
 777 #if SECDB_DEBUGGING
 778                 secdebug("db", "kSecDbRowStep %@", error ? *error : NULL);
 779 #endif
 780                 if (row) {
 781                     __block bool stop = false;
 782                     SecAutoreleaseInvokeWithPool(^{
 783                         row(&stop);
 784                     });
 785                     if (stop)
 786                         return true;
 787                     break;
 788                 }
 789                 SecDbError(SQLITE_ERROR, error, CFSTR("SecDbStep SQLITE_ROW returned without a row handler"));
 790                 return false;
 791             case kSecDbDoneStep:
 792 #if SECDB_DEBUGGING
 793                 secdebug("db", "kSecDbDoneStep %@", error ? *error : NULL);
 794 #endif
 795                 return true;
 796         }
 797     }
 798 }
 799
 800 bool SecDbCheckpoint(SecDbConnectionRef dbconn, CFErrorRef *error)
 801 {
 802     return SecDbConnectionCheckCode(dbconn, sqlite3_wal_checkpoint(dbconn->handle, NULL), error, CFSTR("wal_checkpoint"));
 803 }
 804
 805 static sqlite3 *_SecDbOpenV2(const char *path,
 806                              int flags,
 807                              int useWAL,
 808                              int useRobotVacuum,
 809                              CFErrorRef *error) {
 810     sqlite3 *handle = NULL;
 811     int s3e = sqlite3_open_v2(path, &handle, flags, NULL);
 812     if (s3e) {
 813         if (handle) {
 814             SecDbErrorWithDb(s3e, handle, error, CFSTR("open_v2 \"%s\" 0x%X"), path, flags);
 815             sqlite3_close(handle);
 816             handle = NULL;
 817         } else {
 818             SecDbError(s3e, error, CFSTR("open_v2 \"%s\" 0x%X"), path, flags);
 819         }
 820     } else if (SQLITE_OPEN_READWRITE == (flags & SQLITE_OPEN_READWRITE)) {
 821         if (useRobotVacuum) {
 822 #define SECDB_SQLITE_AUTO_VACUUM_INCREMENTAL 2
 823             sqlite3_stmt *stmt = NULL;
 824             int vacuumMode = -1;
 825
 826             /*
 827              * Setting auto_vacuum = incremental on a database that is not empty requires
 828              * a VACCUUM, so check if the vacuum mode is not INCREMENTAL, and if its not,
 829              * set it to incremental and vacuum.
 830              */
 831
 832             s3e = sqlite3_prepare_v2(handle, "PRAGMA auto_vacuum", -1, &stmt, NULL);
 833             if (s3e == 0) {
 834                 s3e = sqlite3_step(stmt);
 835                 if (s3e == SQLITE_ROW) {
 836                     vacuumMode = sqlite3_column_int(stmt, 0);
 837                 }
 838                 (void)sqlite3_finalize(stmt);
 839             }
 840
 841             if (vacuumMode != SECDB_SQLITE_AUTO_VACUUM_INCREMENTAL) {
 842                 (void)sqlite3_exec(handle, "PRAGMA auto_vacuum = incremental", NULL, NULL, NULL);
 843                 (void)sqlite3_exec(handle, "VACUUM", NULL, NULL, NULL);
 844             }
 845         }
 846         if (useWAL) {
 847             (void)sqlite3_exec(handle, "PRAGMA journal_mode = WAL", NULL, NULL, NULL);
 848         }
 849
 850         // Let SQLite handle timeouts.
 851         sqlite3_busy_timeout(handle, 5*1000);
 852     }
 853     return handle;
 854 }
 855
 856 static bool SecDbOpenV2(SecDbConnectionRef dbconn, const char *path, int flags, CFErrorRef *error) {
 857     return (dbconn->handle = _SecDbOpenV2(path, flags, dbconn->db->useWAL, dbconn->db->useRobotVacuum, error)) != NULL;
 858 }
 859
 860 // This construction lets tests not exit here
 861 static void SecDbProductionCorruptionExitHandler(void)
 862 {
 863     exit(EXIT_FAILURE);
 864 }
 865 void (*SecDbCorruptionExitHandler)(void) = SecDbProductionCorruptionExitHandler;
 866
 867 void SecDbResetCorruptionExitHandler(void)
 868 {
 869     SecDbCorruptionExitHandler = SecDbProductionCorruptionExitHandler;
 870 }
 871
 872 /*
 873  There's not much to do in here because we should only ever be here when
 874  SQLite tells us the DB is corrupt, or the DB is unrecoverable because of
 875  some fatal logic problem. But we can't shoot it dead either due to client
 876  connections. So, first we create a marker to tell ourselves things are bad,
 877  then we'll die. When we come back up we'll notice the marker and remove the DB.
 878  */
 879 static bool SecDbHandleCorrupt(SecDbConnectionRef dbconn, int rc, CFErrorRef *error)
 880 {
 881     if (!dbconn->db->allowRepair) {
 882         SecCFCreateErrorWithFormat(rc, kSecErrnoDomain, NULL, error, NULL,
 883                                    CFSTR("SecDbHandleCorrupt not allowed to repair, handled error: [%d] %s"), rc, strerror(rc));
 884         dbconn->isCorrupted = false;
 885         return false;
 886     }
 887
 888     CFStringPerformWithCString(dbconn->db->db_path, ^(const char *db_path) {
 889         char marker[PATH_MAX+1];
 890         snprintf(marker, sizeof(marker), "%s-iscorrupt", db_path);
 891         struct stat info = {};
 892         if (0 == stat(marker, &info)) {
 893             secerror("SecDbHandleCorrupt: Tried to write corruption marker %s but one already exists", marker);
 894         }
 895
 896         FILE* file = fopen(marker, "w");
 897         if (file == NULL) {
 898             secerror("SecDbHandleCorrupt: Unable (%{darwin.errno}d) to create corruption marker %{public}s", errno, marker);
 899         } else {
 900             fclose(file);
 901         }
 902     });
 903
 904     secwarning("SecDbHandleCorrupt: killing self so that successor might cleanly delete corrupt db");
 905
 906     // Call through function pointer so tests can replace it and call a SecKeychainDbReset instead
 907     SecDbCorruptionExitHandler();
 908     return true;
 909 }
 910
 911 static bool SecDbProcessCorruptionMarker(CFStringRef db_path) {
 912     __block bool ok = true;
 913     CFStringPerformWithCString(db_path, ^(const char *db_path) {
 914         char marker[PATH_MAX+1];
 915         snprintf(marker, sizeof(marker), "%s-iscorrupt", db_path);
 916         struct stat info = {};
 917         int result = stat(marker, &info);
 918         if (result != 0 && errno == ENOENT) {
 919             return;
 920         } else if (result != 0) {
 921             secerror("SecDbSecDbProcessCorruptionMarker: Unable to check for corruption marker: %{darwin.errno}d", errno);
 922             return;
 923         }
 924
 925         secwarning("SecDbSecDbProcessCorruptionMarker: found corruption marker %s", marker);
 926         if (remove(marker)) {
 927             secerror("SecDbSecDbProcessCorruptionMarker: Unable (%{darwin.errno}d) to delete corruption marker", errno);
 928             ok = false;
 929         } else if (remove(db_path) && errno != ENOENT) {    // Not sure how we'd get ENOENT but it would suit us just fine
 930             secerror("SecDbSecDbProcessCorruptionMarker: Unable (%{darwin.errno}d) to delete db %{public}s", errno, db_path);
 931             ok = false;
 932         } else {
 933             secwarning("SecDbSecDbProcessCorruptionMarker: deleted corrupt db %{public}s", db_path);
 934         }
 935     });
 936     return ok;
 937 }
 938
 939 void
 940 SecDbSetCorruptionReset(SecDbRef db, void (^corruptionReset)(void))
 941 {
 942     if (db->corruptionReset) {
 943         Block_release(db->corruptionReset);
 944         db->corruptionReset = NULL;
 945     }
 946     if (corruptionReset) {
 947         db->corruptionReset = Block_copy(corruptionReset);
 948     }
 949 }
 950
 951 static bool SecDbLoggingEnabled(CFStringRef type)
 952 {
 953     CFTypeRef profile = NULL;
 954     bool enabled = false;
 955
 956     if (csr_check(CSR_ALLOW_APPLE_INTERNAL) != 0)
 957         return false;
 958
 959     profile = (CFNumberRef)CFPreferencesCopyValue(CFSTR("SQLProfile"), CFSTR("com.apple.security"), kCFPreferencesAnyUser, kCFPreferencesAnyHost);
 960
 961     if (profile == NULL)
 962         return false;
 963
 964     if (CFGetTypeID(profile) == CFBooleanGetTypeID()) {
 965         enabled = CFBooleanGetValue((CFBooleanRef)profile);
 966     } else if (CFGetTypeID(profile) == CFNumberGetTypeID()) {
 967         int32_t num = 0;
 968         CFNumberGetValue(profile, kCFNumberSInt32Type, &num);
 969         enabled = !!num;
 970     }
 971
 972     CFReleaseSafe(profile);
 973
 974     return enabled;
 975 }
 976
 977 static unsigned
 978 SecDbProfileMask(void)
 979 {
 980     static dispatch_once_t onceToken;
 981     static unsigned profile_mask = 0;
 982
 983     // sudo defaults write /Library/Preferences/com.apple.security SQLProfile -bool true
 984     dispatch_once(&onceToken, ^{
 985         if (SecDbLoggingEnabled(CFSTR("SQLProfile")))
 986             profile_mask = SQLITE_TRACE_PROFILE;
 987 #if DEBUG
 988         profile_mask |= SQLITE_TRACE_STMT;
 989 #else
 990         if (SecDbLoggingEnabled(CFSTR("SQLTrace")))
 991             profile_mask = SQLITE_TRACE_STMT;
 992 #endif
 993         if (SecDbLoggingEnabled(CFSTR("SQLRow")))
 994             profile_mask = SQLITE_TRACE_ROW;
 995         secinfo("#SecDB", "sqlDb: sql trace mask: 0x%08x", profile_mask);
 996     });
 997     return profile_mask;
 998 }
 999
1000 static int
1001 SecDbTraceV2(unsigned mask, void *ctx, void *p, void *x) {
1002     SecDbConnectionRef dbconn __unused = ctx;
1003
1004 #if SECDB_DEBUGGING
1005     const char *trace = "unknown";
1006     char *tofree = NULL;
1007
1008     if (mask == SQLITE_TRACE_PROFILE)
1009         trace = sqlite3_sql(p);
1010     else if (mask == SQLITE_TRACE_STMT) {
1011         trace = sqlite3_sql(p);
1012     } else if (mask == SQLITE_TRACE_ROW) {
1013         trace = tofree = sqlite3_expanded_sql(p);
1014     }
1015
1016     secinfo("#SecDB", "#SecDB %{public}s", trace);
1017
1018     sqlite3_free(tofree);
1019 #endif
1020
1021     return 0;
1022 }
1023
1024
1025 static bool SecDbOpenHandle(SecDbConnectionRef dbconn, bool *created, CFErrorRef *error)
1026 {
1027     __block bool ok = true;
1028
1029     // This is pretty terrible because now what? We know we have a corrupt DB
1030     // and now we can't get rid of it.
1031     if (!SecDbProcessCorruptionMarker(dbconn->db->db_path)) {
1032         SecCFCreateErrorWithFormat(errno, kSecErrnoDomain, NULL, error, NULL, CFSTR("Unable to process corruption marker: %{darwin.errno}d"), errno);
1033         return false;
1034     }
1035
1036     CFStringPerformWithCString(dbconn->db->db_path, ^(const char *db_path) {
1037         int flags = (dbconn->db->readWrite) ? SQLITE_OPEN_READWRITE : SQLITE_OPEN_READONLY;
1038         ok = created && SecDbOpenV2(dbconn, db_path, flags, NULL);
1039         if (!ok) {
1040             ok = true;
1041             if (created) {
1042                 char *tmp = dirname((char *)db_path);
1043                 if (tmp) {
1044                     mode_t omode = dbconn->db->mode;
1045                     if (omode & S_IRUSR) { omode |= S_IXUSR; } // owner can read
1046                     if (omode & S_IRGRP) { omode |= S_IXGRP; } // group can read
1047                     if (omode & S_IROTH) { omode |= S_IXOTH; } // other can read
1048                     int errnum = mkpath_np(tmp, omode);
1049                     if (errnum != 0 && errnum != EEXIST) {
1050                         SecCFCreateErrorWithFormat(errnum, kSecErrnoDomain, NULL, error, NULL,
1051                                                    CFSTR("mkpath_np %s: [%d] %s"), tmp, errnum, strerror(errnum));
1052                         ok = false;
1053                     }
1054                 }
1055             }
1056             // if the enclosing directory is ok, try to create the database.
1057             // this forces us to open it read-write, so we'll need to be the owner here.
1058             ok = ok && SecDbOpenV2(dbconn, db_path, SQLITE_OPEN_READWRITE | SQLITE_OPEN_CREATE, error);
1059             if (ok) {
1060                 chmod(db_path, dbconn->db->mode); // default: 0600 (S_IRUSR | S_IWUSR)
1061                 if (created)
1062                     *created = true;
1063             }
1064         }
1065
1066         if (ok) {
1067             unsigned mask = SecDbProfileMask();
1068             if (mask) {
1069                 (void)sqlite3_trace_v2(dbconn->handle,
1070                                        mask,
1071                                        SecDbTraceV2,
1072                                        dbconn);
1073             }
1074         }
1075     });
1076
1077     return ok;
1078 }
1079
1080 static SecDbConnectionRef
1081 SecDbConnectionCreate(SecDbRef db, bool readOnly, CFErrorRef *error)
1082 {
1083     SecDbConnectionRef dbconn = NULL;
1084
1085     dbconn = CFTypeAllocate(SecDbConnection, struct __OpaqueSecDbConnection, kCFAllocatorDefault);
1086     require(dbconn != NULL, done);
1087
1088     dbconn->db = db;
1089     dbconn->readOnly = readOnly;
1090     dbconn->inTransaction = false;
1091     dbconn->source = kSecDbInvalidTransaction;
1092     dbconn->isCorrupted = false;
1093     dbconn->maybeCorruptedCode = 0;
1094     dbconn->hasIOFailure = false;
1095     dbconn->corruptionError = NULL;
1096     dbconn->handle = NULL;
1097     dbconn->changes = CFArrayCreateMutableForCFTypes(kCFAllocatorDefault);
1098
1099 done:
1100     return dbconn;
1101 }
1102
1103 bool SecDbConnectionIsReadOnly(SecDbConnectionRef dbconn) {
1104     return dbconn->readOnly;
1105 }
1106
1107 static void SecDbConectionSetReadOnly(SecDbConnectionRef dbconn, bool readOnly) {
1108     dbconn->readOnly = readOnly;
1109 }
1110
1111 SecDbConnectionRef SecDbConnectionAcquire(SecDbRef db, bool readOnly, CFErrorRef *error) {
1112     SecDbConnectionRef dbconn = NULL;
1113     SecDbConnectionAcquireRefMigrationSafe(db, readOnly, &dbconn, error);
1114     return dbconn;
1115 }
1116
1117 static void SecDbConnectionConsumeResource(SecDbRef db, bool readOnly) {
1118     if (readOnly) {
1119         dispatch_semaphore_wait(db->readSemaphore, DISPATCH_TIME_FOREVER);
1120     } else {
1121         pthread_mutex_lock(&(db->writeMutex));
1122     }
1123 }
1124
1125 static void SecDbConnectionMakeResourceAvailable(SecDbRef db, bool readOnly) {
1126     if (readOnly) {
1127         dispatch_semaphore_signal(db->readSemaphore);
1128     } else {
1129         pthread_mutex_unlock(&(db->writeMutex));
1130     }
1131 }
1132
1133 bool SecDbConnectionAcquireRefMigrationSafe(SecDbRef db, bool readOnly, SecDbConnectionRef* dbconnRef, CFErrorRef *error)
1134 {
1135     CFRetain(db);
1136 #if SECDB_DEBUGGING
1137     secinfo("dbconn", "acquire %s connection", readOnly ? "ro" : "rw");
1138 #endif
1139     SecDbConnectionConsumeResource(db, readOnly);
1140
1141     __block SecDbConnectionRef dbconn = NULL;
1142     __block bool ok = true;
1143     __block bool ranOpenedHandler = false;
1144
1145     bool (^assignDbConn)(SecDbConnectionRef) = ^bool(SecDbConnectionRef connection) {
1146         dbconn = connection;
1147         if (dbconnRef) {
1148             *dbconnRef = connection;
1149         }
1150
1151         return dbconn != NULL;
1152     };
1153
1154     dispatch_sync(db->queue, ^{
1155         if (!db->didFirstOpen) {
1156             bool didCreate = false;
1157             ok = assignDbConn(SecDbConnectionCreate(db, false, error));
1158             CFErrorRef localError = NULL;
1159             if (ok && !SecDbOpenHandle(dbconn, &didCreate, &localError)) {
1160                 secerror("Unable to create database: %@", localError);
1161                 if (localError && CFEqual(CFErrorGetDomain(localError), kSecDbErrorDomain)) {
1162                     int code = (int)CFErrorGetCode(localError);
1163                     dbconn->isCorrupted = (SQLITE_CORRUPT == code) || (SQLITE_NOTADB == code);
1164                 }
1165                 // If the open failure isn't due to corruption, propagate the error.
1166                 ok = dbconn->isCorrupted;
1167                 if (!ok && error && *error == NULL) {
1168                     *error = localError;
1169                     localError = NULL;
1170                 }
1171             }
1172             CFReleaseNull(localError);
1173
1174             if (ok) {
1175                 db->didFirstOpen = ok = SecDbDidCreateFirstConnection(dbconn, didCreate, error);
1176                 ranOpenedHandler = true;
1177                 SecDbConectionSetReadOnly(dbconn, readOnly);    // first connection always created "rw", so set to real value
1178             } else {
1179                 CFReleaseNull(dbconn);
1180             }
1181         } else {
1182             /* Try to get one from the cache */
1183             CFMutableArrayRef cache = readOnly ? db->idleReadConnections : db->idleWriteConnections;
1184             if (CFArrayGetCount(cache) && !dbconn) {
1185                 if (assignDbConn((SecDbConnectionRef)CFArrayGetValueAtIndex(cache, 0))) {
1186                     CFRetainSafe(dbconn);
1187                 }
1188                 CFArrayRemoveValueAtIndex(cache, 0);
1189             }
1190         }
1191     });
1192
1193     if (ok && !dbconn) {
1194         /* Nothing found in cache, create a new connection */
1195         bool created = false;
1196         if (assignDbConn(SecDbConnectionCreate(db, readOnly, error)) && !SecDbOpenHandle(dbconn, &created, error)) {
1197             CFReleaseNull(dbconn);
1198         }
1199     }
1200
1201     if (dbconn && !ranOpenedHandler && dbconn->db->opened) {
1202         dispatch_sync(db->queue, ^{
1203             if (dbconn->db->callOpenedHandlerForNextConnection) {
1204                 dbconn->db->callOpenedHandlerForNextConnection = false;
1205                 if (!dbconn->db->opened(db, dbconn, false, &dbconn->db->callOpenedHandlerForNextConnection, error)) {
1206                     if (!dbconn->isCorrupted || !SecDbHandleCorrupt(dbconn, 0, error)) {
1207                         CFReleaseNull(dbconn);
1208                     }
1209                 }
1210             }
1211         });
1212     }
1213
1214     if (dbconnRef) {
1215         *dbconnRef = dbconn;
1216     }
1217
1218     if (!dbconn) {
1219         // Caller doesn't get (to use) a connection so the backing synchronization primitive is available again
1220         SecDbConnectionMakeResourceAvailable(db, readOnly);
1221         CFRelease(db);
1222     }
1223
1224     return dbconn ? true : false;
1225 }
1226
1227 void SecDbConnectionRelease(SecDbConnectionRef dbconn) {
1228     if (!dbconn) {
1229         secerror("SecDbConnectionRelease called with NULL dbconn");
1230         return;
1231     }
1232     SecDbRef db = dbconn->db;
1233 #if SECDB_DEBUGGING
1234     secinfo("dbconn", "release %@", dbconn);
1235 #endif
1236
1237     bool readOnly = SecDbConnectionIsReadOnly(dbconn);
1238     dispatch_sync(db->queue, ^{
1239         if (dbconn->hasIOFailure) {
1240             // Something wrong on the file layer (e.g. revoked file descriptor for networked home)
1241             secwarning("SecDbConnectionRelease: IO failure reported in connection, throwing away currently idle caches");
1242             // Any other checked-out connections are beyond our grasp. If they did not have IO failures they'll come back,
1243             // otherwise this branch gets taken more than once and gradually those connections die off
1244             CFArrayRemoveAllValues(db->idleWriteConnections);
1245             CFArrayRemoveAllValues(db->idleReadConnections);
1246         } else {
1247             CFMutableArrayRef cache = readOnly ? db->idleReadConnections : db->idleWriteConnections;
1248             CFIndex count = CFArrayGetCount(cache);
1249             if ((unsigned long)count < (readOnly ? kSecDbMaxReaders : kSecDbMaxWriters)) {
1250                 CFArrayAppendValue(cache, dbconn);
1251             } else {
1252                 secerror("dbconn: did not expect to run out of room in the %s cache when releasing connection", readOnly ? "ro" : "rw");
1253             }
1254         }
1255     });
1256
1257     // Signal after we have put the connection back in the pool of connections
1258     SecDbConnectionMakeResourceAvailable(db, readOnly);
1259     CFRelease(dbconn);
1260     CFRelease(db);
1261 }
1262
1263 void SecDbReleaseAllConnections(SecDbRef db) {
1264     // Force all connections to be removed (e.g. file descriptor no longer valid)
1265     if (!db) {
1266         secerror("called with NULL db");
1267         return;
1268     }
1269     dispatch_sync(db->queue, ^{
1270         CFArrayRemoveAllValues(db->idleReadConnections);
1271         CFArrayRemoveAllValues(db->idleWriteConnections);
1272     });
1273 }
1274
1275 static void onQueueSecDbForceCloseForCache(CFMutableArrayRef cache) {
1276     CFArrayForEach(cache, ^(const void* ptr) {
1277         SecDbConnectionRef connection = (SecDbConnectionRef)ptr;
1278
1279         // this pointer is claimed to be nonretained
1280         connection->db = NULL;
1281
1282         if(connection->handle) {
1283             sqlite3_close(connection->handle);
1284             connection->handle = NULL;
1285         }
1286     });
1287     CFArrayRemoveAllValues(cache);
1288 }
1289
1290 // Please make sure you want to do this. Any use of the outstanding connections to this DB will cause a crash.
1291 void SecDbForceClose(SecDbRef db) {
1292     dispatch_sync(db->queue, ^{
1293         onQueueSecDbForceCloseForCache(db->idleReadConnections);
1294         onQueueSecDbForceCloseForCache(db->idleWriteConnections);
1295     });
1296 }
1297
1298 bool SecDbPerformRead(SecDbRef db, CFErrorRef *error, void (^perform)(SecDbConnectionRef dbconn)) {
1299     SecDbConnectionRef dbconn = SecDbConnectionAcquire(db, true, error);
1300     bool success = false;
1301     if (dbconn) {
1302         perform(dbconn);
1303         success = true;
1304         SecDbConnectionRelease(dbconn);
1305     }
1306     return success;
1307 }
1308
1309 bool SecDbPerformWrite(SecDbRef db, CFErrorRef *error, void (^perform)(SecDbConnectionRef dbconn)) {
1310     if(!db) {
1311         SecError(errSecNotAvailable, error, CFSTR("failed to get a db handle"));
1312         return false;
1313     }
1314     SecDbConnectionRef dbconn = SecDbConnectionAcquire(db, false, error);
1315     bool success = false;
1316     if (dbconn) {
1317         perform(dbconn);
1318         success = true;
1319         SecDbConnectionRelease(dbconn);
1320     }
1321     return success;
1322 }
1323
1324 static CFStringRef
1325 SecDbConnectionCopyFormatDescription(CFTypeRef value, CFDictionaryRef formatOptions)
1326 {
1327     SecDbConnectionRef dbconn = (SecDbConnectionRef)value;
1328     return CFStringCreateWithFormat(kCFAllocatorDefault, NULL, CFSTR("<SecDbConnection %s %s>"),
1329                                     dbconn->readOnly ? "ro" : "rw", dbconn->handle ? "open" : "closed");
1330 }
1331
1332 static void
1333 SecDbConnectionDestroy(CFTypeRef value)
1334 {
1335     SecDbConnectionRef dbconn = (SecDbConnectionRef)value;
1336     if (dbconn->handle) {
1337         int s3e = sqlite3_close(dbconn->handle);
1338         if (s3e != SQLITE_OK) {
1339             secerror("failed to close database connection (%d) for %@: %s", s3e, dbconn->db->db_path, sqlite3_errmsg(dbconn->handle));
1340         }
1341         os_assert(s3e == SQLITE_OK); // Crash now or jetsam later
1342     }
1343     dbconn->db = NULL;
1344     CFReleaseNull(dbconn->changes);
1345     CFReleaseNull(dbconn->corruptionError);
1346
1347 }
1348
1349 void SecDbPerformOnCommitQueue(SecDbConnectionRef dbconn, bool barrier, dispatch_block_t perform) {
1350     if (barrier) {
1351         dispatch_barrier_sync(dbconn->db->commitQueue, ^{
1352             perform();
1353         });
1354     } else {
1355         dispatch_sync(dbconn->db->commitQueue, ^{
1356             perform();
1357         });
1358     }
1359 }
1360
1361 // MARK: -
1362 // MARK: Bind helpers
1363
1364 // Logging binds is very spammy when debug logging is on (~90% of log lines), and isn't often useful.
1365 // Enable this in your local build if you actually want every single SQL variable bind logged for debugging.
1366 #define LOG_SECDB_BINDS 0
1367
1368 bool SecDbBindBlob(sqlite3_stmt *stmt, int param, const void *zData, size_t n, void(*xDel)(void*), CFErrorRef *error) {
1369     if (n > INT_MAX) {
1370         return SecDbErrorWithStmt(SQLITE_TOOBIG, stmt, error,
1371                                   CFSTR("bind_blob[%d]: blob bigger than INT_MAX"), param);
1372     }
1373     bool ok = SecDbErrorWithStmt(sqlite3_bind_blob(stmt, param, zData, (int)n, xDel),
1374                                  stmt, error, CFSTR("bind_blob[%d]"), param);
1375 #if LOG_SECDB_BINDS
1376     secinfo("bind", "bind_blob[%d]: %.*P: %@", param, (int)n, zData, error ? *error : NULL);
1377 #endif
1378     return ok;
1379 }
1380
1381 bool SecDbBindText(sqlite3_stmt *stmt, int param, const char *zData, size_t n, void(*xDel)(void*), CFErrorRef *error) {
1382     if (n > INT_MAX) {
1383         return SecDbErrorWithStmt(SQLITE_TOOBIG, stmt, error,
1384                                   CFSTR("bind_text[%d]: text bigger than INT_MAX"), param);
1385     }
1386     bool ok = SecDbErrorWithStmt(sqlite3_bind_text(stmt, param, zData, (int)n, xDel), stmt, error,
1387                                  CFSTR("bind_text[%d]"), param);
1388 #if LOG_SECDB_BINDS
1389     secinfo("bind", "bind_text[%d]: \"%s\" error: %@", param, zData, error ? *error : NULL);
1390 #endif
1391     return ok;
1392 }
1393
1394 bool SecDbBindDouble(sqlite3_stmt *stmt, int param, double value, CFErrorRef *error) {
1395     bool ok = SecDbErrorWithStmt(sqlite3_bind_double(stmt, param, value), stmt, error,
1396                                  CFSTR("bind_double[%d]"), param);
1397 #if LOG_SECDB_BINDS
1398     secinfo("bind", "bind_double[%d]: %f error: %@", param, value, error ? *error : NULL);
1399 #endif
1400     return ok;
1401 }
1402
1403 bool SecDbBindInt(sqlite3_stmt *stmt, int param, int value, CFErrorRef *error) {
1404     bool ok = SecDbErrorWithStmt(sqlite3_bind_int(stmt, param, value), stmt, error,
1405                                  CFSTR("bind_int[%d]"), param);
1406 #if LOG_SECDB_BINDS
1407     secinfo("bind", "bind_int[%d]: %d error: %@", param, value, error ? *error : NULL);
1408 #endif
1409     return ok;
1410 }
1411
1412 bool SecDbBindInt64(sqlite3_stmt *stmt, int param, sqlite3_int64 value, CFErrorRef *error) {
1413     bool ok = SecDbErrorWithStmt(sqlite3_bind_int64(stmt, param, value), stmt, error,
1414                                  CFSTR("bind_int64[%d]"), param);
1415 #if LOG_SECDB_BINDS
1416     secinfo("bind", "bind_int64[%d]: %lld error: %@", param, value, error ? *error : NULL);
1417 #endif
1418     return ok;
1419 }
1420
1421
1422 /* AUDIT[securityd](done):
1423  value (ok) is a caller provided, non NULL CFTypeRef.
1424  */
1425 bool SecDbBindObject(sqlite3_stmt *stmt, int param, CFTypeRef value, CFErrorRef *error) {
1426     CFTypeID valueId;
1427     __block bool result = false;
1428
1429         /* TODO: Can we use SQLITE_STATIC below everwhere we currently use
1430      SQLITE_TRANSIENT since we finalize the statement before the value
1431      goes out of scope? */
1432     if (!value || (valueId = CFGetTypeID(value)) == CFNullGetTypeID()) {
1433         /* Skip bindings for NULL values.  sqlite3 will interpret unbound
1434          params as NULL which is exactly what we want. */
1435         result = true;
1436     } else if (valueId == CFStringGetTypeID()) {
1437         CFStringPerformWithCStringAndLength(value, ^(const char *cstr, size_t clen) {
1438             result = SecDbBindText(stmt, param, cstr, clen, SQLITE_TRANSIENT, error);
1439         });
1440     } else if (valueId == CFDataGetTypeID()) {
1441         CFIndex len = CFDataGetLength(value);
1442         if (len) {
1443             result = SecDbBindBlob(stmt, param, CFDataGetBytePtr(value),
1444                                    len, SQLITE_TRANSIENT, error);
1445         } else {
1446             result = SecDbBindText(stmt, param, "", 0, SQLITE_TRANSIENT, error);
1447         }
1448     } else if (valueId == CFDateGetTypeID()) {
1449         CFAbsoluteTime abs_time = CFDateGetAbsoluteTime(value);
1450         result = SecDbBindDouble(stmt, param, abs_time, error);
1451     } else if (valueId == CFBooleanGetTypeID()) {
1452         int bval = CFBooleanGetValue(value);
1453         result = SecDbBindInt(stmt, param, bval, error);
1454     } else if (valueId == CFNumberGetTypeID()) {
1455         Boolean convertOk;
1456         if (CFNumberIsFloatType(value)) {
1457             double nval;
1458             convertOk = CFNumberGetValue(value, kCFNumberDoubleType, &nval);
1459             result = SecDbBindDouble(stmt, param, nval, error);
1460         } else {
1461             sqlite_int64 nval64;
1462             convertOk = CFNumberGetValue(value, kCFNumberSInt64Type, &nval64);
1463             if (convertOk) {
1464                 result = SecDbBindInt64(stmt, param, nval64, error);
1465             }
1466         }
1467         if (!convertOk) {
1468             result = SecDbError(SQLITE_INTERNAL, error, CFSTR("bind CFNumberGetValue failed for %@"), value);
1469         }
1470     } else {
1471         if (error) {
1472             CFStringRef valueDesc = CFCopyTypeIDDescription(valueId);
1473             SecDbError(SQLITE_MISMATCH, error, CFSTR("bind unsupported type %@"), valueDesc);
1474             CFReleaseSafe(valueDesc);
1475         }
1476     }
1477
1478         return result;
1479 }
1480
1481 // MARK: -
1482 // MARK: SecDbStatementRef
1483
1484 bool SecDbReset(sqlite3_stmt *stmt, CFErrorRef *error) {
1485     return SecDbErrorWithStmt(sqlite3_reset(stmt), stmt, error, CFSTR("reset"));
1486 }
1487
1488 bool SecDbClearBindings(sqlite3_stmt *stmt, CFErrorRef *error) {
1489     return SecDbErrorWithStmt(sqlite3_clear_bindings(stmt), stmt, error, CFSTR("clear bindings"));
1490 }
1491
1492 bool SecDbFinalize(sqlite3_stmt *stmt, CFErrorRef *error) {
1493     sqlite3 *handle = sqlite3_db_handle(stmt);
1494     int s3e = sqlite3_finalize(stmt);
1495     return s3e == SQLITE_OK ? true : SecDbErrorWithDb(s3e, handle, error, CFSTR("finalize: %p"), stmt);
1496 }
1497
1498 sqlite3_stmt *SecDbPrepareV2(SecDbConnectionRef dbconn, const char *sql, size_t sqlLen, const char **sqlTail, CFErrorRef *error) {
1499     sqlite3 *db = SecDbHandle(dbconn);
1500     if (sqlLen > INT_MAX) {
1501         SecDbErrorWithDb(SQLITE_TOOBIG, db, error, CFSTR("prepare_v2: sql bigger than INT_MAX"));
1502         return NULL;
1503     }
1504     int ntries = 0;
1505     for (;;) {
1506         sqlite3_stmt *stmt = NULL;
1507         int s3e = sqlite3_prepare_v2(db, sql, (int)sqlLen, &stmt, sqlTail);
1508         if (s3e == SQLITE_OK)
1509             return stmt;
1510         else if (!SecDbWaitIfNeeded(dbconn, s3e, NULL, CFSTR("preparev2"), ntries, error))
1511             return NULL;
1512         ntries++;
1513     }
1514 }
1515
1516 static sqlite3_stmt *SecDbCopyStatementWithTailRange(SecDbConnectionRef dbconn, CFStringRef sql, CFRange *sqlTail, CFErrorRef *error) {
1517     __block sqlite3_stmt *stmt = NULL;
1518     if (sql) CFStringPerformWithCStringAndLength(sql, ^(const char *sqlStr, size_t sqlLen) {
1519         const char *tail = NULL;
1520         stmt = SecDbPrepareV2(dbconn, sqlStr, sqlLen, &tail, error);
1521         if (sqlTail && sqlStr < tail && tail < sqlStr + sqlLen) {
1522             sqlTail->location = tail - sqlStr;
1523             sqlTail->length = sqlLen - sqlTail->location;
1524         }
1525     });
1526
1527     return stmt;
1528 }
1529
1530 sqlite3_stmt *SecDbCopyStmt(SecDbConnectionRef dbconn, CFStringRef sql, CFStringRef *tail, CFErrorRef *error) {
1531     // TODO: Add caching and cache lookup of statements
1532     CFRange sqlTail = {};
1533     sqlite3_stmt *stmt = SecDbCopyStatementWithTailRange(dbconn, sql, &sqlTail, error);
1534     if (sqlTail.length > 0) {
1535         CFStringRef excess = CFStringCreateWithSubstring(CFGetAllocator(sql), sql, sqlTail);
1536         if (tail) {
1537             *tail = excess;
1538         } else {
1539             SecDbError(SQLITE_INTERNAL, error,
1540                        CFSTR("prepare_v2: %@ unused sql: %@"),
1541                        sql, excess);
1542             CFReleaseSafe(excess);
1543             SecDbFinalize(stmt, error);
1544             stmt = NULL;
1545         }
1546     }
1547     return stmt;
1548 }
1549
1550 /*
1551  TODO: Could do a hack here with a custom kCFAllocatorNULL allocator for a second CFRuntimeBase inside a SecDbStatement,
1552  TODO: Better yet make a full blow SecDbStatement instance whenever SecDbCopyStmt is called.  Then, when the statement is released, in the Dispose method, we Reset and ClearBindings the sqlite3_stmt * and hand it back to the SecDb with the original CFStringRef for the sql (or hash thereof) as an argument. */
1553 bool SecDbReleaseCachedStmt(SecDbConnectionRef dbconn, CFStringRef sql, sqlite3_stmt *stmt, CFErrorRef *error) {
1554     if (stmt) {
1555         return SecDbFinalize(stmt, error);
1556     }
1557     return true;
1558 }
1559
1560 bool SecDbPrepare(SecDbConnectionRef dbconn, CFStringRef sql, CFErrorRef *error, void(^exec)(sqlite3_stmt *stmt)) {
1561     assert(sql != NULL);
1562     sqlite3_stmt *stmt = SecDbCopyStmt(dbconn, sql, NULL, error);
1563     if (!stmt)
1564         return false;
1565
1566     exec(stmt);
1567     return SecDbReleaseCachedStmt(dbconn, sql, stmt, error);
1568 }
1569
1570 bool SecDbWithSQL(SecDbConnectionRef dbconn, CFStringRef sql, CFErrorRef *error, bool(^perform)(sqlite3_stmt *stmt)) {
1571     bool ok = true;
1572     CFRetain(sql);
1573     while (sql) {
1574         CFStringRef tail = NULL;
1575         if (ok) {
1576             sqlite3_stmt *stmt = SecDbCopyStmt(dbconn, sql, &tail, error);
1577             ok = stmt != NULL;
1578             if (stmt) {
1579                 if (perform) {
1580                     ok = perform(stmt);
1581                 } else {
1582                     // TODO: Use a different error scope here.
1583                     ok = SecError(-50 /* errSecParam */, error, CFSTR("SecDbWithSQL perform block missing"));
1584                 }
1585                 ok &= SecDbReleaseCachedStmt(dbconn, sql, stmt, error);
1586             }
1587         } else {
1588             // TODO We already have an error here we really just want the left over sql in it's userData
1589             ok = SecDbError(SQLITE_ERROR, error, CFSTR("Error with unexecuted sql remaining %@"), sql);
1590         }
1591         CFRelease(sql);
1592         sql = tail;
1593     }
1594     return ok;
1595 }
1596
1597 /* SecDbForEach returns true if all SQLITE_ROW returns of sqlite3_step() return true from the row block.
1598  If the row block returns false and doesn't set an error (to indicate it has reached a limit),
1599  this entire function returns false. In that case no error will be set. */
1600 bool SecDbForEach(SecDbConnectionRef dbconn, sqlite3_stmt *stmt, CFErrorRef *error, bool(^row)(int row_index)) {
1601     bool result = false;
1602     for (int row_ix = 0;;++row_ix) {
1603         if (SecDbConnectionIsReadOnly(dbconn) && !sqlite3_stmt_readonly(stmt)) {
1604             secerror("SecDbForEach: SecDbConnection is readonly but we're about to write: %s", sqlite3_sql(stmt));
1605         }
1606         int s3e = sqlite3_step(stmt);
1607         if (s3e == SQLITE_ROW) {
1608             if (row) {
1609                 if (!row(row_ix)) {
1610                     break;
1611                 }
1612             } else {
1613                 // If we have no row block then getting SQLITE_ROW is an error
1614                 SecDbError(s3e, error,
1615                            CFSTR("step[%d]: %s returned SQLITE_ROW with NULL row block"),
1616                            row_ix, sqlite3_sql(stmt));
1617             }
1618         } else {
1619             if (s3e == SQLITE_DONE) {
1620                 result = true;
1621             } else {
1622                 SecDbConnectionCheckCode(dbconn, s3e, error, CFSTR("SecDbForEach step[%d]"), row_ix);
1623             }
1624             break;
1625         }
1626     }
1627     return result;
1628 }
1629
1630 void SecDbRecordChange(SecDbConnectionRef dbconn, CFTypeRef deleted, CFTypeRef inserted) {
1631     if (!dbconn->db->notifyPhase) return;
1632     CFTypeRef entry = SecDbEventCreateWithComponents(deleted, inserted);
1633     if (entry) {
1634         CFArrayAppendValue(dbconn->changes, entry);
1635         CFRelease(entry);
1636
1637         if (!dbconn->inTransaction) {
1638             secerror("db %@ changed outside txn", dbconn);
1639             // Only notify of DidCommit, since WillCommit code assumes
1640             // we are in a txn.
1641             SecDbOnNotify(dbconn, ^{
1642                 SecDbNotifyPhase(dbconn, kSecDbTransactionDidCommit);
1643             });
1644         }
1645     }
1646 }
1647
1648
1649 CFGiblisFor(SecDbConnection)
1650
1651 //
1652 // SecDbEvent Creation and consumption
1653 //
1654
1655 static SecDbEventRef SecDbEventCreateInsert(CFTypeRef inserted) {
1656     return CFRetainSafe(inserted);
1657 }
1658
1659 static SecDbEventRef SecDbEventCreateDelete(CFTypeRef deleted) {
1660     return CFArrayCreate(kCFAllocatorDefault, &deleted, 1, &kCFTypeArrayCallBacks);
1661 }
1662
1663 static SecDbEventRef SecDbEventCreateUpdate(CFTypeRef deleted, CFTypeRef inserted) {
1664     const void *values[2] = { deleted, inserted };
1665     return CFArrayCreate(kCFAllocatorDefault, values, 2, &kCFTypeArrayCallBacks);
1666 }
1667
1668 SecDbEventRef SecDbEventCreateWithComponents(CFTypeRef deleted, CFTypeRef inserted) {
1669     if (deleted && inserted)
1670         return SecDbEventCreateUpdate(deleted, inserted);
1671     else if (deleted)
1672         return SecDbEventCreateDelete(deleted);
1673     else if (inserted)
1674         return SecDbEventCreateInsert(inserted);
1675     else
1676         return NULL;
1677 }
1678
1679 void SecDbEventTranslateComponents(SecDbEventRef item, CFTypeRef* deleted, CFTypeRef* inserted) {
1680     if(CFGetTypeID(item) == CFArrayGetTypeID()) {
1681         // One item: deletion. Two: update.
1682         CFIndex arraySize = CFArrayGetCount(item);
1683         if(arraySize == 1) {
1684             if(deleted) { *deleted = CFArrayGetValueAtIndex(item, 0); }
1685             if(inserted) { *inserted = NULL; }
1686         } else if(arraySize == 2) {
1687             if(deleted) { *deleted = CFArrayGetValueAtIndex(item, 0); }
1688             if(inserted) { *inserted = CFArrayGetValueAtIndex(item, 1); }
1689         } else {
1690             if(deleted) { *deleted = NULL; }
1691             if(inserted) { *inserted = NULL; }
1692         }
1693     } else {
1694         if(deleted) { *deleted = NULL; }
1695         if(inserted) { *inserted = item; }
1696     }
1697
1698 }
1699
1700 bool SecDbEventGetComponents(SecDbEventRef event, CFTypeRef *deleted, CFTypeRef *inserted, CFErrorRef *error) {
1701     if (isArray(event)) {
1702         CFArrayRef array = event;
1703         switch (CFArrayGetCount(array)) {
1704             case 2:
1705                 *deleted = CFArrayGetValueAtIndex(array, 0);
1706                 *inserted = CFArrayGetValueAtIndex(array, 1);
1707                 break;
1708             case 1:
1709                 *deleted = CFArrayGetValueAtIndex(array, 0);
1710                 *inserted = NULL;
1711                 break;
1712             default:
1713                 SecError(errSecParam, error, NULL, CFSTR("invalid entry in changes array: %@"), array);
1714                 break;
1715         }
1716     } else {
1717         *deleted = NULL;
1718         *inserted = event;
1719     }
1720     return true;
1721 }