trust/trustd/SecRevocationDb.c

   1 /*
   2  * Copyright (c) 2016-2019 Apple Inc. All Rights Reserved.
   3  *
   4  * @APPLE_LICENSE_HEADER_START@
   5  *
   6  * This file contains Original Code and/or Modifications of Original Code
   7  * as defined in and that are subject to the Apple Public Source License
   8  * Version 2.0 (the 'License'). You may not use this file except in
   9  * compliance with the License. Please obtain a copy of the License at
  10  * http://www.opensource.apple.com/apsl/ and read it before using this
  11  * file.
  12  *
  13  * The Original Code and all software distributed under the License are
  14  * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  15  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  16  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
  17  * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
  18  * Please see the License for the specific language governing rights and
  19  * limitations under the License.
  20  *
  21  * @APPLE_LICENSE_HEADER_END@
  22  *
  23  */
  24
  25 /*
  26  *  SecRevocationDb.c
  27  */
  28
  29 #include "trust/trustd/SecRevocationDb.h"
  30 #include "trust/trustd/OTATrustUtilities.h"
  31 #include "trust/trustd/SecRevocationNetworking.h"
  32 #include "trust/trustd/SecTrustLoggingServer.h"
  33 #include <Security/SecCertificateInternal.h>
  34 #include <Security/SecCMS.h>
  35 #include <Security/CMSDecoder.h>
  36 #include <Security/SecFramework.h>
  37 #include <Security/SecInternal.h>
  38 #include <Security/SecPolicyPriv.h>
  39 #include <AssertMacros.h>
  40 #include <stdlib.h>
  41 #include <stdatomic.h>
  42 #include <limits.h>
  43 #include <string.h>
  44 #include <fcntl.h>
  45 #include <sys/stat.h>
  46 #include <errno.h>
  47 #include <dispatch/dispatch.h>
  48 #include <notify.h>
  49 #include <asl.h>
  50 #include <copyfile.h>
  51 #include "utilities/debugging.h"
  52 #include "utilities/sec_action.h"
  53 #include "utilities/sqlutils.h"
  54 #include "utilities/SecAppleAnchorPriv.h"
  55 #include "utilities/iOSforOSX.h"
  56 #include <utilities/SecCFError.h>
  57 #include <utilities/SecCFRelease.h>
  58 #include <utilities/SecCFWrappers.h>
  59 #include <utilities/SecDb.h>
  60 #include <utilities/SecFileLocations.h>
  61 #include <sqlite3.h>
  62 #include <zlib.h>
  63 #include <malloc/malloc.h>
  64 #include <xpc/activity.h>
  65 #include <xpc/private.h>
  66 #include <os/transaction_private.h>
  67 #include <os/variant_private.h>
  68 #include <os/lock.h>
  69
  70 #include <CommonCrypto/CommonDigest.h>
  71 #include <CFNetwork/CFHTTPMessage.h>
  72 #include <CoreFoundation/CFURL.h>
  73 #include <CoreFoundation/CFUtilities.h>
  74
  75 /*
  76  ==============================================================================
  77    CoreFoundation utilities
  78  ==============================================================================
  79 */
  80
  81 static bool hashCFThing(CFTypeRef thing, CC_SHA256_CTX* hash_ctx);
  82
  83 /* comparison function for sorting dictionary keys alphabetically */
  84 static int compareCFStrings(const void *p, const void *q) {
  85     CFStringRef str1 = *(CFStringRef *)p;
  86     CFStringRef str2 = *(CFStringRef *)q;
  87     if (!(isString(str1) && isString(str2))) {
  88         return -1; /* can't compare non-string types */
  89     }
  90     CFComparisonResult result = CFStringCompare(str1, str2, 0);
  91     if (result == kCFCompareLessThan) {
  92         return -1;
  93     } else if (result == kCFCompareGreaterThan) {
  94         return 1;
  95     }
  96     return 0; /* (result == kCFCompareEqualTo) */
  97 }
  98
  99 static bool hashData(CFDataRef data, CC_SHA256_CTX* hash_ctx) {
 100     if (!isData(data)) { return false; }
 101     uint32_t length = (uint32_t)(CFDataGetLength(data) & 0xFFFFFFFF);
 102     uint32_t n = OSSwapInt32(length);
 103     CC_SHA256_Update(hash_ctx, &n, sizeof(uint32_t));
 104     const uint8_t *p = (uint8_t*)CFDataGetBytePtr(data);
 105     if (p) { CC_SHA256_Update(hash_ctx, p, length); }
 106     return (p != NULL);
 107 }
 108
 109 static bool hashString(CFStringRef str, CC_SHA256_CTX* hash_ctx) {
 110     if (!isString(str)) { return false; }
 111     __block bool ok = false;
 112     CFStringPerformWithCString(str, ^(const char *strbuf) {
 113         // hash string length (in bytes, not counting null terminator)
 114         uint32_t c = (uint32_t)(strlen(strbuf) & 0xFFFFFFFF);
 115         uint32_t n = OSSwapInt32(c);
 116         CC_SHA256_Update(hash_ctx, &n, sizeof(uint32_t));
 117         // hash string bytes
 118         CC_SHA256_Update(hash_ctx, strbuf, c);
 119         ok = true;
 120     });
 121     return ok;
 122 }
 123
 124 static bool hashNumber(CFNumberRef num, CC_SHA256_CTX* hash_ctx) {
 125     uint32_t n = 0;
 126     if (!isNumber(num) || !CFNumberGetValue(num, kCFNumberSInt32Type, &n)) {
 127         return false;
 128     }
 129     n = OSSwapInt32(n);
 130     CC_SHA256_Update(hash_ctx, &n, sizeof(uint32_t));
 131     return true;
 132 }
 133
 134 static bool hashBoolean(CFBooleanRef value, CC_SHA256_CTX* hash_ctx) {
 135     if (!isBoolean(value)) { return false; }
 136     uint8_t c = CFBooleanGetValue(value) ? 1 : 0;
 137     CC_SHA256_Update(hash_ctx, &c, sizeof(uint8_t));
 138     return true;
 139 }
 140
 141 static bool hashArray(CFArrayRef array, CC_SHA256_CTX* hash_ctx) {
 142     if (!isArray(array)) { return false; }
 143     CFIndex count = CFArrayGetCount(array);
 144     uint32_t n = OSSwapInt32(count & 0xFFFFFFFF);
 145     CC_SHA256_Update(hash_ctx, &n, sizeof(uint32_t));
 146     __block bool ok = true;
 147     CFArrayForEach(array, ^(const void *thing) {
 148         ok &= hashCFThing(thing, hash_ctx);
 149     });
 150     return ok;
 151 }
 152
 153 static bool hashDictionary(CFDictionaryRef dictionary, CC_SHA256_CTX* hash_ctx) {
 154     if (!isDictionary(dictionary)) { return false; }
 155     CFIndex count = CFDictionaryGetCount(dictionary);
 156     const void **keys = (const void **)malloc(sizeof(void*) * count);
 157     const void **vals = (const void **)malloc(sizeof(void*) * count);
 158     bool ok = (keys && vals);
 159     if (ok) {
 160         CFDictionaryGetKeysAndValues(dictionary, keys, vals);
 161         qsort(keys, count, sizeof(CFStringRef), compareCFStrings);
 162         uint32_t n = OSSwapInt32(count & 0xFFFFFFFF);
 163         CC_SHA256_Update(hash_ctx, &n, sizeof(uint32_t));
 164     }
 165     for (CFIndex idx = 0; ok && idx < count; idx++) {
 166         CFStringRef key = (CFStringRef)keys[idx];
 167         CFTypeRef value = (CFTypeRef)CFDictionaryGetValue(dictionary, key);
 168         ok &= hashString(key, hash_ctx);
 169         ok &= hashCFThing(value, hash_ctx);
 170     }
 171     free(keys);
 172     free(vals);
 173     return ok;
 174 }
 175
 176 static bool hashCFThing(CFTypeRef thing, CC_SHA256_CTX* hash_ctx) {
 177     if (isArray(thing)) {
 178         return hashArray(thing, hash_ctx);
 179     } else if (isDictionary(thing)) {
 180         return hashDictionary(thing, hash_ctx);
 181     } else if (isData(thing)) {
 182         return hashData(thing, hash_ctx);
 183     } else if (isString(thing)) {
 184         return hashString(thing, hash_ctx);
 185     } else if (isNumber(thing)) {
 186         return hashNumber(thing, hash_ctx);
 187     } else if (isBoolean(thing)) {
 188         return hashBoolean(thing, hash_ctx);
 189     }
 190     return false;
 191 }
 192
 193 static double htond(double h) {
 194     /* no-op if big endian */
 195     if (OSHostByteOrder() == OSBigEndian) {
 196         return h;
 197     }
 198     double n;
 199     size_t i=0;
 200     char *hp = (char*)&h;
 201     char *np = (char*)&n;
 202     while (i < sizeof(h)) { np[i] = hp[(sizeof(h)-1)-i]; ++i; }
 203     return n;
 204 }
 205
 206
 207 // MARK: -
 208 // MARK: Valid definitions
 209 /*
 210  ==============================================================================
 211    Valid definitions
 212  ==============================================================================
 213 */
 214
 215 const CFStringRef kValidUpdateProdServer   = CFSTR("valid.apple.com");
 216 const CFStringRef kValidUpdateSeedServer   = CFSTR("valid.apple.com/seed");
 217 const CFStringRef kValidUpdateCarryServer  = CFSTR("valid.apple.com/carry");
 218
 219 static CFStringRef kSecPrefsDomain          = CFSTR("com.apple.security");
 220 static CFStringRef kUpdateServerKey         = CFSTR("ValidUpdateServer");
 221 static CFStringRef kUpdateEnabledKey        = CFSTR("ValidUpdateEnabled");
 222 static CFStringRef kVerifyEnabledKey        = CFSTR("ValidVerifyEnabled");
 223 static CFStringRef kUpdateIntervalKey       = CFSTR("ValidUpdateInterval");
 224 static CFStringRef kBoolTrueKey             = CFSTR("1");
 225 static CFStringRef kBoolFalseKey            = CFSTR("0");
 226
 227 /* constant length of boolean string keys */
 228 #define BOOL_STRING_KEY_LENGTH          1
 229
 230 typedef CF_OPTIONS(CFOptionFlags, SecValidInfoFlags) {
 231     kSecValidInfoComplete               = 1u << 0,
 232     kSecValidInfoCheckOCSP              = 1u << 1,
 233     kSecValidInfoKnownOnly              = 1u << 2,
 234     kSecValidInfoRequireCT              = 1u << 3,
 235     kSecValidInfoAllowlist              = 1u << 4,
 236     kSecValidInfoNoCACheck              = 1u << 5,
 237     kSecValidInfoOverridable            = 1u << 6,
 238     kSecValidInfoDateConstraints        = 1u << 7,
 239     kSecValidInfoNameConstraints        = 1u << 8,
 240     kSecValidInfoPolicyConstraints      = 1u << 9,
 241     kSecValidInfoNoCAv2Check            = 1u << 10,
 242 };
 243
 244 /* minimum update interval */
 245 #define kSecMinUpdateInterval           (60.0 * 5)
 246
 247 /* standard update interval */
 248 #define kSecStdUpdateInterval           (60.0 * 60 * 3)
 249
 250 /* maximum allowed interval */
 251 #define kSecMaxUpdateInterval           (60.0 * 60 * 24 * 7)
 252
 253 #define kSecRevocationBasePath          "/Library/Keychains/crls"
 254 #define kSecRevocationCurUpdateFile     "update-current"
 255 #define kSecRevocationDbFileName        "valid.sqlite3"
 256 #define kSecRevocationDbReplaceFile     ".valid_replace"
 257
 258 #define isDbOwner SecOTAPKIIsSystemTrustd
 259
 260 #define kSecRevocationDbChanged         "com.apple.trustd.valid.db-changed"
 261
 262 /* database schema version
 263    v1 = initial version
 264    v2 = fix for group entry transitions
 265    v3 = handle optional entries in update dictionaries
 266    v4 = add db_format and db_source entries
 267    v5 = add date constraints table, with updated group flags
 268    v6 = explicitly set autovacuum and journal modes at db creation
 269    v7 = add policies column to groups table (policy constraints)
 270
 271    Note: kSecRevocationDbMinSchemaVersion is the lowest version whose
 272    results can be used. This allows revocation results to be obtained
 273    from an existing db before the next update interval occurs, at which
 274    time we'll update to the current version (kSecRevocationDbSchemaVersion).
 275 */
 276 #define kSecRevocationDbSchemaVersion       7  /* current version we support */
 277 #define kSecRevocationDbMinSchemaVersion    7  /* minimum version we can use */
 278
 279 /* update file format
 280 */
 281 CF_ENUM(CFIndex) {
 282     kSecValidUpdateFormatG1               = 1, /* initial version */
 283     kSecValidUpdateFormatG2               = 2, /* signed content, single plist */
 284     kSecValidUpdateFormatG3               = 3  /* signed content, multiple plists */
 285 };
 286
 287 #define kSecRevocationDbUpdateFormat        3  /* current version we support */
 288 #define kSecRevocationDbMinUpdateFormat     2  /* minimum version we can use */
 289
 290 #define kSecRevocationDbCacheSize           100
 291
 292 typedef struct __SecRevocationDb *SecRevocationDbRef;
 293 struct __SecRevocationDb {
 294     SecDbRef db;
 295     dispatch_queue_t update_queue;
 296     bool updateInProgress;
 297     bool unsupportedVersion;
 298     bool changed;
 299     CFMutableArrayRef info_cache_list;
 300     CFMutableDictionaryRef info_cache;
 301     os_unfair_lock info_cache_lock;
 302 };
 303
 304 typedef struct __SecRevocationDbConnection *SecRevocationDbConnectionRef;
 305 struct __SecRevocationDbConnection {
 306     SecRevocationDbRef db;
 307     SecDbConnectionRef dbconn;
 308     CFIndex precommitVersion;
 309     CFIndex precommitDbVersion;
 310     CFIndex precommitInterval;
 311     bool fullUpdate;
 312 };
 313
 314 bool SecRevocationDbVerifyUpdate(void *update, CFIndex length);
 315 bool SecRevocationDbIngestUpdate(SecRevocationDbConnectionRef dbc, CFDictionaryRef update, CFIndex chunkVersion, CFIndex *outVersion, CFErrorRef *error);
 316 bool _SecRevocationDbApplyUpdate(SecRevocationDbConnectionRef dbc, CFDictionaryRef update, CFIndex version, CFErrorRef *error);
 317 CFAbsoluteTime SecRevocationDbComputeNextUpdateTime(CFIndex updateInterval);
 318 bool SecRevocationDbUpdateSchema(SecRevocationDbRef rdb);
 319 CFIndex SecRevocationDbGetUpdateFormat(void);
 320 bool _SecRevocationDbSetUpdateSource(SecRevocationDbConnectionRef dbc, CFStringRef source, CFErrorRef *error);
 321 bool SecRevocationDbSetUpdateSource(SecRevocationDbRef rdb, CFStringRef source);
 322 CFStringRef SecRevocationDbCopyUpdateSource(void);
 323 bool SecRevocationDbSetNextUpdateTime(CFAbsoluteTime nextUpdate, CFErrorRef *error);
 324 CFAbsoluteTime SecRevocationDbGetNextUpdateTime(void);
 325 dispatch_queue_t SecRevocationDbGetUpdateQueue(void);
 326 bool _SecRevocationDbRemoveAllEntries(SecRevocationDbConnectionRef dbc, CFErrorRef *error);
 327 void SecRevocationDbReleaseAllConnections(void);
 328 static bool SecValidUpdateForceReplaceDatabase(void);
 329 static void SecRevocationDbWith(void(^dbJob)(SecRevocationDbRef db));
 330 static bool SecRevocationDbPerformWrite(SecRevocationDbRef rdb, CFErrorRef *error, bool(^writeJob)(SecRevocationDbConnectionRef dbc, CFErrorRef *blockError));
 331 static bool SecRevocationDbPerformRead(SecRevocationDbRef rdb, CFErrorRef *error, bool(^readJob)(SecRevocationDbConnectionRef dbc, CFErrorRef *blockError));
 332 static SecValidInfoFormat _SecRevocationDbGetGroupFormat(SecRevocationDbConnectionRef dbc, int64_t groupId, SecValidInfoFlags *flags, CFDataRef *data, CFDataRef *policies, CFErrorRef *error);
 333 static bool _SecRevocationDbSetUpdateInterval(SecRevocationDbConnectionRef dbc, int64_t interval, CFErrorRef *error);
 334 static int64_t _SecRevocationDbGetUpdateInterval(SecRevocationDbConnectionRef dbc, CFErrorRef *error);
 335 static bool _SecRevocationDbSetVersion(SecRevocationDbConnectionRef dbc, CFIndex version, CFErrorRef *error);
 336 static int64_t _SecRevocationDbGetVersion(SecRevocationDbConnectionRef dbc, CFErrorRef *error);
 337 static int64_t _SecRevocationDbGetSchemaVersion(SecRevocationDbRef rdb, SecRevocationDbConnectionRef dbc, CFErrorRef *error);
 338 static CFArrayRef _SecRevocationDbCopyHashes(SecRevocationDbConnectionRef dbc, CFErrorRef *error);
 339 static bool _SecRevocationDbSetHashes(SecRevocationDbConnectionRef dbc, CFArrayRef hashes, CFErrorRef *error);
 340 static void SecRevocationDbResetCaches(void);
 341 static SecRevocationDbConnectionRef SecRevocationDbConnectionInit(SecRevocationDbRef db, SecDbConnectionRef dbconn, CFErrorRef *error);
 342 static bool SecRevocationDbComputeDigests(SecRevocationDbConnectionRef dbc, CFErrorRef *error);
 343
 344
 345 static CFDataRef copyInflatedData(CFDataRef data) {
 346     if (!data) {
 347         return NULL;
 348     }
 349     z_stream zs;
 350     memset(&zs, 0, sizeof(zs));
 351     /* 32 is a magic value which enables automatic header detection
 352        of gzip or zlib compressed data. */
 353     if (inflateInit2(&zs, 32+MAX_WBITS) != Z_OK) {
 354         return NULL;
 355     }
 356     zs.next_in = (UInt8 *)(CFDataGetBytePtr(data));
 357     zs.avail_in = (uInt)CFDataGetLength(data);
 358
 359     CFMutableDataRef outData = CFDataCreateMutable(NULL, 0);
 360     if (!outData) {
 361         return NULL;
 362     }
 363     CFIndex buf_sz = malloc_good_size(zs.avail_in ? zs.avail_in : 1024 * 4);
 364     unsigned char *buf = malloc(buf_sz);
 365     int rc;
 366     do {
 367         zs.next_out = (Bytef*)buf;
 368         zs.avail_out = (uInt)buf_sz;
 369         rc = inflate(&zs, 0);
 370         CFIndex outLen = CFDataGetLength(outData);
 371         if (outLen < (CFIndex)zs.total_out) {
 372             CFDataAppendBytes(outData, (const UInt8*)buf, (CFIndex)zs.total_out - outLen);
 373         }
 374     } while (rc == Z_OK);
 375
 376     inflateEnd(&zs);
 377     free(buf);
 378     if (rc != Z_STREAM_END) {
 379         CFReleaseSafe(outData);
 380         return NULL;
 381     }
 382     return (CFDataRef)outData;
 383 }
 384
 385 static CFDataRef copyDeflatedData(CFDataRef data) {
 386     if (!data) {
 387         return NULL;
 388     }
 389     z_stream zs;
 390     memset(&zs, 0, sizeof(zs));
 391     if (deflateInit(&zs, Z_BEST_COMPRESSION) != Z_OK) {
 392         return NULL;
 393     }
 394     zs.next_in = (UInt8 *)(CFDataGetBytePtr(data));
 395     zs.avail_in = (uInt)CFDataGetLength(data);
 396
 397     CFMutableDataRef outData = CFDataCreateMutable(NULL, 0);
 398     if (!outData) {
 399         return NULL;
 400     }
 401     CFIndex buf_sz = malloc_good_size(zs.avail_in ? zs.avail_in : 1024 * 4);
 402     unsigned char *buf = malloc(buf_sz);
 403     int rc = Z_BUF_ERROR;
 404     do {
 405         zs.next_out = (Bytef*)buf;
 406         zs.avail_out = (uInt)buf_sz;
 407         rc = deflate(&zs, Z_FINISH);
 408
 409         if (rc == Z_OK || rc == Z_STREAM_END) {
 410             CFIndex buf_used = buf_sz - zs.avail_out;
 411             CFDataAppendBytes(outData, (const UInt8*)buf, buf_used);
 412         }
 413         else if (rc == Z_BUF_ERROR) {
 414             free(buf);
 415             buf_sz = malloc_good_size(buf_sz * 2);
 416             buf = malloc(buf_sz);
 417             if (buf) {
 418                 rc = Z_OK; /* try again with larger buffer */
 419             }
 420         }
 421     } while (rc == Z_OK && zs.avail_in);
 422
 423     deflateEnd(&zs);
 424     free(buf);
 425     if (rc != Z_STREAM_END) {
 426         CFReleaseSafe(outData);
 427         return NULL;
 428     }
 429     return (CFDataRef)outData;
 430 }
 431
 432 /* Read file opens the file, mmaps it and then closes the file. */
 433 int readValidFile(const char *fileName,
 434                     CFDataRef  *bytes) {   // mmapped and returned -- must be munmapped!
 435     int rtn, fd;
 436     const uint8_t *buf = NULL;
 437     struct stat sb;
 438     size_t size = 0;
 439
 440     *bytes = NULL;
 441     fd = open(fileName, O_RDONLY);
 442     if (fd < 0) { return errno; }
 443     rtn = fstat(fd, &sb);
 444     if (rtn) { goto errOut; }
 445     if (sb.st_size > (off_t) ((UINT32_MAX >> 1)-1)) {
 446         rtn = EFBIG;
 447         goto errOut;
 448     }
 449     size = (size_t)sb.st_size;
 450
 451     buf = mmap(NULL, size, PROT_READ, MAP_PRIVATE, fd, 0);
 452     if (!buf || buf == MAP_FAILED) {
 453         rtn = errno;
 454         secerror("unable to map %s (errno %d)", fileName, rtn);
 455         goto errOut;
 456     }
 457
 458     *bytes = CFDataCreateWithBytesNoCopy(NULL, buf, size, kCFAllocatorNull);
 459
 460 errOut:
 461     close(fd);
 462     if(rtn) {
 463         CFReleaseNull(*bytes);
 464         if (buf) {
 465             int unmap_err = munmap((void *)buf, size);
 466             if (unmap_err != 0) {
 467                 secerror("unable to unmap %ld bytes at %p (error %d)", (long)size, buf, rtn);
 468             }
 469         }
 470     }
 471     return rtn;
 472 }
 473
 474 static bool removeFileWithSuffix(const char *basepath, const char *suffix) {
 475     bool result = false;
 476     char *path = NULL;
 477     asprintf(&path, "%s%s", basepath, suffix);
 478     if (path) {
 479         if (remove(path) == -1) {
 480             int error = errno;
 481             if (error == ENOENT) {
 482                 result = true; // not an error if the file did not exist
 483             } else {
 484                 secnotice("validupdate", "remove (%s): %s", path, strerror(error));
 485             }
 486         } else {
 487             result = true;
 488         }
 489         free(path);
 490     }
 491     return result;
 492 }
 493
 494 static CFDataRef CF_RETURNS_RETAINED createPoliciesData(CFArrayRef policies) {
 495     /*
 496      * Given an array of CFNumber values (in the range 0..127),
 497      * allocate and return a CFDataRef representation. Per Valid specification,
 498      * a zero-length array is allowed, meaning no policies are permitted.
 499      */
 500     CFIndex count = (policies) ? CFArrayGetCount(policies) : -1;
 501     if (count < 0 || count > 127) {
 502         return NULL; /* either no constraints, or far more than we expect. */
 503     }
 504     CFDataRef data = NULL;
 505     CFIndex length = 1 + (sizeof(int8_t) * count);
 506     int8_t *bytes = malloc(length);
 507     if (bytes) {
 508         int8_t *p = bytes;
 509         *p++ = (int8_t)(count & 0xFF);
 510         for (CFIndex idx = 0; idx < count; idx++) {
 511             int8_t pval = 0;
 512             CFNumberRef value = (CFNumberRef)CFArrayGetValueAtIndex(policies, idx);
 513             if (isNumber(value)) {
 514                 (void)CFNumberGetValue(value, kCFNumberSInt8Type, &pval);
 515             }
 516             *p++ = pval;
 517         }
 518         data = CFDataCreate(kCFAllocatorDefault, (const UInt8*)bytes, length);
 519     }
 520     free(bytes);
 521     return data;
 522 }
 523
 524 static CFDataRef CF_RETURNS_RETAINED cfToHexData(CFDataRef data, bool prependWildcard) {
 525     if (!isData(data)) { return NULL; }
 526     CFIndex len = CFDataGetLength(data) * 2;
 527     CFMutableStringRef hex = CFStringCreateMutable(NULL, len+1);
 528     static const char* digits[]={
 529         "0","1","2","3","4","5","6","7","8","9","A","B","C","D","E","F"};
 530     if (prependWildcard) {
 531         CFStringAppendCString(hex, "%", 1);
 532     }
 533     const uint8_t* p = CFDataGetBytePtr(data);
 534     for (CFIndex i = 0; i < CFDataGetLength(data); i++) {
 535         CFStringAppendCString(hex, digits[p[i] >> 4], 1);
 536         CFStringAppendCString(hex, digits[p[i] & 0xf], 1);
 537     }
 538     CFDataRef result = CFStringCreateExternalRepresentation(NULL, hex, kCFStringEncodingUTF8, 0);
 539     CFReleaseSafe(hex);
 540     return result;
 541 }
 542
 543 static bool copyFilterComponents(CFDataRef xmlData, CFDataRef * CF_RETURNS_RETAINED xor,
 544                                  CFArrayRef * CF_RETURNS_RETAINED params) {
 545     /*
 546        The 'xmlData' parameter is a flattened XML dictionary,
 547        containing 'xor' and 'params' keys. First order of
 548        business is to reconstitute the blob into components.
 549     */
 550     bool result = false;
 551     CFRetainSafe(xmlData);
 552     CFDataRef propListData = xmlData;
 553     /* Expand data blob if needed */
 554     CFDataRef inflatedData = copyInflatedData(propListData);
 555     if (inflatedData) {
 556         CFReleaseSafe(propListData);
 557         propListData = inflatedData;
 558     }
 559     CFDataRef xorData = NULL;
 560     CFArrayRef paramsArray = NULL;
 561     CFPropertyListRef nto1 = CFPropertyListCreateWithData(kCFAllocatorDefault, propListData, 0, NULL, NULL);
 562     CFReleaseSafe(propListData);
 563     if (nto1) {
 564         xorData = (CFDataRef)CFDictionaryGetValue((CFDictionaryRef)nto1, CFSTR("xor"));
 565         CFRetainSafe(xorData);
 566         paramsArray = (CFArrayRef)CFDictionaryGetValue((CFDictionaryRef)nto1, CFSTR("params"));
 567         CFRetainSafe(paramsArray);
 568         CFReleaseSafe(nto1);
 569     }
 570     result = (xorData && paramsArray);
 571     if (xor) {
 572         *xor = xorData;
 573     } else {
 574         CFReleaseSafe(xorData);
 575     }
 576     if (params) {
 577         *params = paramsArray;
 578     } else {
 579         CFReleaseSafe(paramsArray);
 580     }
 581     return result;
 582 }
 583
 584 // MARK: -
 585 // MARK: SecValidUpdate
 586 /*
 587  ==============================================================================
 588    SecValidUpdate
 589  ==============================================================================
 590 */
 591
 592 CFAbsoluteTime gUpdateStarted = 0.0;
 593 CFAbsoluteTime gNextUpdate = 0.0;
 594 static CFIndex gUpdateInterval = 0;
 595 static CFIndex gLastVersion = 0;
 596
 597 /* Update Format:
 598     1. The length of the signed data, as a 4-byte integer in network byte order.
 599     2. The signed data, which consists of:
 600         a. A 4-byte integer in network byte order, the count of plists to follow; and then for each plist:
 601             i. A 4-byte integer, the length of each plist
 602             ii. A plist, in binary form
 603         b. There may be other data after the plists in the signed data, described by a future version of this specification.
 604     3. The length of the following CMS blob, as a 4-byte integer in network byte order.
 605     4. A detached CMS signature of the signed data described above.
 606     5. There may be additional data after the CMS blob, described by a future version of this specification.
 607
 608    Note: the difference between g2 and g3 format is the addition of the 4-byte count in (2a).
 609 */
 610 static bool SecValidUpdateProcessData(SecRevocationDbConnectionRef dbc, CFIndex format, CFDataRef updateData, CFErrorRef *error) {
 611     bool result = false;
 612     if (!updateData || format < 2) {
 613         SecError(errSecParam, error, CFSTR("SecValidUpdateProcessData: invalid update format"));
 614         return result;
 615     }
 616     CFIndex version = 0;
 617     CFIndex interval = 0;
 618     const UInt8* p = CFDataGetBytePtr(updateData);
 619     size_t bytesRemaining = (p) ? (size_t)CFDataGetLength(updateData) : 0;
 620     /* make sure there is enough data to contain length and count */
 621     if (bytesRemaining < ((CFIndex)sizeof(uint32_t) * 2)) {
 622         secinfo("validupdate", "Skipping property list creation (length %ld is too short)", (long)bytesRemaining);
 623         SecError(errSecParam, error, CFSTR("SecValidUpdateProcessData: data length is too short"));
 624         return result;
 625     }
 626     /* get length of signed data */
 627     uint32_t dataLength = OSSwapInt32(*((uint32_t *)p));
 628     bytesRemaining -= sizeof(uint32_t);
 629     p += sizeof(uint32_t);
 630
 631     /* get plist count (G3 format and later) */
 632     uint32_t plistCount = 1;
 633     uint32_t plistTotal = 1;
 634     if (format > kSecValidUpdateFormatG2) {
 635         plistCount = OSSwapInt32(*((uint32_t *)p));
 636         plistTotal = plistCount;
 637         bytesRemaining -= sizeof(uint32_t);
 638         p += sizeof(uint32_t);
 639     }
 640     if (dataLength > bytesRemaining) {
 641         secinfo("validupdate", "Skipping property list creation (dataLength=%ld, bytesRemaining=%ld)",
 642                 (long)dataLength, (long)bytesRemaining);
 643         SecError(errSecParam, error, CFSTR("SecValidUpdateProcessData: data longer than expected"));
 644         return result;
 645     }
 646
 647     /* process each chunked plist */
 648     bool ok = true;
 649     CFErrorRef localError = NULL;
 650     uint32_t plistProcessed = 0;
 651     while (plistCount > 0 && bytesRemaining > 0) {
 652         CFPropertyListRef propertyList = NULL;
 653         uint32_t plistLength = dataLength;
 654         if (format > kSecValidUpdateFormatG2) {
 655             plistLength = OSSwapInt32(*((uint32_t *)p));
 656             bytesRemaining -= sizeof(uint32_t);
 657             p += sizeof(uint32_t);
 658         }
 659         --plistCount;
 660         ++plistProcessed;
 661
 662         if (plistLength <= bytesRemaining) {
 663             CFDataRef data = CFDataCreateWithBytesNoCopy(NULL, p, plistLength, kCFAllocatorNull);
 664             propertyList = CFPropertyListCreateWithData(NULL, data, kCFPropertyListImmutable, NULL, NULL);
 665             CFReleaseNull(data);
 666         }
 667         if (isDictionary(propertyList)) {
 668             secdebug("validupdate", "Ingesting plist chunk %u of %u, length: %u",
 669                     plistProcessed, plistTotal, plistLength);
 670             CFIndex curVersion = -1;
 671             ok = ok && SecRevocationDbIngestUpdate(dbc, (CFDictionaryRef)propertyList, version, &curVersion, &localError);
 672             if (plistProcessed == 1) {
 673                 dbc->precommitVersion = version = curVersion;
 674                 // get server-provided interval
 675                 CFTypeRef value = (CFNumberRef)CFDictionaryGetValue((CFDictionaryRef)propertyList,
 676                                                                     CFSTR("check-again"));
 677                 if (isNumber(value)) {
 678                     CFNumberGetValue((CFNumberRef)value, kCFNumberCFIndexType, &interval);
 679                 }
 680                 // get server-provided hash list
 681                 value = (CFArrayRef)CFDictionaryGetValue((CFDictionaryRef)propertyList,
 682                                                          CFSTR("hash"));
 683                 ok = _SecRevocationDbSetHashes(dbc, (CFArrayRef)value, &localError);
 684             }
 685             if (ok && curVersion < 0) {
 686                 plistCount = 0; // we already had this version; skip remaining plists
 687                 result = true;
 688             }
 689         } else {
 690             secinfo("validupdate", "Failed to deserialize update chunk %u of %u",
 691                     plistProcessed, plistTotal);
 692             SecError(errSecParam, error, CFSTR("SecValidUpdateProcessData: failed to get update chunk"));
 693             if (plistProcessed == 1) {
 694                 gNextUpdate = SecRevocationDbComputeNextUpdateTime(0);
 695             }
 696         }
 697         /* All finished with this property list */
 698         CFReleaseSafe(propertyList);
 699
 700         bytesRemaining -= plistLength;
 701         p += plistLength;
 702     }
 703
 704     if (ok && version > 0) {
 705         secdebug("validupdate", "Update received: v%ld", (long)version);
 706         gLastVersion = version;
 707         gNextUpdate = SecRevocationDbComputeNextUpdateTime(interval);
 708         secdebug("validupdate", "Next update time: %f", gNextUpdate);
 709         result = true;
 710     }
 711
 712     (void) CFErrorPropagate(localError, error);
 713     return result;
 714 }
 715
 716 void SecValidUpdateVerifyAndIngest(CFDataRef updateData, CFStringRef updateServer, bool fullUpdate) {
 717     if (!updateData) {
 718         secnotice("validupdate", "invalid update data");
 719         return;
 720     }
 721     /* Verify CMS signature on signed data */
 722     if (!SecRevocationDbVerifyUpdate((void *)CFDataGetBytePtr(updateData), CFDataGetLength(updateData))) {
 723         secerror("failed to verify valid update");
 724         TrustdHealthAnalyticsLogErrorCode(TAEventValidUpdate, TAFatalError, errSecVerifyFailed);
 725         return;
 726     }
 727     /* Read current update source from database. */
 728     CFStringRef dbSource = SecRevocationDbCopyUpdateSource();
 729     if (dbSource && updateServer && (kCFCompareEqualTo != CFStringCompare(dbSource, updateServer,
 730         kCFCompareCaseInsensitive))) {
 731         secnotice("validupdate", "switching db source from \"%@\" to \"%@\"", dbSource, updateServer);
 732     }
 733     CFReleaseNull(dbSource);
 734
 735     /* Ingest the update. This is now performed under a single immediate write transaction,
 736        so other writers are blocked (but not other readers), and the changes can be rolled back
 737        in their entirety if any error occurs. */
 738     __block bool ok = true;
 739     __block CFErrorRef localError = NULL;
 740     SecRevocationDbWith(^(SecRevocationDbRef rdb) {
 741         ok &= SecRevocationDbPerformWrite(rdb, &localError, ^bool(SecRevocationDbConnectionRef dbc, CFErrorRef *blockError) {
 742             if (fullUpdate) {
 743                 /* Must completely replace existing database contents */
 744                 secdebug("validupdate", "starting to process full update; clearing database");
 745                 ok = ok && _SecRevocationDbRemoveAllEntries(dbc, blockError);
 746                 ok = ok && _SecRevocationDbSetUpdateSource(dbc, updateServer, blockError);
 747                 dbc->precommitVersion = 0;
 748                 dbc->fullUpdate = true;
 749             }
 750             CFIndex startingVersion = dbc->precommitVersion;
 751             ok = ok && SecValidUpdateProcessData(dbc, kSecValidUpdateFormatG3, updateData, blockError);
 752             rdb->changed = ok && (startingVersion < dbc->precommitVersion);
 753             if (!ok) {
 754                 secerror("failed to process valid update: %@", blockError ? *blockError : NULL);
 755                 TrustdHealthAnalyticsLogErrorCode(TAEventValidUpdate, TAFatalError, errSecDecode);
 756             } else {
 757                 TrustdHealthAnalyticsLogSuccess(TAEventValidUpdate);
 758             }
 759             return ok;
 760         });
 761         if (rdb->changed) {
 762             rdb->changed = false;
 763             bool verifyEnabled = false;
 764             CFBooleanRef value = (CFBooleanRef)CFPreferencesCopyValue(kVerifyEnabledKey,
 765                                  kSecPrefsDomain, kCFPreferencesAnyUser, kCFPreferencesCurrentHost);
 766             if (isBoolean(value)) {
 767                 verifyEnabled = CFBooleanGetValue(value);
 768             }
 769             CFReleaseNull(value);
 770             if (verifyEnabled) {
 771                 /* compute and verify database content hashes */
 772                 ok = ok && SecRevocationDbPerformRead(rdb, &localError, ^bool(SecRevocationDbConnectionRef dbc, CFErrorRef *blockError) {
 773                     ok = ok && SecRevocationDbComputeDigests(dbc, blockError);
 774                     if (!ok) {
 775                         /* digests failed to verify, so roll back to known-good snapshot */
 776                         (void) SecValidUpdateForceReplaceDatabase();
 777                     }
 778                     return ok;
 779                 });
 780             }
 781             /* signal other trustd instances that the database has been updated */
 782             notify_post(kSecRevocationDbChanged);
 783         }
 784     });
 785
 786     /* remember next update time in case of restart (separate write transaction) */
 787     (void) SecRevocationDbSetNextUpdateTime(gNextUpdate, NULL);
 788
 789     CFReleaseSafe(localError);
 790 }
 791
 792 static bool SecValidUpdateForceReplaceDatabase(void) {
 793     __block bool result = false;
 794
 795     // write semaphore file that we will pick up when we next launch
 796     WithPathInRevocationInfoDirectory(CFSTR(kSecRevocationDbReplaceFile), ^(const char *utf8String) {
 797         struct stat sb;
 798         int fd = open(utf8String, O_WRONLY | O_CREAT, DEFFILEMODE);
 799         if (fd == -1 || fstat(fd, &sb)) {
 800             secnotice("validupdate", "unable to write %s (error %d)", utf8String, errno);
 801         } else {
 802             result = true;
 803         }
 804         if (fd >= 0) {
 805             close(fd);
 806         }
 807     });
 808     if (result) {
 809         // exit as gracefully as possible so we can replace the database
 810         secnotice("validupdate", "process exiting to replace db file");
 811         dispatch_after(dispatch_time(DISPATCH_TIME_NOW, 3ull*NSEC_PER_SEC), dispatch_get_global_queue(DISPATCH_QUEUE_PRIORITY_DEFAULT, 0), ^{
 812             xpc_transaction_exit_clean();
 813         });
 814     }
 815     return result;
 816 }
 817
 818 static bool SecValidUpdateSatisfiedLocally(CFStringRef server, CFIndex version, bool safeToReplace) {
 819     __block bool result = false;
 820     SecOTAPKIRef otapkiRef = NULL;
 821     bool relaunching = false;
 822     static int sNumLocalUpdates = 0;
 823
 824     // if we've replaced the database with a local asset twice in a row,
 825     // something is wrong with it. Get this update from the server.
 826     if (sNumLocalUpdates > 1) {
 827         secdebug("validupdate", "%d consecutive db resets, ignoring local asset", sNumLocalUpdates);
 828         goto updateExit;
 829     }
 830
 831     // if a non-production server is specified, we will not be able to use a
 832     // local production asset since its update sequence will be different.
 833     if (kCFCompareEqualTo != CFStringCompare(server, kValidUpdateProdServer,
 834         kCFCompareCaseInsensitive)) {
 835         secdebug("validupdate", "non-production server specified, ignoring local asset");
 836         goto updateExit;
 837     }
 838
 839     // check static database asset(s)
 840     otapkiRef = SecOTAPKICopyCurrentOTAPKIRef();
 841     if (!otapkiRef) {
 842         goto updateExit;
 843     }
 844     CFIndex assetVersion = SecOTAPKIGetValidSnapshotVersion(otapkiRef);
 845     CFIndex assetFormat = SecOTAPKIGetValidSnapshotFormat(otapkiRef);
 846     // version <= 0 means the database is invalid or empty.
 847     // version > 0 means we have some version, but we need to see if a
 848     // newer version is available as a local asset.
 849     if (assetVersion <= version || assetFormat < kSecValidUpdateFormatG3) {
 850         // asset is not newer than ours, or its version is unknown
 851         goto updateExit;
 852     }
 853
 854     // replace database only if safe to do so (i.e. called at startup)
 855     if (!safeToReplace) {
 856         relaunching = SecValidUpdateForceReplaceDatabase();
 857         goto updateExit;
 858     }
 859
 860     // try to copy uncompressed database asset, if available
 861     const char *validDbPathBuf = SecOTAPKIGetValidDatabaseSnapshot(otapkiRef);
 862     if (validDbPathBuf) {
 863         WithPathInRevocationInfoDirectory(CFSTR(kSecRevocationDbFileName), ^(const char *path) {
 864             secdebug("validupdate", "will copy data from \"%s\"", validDbPathBuf);
 865             copyfile_state_t state = copyfile_state_alloc();
 866             int retval = copyfile(validDbPathBuf, path, state, COPYFILE_DATA);
 867             copyfile_state_free(state);
 868             if (retval < 0) {
 869                 secnotice("validupdate", "copyfile error %d", retval);
 870             } else {
 871                 result = true;
 872             }
 873         });
 874     }
 875
 876 updateExit:
 877     CFReleaseNull(otapkiRef);
 878     if (result) {
 879         sNumLocalUpdates++;
 880         gLastVersion = SecRevocationDbGetVersion();
 881         // note: snapshot should already have latest schema and production source,
 882         // but set it here anyway so we don't keep trying to replace the db.
 883         SecRevocationDbWith(^(SecRevocationDbRef db) {
 884             (void)SecRevocationDbSetUpdateSource(db, server);
 885             (void)SecRevocationDbUpdateSchema(db);
 886         });
 887         gUpdateStarted = 0;
 888         secdebug("validupdate", "local update to g%ld/v%ld complete at %f",
 889                  (long)SecRevocationDbGetUpdateFormat(), (long)gLastVersion,
 890                  (double)CFAbsoluteTimeGetCurrent());
 891     } else {
 892         sNumLocalUpdates = 0; // reset counter
 893     }
 894     if (relaunching) {
 895         // request is locally satisfied; don't schedule a network update
 896         result = true;
 897     }
 898     return result;
 899 }
 900
 901 static bool SecValidUpdateSchedule(bool updateEnabled, CFStringRef server, CFIndex version) {
 902     /* Check if we have a later version available locally */
 903     if (SecValidUpdateSatisfiedLocally(server, version, false)) {
 904         return true;
 905     }
 906
 907     /* If update not permitted return */
 908     if (!updateEnabled) {
 909         return false;
 910     }
 911
 912 #if !TARGET_OS_BRIDGE
 913     /* Schedule as a maintenance task */
 914     secdebug("validupdate", "will fetch v%lu from \"%@\"", (unsigned long)version, server);
 915     return SecValidUpdateRequest(SecRevocationDbGetUpdateQueue(), server, version);
 916 #else
 917     return false;
 918 #endif
 919 }
 920
 921 static CFStringRef SecRevocationDbGetDefaultServer(void) {
 922 #if !TARGET_OS_WATCH && !TARGET_OS_BRIDGE
 923     #if RC_SEED_BUILD
 924         CFStringRef defaultServer = kValidUpdateSeedServer;
 925     #else // !RC_SEED_BUILD
 926         CFStringRef defaultServer = kValidUpdateProdServer;
 927     #endif // !RC_SEED_BUILD
 928         if (os_variant_has_internal_diagnostics("com.apple.security")) {
 929             defaultServer = kValidUpdateCarryServer;
 930         }
 931         return defaultServer;
 932 #else // TARGET_OS_WATCH || TARGET_OS_BRIDGE
 933     /* Because watchOS and bridgeOS can't update over the air, we should
 934      * always use the prod server so that the valid database built into the
 935      * image is used. */
 936     return kValidUpdateProdServer;
 937 #endif
 938 }
 939
 940 void SecRevocationDbInitialize() {
 941     if (!isDbOwner()) { return; }
 942     os_transaction_t transaction = os_transaction_create("com.apple.trustd.valid.initialize");
 943     __block bool initializeDb = false;
 944
 945     /* create base path if it doesn't exist */
 946     WithPathInRevocationInfoDirectory(NULL, ^(const char *utf8String) {
 947         (void)mkpath_np(utf8String, 0755);
 948     });
 949
 950     /* check semaphore file */
 951     WithPathInRevocationInfoDirectory(CFSTR(kSecRevocationDbReplaceFile), ^(const char *path) {
 952         struct stat sb;
 953         if (stat(path, &sb) == 0) {
 954             initializeDb = true; /* file was found, so we will replace the database */
 955             if (remove(path) == -1) {
 956                 int error = errno;
 957                 secnotice("validupdate", "remove (%s): %s", path, strerror(error));
 958             }
 959         }
 960     });
 961
 962     /* check database */
 963     WithPathInRevocationInfoDirectory(CFSTR(kSecRevocationDbFileName), ^(const char *path) {
 964         if (initializeDb) {
 965             /* remove old database file(s) */
 966             (void)removeFileWithSuffix(path, "");
 967             (void)removeFileWithSuffix(path, "-journal");
 968             (void)removeFileWithSuffix(path, "-shm");
 969             (void)removeFileWithSuffix(path, "-wal");
 970         }
 971         else {
 972             struct stat sb;
 973             if (stat(path, &sb) == -1) {
 974                 initializeDb = true; /* file not found, so we will create the database */
 975             }
 976         }
 977     });
 978
 979     if (!initializeDb) {
 980         os_release(transaction);
 981         return; /* database exists and doesn't need replacing */
 982     }
 983
 984     /* initialize database from local asset */
 985     CFTypeRef value = (CFStringRef)CFPreferencesCopyValue(kUpdateServerKey, kSecPrefsDomain, kCFPreferencesAnyUser, kCFPreferencesCurrentHost);
 986     CFStringRef server = (isString(value)) ? (CFStringRef)value : (CFStringRef)SecRevocationDbGetDefaultServer();
 987     CFIndex version = 0;
 988     secnotice("validupdate", "initializing database");
 989     if (!SecValidUpdateSatisfiedLocally(server, version, true)) {
 990 #if !TARGET_OS_BRIDGE
 991         /* Schedule full update as a maintenance task */
 992         (void)SecValidUpdateRequest(SecRevocationDbGetUpdateQueue(), server, version);
 993 #endif
 994     }
 995     CFReleaseSafe(value);
 996     os_release(transaction);
 997 }
 998
 999
1000 // MARK: -
1001 // MARK: SecValidInfoRef
1002 /*
1003  ==============================================================================
1004    SecValidInfoRef
1005  ==============================================================================
1006 */
1007
1008 CFGiblisWithCompareFor(SecValidInfo);
1009
1010 static SecValidInfoRef SecValidInfoCreate(SecValidInfoFormat format,
1011                                           CFOptionFlags flags,
1012                                           bool isOnList,
1013                                           CFDataRef certHash,
1014                                           CFDataRef issuerHash,
1015                                           CFDataRef anchorHash,
1016                                           CFDateRef notBeforeDate,
1017                                           CFDateRef notAfterDate,
1018                                           CFDataRef nameConstraints,
1019                                           CFDataRef policyConstraints) {
1020     SecValidInfoRef validInfo;
1021     validInfo = CFTypeAllocate(SecValidInfo, struct __SecValidInfo, kCFAllocatorDefault);
1022     if (!validInfo) { return NULL; }
1023
1024     CFRetainSafe(certHash);
1025     CFRetainSafe(issuerHash);
1026     CFRetainSafe(anchorHash);
1027     CFRetainSafe(notBeforeDate);
1028     CFRetainSafe(notAfterDate);
1029     CFRetainSafe(nameConstraints);
1030     CFRetainSafe(policyConstraints);
1031
1032     validInfo->format = format;
1033     validInfo->certHash = certHash;
1034     validInfo->issuerHash = issuerHash;
1035     validInfo->anchorHash = anchorHash;
1036     validInfo->isOnList = isOnList;
1037     validInfo->valid = (flags & kSecValidInfoAllowlist);
1038     validInfo->complete = (flags & kSecValidInfoComplete);
1039     validInfo->checkOCSP = (flags & kSecValidInfoCheckOCSP);
1040     validInfo->knownOnly = (flags & kSecValidInfoKnownOnly);
1041     validInfo->requireCT = (flags & kSecValidInfoRequireCT);
1042     validInfo->noCACheck = (flags & kSecValidInfoNoCAv2Check);
1043     validInfo->overridable = (flags & kSecValidInfoOverridable);
1044     validInfo->hasDateConstraints = (flags & kSecValidInfoDateConstraints);
1045     validInfo->hasNameConstraints = (flags & kSecValidInfoNameConstraints);
1046     validInfo->hasPolicyConstraints = (flags & kSecValidInfoPolicyConstraints);
1047     validInfo->notBeforeDate = notBeforeDate;
1048     validInfo->notAfterDate = notAfterDate;
1049     validInfo->nameConstraints = nameConstraints;
1050     validInfo->policyConstraints = policyConstraints;
1051
1052     return validInfo;
1053 }
1054
1055 static void SecValidInfoDestroy(CFTypeRef cf) {
1056     SecValidInfoRef validInfo = (SecValidInfoRef)cf;
1057     if (validInfo) {
1058         CFReleaseNull(validInfo->certHash);
1059         CFReleaseNull(validInfo->issuerHash);
1060         CFReleaseNull(validInfo->anchorHash);
1061         CFReleaseNull(validInfo->notBeforeDate);
1062         CFReleaseNull(validInfo->notAfterDate);
1063         CFReleaseNull(validInfo->nameConstraints);
1064         CFReleaseNull(validInfo->policyConstraints);
1065     }
1066 }
1067
1068 void SecValidInfoSetAnchor(SecValidInfoRef validInfo, SecCertificateRef anchor) {
1069     if (!validInfo) {
1070         return;
1071     }
1072     CFDataRef anchorHash = NULL;
1073     if (anchor) {
1074         anchorHash = SecCertificateCopySHA256Digest(anchor);
1075
1076         /* clear no-ca flag for anchors where we want OCSP checked [32523118] */
1077         if (SecIsAppleTrustAnchor(anchor, 0)) {
1078             validInfo->noCACheck = false;
1079         }
1080     }
1081     CFReleaseNull(validInfo->anchorHash);
1082     validInfo->anchorHash = anchorHash;
1083 }
1084
1085 static Boolean SecValidInfoCompare(CFTypeRef a, CFTypeRef b) {
1086     SecValidInfoRef validInfoA = (SecValidInfoRef)a;
1087     SecValidInfoRef validInfoB = (SecValidInfoRef)b;
1088     if (validInfoA == validInfoB) {
1089         return true;
1090     }
1091     if (!validInfoA || !validInfoB ||
1092         (CFGetTypeID(a) != SecValidInfoGetTypeID()) ||
1093         (CFGetTypeID(b) != SecValidInfoGetTypeID())) {
1094         return false;
1095     }
1096     return CFEqualSafe(validInfoA->certHash, validInfoB->certHash) && CFEqualSafe(validInfoA->issuerHash, validInfoB->issuerHash);
1097 }
1098
1099 static CFStringRef SecValidInfoCopyFormatDescription(CFTypeRef cf, CFDictionaryRef formatOptions) {
1100     SecValidInfoRef validInfo = (SecValidInfoRef)cf;
1101     CFStringRef certHash = CFDataCopyHexString(validInfo->certHash);
1102     CFStringRef issuerHash = CFDataCopyHexString(validInfo->issuerHash);
1103     CFStringRef desc = CFStringCreateWithFormat(NULL, formatOptions, CFSTR("validInfo certHash: %@ issuerHash: %@"), certHash, issuerHash);
1104     CFReleaseNull(certHash);
1105     CFReleaseNull(issuerHash);
1106     return desc;
1107 }
1108
1109
1110 // MARK: -
1111 // MARK: SecRevocationDb
1112 /*
1113  ==============================================================================
1114    SecRevocationDb
1115  ==============================================================================
1116 */
1117
1118 /* SecRevocationDbCheckNextUpdate returns true if we dispatched an
1119    update request, otherwise false.
1120 */
1121 static bool _SecRevocationDbCheckNextUpdate(void) {
1122     // are we the db owner instance?
1123     if (!isDbOwner()) {
1124         return false;
1125     }
1126     CFTypeRef value = NULL;
1127
1128     // is it time to check?
1129     CFAbsoluteTime now = CFAbsoluteTimeGetCurrent();
1130     CFAbsoluteTime minNextUpdate = now + gUpdateInterval;
1131     gUpdateStarted = now;
1132
1133     if (0 == gNextUpdate) {
1134         // first time we're called, check if we have a saved nextUpdate value
1135         gNextUpdate = SecRevocationDbGetNextUpdateTime();
1136         minNextUpdate = now;
1137         if (gNextUpdate < minNextUpdate) {
1138             gNextUpdate = minNextUpdate;
1139         }
1140         // allow pref to override update interval, if it exists
1141         CFIndex interval = -1;
1142         value = (CFNumberRef)CFPreferencesCopyValue(kUpdateIntervalKey, kSecPrefsDomain, kCFPreferencesAnyUser, kCFPreferencesCurrentHost);
1143         if (isNumber(value)) {
1144             if (CFNumberGetValue((CFNumberRef)value, kCFNumberCFIndexType, &interval)) {
1145                 if (interval < kSecMinUpdateInterval) {
1146                     interval = kSecMinUpdateInterval;
1147                 } else if (interval > kSecMaxUpdateInterval) {
1148                     interval = kSecMaxUpdateInterval;
1149                 }
1150             }
1151         }
1152         CFReleaseNull(value);
1153         gUpdateInterval = kSecStdUpdateInterval;
1154         if (interval > 0) {
1155             gUpdateInterval = interval;
1156         }
1157         // pin next update time to the preferred update interval
1158         if (gNextUpdate > (gUpdateStarted + gUpdateInterval)) {
1159             gNextUpdate = gUpdateStarted + gUpdateInterval;
1160         }
1161         secdebug("validupdate", "next update at %f (in %f seconds)",
1162                  (double)gUpdateStarted, (double)gNextUpdate-gUpdateStarted);
1163     }
1164     if (gNextUpdate > now) {
1165         gUpdateStarted = 0;
1166         return false;
1167     }
1168     secnotice("validupdate", "starting update");
1169
1170     // set minimum next update time here in case we can't get an update
1171     gNextUpdate = minNextUpdate;
1172
1173     // determine which server to query
1174     CFStringRef server;
1175     value = (CFStringRef)CFPreferencesCopyValue(kUpdateServerKey, kSecPrefsDomain, kCFPreferencesAnyUser, kCFPreferencesCurrentHost);
1176     if (isString(value)) {
1177         server = (CFStringRef) CFRetain(value);
1178     } else {
1179         server = (CFStringRef) CFRetain(SecRevocationDbGetDefaultServer());
1180     }
1181     CFReleaseNull(value);
1182
1183     // determine version of our current database
1184     CFIndex version = SecRevocationDbGetVersion();
1185     secdebug("validupdate", "got version %ld from db", (long)version);
1186     if (version <= 0) {
1187         if (gLastVersion > 0) {
1188             secdebug("validupdate", "error getting version; using last good version: %ld", (long)gLastVersion);
1189         }
1190         version = gLastVersion;
1191     }
1192
1193     // determine source of our current database
1194     // (if this ever changes, we will need to reload the db)
1195     CFStringRef db_source = SecRevocationDbCopyUpdateSource();
1196     if (!db_source) {
1197         db_source = (CFStringRef) CFRetain(kValidUpdateProdServer);
1198     }
1199
1200     // determine whether we need to recreate the database
1201     CFIndex db_version = SecRevocationDbGetSchemaVersion();
1202     CFIndex db_format = SecRevocationDbGetUpdateFormat();
1203     if (db_version < kSecRevocationDbSchemaVersion ||
1204         db_format < kSecRevocationDbUpdateFormat ||
1205         kCFCompareEqualTo != CFStringCompare(server, db_source, kCFCompareCaseInsensitive)) {
1206         // we need to fully rebuild the db contents, so we set our version to 0.
1207         version = gLastVersion = 0;
1208     }
1209
1210     // determine whether update fetching is enabled
1211 #if (__MAC_OS_X_VERSION_MIN_REQUIRED >= 101300 || __IPHONE_OS_VERSION_MIN_REQUIRED >= 110000)
1212     bool updateEnabled = true; // macOS 10.13 or iOS 11.0
1213 #else
1214     bool updateEnabled = false;
1215 #endif
1216     value = (CFBooleanRef)CFPreferencesCopyValue(kUpdateEnabledKey, kSecPrefsDomain, kCFPreferencesAnyUser, kCFPreferencesCurrentHost);
1217     if (isBoolean(value)) {
1218         updateEnabled = CFBooleanGetValue((CFBooleanRef)value);
1219     }
1220     CFReleaseNull(value);
1221
1222     // Schedule maintenance work
1223     bool result = SecValidUpdateSchedule(updateEnabled, server, version);
1224     CFReleaseNull(server);
1225     CFReleaseNull(db_source);
1226     return result;
1227 }
1228
1229 void SecRevocationDbCheckNextUpdate(void) {
1230     static dispatch_once_t once;
1231     static sec_action_t action;
1232
1233     dispatch_once(&once, ^{
1234         dispatch_queue_t update_queue = SecRevocationDbGetUpdateQueue();
1235         action = sec_action_create_with_queue(update_queue, "update_check", kSecMinUpdateInterval);
1236         sec_action_set_handler(action, ^{
1237             os_transaction_t transaction = os_transaction_create("com.apple.trustd.valid.checkNextUpdate");
1238             (void)_SecRevocationDbCheckNextUpdate();
1239             os_release(transaction);
1240         });
1241     });
1242     sec_action_perform(action);
1243 }
1244
1245 /*  This function verifies an update, in this format:
1246     1) unsigned 32-bit network-byte-order length of binary plist
1247     2) binary plist data
1248     3) unsigned 32-bit network-byte-order length of CMS message
1249     4) CMS message (containing certificates and signature over binary plist)
1250
1251     The length argument is the total size of the packed update data.
1252 */
1253 bool SecRevocationDbVerifyUpdate(void *update, CFIndex length) {
1254     if (!update || length <= (CFIndex)sizeof(uint32_t)) {
1255         return false;
1256     }
1257     uint32_t plistLength = OSSwapInt32(*((uint32_t *)update));
1258     if ((plistLength + (CFIndex)(sizeof(uint32_t)*2)) > (uint64_t) length) {
1259         secdebug("validupdate", "ERROR: reported plist length (%lu)+%lu exceeds total length (%lu)\n",
1260                 (unsigned long)plistLength, (unsigned long)sizeof(uint32_t)*2, (unsigned long)length);
1261         return false;
1262     }
1263     uint8_t *plistData = (uint8_t *)update + sizeof(uint32_t);
1264     uint8_t *sigData = (uint8_t *)plistData + plistLength;
1265     uint32_t sigLength = OSSwapInt32(*((uint32_t *)sigData));
1266     sigData += sizeof(uint32_t);
1267     if ((plistLength + sigLength + (CFIndex)(sizeof(uint32_t) * 2)) != (uint64_t) length) {
1268         secdebug("validupdate", "ERROR: reported lengths do not add up to total length\n");
1269         return false;
1270     }
1271
1272     OSStatus status = 0;
1273     CMSSignerStatus signerStatus;
1274     CMSDecoderRef cms = NULL;
1275     SecPolicyRef policy = NULL;
1276     SecTrustRef trust = NULL;
1277     CFDataRef content = NULL;
1278
1279     if ((content = CFDataCreateWithBytesNoCopy(kCFAllocatorDefault,
1280         (const UInt8 *)plistData, (CFIndex)plistLength, kCFAllocatorNull)) == NULL) {
1281         secdebug("validupdate", "CFDataCreateWithBytesNoCopy failed (%ld bytes)\n", (long)plistLength);
1282         return false;
1283     }
1284
1285     if ((status = CMSDecoderCreate(&cms)) != errSecSuccess) {
1286         secdebug("validupdate", "CMSDecoderCreate failed with error %d\n", (int)status);
1287         goto verifyExit;
1288     }
1289     if ((status = CMSDecoderUpdateMessage(cms, sigData, sigLength)) != errSecSuccess) {
1290         secdebug("validupdate", "CMSDecoderUpdateMessage failed with error %d\n", (int)status);
1291         goto verifyExit;
1292     }
1293     if ((status = CMSDecoderSetDetachedContent(cms, content)) != errSecSuccess) {
1294         secdebug("validupdate", "CMSDecoderSetDetachedContent failed with error %d\n", (int)status);
1295         goto verifyExit;
1296     }
1297     if ((status = CMSDecoderFinalizeMessage(cms)) != errSecSuccess) {
1298         secdebug("validupdate", "CMSDecoderFinalizeMessage failed with error %d\n", (int)status);
1299         goto verifyExit;
1300     }
1301
1302     policy = SecPolicyCreateApplePinned(CFSTR("ValidUpdate"), // kSecPolicyNameAppleValidUpdate
1303                 CFSTR("1.2.840.113635.100.6.2.10"), // System Integration 2 Intermediate Certificate
1304                 CFSTR("1.2.840.113635.100.6.51"));  // Valid update signing OID
1305
1306     // Check that the first signer actually signed this message.
1307     if ((status = CMSDecoderCopySignerStatus(cms, 0, policy,
1308             false, &signerStatus, &trust, NULL)) != errSecSuccess) {
1309         secdebug("validupdate", "CMSDecoderCopySignerStatus failed with error %d\n", (int)status);
1310         goto verifyExit;
1311     }
1312     // Make sure the signature verifies against the detached content
1313     if (signerStatus != kCMSSignerValid) {
1314         secdebug("validupdate", "ERROR: signature did not verify (signer status %d)\n", (int)signerStatus);
1315         status = errSecInvalidSignature;
1316         goto verifyExit;
1317     }
1318     // Make sure the signing certificate is valid for the specified policy
1319     SecTrustResultType trustResult = kSecTrustResultInvalid;
1320     status = SecTrustEvaluate(trust, &trustResult);
1321     if (status != errSecSuccess) {
1322         secdebug("validupdate", "SecTrustEvaluate failed with error %d (trust=%p)\n", (int)status, (void *)trust);
1323     } else if (!(trustResult == kSecTrustResultUnspecified || trustResult == kSecTrustResultProceed)) {
1324         secdebug("validupdate", "SecTrustEvaluate failed with trust result %d\n", (int)trustResult);
1325         status = errSecVerificationFailure;
1326         goto verifyExit;
1327     }
1328
1329 verifyExit:
1330     CFReleaseSafe(content);
1331     CFReleaseSafe(trust);
1332     CFReleaseSafe(policy);
1333     CFReleaseSafe(cms);
1334
1335     return (status == errSecSuccess);
1336 }
1337
1338 CFAbsoluteTime SecRevocationDbComputeNextUpdateTime(CFIndex updateInterval) {
1339     CFIndex interval = updateInterval;
1340     // try to use interval preference if it exists
1341     CFTypeRef value = (CFNumberRef)CFPreferencesCopyValue(kUpdateIntervalKey, kSecPrefsDomain, kCFPreferencesAnyUser, kCFPreferencesCurrentHost);
1342     if (isNumber(value)) {
1343         CFNumberGetValue((CFNumberRef)value, kCFNumberCFIndexType, &interval);
1344     }
1345     CFReleaseNull(value);
1346
1347     if (interval <= 0) {
1348         interval = kSecStdUpdateInterval;
1349     }
1350
1351     // sanity check
1352     if (interval < kSecMinUpdateInterval) {
1353         interval = kSecMinUpdateInterval;
1354     } else if (interval > kSecMaxUpdateInterval) {
1355         interval = kSecMaxUpdateInterval;
1356     }
1357
1358     // compute randomization factor, between 0 and 50% of the interval
1359     CFIndex fuzz = arc4random() % (long)(interval/2.0);
1360     CFAbsoluteTime nextUpdate =  CFAbsoluteTimeGetCurrent() + interval + fuzz;
1361     secdebug("validupdate", "next update in %ld seconds", (long)(interval + fuzz));
1362     return nextUpdate;
1363 }
1364
1365 void SecRevocationDbComputeAndSetNextUpdateTime(void) {
1366     gNextUpdate = SecRevocationDbComputeNextUpdateTime(0);
1367     (void) SecRevocationDbSetNextUpdateTime(gNextUpdate, NULL);
1368     gUpdateStarted = 0; /* no update is currently in progress */
1369 }
1370
1371 bool SecRevocationDbIngestUpdate(SecRevocationDbConnectionRef dbc, CFDictionaryRef update, CFIndex chunkVersion, CFIndex *outVersion, CFErrorRef *error) {
1372     bool ok = false;
1373     CFIndex version = 0;
1374     CFErrorRef localError = NULL;
1375     if (!update) {
1376         SecError(errSecParam, &localError, CFSTR("SecRevocationDbIngestUpdate: invalid update parameter"));
1377         goto setVersionAndExit;
1378     }
1379     CFTypeRef value = (CFNumberRef)CFDictionaryGetValue(update, CFSTR("version"));
1380     if (isNumber(value)) {
1381         if (!CFNumberGetValue((CFNumberRef)value, kCFNumberCFIndexType, &version)) {
1382             version = 0;
1383         }
1384     }
1385     if (version == 0) {
1386         // only the first chunk will have a version, so the second and
1387         // subsequent chunks will need to pass it in chunkVersion.
1388         version = chunkVersion;
1389     }
1390     // check precommitted version since update hasn't been committed yet
1391     CFIndex curVersion = dbc->precommitVersion;
1392     if (version > curVersion || chunkVersion > 0) {
1393         ok = _SecRevocationDbApplyUpdate(dbc, update, version, &localError);
1394         secdebug("validupdate", "_SecRevocationDbApplyUpdate=%s, v%ld, precommit=%ld, full=%s",
1395                  (ok) ? "1" : "0", (long)version, (long)dbc->precommitVersion,
1396                  (dbc->fullUpdate) ? "1" : "0");
1397     } else {
1398         secdebug("validupdate", "we have v%ld, skipping update to v%ld",
1399                  (long)curVersion, (long)version);
1400         version = -1; // invalid, so we know to skip subsequent chunks
1401         ok = true; // this is not an error condition
1402     }
1403 setVersionAndExit:
1404     if (outVersion) {
1405         *outVersion = version;
1406     }
1407     (void) CFErrorPropagate(localError, error);
1408     return ok;
1409 }
1410
1411
1412 /* Database schema */
1413
1414 /* admin table holds these key-value (or key-ival) pairs:
1415  'version' (integer)    // version of database content
1416  'check_again' (double) // CFAbsoluteTime of next check (optional)
1417  'db_version' (integer) // version of database schema
1418  'db_hash' (blob)       // SHA-256 database hash
1419  --> entries in admin table are unique by text key
1420
1421  issuers table holds map of issuing CA hashes to group identifiers:
1422  groupid (integer)     // associated group identifier in group ID table
1423  issuer_hash (blob)    // SHA-256 hash of issuer certificate (primary key)
1424  --> entries in issuers table are unique by issuer_hash;
1425  multiple issuer entries may have the same groupid!
1426
1427  groups table holds records with these attributes:
1428  groupid (integer)     // ordinal ID associated with this group entry
1429  flags (integer)       // a bitmask of the following values:
1430    kSecValidInfoComplete    (0x00000001) set if we have all revocation info for this issuer group
1431    kSecValidInfoCheckOCSP   (0x00000002) set if must check ocsp for certs from this issuer group
1432    kSecValidInfoKnownOnly   (0x00000004) set if any CA from this issuer group must be in database
1433    kSecValidInfoRequireCT   (0x00000008) set if all certs from this issuer group must have SCTs
1434    kSecValidInfoAllowlist   (0x00000010) set if this entry describes valid certs (i.e. is allowed)
1435    kSecValidInfoNoCACheck   (0x00000020) set if this entry does not require an OCSP check to accept (deprecated)
1436    kSecValidInfoOverridable (0x00000040) set if the trust status is recoverable and can be overridden
1437    kSecValidInfoDateConstraints (0x00000080) set if this group has not-before or not-after constraints
1438    kSecValidInfoNameConstraints (0x00000100) set if this group has name constraints in database
1439    kSecValidInfoPolicyConstraints (0x00000200) set if this group has policy constraints in database
1440    kSecValidInfoNoCAv2Check (0x00000400) set if this entry does not require an OCSP check to accept
1441  format (integer)      // an integer describing format of entries:
1442    kSecValidInfoFormatUnknown (0) unknown format
1443    kSecValidInfoFormatSerial  (1) serial number, not greater than 20 bytes in length
1444    kSecValidInfoFormatSHA256  (2) SHA-256 hash, 32 bytes in length
1445    kSecValidInfoFormatNto1    (3) filter data blob of arbitrary length
1446  data (blob)           // Bloom filter data if format is 'nto1', otherwise NULL
1447  policies (blob)       // NULL, or uint8_t count value followed by array of int8_t policy values
1448  --> entries in groups table are unique by groupid
1449
1450  serials table holds serial number blobs with these attributes:
1451  groupid (integer)     // identifier for issuer group in the groups table
1452  serial (blob)         // serial number
1453  --> entries in serials table are unique by serial and groupid
1454
1455  hashes table holds SHA-256 hashes of certificates with these attributes:
1456  groupid (integer)     // identifier for issuer group in the groups table
1457  sha256 (blob)         // SHA-256 hash of subject certificate
1458  --> entries in hashes table are unique by sha256 and groupid
1459
1460  dates table holds notBefore and notAfter dates (as CFAbsoluteTime) with these attributes:
1461  groupid (integer)     // identifier for issuer group in the groups table (primary key)
1462  notbefore (real)      // issued certs are invalid if their notBefore is prior to this date
1463  notafter (real)       // issued certs are invalid after this date (or their notAfter, if earlier)
1464  --> entries in dates table are unique by groupid, and only exist if kSecValidInfoDateConstraints is true
1465
1466  */
1467 #define createTablesSQL   CFSTR("CREATE TABLE IF NOT EXISTS admin(" \
1468                                     "key TEXT PRIMARY KEY NOT NULL," \
1469                                     "ival INTEGER NOT NULL," \
1470                                     "value BLOB" \
1471                                 ");" \
1472                                 "CREATE TABLE IF NOT EXISTS issuers(" \
1473                                     "groupid INTEGER NOT NULL," \
1474                                     "issuer_hash BLOB PRIMARY KEY NOT NULL" \
1475                                 ");" \
1476                                 "CREATE INDEX IF NOT EXISTS issuer_idx ON issuers(issuer_hash);" \
1477                                 "CREATE TABLE IF NOT EXISTS groups(" \
1478                                     "groupid INTEGER PRIMARY KEY AUTOINCREMENT," \
1479                                     "flags INTEGER," \
1480                                     "format INTEGER," \
1481                                     "data BLOB," \
1482                                     "policies BLOB" \
1483                                 ");" \
1484                                 "CREATE TABLE IF NOT EXISTS serials(" \
1485                                     "groupid INTEGER NOT NULL," \
1486                                     "serial BLOB NOT NULL," \
1487                                     "UNIQUE(groupid,serial)" \
1488                                 ");" \
1489                                 "CREATE TABLE IF NOT EXISTS hashes(" \
1490                                     "groupid INTEGER NOT NULL," \
1491                                     "sha256 BLOB NOT NULL," \
1492                                     "UNIQUE(groupid,sha256)" \
1493                                 ");" \
1494                                 "CREATE TABLE IF NOT EXISTS dates(" \
1495                                     "groupid INTEGER PRIMARY KEY NOT NULL," \
1496                                     "notbefore REAL," \
1497                                     "notafter REAL" \
1498                                 ");" \
1499                                 "CREATE TRIGGER IF NOT EXISTS group_del " \
1500                                     "BEFORE DELETE ON groups FOR EACH ROW " \
1501                                     "BEGIN " \
1502                                         "DELETE FROM serials WHERE groupid=OLD.groupid; " \
1503                                         "DELETE FROM hashes WHERE groupid=OLD.groupid; " \
1504                                         "DELETE FROM issuers WHERE groupid=OLD.groupid; " \
1505                                         "DELETE FROM dates WHERE groupid=OLD.groupid; " \
1506                                     "END;")
1507
1508 #define selectGroupIdSQL  CFSTR("SELECT DISTINCT groupid " \
1509     "FROM issuers WHERE issuer_hash=?")
1510 #define selectVersionSQL CFSTR("SELECT ival FROM admin " \
1511     "WHERE key='version'")
1512 #define selectDbVersionSQL CFSTR("SELECT ival FROM admin " \
1513     "WHERE key='db_version'")
1514 #define selectDbFormatSQL CFSTR("SELECT ival FROM admin " \
1515     "WHERE key='db_format'")
1516 #define selectDbHashSQL CFSTR("SELECT value FROM admin " \
1517     "WHERE key='db_hash'")
1518 #define selectDbSourceSQL CFSTR("SELECT value FROM admin " \
1519     "WHERE key='db_source'")
1520 #define selectNextUpdateSQL CFSTR("SELECT value FROM admin " \
1521     "WHERE key='check_again'")
1522 #define selectUpdateIntervalSQL CFSTR("SELECT ival FROM admin " \
1523     "WHERE key='interval'")
1524 #define selectGroupRecordSQL CFSTR("SELECT flags,format,data,policies " \
1525     "FROM groups WHERE groupid=?")
1526 #define selectSerialRecordSQL CFSTR("SELECT rowid FROM serials " \
1527     "WHERE groupid=? AND serial=?")
1528 #define selectDateRecordSQL CFSTR("SELECT notbefore,notafter FROM " \
1529     "dates WHERE groupid=?")
1530 #define selectHashRecordSQL CFSTR("SELECT rowid FROM hashes " \
1531     "WHERE groupid=? AND sha256=?")
1532 #define insertAdminRecordSQL CFSTR("INSERT OR REPLACE INTO admin " \
1533     "(key,ival,value) VALUES (?,?,?)")
1534 #define insertIssuerRecordSQL CFSTR("INSERT OR REPLACE INTO issuers " \
1535     "(groupid,issuer_hash) VALUES (?,?)")
1536 #define insertGroupRecordSQL CFSTR("INSERT OR REPLACE INTO groups " \
1537     "(groupid,flags,format,data,policies) VALUES (?,?,?,?,?)")
1538 #define insertSerialRecordSQL CFSTR("INSERT OR REPLACE INTO serials " \
1539     "(groupid,serial) VALUES (?,?)")
1540 #define deleteSerialRecordSQL CFSTR("DELETE FROM serials " \
1541     "WHERE groupid=? AND hex(serial) LIKE ?")
1542 #define insertSha256RecordSQL CFSTR("INSERT OR REPLACE INTO hashes " \
1543     "(groupid,sha256) VALUES (?,?)")
1544 #define deleteSha256RecordSQL CFSTR("DELETE FROM hashes " \
1545     "WHERE groupid=? AND hex(sha256) LIKE ?")
1546 #define insertDateRecordSQL CFSTR("INSERT OR REPLACE INTO dates " \
1547     "(groupid,notbefore,notafter) VALUES (?,?,?)")
1548 #define deleteGroupRecordSQL CFSTR("DELETE FROM groups " \
1549     "WHERE groupid=?")
1550 #define deleteGroupIssuersSQL CFSTR("DELETE FROM issuers " \
1551     "WHERE groupid=?")
1552 #define addPoliciesColumnSQL CFSTR("ALTER TABLE groups " \
1553     "ADD COLUMN policies BLOB")
1554 #define updateGroupPoliciesSQL CFSTR("UPDATE OR IGNORE groups " \
1555     "SET policies=? WHERE groupid=?")
1556
1557 #define updateConstraintsTablesSQL CFSTR("" \
1558 "CREATE TABLE IF NOT EXISTS dates(" \
1559     "groupid INTEGER PRIMARY KEY NOT NULL," \
1560     "notbefore REAL," \
1561     "notafter REAL" \
1562 ");")
1563
1564 #define updateGroupDeleteTriggerSQL CFSTR("" \
1565     "DROP TRIGGER IF EXISTS group_del;" \
1566     "CREATE TRIGGER group_del BEFORE DELETE ON groups FOR EACH ROW " \
1567     "BEGIN " \
1568         "DELETE FROM serials WHERE groupid=OLD.groupid; " \
1569         "DELETE FROM hashes WHERE groupid=OLD.groupid; " \
1570         "DELETE FROM issuers WHERE groupid=OLD.groupid; " \
1571         "DELETE FROM dates WHERE groupid=OLD.groupid; " \
1572     "END;")
1573
1574 #define deleteAllEntriesSQL CFSTR("" \
1575     "DELETE FROM groups; " \
1576     "DELETE FROM admin WHERE key='version'; " \
1577     "DELETE FROM sqlite_sequence")
1578
1579
1580 /* Database management */
1581
1582 static SecDbRef SecRevocationDbCreate(CFStringRef path) {
1583     /* only the db owner should open a read-write connection. */
1584     __block bool readWrite = isDbOwner();
1585     mode_t mode = 0644;
1586
1587     SecDbRef result = SecDbCreate(path, mode, readWrite, false, true, true, 1, ^bool (SecDbRef db, SecDbConnectionRef dbconn, bool didCreate, bool *callMeAgainForNextConnection, CFErrorRef *error) {
1588         __block bool ok = true;
1589         if (readWrite) {
1590             ok &= SecDbTransaction(dbconn, kSecDbExclusiveTransactionType, error, ^(bool *commit) {
1591                 /* Create all database tables, indexes, and triggers.
1592                  * SecDbOpen will set auto_vacuum and journal_mode for us before we get called back.*/
1593                 ok = ok && SecDbExec(dbconn, createTablesSQL, error);
1594                 *commit = ok;
1595             });
1596         }
1597         if (!ok || (error && *error)) {
1598             CFIndex errCode = errSecInternalComponent;
1599             if (error && *error) {
1600                 errCode = CFErrorGetCode(*error);
1601             }
1602             secerror("%s failed: %@", didCreate ? "Create" : "Open", error ? *error : NULL);
1603             TrustdHealthAnalyticsLogErrorCodeForDatabase(TARevocationDb, TAOperationCreate, TAFatalError, errCode);
1604         }
1605         return ok;
1606     });
1607
1608     return result;
1609 }
1610
1611 static dispatch_once_t kSecRevocationDbOnce;
1612 static SecRevocationDbRef kSecRevocationDb = NULL;
1613
1614 static SecRevocationDbRef SecRevocationDbInit(CFStringRef db_name) {
1615     SecRevocationDbRef rdb;
1616     dispatch_queue_attr_t attr;
1617
1618     require(rdb = (SecRevocationDbRef)malloc(sizeof(struct __SecRevocationDb)), errOut);
1619     rdb->db = NULL;
1620     rdb->update_queue = NULL;
1621     rdb->updateInProgress = false;
1622     rdb->unsupportedVersion = false;
1623     rdb->changed = false;
1624
1625     require(rdb->db = SecRevocationDbCreate(db_name), errOut);
1626     attr = dispatch_queue_attr_make_with_qos_class(DISPATCH_QUEUE_SERIAL, QOS_CLASS_BACKGROUND, 0);
1627     attr = dispatch_queue_attr_make_with_autorelease_frequency(attr, DISPATCH_AUTORELEASE_FREQUENCY_WORK_ITEM);
1628     require(rdb->update_queue = dispatch_queue_create(NULL, attr), errOut);
1629     require(rdb->info_cache_list = CFArrayCreateMutable(NULL, 0, &kCFTypeArrayCallBacks), errOut);
1630     require(rdb->info_cache = CFDictionaryCreateMutable(NULL, 0, &kCFTypeDictionaryKeyCallBacks, &kCFTypeDictionaryValueCallBacks), errOut);
1631     rdb->info_cache_lock = OS_UNFAIR_LOCK_INIT;
1632
1633     if (!isDbOwner()) {
1634         /* register for changes signaled by the db owner instance */
1635         int out_token = 0;
1636         notify_register_dispatch(kSecRevocationDbChanged, &out_token, rdb->update_queue, ^(int __unused token) {
1637             secnotice("validupdate", "Got notification of database change");
1638             SecRevocationDbResetCaches();
1639         });
1640     }
1641     return rdb;
1642
1643 errOut:
1644     secdebug("validupdate", "Failed to create db at \"%@\"", db_name);
1645     if (rdb) {
1646         if (rdb->update_queue) {
1647             dispatch_release(rdb->update_queue);
1648         }
1649         CFReleaseSafe(rdb->db);
1650         free(rdb);
1651     }
1652     return NULL;
1653 }
1654
1655 static CFStringRef SecRevocationDbCopyPath(void) {
1656     CFURLRef revDbURL = NULL;
1657     CFStringRef revInfoRelPath = NULL;
1658     if ((revInfoRelPath = CFStringCreateWithFormat(NULL, NULL, CFSTR("%s"), kSecRevocationDbFileName)) != NULL) {
1659         revDbURL = SecCopyURLForFileInRevocationInfoDirectory(revInfoRelPath);
1660     }
1661     CFReleaseSafe(revInfoRelPath);
1662
1663     CFStringRef revDbPath = NULL;
1664     if (revDbURL) {
1665         revDbPath = CFURLCopyFileSystemPath(revDbURL, kCFURLPOSIXPathStyle);
1666         CFRelease(revDbURL);
1667     }
1668     return revDbPath;
1669 }
1670
1671 static void SecRevocationDbWith(void(^dbJob)(SecRevocationDbRef db)) {
1672     dispatch_once(&kSecRevocationDbOnce, ^{
1673         CFStringRef dbPath = SecRevocationDbCopyPath();
1674         if (dbPath) {
1675             kSecRevocationDb = SecRevocationDbInit(dbPath);
1676             CFRelease(dbPath);
1677             if (kSecRevocationDb && isDbOwner()) {
1678                 /* check and update schema immediately after database is opened */
1679                 SecRevocationDbUpdateSchema(kSecRevocationDb);
1680             }
1681         }
1682     });
1683     // Do pre job run work here (cancel idle timers etc.)
1684     if (kSecRevocationDb->updateInProgress) {
1685         return; // this would block since SecDb has an exclusive transaction lock
1686     }
1687     dbJob(kSecRevocationDb);
1688     // Do post job run work here (gc timer, etc.)
1689 }
1690
1691 static bool SecRevocationDbPerformWrite(SecRevocationDbRef rdb, CFErrorRef *error,
1692                                         bool(^writeJob)(SecRevocationDbConnectionRef dbc, CFErrorRef *blockError)) {
1693     __block bool ok = true;
1694     __block CFErrorRef localError = NULL;
1695
1696     ok &= SecDbPerformWrite(rdb->db, &localError, ^(SecDbConnectionRef dbconn) {
1697         ok &= SecDbTransaction(dbconn, kSecDbImmediateTransactionType, &localError, ^(bool *commit) {
1698             SecRevocationDbConnectionRef dbc = SecRevocationDbConnectionInit(rdb, dbconn, &localError);
1699             ok = ok && writeJob(dbc, &localError);
1700             *commit = ok;
1701             free(dbc);
1702         });
1703     });
1704     ok &= CFErrorPropagate(localError, error);
1705     return ok;
1706 }
1707
1708 static bool SecRevocationDbPerformRead(SecRevocationDbRef rdb, CFErrorRef *error,
1709                                        bool(^readJob)(SecRevocationDbConnectionRef dbc, CFErrorRef *blockError)) {
1710     __block CFErrorRef localError = NULL;
1711     __block bool ok = true;
1712
1713     ok &= SecDbPerformRead(rdb->db, &localError, ^(SecDbConnectionRef dbconn) {
1714         SecRevocationDbConnectionRef dbc = SecRevocationDbConnectionInit(rdb, dbconn, &localError);
1715         ok = ok && readJob(dbc, &localError);
1716         free(dbc);
1717     });
1718     ok &= CFErrorPropagate(localError, error);
1719     return ok;
1720 }
1721
1722 static SecRevocationDbConnectionRef SecRevocationDbConnectionInit(SecRevocationDbRef db, SecDbConnectionRef dbconn, CFErrorRef *error) {
1723     SecRevocationDbConnectionRef dbc = NULL;
1724     CFErrorRef localError = NULL;
1725
1726     dbc = (SecRevocationDbConnectionRef)malloc(sizeof(struct __SecRevocationDbConnection));
1727     if (dbc) {
1728         dbc->db = db;
1729         dbc->dbconn = dbconn;
1730         dbc->precommitVersion = (CFIndex)_SecRevocationDbGetVersion(dbc, &localError);
1731         dbc->precommitDbVersion = (CFIndex)_SecRevocationDbGetSchemaVersion(db, dbc, &localError);
1732         dbc->precommitInterval = 0; /* set only if we are explicitly given a new value */
1733         dbc->fullUpdate = false;
1734     }
1735     (void) CFErrorPropagate(localError, error);
1736     return dbc;
1737 }
1738
1739 static CF_RETURNS_RETAINED CFDataRef createCacheKey(CFDataRef certHash, CFDataRef issuerHash) {
1740     CFMutableDataRef concat = CFDataCreateMutableCopy(NULL, 0, certHash);
1741     CFDataAppend(concat, issuerHash);
1742     CFDataRef result = SecSHA256DigestCreateFromData(NULL, concat);
1743     CFReleaseNull(concat);
1744     return result;
1745 }
1746
1747 static CF_RETURNS_RETAINED SecValidInfoRef SecRevocationDbCacheRead(SecRevocationDbRef db,
1748                                                                      SecCertificateRef certificate,
1749                                                                      CFDataRef issuerHash) {
1750     if (!db) {
1751         return NULL;
1752     }
1753     SecValidInfoRef result = NULL;
1754     if (!db || !db->info_cache || !db->info_cache_list) {
1755         return result;
1756     }
1757     CFIndex ix = kCFNotFound;
1758     CFDataRef certHash = SecCertificateCopySHA256Digest(certificate);
1759     CFDataRef cacheKey = createCacheKey(certHash, issuerHash);
1760
1761     os_unfair_lock_lock(&db->info_cache_lock); // grab the cache lock before using the cache
1762     if (0 <= (ix = CFArrayGetFirstIndexOfValue(db->info_cache_list,
1763                                                CFRangeMake(0, CFArrayGetCount(db->info_cache_list)),
1764                                                cacheKey))) {
1765         result = (SecValidInfoRef)CFDictionaryGetValue(db->info_cache, cacheKey);
1766         // Verify this really is the right result
1767         if (CFEqualSafe(result->certHash, certHash) && CFEqualSafe(result->issuerHash, issuerHash)) {
1768             // Cache hit. Move the entry to the bottom of the list.
1769             CFArrayRemoveValueAtIndex(db->info_cache_list, ix);
1770             CFArrayAppendValue(db->info_cache_list, cacheKey);
1771             secdebug("validcache", "cache hit: %@", cacheKey);
1772         } else {
1773             // Just remove this bad entry
1774             CFArrayRemoveValueAtIndex(db->info_cache_list, ix);
1775             CFDictionaryRemoveValue(db->info_cache, cacheKey);
1776             secdebug("validcache", "cache remove bad: %@", cacheKey);
1777             secnotice("validcache", "found a bad valid info cache entry at %ld", (long)ix);
1778         }
1779     }
1780     CFRetainSafe(result);
1781     os_unfair_lock_unlock(&db->info_cache_lock);
1782     CFReleaseSafe(certHash);
1783     CFReleaseSafe(cacheKey);
1784     return result;
1785 }
1786
1787 static void SecRevocationDbCacheWrite(SecRevocationDbRef db,
1788                                        SecValidInfoRef validInfo) {
1789     if (!db || !validInfo || !db->info_cache || !db->info_cache_list) {
1790         return;
1791     }
1792
1793     CFDataRef cacheKey = createCacheKey(validInfo->certHash, validInfo->issuerHash);
1794
1795     os_unfair_lock_lock(&db->info_cache_lock); // grab the cache lock before using the cache
1796     // check to make sure another thread didn't add this entry to the cache already
1797     if (0 > CFArrayGetFirstIndexOfValue(db->info_cache_list,
1798                                         CFRangeMake(0, CFArrayGetCount(db->info_cache_list)),
1799                                         cacheKey)) {
1800         CFDictionaryAddValue(db->info_cache, cacheKey, validInfo);
1801         if (kSecRevocationDbCacheSize <= CFArrayGetCount(db->info_cache_list)) {
1802             // Remove least recently used cache entry.
1803             secdebug("validcache", "cache remove stale: %@", CFArrayGetValueAtIndex(db->info_cache_list, 0));
1804             CFDictionaryRemoveValue(db->info_cache, CFArrayGetValueAtIndex(db->info_cache_list, 0));
1805             CFArrayRemoveValueAtIndex(db->info_cache_list, 0);
1806         }
1807         CFArrayAppendValue(db->info_cache_list, cacheKey);
1808         secdebug("validcache", "cache add: %@", cacheKey);
1809     }
1810     os_unfair_lock_unlock(&db->info_cache_lock);
1811     CFReleaseNull(cacheKey);
1812 }
1813
1814 static void SecRevocationDbCachePurge(SecRevocationDbRef db) {
1815     if (!db || !db->info_cache || !db->info_cache_list) {
1816         return;
1817     }
1818
1819     /* grab the cache lock and clear all entries */
1820     os_unfair_lock_lock(&db->info_cache_lock);
1821     CFArrayRemoveAllValues(db->info_cache_list);
1822     CFDictionaryRemoveAllValues(db->info_cache);
1823     secdebug("validcache", "cache purge");
1824     os_unfair_lock_unlock(&db->info_cache_lock);
1825 }
1826
1827 static int64_t _SecRevocationDbGetUpdateInterval(SecRevocationDbConnectionRef dbc, CFErrorRef *error) {
1828     /* look up interval entry in admin table; returns -1 on error */
1829     __block int64_t interval = -1;
1830     __block bool ok = true;
1831     __block CFErrorRef localError = NULL;
1832
1833     ok = ok && SecDbWithSQL(dbc->dbconn, selectUpdateIntervalSQL, &localError, ^bool(sqlite3_stmt *selectInterval) {
1834         ok = ok && SecDbStep(dbc->dbconn, selectInterval, &localError, ^void(bool *stop) {
1835             interval = sqlite3_column_int64(selectInterval, 0);
1836             *stop = true;
1837         });
1838         return ok;
1839     });
1840     if (!ok || localError) {
1841         secerror("_SecRevocationDbGetUpdateInterval failed: %@", localError);
1842         TrustdHealthAnalyticsLogErrorCodeForDatabase(TARevocationDb, TAOperationRead, TAFatalError,
1843                                                      localError ? CFErrorGetCode(localError) : errSecInternalComponent);
1844     }
1845     (void) CFErrorPropagate(localError, error);
1846     return interval;
1847 }
1848
1849 static bool _SecRevocationDbSetUpdateInterval(SecRevocationDbConnectionRef dbc, int64_t interval, CFErrorRef *error) {
1850     secdebug("validupdate", "setting interval to %lld", interval);
1851
1852     __block CFErrorRef localError = NULL;
1853     __block bool ok = (dbc != NULL);
1854     ok = ok && SecDbWithSQL(dbc->dbconn, insertAdminRecordSQL, &localError, ^bool(sqlite3_stmt *insertInterval) {
1855         const char *intervalKey = "interval";
1856         ok = ok && SecDbBindText(insertInterval, 1, intervalKey, strlen(intervalKey),
1857                             SQLITE_TRANSIENT, &localError);
1858         ok = ok && SecDbBindInt64(insertInterval, 2,
1859                              (sqlite3_int64)interval, &localError);
1860         ok = ok && SecDbStep(dbc->dbconn, insertInterval, &localError, NULL);
1861         return ok;
1862     });
1863     if (!ok || localError) {
1864         secerror("_SecRevocationDbSetUpdateInterval failed: %@", localError);
1865         TrustdHealthAnalyticsLogErrorCodeForDatabase(TARevocationDb, TAOperationWrite, TAFatalError,
1866                                                      localError ? CFErrorGetCode(localError) : errSecInternalComponent);
1867     }
1868     (void) CFErrorPropagate(localError, error);
1869     return ok;
1870 }
1871
1872 static CFArrayRef _SecRevocationDbCopyHashes(SecRevocationDbConnectionRef dbc, CFErrorRef *error) {
1873     /* return a retained copy of the db_hash array stored in the admin table; or NULL on error */
1874     __block CFMutableArrayRef hashes = CFArrayCreateMutable(kCFAllocatorDefault, 0, &kCFTypeArrayCallBacks);
1875     __block bool ok = (dbc && hashes);
1876     __block CFErrorRef localError = NULL;
1877
1878     ok = ok && SecDbWithSQL(dbc->dbconn, selectDbHashSQL, &localError, ^bool(sqlite3_stmt *selectDbHash) {
1879         ok = ok && SecDbStep(dbc->dbconn, selectDbHash, &localError, ^void(bool *stop) {
1880             uint8_t *p = (uint8_t *)sqlite3_column_blob(selectDbHash, 0);
1881             uint64_t len = sqlite3_column_bytes(selectDbHash, 0);
1882             CFIndex hashLen = CC_SHA256_DIGEST_LENGTH;
1883             while (p && len >= (uint64_t)hashLen) {
1884                 CFDataRef hash = CFDataCreate(kCFAllocatorDefault, (const UInt8 *)p, hashLen);
1885                 if (hash) {
1886                     CFArrayAppendValue(hashes, hash);
1887                     CFReleaseNull(hash);
1888                 }
1889                 len -= hashLen;
1890                 p += hashLen;
1891             }
1892             *stop = true;
1893         });
1894         return ok;
1895     });
1896     if (!ok || localError) {
1897         CFReleaseNull(hashes);
1898     }
1899     (void) CFErrorPropagate(localError, error);
1900     return hashes;
1901 }
1902
1903 static bool _SecRevocationDbSetHashes(SecRevocationDbConnectionRef dbc, CFArrayRef hashes, CFErrorRef *error) {
1904     /* flatten and store db_hash array in the admin table */
1905     __block CFErrorRef localError = NULL;
1906     __block bool ok = (dbc && hashes);
1907
1908     ok = ok && SecDbWithSQL(dbc->dbconn, insertAdminRecordSQL, &localError, ^bool(sqlite3_stmt *insertHashes) {
1909         CFIndex count = CFArrayGetCount(hashes);
1910         CFIndex hashLen = CC_SHA256_DIGEST_LENGTH, dataLen = hashLen * count;
1911         uint8_t *dataPtr = (uint8_t *)calloc(dataLen, 1);
1912         uint8_t *p = dataPtr;
1913         for (CFIndex idx = 0; idx < count && p; idx++) {
1914             CFDataRef hash = CFArrayGetValueAtIndex(hashes, idx);
1915             uint8_t *h = (hash) ? (uint8_t *)CFDataGetBytePtr(hash) : NULL;
1916             if (h && CFDataGetLength(hash) == hashLen) { memcpy(p, h, hashLen); }
1917             p += hashLen;
1918         }
1919         const char *hashKey = "db_hash";
1920         ok = ok && SecDbBindText(insertHashes, 1, hashKey, strlen(hashKey),
1921                                  SQLITE_TRANSIENT, &localError);
1922         ok = ok && SecDbBindInt64(insertHashes, 2,
1923                                   (sqlite3_int64)0, &localError);
1924         ok = ok && SecDbBindBlob(insertHashes, 3,
1925                                  dataPtr, dataLen,
1926                                  SQLITE_TRANSIENT, &localError);
1927         ok = ok && SecDbStep(dbc->dbconn, insertHashes, &localError, NULL);
1928         free(dataPtr);
1929         return ok;
1930     });
1931     if (!ok || localError) {
1932     }
1933     (void) CFErrorPropagate(localError, error);
1934     return ok;
1935 }
1936
1937 static int64_t _SecRevocationDbGetVersion(SecRevocationDbConnectionRef dbc, CFErrorRef *error) {
1938     /* look up version entry in admin table; returns -1 on error */
1939     __block int64_t version = -1;
1940     __block bool ok = (dbc != NULL);
1941     __block CFErrorRef localError = NULL;
1942
1943     ok = ok && SecDbWithSQL(dbc->dbconn, selectVersionSQL, &localError, ^bool(sqlite3_stmt *selectVersion) {
1944         ok = ok && SecDbStep(dbc->dbconn, selectVersion, &localError, ^void(bool *stop) {
1945             version = sqlite3_column_int64(selectVersion, 0);
1946             *stop = true;
1947         });
1948         return ok;
1949     });
1950     if (!ok || localError) {
1951         secerror("_SecRevocationDbGetVersion failed: %@", localError);
1952         TrustdHealthAnalyticsLogErrorCodeForDatabase(TARevocationDb, TAOperationRead, TAFatalError,
1953                                                      localError ? CFErrorGetCode(localError) : errSecInternalComponent);
1954     }
1955     (void) CFErrorPropagate(localError, error);
1956     return version;
1957 }
1958
1959 static bool _SecRevocationDbSetVersion(SecRevocationDbConnectionRef dbc, CFIndex version, CFErrorRef *error) {
1960     secdebug("validupdate", "setting version to %ld", (long)version);
1961
1962     __block CFErrorRef localError = NULL;
1963     __block bool ok = (dbc != NULL);
1964     ok = ok && SecDbWithSQL(dbc->dbconn, insertAdminRecordSQL, &localError, ^bool(sqlite3_stmt *insertVersion) {
1965         const char *versionKey = "version";
1966         ok = ok && SecDbBindText(insertVersion, 1, versionKey, strlen(versionKey),
1967                             SQLITE_TRANSIENT, &localError);
1968         ok = ok && SecDbBindInt64(insertVersion, 2,
1969                              (sqlite3_int64)version, &localError);
1970         ok = ok && SecDbStep(dbc->dbconn, insertVersion, &localError, NULL);
1971         return ok;
1972     });
1973     if (!ok || localError) {
1974         secerror("_SecRevocationDbSetVersion failed: %@", localError);
1975         TrustdHealthAnalyticsLogErrorCodeForDatabase(TARevocationDb, TAOperationWrite, TAFatalError,
1976                                                      localError ? CFErrorGetCode(localError) : errSecInternalComponent);
1977     }
1978     (void) CFErrorPropagate(localError, error);
1979     return ok;
1980 }
1981
1982 static int64_t _SecRevocationDbReadSchemaVersionFromDb(SecRevocationDbConnectionRef dbc, CFErrorRef *error) {
1983     /* look up db_version entry in admin table; returns -1 on error */
1984     __block int64_t db_version = -1;
1985     __block bool ok = (dbc != NULL);
1986     __block CFErrorRef localError = NULL;
1987
1988     ok = ok && SecDbWithSQL(dbc->dbconn, selectDbVersionSQL, &localError, ^bool(sqlite3_stmt *selectDbVersion) {
1989         ok = ok && SecDbStep(dbc->dbconn, selectDbVersion, &localError, ^void(bool *stop) {
1990             db_version = sqlite3_column_int64(selectDbVersion, 0);
1991             *stop = true;
1992         });
1993         return ok;
1994     });
1995     if (!ok || localError) {
1996         secerror("_SecRevocationDbReadSchemaVersionFromDb failed: %@", localError);
1997         TrustdHealthAnalyticsLogErrorCodeForDatabase(TARevocationDb, TAOperationRead, TAFatalError,
1998                                                      localError ? CFErrorGetCode(localError) : errSecInternalComponent);
1999     }
2000     (void) CFErrorPropagate(localError, error);
2001     return db_version;
2002 }
2003
2004 static _Atomic int64_t gSchemaVersion = -1;
2005 static int64_t _SecRevocationDbGetSchemaVersion(SecRevocationDbRef rdb, SecRevocationDbConnectionRef dbc, CFErrorRef *error) {
2006     static dispatch_once_t onceToken;
2007     dispatch_once(&onceToken, ^{
2008         if (dbc) {
2009             atomic_init(&gSchemaVersion, _SecRevocationDbReadSchemaVersionFromDb(dbc, error));
2010         } else {
2011             (void) SecRevocationDbPerformRead(rdb, error, ^bool(SecRevocationDbConnectionRef dbc, CFErrorRef *blockError) {
2012                 atomic_init(&gSchemaVersion, _SecRevocationDbReadSchemaVersionFromDb(dbc, blockError));
2013                 return true;
2014             });
2015         }
2016     });
2017     if (atomic_load(&gSchemaVersion) == -1) {
2018         /* Initial read(s) failed. Try to read the schema version again. */
2019         if (dbc) {
2020             atomic_store(&gSchemaVersion, _SecRevocationDbReadSchemaVersionFromDb(dbc, error));
2021         } else {
2022             (void) SecRevocationDbPerformRead(rdb, error, ^bool(SecRevocationDbConnectionRef dbc, CFErrorRef *blockError) {
2023                 atomic_store(&gSchemaVersion, _SecRevocationDbReadSchemaVersionFromDb(dbc, blockError));
2024                 return true;
2025             });
2026         }
2027     }
2028     return atomic_load(&gSchemaVersion);
2029 }
2030
2031 static void SecRevocationDbResetCaches(void) {
2032     SecRevocationDbWith(^(SecRevocationDbRef db) {
2033         db->unsupportedVersion = false;
2034         db->changed = false;
2035         (void) SecRevocationDbPerformRead(db, NULL, ^bool(SecRevocationDbConnectionRef dbc, CFErrorRef *blockError) {
2036             atomic_store(&gSchemaVersion, _SecRevocationDbReadSchemaVersionFromDb(dbc, blockError));
2037             return true;
2038         });
2039         SecRevocationDbCachePurge(db);
2040     });
2041 }
2042
2043 static bool _SecRevocationDbSetSchemaVersion(SecRevocationDbConnectionRef dbc, CFIndex dbversion, CFErrorRef *error) {
2044     if (dbversion > 0) {
2045         int64_t db_version = (dbc) ? dbc->precommitDbVersion : -1;
2046         if (db_version >= dbversion) {
2047             return true; /* requested schema is earlier than current schema */
2048         }
2049     }
2050     secdebug("validupdate", "setting db_version to %ld", (long)dbversion);
2051
2052     __block CFErrorRef localError = NULL;
2053     __block bool ok = (dbc != NULL);
2054     ok = ok && SecDbWithSQL(dbc->dbconn, insertAdminRecordSQL, &localError, ^bool(sqlite3_stmt *insertDbVersion) {
2055         const char *dbVersionKey = "db_version";
2056         ok = ok && SecDbBindText(insertDbVersion, 1, dbVersionKey, strlen(dbVersionKey),
2057                             SQLITE_TRANSIENT, &localError);
2058         ok = ok && SecDbBindInt64(insertDbVersion, 2,
2059                              (sqlite3_int64)dbversion, &localError);
2060         ok = ok && SecDbStep(dbc->dbconn, insertDbVersion, &localError, NULL);
2061         return ok;
2062     });
2063     if (!ok || localError) {
2064         secerror("_SecRevocationDbSetSchemaVersion failed: %@", localError);
2065         TrustdHealthAnalyticsLogErrorCodeForDatabase(TARevocationDb, TAOperationWrite, TAFatalError,
2066                                                      localError ? CFErrorGetCode(localError) : errSecInternalComponent);
2067     } else {
2068         dbc->db->changed = true; /* will notify clients of this change */
2069         dbc->db->unsupportedVersion = false;
2070         dbc->precommitDbVersion = dbversion;
2071         atomic_store(&gSchemaVersion, (int64_t)dbversion);
2072     }
2073     CFReleaseSafe(localError);
2074     return ok;
2075 }
2076
2077 static bool _SecRevocationDbUpdateSchema(SecRevocationDbConnectionRef dbc, CFErrorRef *error) {
2078     __block CFErrorRef localError = NULL;
2079     __block bool ok = (dbc != NULL);
2080     __block int64_t db_version = (dbc) ? dbc->precommitDbVersion : 0;
2081     if (db_version >= kSecRevocationDbSchemaVersion) {
2082         return ok; /* schema version already up to date */
2083     }
2084     secdebug("validupdate", "updating db schema from v%lld to v%lld",
2085             (long long)db_version, (long long)kSecRevocationDbSchemaVersion);
2086
2087     if (ok && db_version < 5) {
2088         /* apply v5 changes (add dates table and replace trigger) */
2089         ok &= SecDbWithSQL(dbc->dbconn, updateConstraintsTablesSQL, &localError, ^bool(sqlite3_stmt *updateTables) {
2090             ok = SecDbStep(dbc->dbconn, updateTables, &localError, NULL);
2091             return ok;
2092         });
2093         ok &= SecDbWithSQL(dbc->dbconn, updateGroupDeleteTriggerSQL, &localError, ^bool(sqlite3_stmt *updateTrigger) {
2094             ok = SecDbStep(dbc->dbconn, updateTrigger, &localError, NULL);
2095             return ok;
2096         });
2097         secdebug("validupdate", "applied schema update to v5 (%s)", (ok) ? "ok" : "failed!");
2098     }
2099     if (ok && db_version < 6) {
2100         /* apply v6 changes (the SecDb layer will update autovacuum mode if needed, so we don't execute
2101            any SQL here, but we do want the database to be replaced in case transaction scope problems
2102            with earlier versions caused missing entries.) */
2103         secdebug("validupdate", "applied schema update to v6 (%s)", (ok) ? "ok" : "failed!");
2104         if (db_version > 0) {
2105             SecValidUpdateForceReplaceDatabase();
2106         }
2107     }
2108     if (ok && db_version < 7) {
2109         /* apply v7 changes (add policies column in groups table) */
2110         ok &= SecDbWithSQL(dbc->dbconn, addPoliciesColumnSQL, &localError, ^bool(sqlite3_stmt *addPoliciesColumn) {
2111             ok = SecDbStep(dbc->dbconn, addPoliciesColumn, &localError, NULL);
2112             return ok;
2113         });
2114         secdebug("validupdate", "applied schema update to v7 (%s)", (ok) ? "ok" : "failed!");
2115     }
2116
2117     if (!ok) {
2118         secerror("_SecRevocationDbUpdateSchema failed: %@", localError);
2119     } else {
2120         ok = ok && _SecRevocationDbSetSchemaVersion(dbc, kSecRevocationDbSchemaVersion, &localError);
2121     }
2122     (void) CFErrorPropagate(localError, error);
2123     return ok;
2124 }
2125
2126 bool SecRevocationDbUpdateSchema(SecRevocationDbRef rdb) {
2127     /* note: this function assumes it is called only by the database owner.
2128        non-owner (read-only) clients will fail if changes to the db are needed. */
2129     if (!rdb || !rdb->db) {
2130         return false;
2131     }
2132     __block bool ok = true;
2133     __block CFErrorRef localError = NULL;
2134     ok &= SecRevocationDbPerformWrite(rdb, &localError, ^bool(SecRevocationDbConnectionRef dbc, CFErrorRef *blockError) {
2135         return _SecRevocationDbUpdateSchema(dbc, blockError);
2136     });
2137     CFReleaseSafe(localError);
2138     return ok;
2139 }
2140
2141 static int64_t _SecRevocationDbGetUpdateFormat(SecRevocationDbConnectionRef dbc, CFErrorRef *error) {
2142     /* look up db_format entry in admin table; returns -1 on error */
2143     __block int64_t db_format = -1;
2144     __block bool ok = (dbc != NULL);
2145     __block CFErrorRef localError = NULL;
2146
2147     ok = ok && SecDbWithSQL(dbc->dbconn, selectDbFormatSQL, &localError, ^bool(sqlite3_stmt *selectDbFormat) {
2148         ok &= SecDbStep(dbc->dbconn, selectDbFormat, &localError, ^void(bool *stop) {
2149             db_format = sqlite3_column_int64(selectDbFormat, 0);
2150             *stop = true;
2151         });
2152         return ok;
2153     });
2154     if (!ok || localError) {
2155         secerror("_SecRevocationDbGetUpdateFormat failed: %@", localError);
2156         TrustdHealthAnalyticsLogErrorCodeForDatabase(TARevocationDb, TAOperationRead, TAFatalError,
2157                                                      localError ? CFErrorGetCode(localError) : errSecInternalComponent);
2158     }
2159     (void) CFErrorPropagate(localError, error);
2160     return db_format;
2161 }
2162
2163 static bool _SecRevocationDbSetUpdateFormat(SecRevocationDbConnectionRef dbc, CFIndex dbformat, CFErrorRef *error) {
2164     secdebug("validupdate", "setting db_format to %ld", (long)dbformat);
2165
2166     __block CFErrorRef localError = NULL;
2167     __block bool ok = (dbc != NULL);
2168     ok = ok && SecDbWithSQL(dbc->dbconn, insertAdminRecordSQL, &localError, ^bool(sqlite3_stmt *insertDbFormat) {
2169         const char *dbFormatKey = "db_format";
2170         ok = ok && SecDbBindText(insertDbFormat, 1, dbFormatKey, strlen(dbFormatKey),
2171                             SQLITE_TRANSIENT, &localError);
2172         ok = ok && SecDbBindInt64(insertDbFormat, 2,
2173                              (sqlite3_int64)dbformat, &localError);
2174         ok = ok && SecDbStep(dbc->dbconn, insertDbFormat, &localError, NULL);
2175         return ok;
2176     });
2177     if (!ok || localError) {
2178         secerror("_SecRevocationDbSetUpdateFormat failed: %@", localError);
2179         TrustdHealthAnalyticsLogErrorCodeForDatabase(TARevocationDb, TAOperationWrite, TAFatalError,
2180                                                      localError ? CFErrorGetCode(localError) : errSecInternalComponent);
2181     } else {
2182         dbc->db->changed = true; /* will notify clients of this change */
2183         dbc->db->unsupportedVersion = false;
2184     }
2185     (void) CFErrorPropagate(localError, error);
2186     return ok;
2187 }
2188
2189 static CFStringRef _SecRevocationDbCopyUpdateSource(SecRevocationDbConnectionRef dbc, CFErrorRef *error) {
2190     /* look up db_source entry in admin table; returns NULL on error */
2191     __block CFStringRef updateSource = NULL;
2192     __block bool ok = (dbc != NULL);
2193     __block CFErrorRef localError = NULL;
2194
2195     ok = ok && SecDbWithSQL(dbc->dbconn, selectDbSourceSQL, &localError, ^bool(sqlite3_stmt *selectDbSource) {
2196         ok &= SecDbStep(dbc->dbconn, selectDbSource, &localError, ^void(bool *stop) {
2197             const UInt8 *p = (const UInt8 *)sqlite3_column_blob(selectDbSource, 0);
2198             if (p != NULL) {
2199                 CFIndex length = (CFIndex)sqlite3_column_bytes(selectDbSource, 0);
2200                 if (length > 0) {
2201                     updateSource = CFStringCreateWithBytes(kCFAllocatorDefault, p, length, kCFStringEncodingUTF8, false);
2202                 }
2203             }
2204             *stop = true;
2205         });
2206         return ok;
2207     });
2208     if (!ok || localError) {
2209         secerror("_SecRevocationDbCopyUpdateSource failed: %@", localError);
2210         TrustdHealthAnalyticsLogErrorCodeForDatabase(TARevocationDb, TAOperationRead, TAFatalError,
2211                                                      localError ? CFErrorGetCode(localError) : errSecInternalComponent);
2212     }
2213     (void) CFErrorPropagate(localError, error);
2214     return updateSource;
2215 }
2216
2217 bool _SecRevocationDbSetUpdateSource(SecRevocationDbConnectionRef dbc, CFStringRef updateSource, CFErrorRef *error) {
2218     if (!updateSource) {
2219         secerror("_SecRevocationDbSetUpdateSource failed: %d", errSecParam);
2220         return false;
2221     }
2222     __block char buffer[256];
2223     __block const char *updateSourceCStr = CFStringGetCStringPtr(updateSource, kCFStringEncodingUTF8);
2224     if (!updateSourceCStr) {
2225         if (CFStringGetCString(updateSource, buffer, 256, kCFStringEncodingUTF8)) {
2226             updateSourceCStr = buffer;
2227         }
2228     }
2229     if (!updateSourceCStr) {
2230         secerror("_SecRevocationDbSetUpdateSource failed: unable to get UTF-8 encoding");
2231         return false;
2232     }
2233     secdebug("validupdate", "setting update source to \"%s\"", updateSourceCStr);
2234
2235     __block CFErrorRef localError = NULL;
2236     __block bool ok = (dbc != NULL);
2237     ok = ok && SecDbWithSQL(dbc->dbconn, insertAdminRecordSQL, &localError, ^bool(sqlite3_stmt *insertRecord) {
2238         const char *dbSourceKey = "db_source";
2239         ok = ok && SecDbBindText(insertRecord, 1, dbSourceKey, strlen(dbSourceKey),
2240                            SQLITE_TRANSIENT, &localError);
2241         ok = ok && SecDbBindInt64(insertRecord, 2,
2242                             (sqlite3_int64)0, &localError);
2243         ok = ok && SecDbBindBlob(insertRecord, 3,
2244                            updateSourceCStr, strlen(updateSourceCStr),
2245                            SQLITE_TRANSIENT, &localError);
2246         ok = ok && SecDbStep(dbc->dbconn, insertRecord, &localError, NULL);
2247         return ok;
2248     });
2249     if (!ok || localError) {
2250         secerror("_SecRevocationDbSetUpdateSource failed: %@", localError);
2251         TrustdHealthAnalyticsLogErrorCodeForDatabase(TARevocationDb, TAOperationWrite, TAFatalError,
2252                                                      localError ? CFErrorGetCode(localError) : errSecInternalComponent);
2253     }
2254     (void) CFErrorPropagate(localError, error);
2255     CFReleaseSafe(localError);
2256     return ok;
2257 }
2258
2259 bool SecRevocationDbSetUpdateSource(SecRevocationDbRef rdb, CFStringRef updateSource) {
2260     /* note: this function assumes it is called only by the database owner.
2261        non-owner (read-only) clients will fail if changes to the db are needed. */
2262     if (!rdb || !rdb->db) {
2263         return false;
2264     }
2265     CFErrorRef localError = NULL;
2266     bool ok = true;
2267     ok &= SecRevocationDbPerformWrite(rdb, &localError, ^bool(SecRevocationDbConnectionRef dbc, CFErrorRef *error) {
2268         return _SecRevocationDbSetUpdateSource(dbc, updateSource, error);
2269     });
2270     CFReleaseSafe(localError);
2271     return ok;
2272 }
2273
2274 static CFAbsoluteTime _SecRevocationDbGetNextUpdateTime(SecRevocationDbConnectionRef dbc, CFErrorRef *error) {
2275     /* look up check_again entry in admin table; returns 0 on error */
2276     __block CFAbsoluteTime nextUpdate = 0;
2277     __block bool ok = (dbc != NULL);
2278     __block CFErrorRef localError = NULL;
2279
2280     ok = ok && SecDbWithSQL(dbc->dbconn, selectNextUpdateSQL, &localError, ^bool(sqlite3_stmt *selectNextUpdate) {
2281         ok &= SecDbStep(dbc->dbconn, selectNextUpdate, &localError, ^void(bool *stop) {
2282             CFAbsoluteTime *p = (CFAbsoluteTime *)sqlite3_column_blob(selectNextUpdate, 0);
2283             if (p != NULL) {
2284                 if (sizeof(CFAbsoluteTime) == sqlite3_column_bytes(selectNextUpdate, 0)) {
2285                     nextUpdate = *p;
2286                 }
2287             }
2288             *stop = true;
2289         });
2290         return ok;
2291     });
2292     if (!ok || localError) {
2293         secerror("_SecRevocationDbGetNextUpdateTime failed: %@", localError);
2294         TrustdHealthAnalyticsLogErrorCodeForDatabase(TARevocationDb, TAOperationRead, TAFatalError,
2295                                                      localError ? CFErrorGetCode(localError) : errSecInternalComponent);
2296     }
2297     (void) CFErrorPropagate(localError, error);
2298     return nextUpdate;
2299 }
2300
2301 static bool _SecRevocationDbSetNextUpdateTime(SecRevocationDbConnectionRef dbc, CFAbsoluteTime nextUpdate, CFErrorRef *error){
2302     secdebug("validupdate", "setting next update to %f", (double)nextUpdate);
2303
2304     __block CFErrorRef localError = NULL;
2305     __block bool ok = (dbc != NULL);
2306     ok = ok && SecDbWithSQL(dbc->dbconn, insertAdminRecordSQL, &localError, ^bool(sqlite3_stmt *insertRecord) {
2307         const char *nextUpdateKey = "check_again";
2308         ok = ok && SecDbBindText(insertRecord, 1, nextUpdateKey, strlen(nextUpdateKey),
2309                             SQLITE_TRANSIENT, &localError);
2310         ok = ok && SecDbBindInt64(insertRecord, 2,
2311                              (sqlite3_int64)0, &localError);
2312         ok = ok && SecDbBindBlob(insertRecord, 3,
2313                             &nextUpdate, sizeof(CFAbsoluteTime),
2314                             SQLITE_TRANSIENT, &localError);
2315         ok = ok && SecDbStep(dbc->dbconn, insertRecord, &localError, NULL);
2316         return ok;
2317     });
2318     if (!ok || localError) {
2319         secerror("_SecRevocationDbSetNextUpdate failed: %@", localError);
2320         TrustdHealthAnalyticsLogErrorCodeForDatabase(TARevocationDb, TAOperationWrite, TAFatalError,
2321                                                      localError ? CFErrorGetCode(localError) : errSecInternalComponent);
2322     }
2323     (void) CFErrorPropagate(localError, error);
2324     return ok;
2325 }
2326
2327 bool _SecRevocationDbRemoveAllEntries(SecRevocationDbConnectionRef dbc, CFErrorRef *error) {
2328     /* clear out the contents of the database and start fresh */
2329     bool ok = (dbc != NULL);
2330     CFErrorRef localError = NULL;
2331
2332     /* _SecRevocationDbUpdateSchema was called when db was opened, so no need to do it again. */
2333
2334     /* delete all entries */
2335     ok = ok && SecDbExec(dbc->dbconn, deleteAllEntriesSQL, &localError);
2336     secnotice("validupdate", "resetting database, result: %d (expected 1)", (ok) ? 1 : 0);
2337
2338     /* one more thing: update the schema version and format to current */
2339     ok = ok && _SecRevocationDbSetSchemaVersion(dbc, kSecRevocationDbSchemaVersion, &localError);
2340     ok = ok && _SecRevocationDbSetUpdateFormat(dbc, kSecRevocationDbUpdateFormat, &localError);
2341
2342     if (!ok || localError) {
2343         secerror("_SecRevocationDbRemoveAllEntries failed: %@", localError);
2344         TrustdHealthAnalyticsLogErrorCodeForDatabase(TARevocationDb, TAOperationWrite, TAFatalError,
2345                                                      localError ? CFErrorGetCode(localError) : errSecInternalComponent);
2346     }
2347     (void) CFErrorPropagate(localError, error);
2348     return ok;
2349 }
2350
2351 static bool _SecRevocationDbUpdateIssuers(SecRevocationDbConnectionRef dbc, int64_t groupId, CFArrayRef issuers, CFErrorRef *error) {
2352     /* insert or replace issuer records in issuers table */
2353     if (!issuers || groupId < 0) {
2354         return false; /* must have something to insert, and a group to associate with it */
2355     }
2356     __block bool ok = (dbc != NULL);
2357     __block CFErrorRef localError = NULL;
2358     if (isArray(issuers)) {
2359         CFIndex issuerIX, issuerCount = CFArrayGetCount(issuers);
2360         for (issuerIX=0; issuerIX<issuerCount && ok; issuerIX++) {
2361             CFDataRef hash = (CFDataRef)CFArrayGetValueAtIndex(issuers, issuerIX);
2362             if (!hash) { continue; }
2363             ok = ok && SecDbWithSQL(dbc->dbconn, insertIssuerRecordSQL, &localError, ^bool(sqlite3_stmt *insertIssuer) {
2364                 ok = ok && SecDbBindInt64(insertIssuer, 1,
2365                                      groupId, &localError);
2366                 ok = ok && SecDbBindBlob(insertIssuer, 2,
2367                                     CFDataGetBytePtr(hash),
2368                                     CFDataGetLength(hash),
2369                                     SQLITE_TRANSIENT, &localError);
2370                 /* Execute the insert statement for this issuer record. */
2371                 ok = ok && SecDbStep(dbc->dbconn, insertIssuer, &localError, NULL);
2372                 return ok;
2373             });
2374         }
2375     }
2376     if (!ok || localError) {
2377         secerror("_SecRevocationDbUpdateIssuers failed: %@", localError);
2378         TrustdHealthAnalyticsLogErrorCodeForDatabase(TARevocationDb, TAOperationWrite, TAFatalError,
2379                                                      localError ? CFErrorGetCode(localError) : errSecInternalComponent);
2380     }
2381     (void) CFErrorPropagate(localError, error);
2382     return ok;
2383 }
2384
2385 static SecValidInfoFormat _SecRevocationDbGetGroupFormatForData(SecRevocationDbConnectionRef dbc, int64_t groupId, CFDataRef data) {
2386     /* determine existing format if groupId is supplied and this is a partial update,
2387        otherwise return the expected format for the given data. */
2388     SecValidInfoFormat format = kSecValidInfoFormatUnknown;
2389     if (groupId >= 0 && !dbc->fullUpdate) {
2390         format = _SecRevocationDbGetGroupFormat(dbc, groupId, NULL, NULL, NULL, NULL);
2391     }
2392     if (format == kSecValidInfoFormatUnknown && data != NULL) {
2393         /* group doesn't exist, so determine format based on length of specified data.
2394            len <= 20 is a serial number (actually, <=37, but != 32.)
2395            len==32 is a sha256 hash. otherwise: nto1. */
2396         CFIndex length = CFDataGetLength(data);
2397         if (length == 32) {
2398             format = kSecValidInfoFormatSHA256;
2399         } else if (length <= 37) {
2400             format = kSecValidInfoFormatSerial;
2401         } else if (length > 0) {
2402             format = kSecValidInfoFormatNto1;
2403         }
2404     }
2405     return format;
2406 }
2407
2408 static bool _SecRevocationDbUpdateIssuerData(SecRevocationDbConnectionRef dbc, int64_t groupId, CFDictionaryRef dict, CFErrorRef *error) {
2409     /* update/delete records in serials or hashes table. */
2410     if (!dict || groupId < 0) {
2411         return false; /* must have something to insert, and a group to associate with it */
2412     }
2413     __block bool ok = (dbc != NULL);
2414     __block CFErrorRef localError = NULL;
2415     /* process deletions */
2416     CFArrayRef deleteArray = (CFArrayRef)CFDictionaryGetValue(dict, CFSTR("delete"));
2417     if (isArray(deleteArray)) {
2418         SecValidInfoFormat format = kSecValidInfoFormatUnknown;
2419         CFIndex processed=0, identifierIX, identifierCount = CFArrayGetCount(deleteArray);
2420         for (identifierIX=0; identifierIX<identifierCount; identifierIX++) {
2421             CFDataRef identifierData = (CFDataRef)CFArrayGetValueAtIndex(deleteArray, identifierIX);
2422             if (!identifierData) { continue; }
2423             if (format == kSecValidInfoFormatUnknown) {
2424                 format = _SecRevocationDbGetGroupFormatForData(dbc, groupId, identifierData);
2425             }
2426             CFStringRef sql = NULL;
2427             if (format == kSecValidInfoFormatSerial) {
2428                 sql = deleteSerialRecordSQL;
2429             } else if (format == kSecValidInfoFormatSHA256) {
2430                 sql = deleteSha256RecordSQL;
2431             }
2432             if (!sql) { continue; }
2433
2434             ok = ok && SecDbWithSQL(dbc->dbconn, sql, &localError, ^bool(sqlite3_stmt *deleteIdentifier) {
2435                 /* (groupid,serial|sha256) */
2436                 CFDataRef hexData = cfToHexData(identifierData, true);
2437                 if (!hexData) { return false; }
2438                 ok = ok && SecDbBindInt64(deleteIdentifier, 1,
2439                                      groupId, &localError);
2440                 ok = ok && SecDbBindBlob(deleteIdentifier, 2,
2441                                     CFDataGetBytePtr(hexData),
2442                                     CFDataGetLength(hexData),
2443                                     SQLITE_TRANSIENT, &localError);
2444                 /* Execute the delete statement for the identifier record. */
2445                 ok = ok && SecDbStep(dbc->dbconn, deleteIdentifier, &localError, NULL);
2446                 CFReleaseSafe(hexData);
2447                 return ok;
2448             });
2449             if (ok) { ++processed; }
2450         }
2451 #if VERBOSE_LOGGING
2452         secdebug("validupdate", "Processed %ld of %ld deletions for group %lld, result=%s",
2453                  processed, identifierCount, groupId, (ok) ? "true" : "false");
2454 #endif
2455     }
2456     /* process additions */
2457     CFArrayRef addArray = (CFArrayRef)CFDictionaryGetValue(dict, CFSTR("add"));
2458     if (isArray(addArray)) {
2459         SecValidInfoFormat format = kSecValidInfoFormatUnknown;
2460         CFIndex processed=0, identifierIX, identifierCount = CFArrayGetCount(addArray);
2461         for (identifierIX=0; identifierIX<identifierCount; identifierIX++) {
2462             CFDataRef identifierData = (CFDataRef)CFArrayGetValueAtIndex(addArray, identifierIX);
2463             if (!identifierData) { continue; }
2464             if (format == kSecValidInfoFormatUnknown) {
2465                 format = _SecRevocationDbGetGroupFormatForData(dbc, groupId, identifierData);
2466             }
2467             CFStringRef sql = NULL;
2468             if (format == kSecValidInfoFormatSerial) {
2469                 sql = insertSerialRecordSQL;
2470             } else if (format == kSecValidInfoFormatSHA256) {
2471                 sql = insertSha256RecordSQL;
2472             }
2473             if (!sql) { continue; }
2474
2475             ok = ok && SecDbWithSQL(dbc->dbconn, sql, &localError, ^bool(sqlite3_stmt *insertIdentifier) {
2476                 /* rowid,(groupid,serial|sha256) */
2477                 /* rowid is autoincremented and we never set it directly */
2478                 ok = ok && SecDbBindInt64(insertIdentifier, 1,
2479                                      groupId, &localError);
2480                 ok = ok && SecDbBindBlob(insertIdentifier, 2,
2481                                     CFDataGetBytePtr(identifierData),
2482                                     CFDataGetLength(identifierData),
2483                                     SQLITE_TRANSIENT, &localError);
2484                 /* Execute the insert statement for the identifier record. */
2485                 ok = ok && SecDbStep(dbc->dbconn, insertIdentifier, &localError, NULL);
2486                 return ok;
2487             });
2488             if (ok) { ++processed; }
2489         }
2490 #if VERBOSE_LOGGING
2491         secdebug("validupdate", "Processed %ld of %ld additions for group %lld, result=%s",
2492                  processed, identifierCount, groupId, (ok) ? "true" : "false");
2493 #endif
2494     }
2495     if (!ok || localError) {
2496         secerror("_SecRevocationDbUpdatePerIssuerData failed: %@", localError);
2497         TrustdHealthAnalyticsLogErrorCodeForDatabase(TARevocationDb, TAOperationWrite, TAFatalError,
2498                                                      localError ? CFErrorGetCode(localError) : errSecInternalComponent);
2499     }
2500     (void) CFErrorPropagate(localError, error);
2501     return ok;
2502 }
2503
2504 static bool _SecRevocationDbCopyDateConstraints(SecRevocationDbConnectionRef dbc,
2505     int64_t groupId, CFDateRef *notBeforeDate, CFDateRef *notAfterDate, CFErrorRef *error) {
2506     /* return true if one or both date constraints exist for a given groupId.
2507        the actual constraints are optionally returned in output CFDateRef parameters.
2508        caller is responsible for releasing date and error parameters, if provided.
2509     */
2510     __block bool ok = (dbc != NULL);
2511     __block CFDateRef localNotBefore = NULL;
2512     __block CFDateRef localNotAfter = NULL;
2513     __block CFErrorRef localError = NULL;
2514
2515     ok = ok && SecDbWithSQL(dbc->dbconn, selectDateRecordSQL, &localError, ^bool(sqlite3_stmt *selectDates) {
2516         /* (groupid,notbefore,notafter) */
2517         ok &= SecDbBindInt64(selectDates, 1, groupId, &localError);
2518         ok = ok && SecDbStep(dbc->dbconn, selectDates, &localError, ^(bool *stop) {
2519             /* if column has no value, its type will be SQLITE_NULL */
2520             if (SQLITE_NULL != sqlite3_column_type(selectDates, 0)) {
2521                 CFAbsoluteTime nb = (CFAbsoluteTime)sqlite3_column_double(selectDates, 0);
2522                 localNotBefore = CFDateCreate(NULL, nb);
2523             }
2524             if (SQLITE_NULL != sqlite3_column_type(selectDates, 1)) {
2525                 CFAbsoluteTime na = (CFAbsoluteTime)sqlite3_column_double(selectDates, 1);
2526                 localNotAfter = CFDateCreate(NULL, na);
2527             }
2528         });
2529         return ok;
2530     });
2531     /* must have at least one date constraint to return true.
2532        since date constraints are optional, not finding any should not log an error. */
2533     ok = ok && !localError && (localNotBefore != NULL || localNotAfter != NULL);
2534     if (localError) {
2535         secerror("_SecRevocationDbCopyDateConstraints failed: %@", localError);
2536         TrustdHealthAnalyticsLogErrorCodeForDatabase(TARevocationDb, TAOperationRead, TAFatalError,
2537                                                      localError ? CFErrorGetCode(localError) : errSecInternalComponent);
2538     }
2539     if (!ok) {
2540         CFReleaseNull(localNotBefore);
2541         CFReleaseNull(localNotAfter);
2542     }
2543     if (notBeforeDate) {
2544         *notBeforeDate = localNotBefore;
2545     } else {
2546         CFReleaseSafe(localNotBefore);
2547     }
2548     if (notAfterDate) {
2549         *notAfterDate = localNotAfter;
2550     } else {
2551         CFReleaseSafe(localNotAfter);
2552     }
2553
2554     (void) CFErrorPropagate(localError, error);
2555     return ok;
2556 }
2557
2558 static bool _SecRevocationDbUpdateDateConstraints(SecRevocationDbConnectionRef dbc, int64_t groupId, CFDictionaryRef dict, CFErrorRef *error) {
2559     /* Called only from _SecRevocationDbUpdateIssuerConstraints.
2560        Function assumes that the caller has checked the input arguments.
2561     */
2562     __block bool ok = true;
2563     __block CFErrorRef localError = NULL;
2564     __block CFAbsoluteTime notBefore = -3155760000.0; /* default: 1901-01-01 00:00:00-0000 */
2565     __block CFAbsoluteTime notAfter = 31556908800.0;  /* default: 3001-01-01 00:00:00-0000 */
2566
2567     CFDateRef notBeforeDate = (CFDateRef)CFDictionaryGetValue(dict, CFSTR("not-before"));
2568     CFDateRef notAfterDate = (CFDateRef)CFDictionaryGetValue(dict, CFSTR("not-after"));
2569
2570     if (isDate(notBeforeDate)) {
2571         notBefore = CFDateGetAbsoluteTime(notBeforeDate);
2572     } else {
2573         notBeforeDate = NULL;
2574     }
2575     if (isDate(notAfterDate)) {
2576         notAfter = CFDateGetAbsoluteTime(notAfterDate);
2577     } else {
2578         notAfterDate = NULL;
2579     }
2580     if (!(notBeforeDate || notAfterDate)) {
2581         return ok; /* no dates supplied, so we have nothing to update for this issuer */
2582     }
2583
2584     if (!(notBeforeDate && notAfterDate) && !dbc->fullUpdate) {
2585         /* only one date was supplied, so check for existing date constraints */
2586         CFDateRef curNotBeforeDate = NULL;
2587         CFDateRef curNotAfterDate = NULL;
2588         if (_SecRevocationDbCopyDateConstraints(dbc, groupId, &curNotBeforeDate,
2589                                                        &curNotAfterDate, &localError)) {
2590             if (!notBeforeDate) {
2591                 notBeforeDate = curNotBeforeDate;
2592                 notBefore = CFDateGetAbsoluteTime(notBeforeDate);
2593             } else {
2594                 CFReleaseSafe(curNotBeforeDate);
2595             }
2596             if (!notAfterDate) {
2597                 notAfterDate = curNotAfterDate;
2598                 notAfter = CFDateGetAbsoluteTime(notAfterDate);
2599             } else {
2600                 CFReleaseSafe(curNotAfterDate);
2601             }
2602         }
2603     }
2604     ok = ok && SecDbWithSQL(dbc->dbconn, insertDateRecordSQL, &localError, ^bool(sqlite3_stmt *insertDate) {
2605         /* (groupid,notbefore,notafter) */
2606         ok = ok && SecDbBindInt64(insertDate, 1, groupId, &localError);
2607         ok = ok && SecDbBindDouble(insertDate, 2, notBefore, &localError);
2608         ok = ok && SecDbBindDouble(insertDate, 3, notAfter, &localError);
2609         ok = ok && SecDbStep(dbc->dbconn, insertDate, &localError, NULL);
2610         return ok;
2611     });
2612
2613     if (!ok || localError) {
2614         secinfo("validupdate", "_SecRevocationDbUpdateDateConstraints failed (ok=%s, localError=%@)",
2615                 (ok) ? "1" : "0", localError);
2616         TrustdHealthAnalyticsLogErrorCodeForDatabase(TARevocationDb, TAOperationWrite, TAFatalError,
2617                                                      localError ? CFErrorGetCode(localError) : errSecInternalComponent);
2618     }
2619     (void) CFErrorPropagate(localError, error);
2620     return ok;
2621 }
2622
2623 static bool _SecRevocationDbUpdatePolicyConstraints(SecRevocationDbConnectionRef dbc, int64_t groupId, CFDictionaryRef dict, CFErrorRef *error) {
2624     /* Called only from _SecRevocationDbUpdateIssuerConstraints.
2625        Function assumes that the caller has checked the input arguments.
2626     */
2627     __block bool ok = true;
2628     __block CFErrorRef localError = NULL;
2629     CFArrayRef policies = (CFArrayRef)CFDictionaryGetValue(dict, CFSTR("policies"));
2630     if (!isArray(policies)) {
2631         return ok; /* no policies supplied, so nothing to update for this issuer */
2632     }
2633
2634     __block CFDataRef data = createPoliciesData(policies);
2635     ok = data && SecDbWithSQL(dbc->dbconn, updateGroupPoliciesSQL, &localError, ^bool(sqlite3_stmt *updatePolicies) {
2636         /* (policies,groupid) */
2637         ok = ok && SecDbBindBlob(updatePolicies, 1,
2638                                  CFDataGetBytePtr(data),
2639                                  CFDataGetLength(data),
2640                                  SQLITE_TRANSIENT, &localError);
2641         ok = ok && SecDbBindInt64(updatePolicies, 2, groupId, &localError);
2642         ok = ok && SecDbStep(dbc->dbconn, updatePolicies, &localError, NULL);
2643         return ok;
2644     });
2645     CFReleaseSafe(data);
2646
2647     if (!ok || localError) {
2648         secinfo("validupdate", "_SecRevocationDbUpdatePolicyConstraints failed (ok=%s, localError=%@)",
2649                 (ok) ? "1" : "0", localError);
2650         TrustdHealthAnalyticsLogErrorCodeForDatabase(TARevocationDb, TAOperationWrite, TAFatalError,
2651                                                      localError ? CFErrorGetCode(localError) : errSecInternalComponent);
2652     }
2653     (void) CFErrorPropagate(localError, error);
2654     return ok;
2655 }
2656
2657 static bool _SecRevocationDbUpdateNameConstraints(SecRevocationDbConnectionRef dbc, int64_t groupId, CFDictionaryRef dict, CFErrorRef *error) {
2658     /* Called only from _SecRevocationDbUpdateIssuerConstraints.
2659        Function assumes that the caller has checked the input arguments.
2660     */
2661
2662     /* %%% (TBI:9254570) update name constraint entries here */
2663     return true;
2664 }
2665
2666 static bool _SecRevocationDbUpdateIssuerConstraints(SecRevocationDbConnectionRef dbc, int64_t groupId, CFDictionaryRef dict, CFErrorRef *error) {
2667     /* check input arguments */
2668     if (!dbc || !dict || groupId < 0) {
2669         return false;
2670     }
2671     bool ok = true;
2672     ok = ok && _SecRevocationDbUpdateDateConstraints(dbc, groupId, dict, error);
2673     ok = ok && _SecRevocationDbUpdateNameConstraints(dbc, groupId, dict, error);
2674     ok = ok && _SecRevocationDbUpdatePolicyConstraints(dbc, groupId, dict, error);
2675     return ok;
2676 }
2677
2678 static SecValidInfoFormat _SecRevocationDbGetGroupFormat(SecRevocationDbConnectionRef dbc,
2679     int64_t groupId, SecValidInfoFlags *flags, CFDataRef *data, CFDataRef *policies, CFErrorRef *error) {
2680     /* return group record fields for a given groupId.
2681        on success, returns a non-zero format type, and other field values in optional output parameters.
2682        caller is responsible for releasing data, policies, and error parameters, if provided.
2683     */
2684     __block bool ok = (dbc != NULL);
2685     __block SecValidInfoFormat format = 0;
2686     __block CFErrorRef localError = NULL;
2687
2688     /* Select the group record to determine flags and format. */
2689     ok = ok && SecDbWithSQL(dbc->dbconn, selectGroupRecordSQL, &localError, ^bool(sqlite3_stmt *selectGroup) {
2690         ok = ok && SecDbBindInt64(selectGroup, 1, groupId, &localError);
2691         ok = ok && SecDbStep(dbc->dbconn, selectGroup, &localError, ^(bool *stop) {
2692             if (flags) {
2693                 *flags = (SecValidInfoFlags)sqlite3_column_int(selectGroup, 0);
2694             }
2695             format = (SecValidInfoFormat)sqlite3_column_int(selectGroup, 1);
2696             if (data) {
2697                 //%%% stream the data from the db into a streamed decompression <rdar://32142637>
2698                 uint8_t *p = (uint8_t *)sqlite3_column_blob(selectGroup, 2);
2699                 if (p != NULL && format == kSecValidInfoFormatNto1) {
2700                     CFIndex length = (CFIndex)sqlite3_column_bytes(selectGroup, 2);
2701                     *data = CFDataCreate(kCFAllocatorDefault, p, length);
2702                 }
2703             }
2704             if (policies) {
2705                 uint8_t *p = (uint8_t *)sqlite3_column_blob(selectGroup, 3);
2706                 if (p != NULL) {
2707                     CFIndex length = (CFIndex)sqlite3_column_bytes(selectGroup, 3);
2708                     *policies = CFDataCreate(kCFAllocatorDefault, p, length);
2709                 }
2710             }
2711         });
2712         return ok;
2713     });
2714     if (!ok || localError) {
2715         secdebug("validupdate", "GetGroupFormat for groupId %lu failed", (unsigned long)groupId);
2716         TrustdHealthAnalyticsLogErrorCodeForDatabase(TARevocationDb, TAOperationRead, TAFatalError,
2717                                                      localError ? CFErrorGetCode(localError) : errSecInternalComponent);
2718         format = kSecValidInfoFormatUnknown;
2719     }
2720     (void) CFErrorPropagate(localError, error);
2721     if (!(format > kSecValidInfoFormatUnknown)) {
2722         secdebug("validupdate", "GetGroupFormat: got format %d for groupId %lld", format, (long long)groupId);
2723     }
2724     return format;
2725 }
2726
2727 static bool _SecRevocationDbUpdateFlags(CFDictionaryRef dict, CFStringRef key, SecValidInfoFlags mask, SecValidInfoFlags *flags) {
2728     /* If a boolean value exists in the given dictionary for the given key,
2729        or an explicit "1" or "0" is specified as the key string,
2730        set or clear the corresponding bit(s) defined by the mask argument.
2731        Function returns true if the flags value was changed, false otherwise.
2732     */
2733     if (!isDictionary(dict) || !isString(key) || !flags) {
2734         return false;
2735     }
2736     bool hasValue = false, newValue = false, result = false;
2737     CFTypeRef value = (CFBooleanRef)CFDictionaryGetValue(dict, key);
2738     if (isBoolean(value)) {
2739         newValue = CFBooleanGetValue((CFBooleanRef)value);
2740         hasValue = true;
2741     } else if (BOOL_STRING_KEY_LENGTH == CFStringGetLength(key)) {
2742         if (CFStringCompare(key, kBoolTrueKey, 0) == kCFCompareEqualTo) {
2743             hasValue = newValue = true;
2744         } else if (CFStringCompare(key, kBoolFalseKey, 0) == kCFCompareEqualTo) {
2745             hasValue = true;
2746         }
2747     }
2748     if (hasValue) {
2749         SecValidInfoFlags oldFlags = *flags;
2750         if (newValue) {
2751             *flags |= mask;
2752         } else {
2753             *flags &= ~(mask);
2754         }
2755         result = (*flags != oldFlags);
2756     }
2757     return result;
2758 }
2759
2760 static bool _SecRevocationDbUpdateFilter(CFDictionaryRef dict, CFDataRef oldData, CFDataRef * __nonnull CF_RETURNS_RETAINED xmlData) {
2761     /* If xor and/or params values exist in the given dictionary, create a new
2762        property list containing the updated values, and return as a flattened
2763        data blob in the xmlData output parameter (note: caller must release.)
2764        Function returns true if there is new xmlData to save, false otherwise.
2765     */
2766     bool result = false;
2767     bool xorProvided = false;
2768     bool paramsProvided = false;
2769     bool missingData = false;
2770
2771     if (!dict || !xmlData) {
2772         return result; /* no-op if no dictionary is provided, or no way to update the data */
2773     }
2774     *xmlData = NULL;
2775     CFDataRef xorCurrent = NULL;
2776     CFDataRef xorUpdate = (CFDataRef)CFDictionaryGetValue(dict, CFSTR("xor"));
2777     if (isData(xorUpdate)) {
2778         xorProvided = true;
2779     }
2780     CFArrayRef paramsCurrent = NULL;
2781     CFArrayRef paramsUpdate = (CFArrayRef)CFDictionaryGetValue(dict, CFSTR("params"));
2782     if (isArray(paramsUpdate)) {
2783         paramsProvided = true;
2784     }
2785     if (!(xorProvided || paramsProvided)) {
2786         return result; /* nothing to update, so we can bail out here. */
2787     }
2788
2789     CFPropertyListRef nto1Current = NULL;
2790     CFMutableDictionaryRef nto1Update = CFDictionaryCreateMutable(kCFAllocatorDefault, 0,
2791                                                                   &kCFTypeDictionaryKeyCallBacks,
2792                                                                   &kCFTypeDictionaryValueCallBacks);
2793     if (!nto1Update) {
2794         return result;
2795     }
2796
2797     /* turn old data into property list */
2798     CFDataRef data = (CFDataRef)CFRetainSafe(oldData);
2799     CFDataRef inflatedData = copyInflatedData(data);
2800     if (inflatedData) {
2801         CFReleaseSafe(data);
2802         data = inflatedData;
2803     }
2804     if (data) {
2805         nto1Current = CFPropertyListCreateWithData(kCFAllocatorDefault, data, 0, NULL, NULL);
2806         CFReleaseSafe(data);
2807     }
2808     if (nto1Current) {
2809         xorCurrent = (CFDataRef)CFDictionaryGetValue((CFDictionaryRef)nto1Current, CFSTR("xor"));
2810         paramsCurrent = (CFArrayRef)CFDictionaryGetValue((CFDictionaryRef)nto1Current, CFSTR("params"));
2811     }
2812
2813     /* set current or updated xor data in new property list */
2814     if (xorProvided) {
2815         CFDataRef xorNew = NULL;
2816         if (xorCurrent) {
2817             CFIndex xorUpdateLen = CFDataGetLength(xorUpdate);
2818             CFMutableDataRef xor = CFDataCreateMutableCopy(NULL, 0, xorCurrent);
2819             if (xor && xorUpdateLen > 0) {
2820                 /* truncate or zero-extend data to match update size */
2821                 CFDataSetLength(xor, xorUpdateLen);
2822                 /* exclusive-or update bytes over the existing data */
2823                 UInt8 *xorP = (UInt8 *)CFDataGetMutableBytePtr(xor);
2824                 UInt8 *updP = (UInt8 *)CFDataGetBytePtr(xorUpdate);
2825                 if (xorP && updP) {
2826                     for (int idx = 0; idx < xorUpdateLen; idx++) {
2827                         xorP[idx] = xorP[idx] ^ updP[idx];
2828                     }
2829                 }
2830             }
2831             xorNew = (CFDataRef)xor;
2832         } else {
2833             xorNew = (CFDataRef)CFRetainSafe(xorUpdate);
2834         }
2835         if (xorNew) {
2836             CFDictionaryAddValue(nto1Update, CFSTR("xor"), xorNew);
2837             CFReleaseSafe(xorNew);
2838         } else {
2839             secdebug("validupdate", "Failed to get updated filter data");
2840             missingData = true;
2841         }
2842     } else if (xorCurrent) {
2843         /* not provided, so use existing xor value */
2844         CFDictionaryAddValue(nto1Update, CFSTR("xor"), xorCurrent);
2845     } else {
2846         secdebug("validupdate", "Failed to get current filter data");
2847         missingData = true;
2848     }
2849
2850     /* set current or updated params in new property list */
2851     if (paramsProvided) {
2852         CFDictionaryAddValue(nto1Update, CFSTR("params"), paramsUpdate);
2853     } else if (paramsCurrent) {
2854         /* not provided, so use existing params value */
2855         CFDictionaryAddValue(nto1Update, CFSTR("params"), paramsCurrent);
2856     } else {
2857         /* missing params: neither provided nor existing */
2858         secdebug("validupdate", "Failed to get current filter params");
2859         missingData = true;
2860     }
2861
2862     CFReleaseSafe(nto1Current);
2863     if (!missingData) {
2864         *xmlData = CFPropertyListCreateData(kCFAllocatorDefault, nto1Update,
2865                                             kCFPropertyListXMLFormat_v1_0,
2866                                             0, NULL);
2867         result = (*xmlData != NULL);
2868     }
2869     CFReleaseSafe(nto1Update);
2870
2871     /* compress the xmlData blob, if possible */
2872     if (result) {
2873         CFDataRef deflatedData = copyDeflatedData(*xmlData);
2874         if (deflatedData) {
2875             if (CFDataGetLength(deflatedData) < CFDataGetLength(*xmlData)) {
2876                 CFRelease(*xmlData);
2877                 *xmlData = deflatedData;
2878             } else {
2879                 CFRelease(deflatedData);
2880             }
2881         }
2882     }
2883     return result;
2884 }
2885
2886
2887 static int64_t _SecRevocationDbUpdateGroup(SecRevocationDbConnectionRef dbc, int64_t groupId, CFDictionaryRef dict, CFErrorRef *error) {
2888     /* insert group record for a given groupId.
2889        if the specified groupId is < 0, a new group entry is created.
2890        returns the groupId on success, or -1 on failure.
2891      */
2892     if (!dict) {
2893         return groupId; /* no-op if no dictionary is provided */
2894     }
2895
2896     __block int64_t result = -1;
2897     __block bool ok = (dbc != NULL);
2898     __block bool isFormatChange = false;
2899     __block CFErrorRef localError = NULL;
2900
2901     __block SecValidInfoFlags flags = 0;
2902     __block SecValidInfoFormat format = kSecValidInfoFormatUnknown;
2903     __block SecValidInfoFormat formatUpdate = kSecValidInfoFormatUnknown;
2904     __block CFDataRef data = NULL;
2905     __block CFDataRef policies = NULL;
2906
2907     if (groupId >= 0) {
2908         /* fetch the flags and data for an existing group record, in case some are being changed. */
2909         if (ok) {
2910             format = _SecRevocationDbGetGroupFormat(dbc, groupId, &flags, &data, &policies, NULL);
2911         }
2912         if (format == kSecValidInfoFormatUnknown) {
2913             secdebug("validupdate", "existing group %lld has unknown format %d, flags=0x%lx",
2914                      (long long)groupId, format, flags);
2915             //%%% clean up by deleting all issuers with this groupId, then the group record,
2916             // or just force a full update? note: we can get here if we fail to bind the
2917             // format value in the prepared SQL statement below.
2918             return -1;
2919         }
2920     }
2921     CFTypeRef value = (CFStringRef)CFDictionaryGetValue(dict, CFSTR("format"));
2922     if (isString(value)) {
2923         if (CFStringCompare((CFStringRef)value, CFSTR("serial"), 0) == kCFCompareEqualTo) {
2924             formatUpdate = kSecValidInfoFormatSerial;
2925         } else if (CFStringCompare((CFStringRef)value, CFSTR("sha256"), 0) == kCFCompareEqualTo) {
2926             formatUpdate = kSecValidInfoFormatSHA256;
2927         } else if (CFStringCompare((CFStringRef)value, CFSTR("nto1"), 0) == kCFCompareEqualTo) {
2928             formatUpdate = kSecValidInfoFormatNto1;
2929         }
2930     }
2931     /* if format value is explicitly supplied, then this is effectively a new group entry. */
2932     isFormatChange = (formatUpdate > kSecValidInfoFormatUnknown &&
2933                       formatUpdate != format &&
2934                       groupId >= 0);
2935
2936     if (isFormatChange) {
2937         secdebug("validupdate", "group %lld format change from %d to %d",
2938                  (long long)groupId, format, formatUpdate);
2939         /* format of an existing group is changing; delete the group first.
2940            this should ensure that all entries referencing the old groupid are deleted.
2941         */
2942         ok = ok && SecDbWithSQL(dbc->dbconn, deleteGroupRecordSQL, &localError, ^bool(sqlite3_stmt *deleteResponse) {
2943             ok = ok && SecDbBindInt64(deleteResponse, 1, groupId, &localError);
2944             /* Execute the delete statement. */
2945             ok = ok && SecDbStep(dbc->dbconn, deleteResponse, &localError, NULL);
2946             return ok;
2947         });
2948     }
2949     ok = ok && SecDbWithSQL(dbc->dbconn, insertGroupRecordSQL, &localError, ^bool(sqlite3_stmt *insertGroup) {
2950         /* (groupid,flags,format,data,policies) */
2951         /* groups.groupid */
2952         if (ok && (!isFormatChange) && (groupId >= 0)) {
2953             /* bind to existing groupId row if known, otherwise will insert and autoincrement */
2954             ok = SecDbBindInt64(insertGroup, 1, groupId, &localError);
2955             if (!ok) {
2956                 secdebug("validupdate", "failed to set groupId %lld", (long long)groupId);
2957             }
2958         }
2959         /* groups.flags */
2960         if (ok) {
2961             (void)_SecRevocationDbUpdateFlags(dict, CFSTR("complete"), kSecValidInfoComplete, &flags);
2962             (void)_SecRevocationDbUpdateFlags(dict, CFSTR("check-ocsp"), kSecValidInfoCheckOCSP, &flags);
2963             (void)_SecRevocationDbUpdateFlags(dict, CFSTR("known-intermediates-only"), kSecValidInfoKnownOnly, &flags);
2964             (void)_SecRevocationDbUpdateFlags(dict, CFSTR("require-ct"), kSecValidInfoRequireCT, &flags);
2965             (void)_SecRevocationDbUpdateFlags(dict, CFSTR("valid"), kSecValidInfoAllowlist, &flags);
2966             (void)_SecRevocationDbUpdateFlags(dict, CFSTR("no-ca"), kSecValidInfoNoCACheck, &flags);
2967             (void)_SecRevocationDbUpdateFlags(dict, CFSTR("no-ca-v2"), kSecValidInfoNoCAv2Check, &flags);
2968             (void)_SecRevocationDbUpdateFlags(dict, CFSTR("overridable"), kSecValidInfoOverridable, &flags);
2969
2970             /* date constraints exist if either "not-before" or "not-after" keys are found */
2971             CFTypeRef notBeforeValue = (CFDateRef)CFDictionaryGetValue(dict, CFSTR("not-before"));
2972             CFTypeRef notAfterValue = (CFDateRef)CFDictionaryGetValue(dict, CFSTR("not-after"));
2973             if (isDate(notBeforeValue) || isDate(notAfterValue)) {
2974                 (void)_SecRevocationDbUpdateFlags(dict, kBoolTrueKey, kSecValidInfoDateConstraints, &flags);
2975                 /* Note that the spec defines not-before and not-after dates as optional, such that
2976                    not providing one does not change the database contents. Therefore, we can never clear
2977                    this flag; either a new date entry will be supplied, or a format change will cause
2978                    the entire group entry to be deleted. */
2979             }
2980             /* policy constraints exist if "policies" key is found */
2981             CFTypeRef policiesValue = (CFArrayRef)CFDictionaryGetValue(dict, CFSTR("policies"));
2982             if (isArray(policiesValue)) {
2983                 (void)_SecRevocationDbUpdateFlags(dict, kBoolTrueKey, kSecValidInfoPolicyConstraints, &flags);
2984                 /* As above, not providing this value in an update does not change the existing state,
2985                    so we never need to clear this flag once it is set. */
2986             }
2987
2988             /* %%% (TBI:9254570) name constraints don't exist yet */
2989             (void)_SecRevocationDbUpdateFlags(dict, kBoolFalseKey, kSecValidInfoNameConstraints, &flags);
2990
2991             ok = SecDbBindInt(insertGroup, 2, (int)flags, &localError);
2992             if (!ok) {
2993                 secdebug("validupdate", "failed to set flags (%lu) for groupId %lld", flags, (long long)groupId);
2994             }
2995         }
2996         /* groups.format */
2997         if (ok) {
2998             SecValidInfoFormat formatValue = format;
2999             if (formatUpdate > kSecValidInfoFormatUnknown) {
3000                 formatValue = formatUpdate;
3001             }
3002             ok = SecDbBindInt(insertGroup, 3, (int)formatValue, &localError);
3003             if (!ok) {
3004                 secdebug("validupdate", "failed to set format (%d) for groupId %lld", formatValue, (long long)groupId);
3005             }
3006         }
3007         /* groups.data */
3008         CFDataRef xmlData = NULL;
3009         if (ok) {
3010             bool hasFilter = ((formatUpdate == kSecValidInfoFormatNto1) ||
3011                               (formatUpdate == kSecValidInfoFormatUnknown &&
3012                                format == kSecValidInfoFormatNto1));
3013             if (hasFilter) {
3014                 CFDataRef dataValue = data; /* use existing data */
3015                 if (_SecRevocationDbUpdateFilter(dict, data, &xmlData)) {
3016                     dataValue = xmlData; /* use updated data */
3017                 }
3018                 if (dataValue) {
3019                     ok = SecDbBindBlob(insertGroup, 4,
3020                                        CFDataGetBytePtr(dataValue),
3021                                        CFDataGetLength(dataValue),
3022                                        SQLITE_TRANSIENT, &localError);
3023                 }
3024                 if (!ok) {
3025                     secdebug("validupdate", "failed to set data for groupId %lld",
3026                              (long long)groupId);
3027                 }
3028             }
3029             /* else there is no data, so NULL is implicitly bound to column 4 */
3030         }
3031         /* groups.policies */
3032         CFDataRef newPoliciesData = NULL;
3033         if (ok) {
3034             CFDataRef policiesValue = policies; /* use existing policies */
3035             newPoliciesData = createPoliciesData((CFArrayRef)CFDictionaryGetValue(dict, CFSTR("policies")));
3036             if (newPoliciesData) {
3037                 policiesValue = newPoliciesData; /* use updated policies */
3038             }
3039             if (policiesValue) {
3040                 ok = SecDbBindBlob(insertGroup, 5,
3041                                    CFDataGetBytePtr(policiesValue),
3042                                    CFDataGetLength(policiesValue),
3043                                    SQLITE_TRANSIENT, &localError);
3044             }
3045             /* else there is no policy data, so NULL is implicitly bound to column 5 */
3046             if (!ok) {
3047                 secdebug("validupdate", "failed to set policies for groupId %lld",
3048                          (long long)groupId);
3049             }
3050         }
3051
3052         /* Execute the insert statement for the group record. */
3053         if (ok) {
3054             ok = SecDbStep(dbc->dbconn, insertGroup, &localError, NULL);
3055             if (!ok) {
3056                 secdebug("validupdate", "failed to execute insertGroup statement for groupId %lld",
3057                          (long long)groupId);
3058             }
3059             result = (int64_t)sqlite3_last_insert_rowid(SecDbHandle(dbc->dbconn));
3060         }
3061         if (!ok) {
3062             secdebug("validupdate", "failed to insert group %lld", (long long)result);
3063         }
3064         /* Clean up temporary allocations made in this block. */
3065         CFReleaseSafe(xmlData);
3066         CFReleaseSafe(newPoliciesData);
3067         return ok;
3068     });
3069
3070     CFReleaseSafe(data);
3071     CFReleaseSafe(policies);
3072
3073     if (!ok || localError) {
3074         secerror("_SecRevocationDbUpdateGroup failed: %@", localError);
3075         TrustdHealthAnalyticsLogErrorCodeForDatabase(TARevocationDb, TAOperationWrite, TAFatalError,
3076                                                      localError ? CFErrorGetCode(localError) : errSecInternalComponent);
3077     }
3078     (void) CFErrorPropagate(localError, error);
3079     return result;
3080 }
3081
3082 static int64_t _SecRevocationDbGroupIdForIssuerHash(SecRevocationDbConnectionRef dbc, CFDataRef hash, CFErrorRef *error) {
3083     /* look up issuer hash in issuers table to get groupid, if it exists */
3084     __block int64_t groupId = -1;
3085     __block bool ok = (dbc != NULL);
3086     __block CFErrorRef localError = NULL;
3087
3088     if (!hash) {
3089         secdebug("validupdate", "failed to get hash (%@)", hash);
3090     }
3091     require(hash && dbc, errOut);
3092
3093     /* This is the starting point for any lookup; find a group id for the given issuer hash.
3094        Before we do that, need to verify the current db_version. We cannot use results from a
3095        database created with a schema version older than the minimum supported version.
3096        However, we may be able to use results from a newer version. At the next database
3097        update interval, if the existing schema is old, we'll be removing and recreating
3098        the database contents with the current schema version.
3099     */
3100     int64_t db_version = _SecRevocationDbGetSchemaVersion(dbc->db, dbc, NULL);
3101     if (db_version < kSecRevocationDbMinSchemaVersion) {
3102         if (!dbc->db->unsupportedVersion) {
3103             secdebug("validupdate", "unsupported db_version: %lld", (long long)db_version);
3104             dbc->db->unsupportedVersion = true; /* only warn once for a given unsupported version */
3105         }
3106     }
3107     require_quiet(db_version >= kSecRevocationDbMinSchemaVersion, errOut);
3108
3109     /* Look up provided issuer_hash in the issuers table.
3110     */
3111     ok = ok && SecDbWithSQL(dbc->dbconn, selectGroupIdSQL, &localError, ^bool(sqlite3_stmt *selectGroupId) {
3112         ok &= SecDbBindBlob(selectGroupId, 1, CFDataGetBytePtr(hash), CFDataGetLength(hash), SQLITE_TRANSIENT, &localError);
3113         ok &= SecDbStep(dbc->dbconn, selectGroupId, &localError, ^(bool *stopGroupId) {
3114             groupId = sqlite3_column_int64(selectGroupId, 0);
3115         });
3116         return ok;
3117     });
3118
3119 errOut:
3120     if (!ok || localError) {
3121         secerror("_SecRevocationDbGroupIdForIssuerHash failed: %@", localError);
3122         TrustdHealthAnalyticsLogErrorCodeForDatabase(TARevocationDb, TAOperationRead, TAFatalError,
3123                                                      localError ? CFErrorGetCode(localError) : errSecInternalComponent);
3124     }
3125     (void) CFErrorPropagate(localError, error);
3126     return groupId;
3127 }
3128
3129 static bool _SecRevocationDbApplyGroupDelete(SecRevocationDbConnectionRef dbc, CFDataRef issuerHash, CFErrorRef *error) {
3130     /* delete group associated with the given issuer;
3131        schema trigger will delete associated issuers, serials, and hashes. */
3132     __block int64_t groupId = -1;
3133     __block bool ok = (dbc != NULL);
3134     __block CFErrorRef localError = NULL;
3135
3136     if (ok) {
3137         groupId = _SecRevocationDbGroupIdForIssuerHash(dbc, issuerHash, &localError);
3138     }
3139     if (groupId < 0) {
3140         if (!localError) {
3141             SecError(errSecParam, &localError, CFSTR("group not found for issuer"));
3142         }
3143         ok = false;
3144     }
3145     ok = ok && SecDbWithSQL(dbc->dbconn, deleteGroupRecordSQL, &localError, ^bool(sqlite3_stmt *deleteResponse) {
3146         ok &= SecDbBindInt64(deleteResponse, 1, groupId, &localError);
3147         /* Execute the delete statement. */
3148         ok = ok && SecDbStep(dbc->dbconn, deleteResponse, &localError, NULL);
3149         return ok;
3150     });
3151     if (!ok || localError) {
3152         secerror("_SecRevocationDbApplyGroupDelete failed: %@", localError);
3153         TrustdHealthAnalyticsLogErrorCodeForDatabase(TARevocationDb, TAOperationWrite, TAFatalError,
3154                                                      localError ? CFErrorGetCode(localError) : errSecInternalComponent);
3155     }
3156     (void) CFErrorPropagate(localError, error);
3157     return ok;
3158 }
3159
3160 static bool _SecRevocationDbApplyGroupUpdate(SecRevocationDbConnectionRef dbc, CFDictionaryRef dict, CFErrorRef *error) {
3161     /* process one issuer group's update dictionary */
3162     __block int64_t groupId = -1;
3163     __block bool ok = (dbc != NULL);
3164     __block CFErrorRef localError = NULL;
3165
3166     CFArrayRef issuers = (dict) ? (CFArrayRef)CFDictionaryGetValue(dict, CFSTR("issuer-hash")) : NULL;
3167     /* if this is not a full update, then look for existing group id */
3168     if (ok && isArray(issuers) && !dbc->fullUpdate) {
3169         CFIndex issuerIX, issuerCount = CFArrayGetCount(issuers);
3170         /* while we have issuers and haven't found a matching group id */
3171         for (issuerIX=0; issuerIX<issuerCount && groupId < 0; issuerIX++) {
3172             CFDataRef hash = (CFDataRef)CFArrayGetValueAtIndex(issuers, issuerIX);
3173             if (!hash) { continue; }
3174             groupId = _SecRevocationDbGroupIdForIssuerHash(dbc, hash, &localError);
3175         }
3176         if (groupId >= 0) {
3177             /* according to the spec, we must replace all existing issuers with
3178                the new issuers list, so delete all issuers in the group first. */
3179             ok = ok && SecDbWithSQL(dbc->dbconn, deleteGroupIssuersSQL, &localError, ^bool(sqlite3_stmt *deleteIssuers) {
3180                 ok = ok && SecDbBindInt64(deleteIssuers, 1, groupId, &localError);
3181                 ok = ok && SecDbStep(dbc->dbconn, deleteIssuers, &localError, NULL);
3182                 return ok;
3183             });
3184         }
3185     }
3186     /* create or update the group entry */
3187     if (ok) {
3188         groupId = _SecRevocationDbUpdateGroup(dbc, groupId, dict, &localError);
3189     }
3190     if (groupId < 0) {
3191         secdebug("validupdate", "failed to get groupId");
3192         ok = false;
3193     } else {
3194         /* create or update issuer entries, now that we know the group id */
3195         ok = ok && _SecRevocationDbUpdateIssuers(dbc, groupId, issuers, &localError);
3196         /* create or update entries in serials or hashes tables */
3197         ok = ok && _SecRevocationDbUpdateIssuerData(dbc, groupId, dict, &localError);
3198         /* create or update entries in dates/names/policies tables */
3199         ok = ok && _SecRevocationDbUpdateIssuerConstraints(dbc, groupId, dict, &localError);
3200     }
3201
3202     (void) CFErrorPropagate(localError, error);
3203     return ok;
3204 }
3205
3206 bool _SecRevocationDbApplyUpdate(SecRevocationDbConnectionRef dbc, CFDictionaryRef update, CFIndex version, CFErrorRef *error) {
3207     /* process entire update dictionary */
3208     if (!dbc || !dbc->db || !update) {
3209         secerror("_SecRevocationDbApplyUpdate failed: invalid args");
3210         SecError(errSecParam, error, CFSTR("_SecRevocationDbApplyUpdate: invalid db or update parameter"));
3211         return false;
3212     }
3213
3214     CFDictionaryRef localUpdate = (CFDictionaryRef)CFRetainSafe(update);
3215     CFErrorRef localError = NULL;
3216     bool ok = true;
3217
3218     CFTypeRef value = NULL;
3219     CFIndex deleteCount = 0;
3220     CFIndex updateCount = 0;
3221
3222     dbc->db->updateInProgress = true;
3223
3224     /* check whether this is a full update */
3225     value = (CFBooleanRef)CFDictionaryGetValue(update, CFSTR("full"));
3226     if (isBoolean(value) && CFBooleanGetValue((CFBooleanRef)value)) {
3227         /* clear the database before processing a full update */
3228         dbc->fullUpdate = true;
3229         secdebug("validupdate", "update has \"full\" attribute; clearing database");
3230         ok = ok && _SecRevocationDbRemoveAllEntries(dbc, &localError);
3231     }
3232
3233     /* process 'delete' list */
3234     value = (CFArrayRef)CFDictionaryGetValue(localUpdate, CFSTR("delete"));
3235     if (isArray(value)) {
3236         deleteCount = CFArrayGetCount((CFArrayRef)value);
3237         secdebug("validupdate", "processing %ld deletes", (long)deleteCount);
3238         for (CFIndex deleteIX=0; deleteIX<deleteCount; deleteIX++) {
3239             CFDataRef issuerHash = (CFDataRef)CFArrayGetValueAtIndex((CFArrayRef)value, deleteIX);
3240             if (isData(issuerHash)) {
3241                 ok = ok && _SecRevocationDbApplyGroupDelete(dbc, issuerHash, &localError);
3242             } else {
3243                 secdebug("validupdate", "skipping delete %ld (hash is not a data value)", (long)deleteIX);
3244             }
3245         }
3246     }
3247
3248     /* process 'update' list */
3249     value = (CFArrayRef)CFDictionaryGetValue(localUpdate, CFSTR("update"));
3250     if (isArray(value)) {
3251         updateCount = CFArrayGetCount((CFArrayRef)value);
3252         secdebug("validupdate", "processing %ld updates", (long)updateCount);
3253         for (CFIndex updateIX=0; updateIX<updateCount; updateIX++) {
3254             CFDictionaryRef dict = (CFDictionaryRef)CFArrayGetValueAtIndex((CFArrayRef)value, updateIX);
3255             if (isDictionary(dict)) {
3256                 ok = ok && _SecRevocationDbApplyGroupUpdate(dbc, dict, &localError);
3257             } else {
3258                 secdebug("validupdate", "skipping update %ld (not a dictionary)", (long)updateIX);
3259             }
3260         }
3261     }
3262     CFReleaseSafe(localUpdate);
3263
3264     /* set version */
3265     ok = ok && _SecRevocationDbSetVersion(dbc, version, &localError);
3266
3267     /* set interval if not already set, or changed */
3268     int64_t interval = _SecRevocationDbGetUpdateInterval(dbc, NULL);
3269     if (interval != dbc->precommitInterval) {
3270         interval = (dbc->precommitInterval > 0) ? dbc->precommitInterval : kSecStdUpdateInterval;
3271         ok = ok && _SecRevocationDbSetUpdateInterval(dbc, interval, &localError);
3272     }
3273
3274     /* set db_version if not already set */
3275     int64_t db_version = _SecRevocationDbGetSchemaVersion(dbc->db, dbc, NULL);
3276     if (db_version <= 0) {
3277         ok = ok && _SecRevocationDbSetSchemaVersion(dbc, kSecRevocationDbSchemaVersion, &localError);
3278     }
3279
3280     /* set db_format if not already set */
3281     int64_t db_format = _SecRevocationDbGetUpdateFormat(dbc, NULL);
3282     if (db_format <= 0) {
3283         ok = ok && _SecRevocationDbSetUpdateFormat(dbc, kSecRevocationDbUpdateFormat, &localError);
3284     }
3285
3286     /* purge the in-memory cache */
3287     SecRevocationDbCachePurge(dbc->db);
3288
3289     dbc->db->updateInProgress = false;
3290
3291     (void) CFErrorPropagate(localError, error);
3292     return ok;
3293 }
3294
3295 static bool _SecRevocationDbSerialInGroup(SecRevocationDbConnectionRef dbc,
3296                                           CFDataRef serial,
3297                                           int64_t groupId,
3298                                           CFErrorRef *error) {
3299     __block bool result = false;
3300     __block bool ok = true;
3301     __block CFErrorRef localError = NULL;
3302     require(dbc && serial, errOut);
3303     ok &= SecDbWithSQL(dbc->dbconn, selectSerialRecordSQL, &localError, ^bool(sqlite3_stmt *selectSerial) {
3304         ok &= SecDbBindInt64(selectSerial, 1, groupId, &localError);
3305         ok &= SecDbBindBlob(selectSerial, 2, CFDataGetBytePtr(serial),
3306                             CFDataGetLength(serial), SQLITE_TRANSIENT, &localError);
3307         ok &= SecDbStep(dbc->dbconn, selectSerial, &localError, ^(bool *stop) {
3308             int64_t foundRowId = (int64_t)sqlite3_column_int64(selectSerial, 0);
3309             result = (foundRowId > 0);
3310         });
3311         return ok;
3312     });
3313
3314 errOut:
3315     if (!ok || localError) {
3316         secerror("_SecRevocationDbSerialInGroup failed: %@", localError);
3317         TrustdHealthAnalyticsLogErrorCodeForDatabase(TARevocationDb, TAOperationRead, TAFatalError,
3318                                                      localError ? CFErrorGetCode(localError) : errSecInternalComponent);
3319     }
3320     (void) CFErrorPropagate(localError, error);
3321     return result;
3322 }
3323
3324 static bool _SecRevocationDbCertHashInGroup(SecRevocationDbConnectionRef dbc,
3325                                             CFDataRef certHash,
3326                                             int64_t groupId,
3327                                             CFErrorRef *error) {
3328     __block bool result = false;
3329     __block bool ok = true;
3330     __block CFErrorRef localError = NULL;
3331     require(dbc && certHash, errOut);
3332     ok &= SecDbWithSQL(dbc->dbconn, selectHashRecordSQL, &localError, ^bool(sqlite3_stmt *selectHash) {
3333         ok &= SecDbBindInt64(selectHash, 1, groupId, &localError);
3334         ok = SecDbBindBlob(selectHash, 2, CFDataGetBytePtr(certHash),
3335                            CFDataGetLength(certHash), SQLITE_TRANSIENT, &localError);
3336         ok &= SecDbStep(dbc->dbconn, selectHash, &localError, ^(bool *stop) {
3337             int64_t foundRowId = (int64_t)sqlite3_column_int64(selectHash, 0);
3338             result = (foundRowId > 0);
3339         });
3340         return ok;
3341     });
3342
3343 errOut:
3344     if (!ok || localError) {
3345         secerror("_SecRevocationDbCertHashInGroup failed: %@", localError);
3346         TrustdHealthAnalyticsLogErrorCodeForDatabase(TARevocationDb, TAOperationRead, TAFatalError,
3347                                                      localError ? CFErrorGetCode(localError) : errSecInternalComponent);
3348     }
3349     (void) CFErrorPropagate(localError, error);
3350     return result;
3351 }
3352
3353 static bool _SecRevocationDbSerialInFilter(SecRevocationDbConnectionRef dbc,
3354                                            CFDataRef serialData,
3355                                            CFDataRef xmlData) {
3356     /* N-To-1 filter implementation.
3357        The 'xmlData' parameter is a flattened XML dictionary,
3358        containing 'xor' and 'params' keys. First order of
3359        business is to reconstitute the blob into components.
3360     */
3361     bool result = false;
3362     CFRetainSafe(xmlData);
3363     CFDataRef propListData = xmlData;
3364     /* Expand data blob if needed */
3365     CFDataRef inflatedData = copyInflatedData(propListData);
3366     if (inflatedData) {
3367         CFReleaseSafe(propListData);
3368         propListData = inflatedData;
3369     }
3370     CFDataRef xor = NULL;
3371     CFArrayRef params = NULL;
3372     CFPropertyListRef nto1 = CFPropertyListCreateWithData(kCFAllocatorDefault, propListData, 0, NULL, NULL);
3373     if (nto1) {
3374         xor = (CFDataRef)CFDictionaryGetValue((CFDictionaryRef)nto1, CFSTR("xor"));
3375         params = (CFArrayRef)CFDictionaryGetValue((CFDictionaryRef)nto1, CFSTR("params"));
3376     }
3377     uint8_t *hash = (xor) ? (uint8_t*)CFDataGetBytePtr(xor) : NULL;
3378     CFIndex hashLen = (hash) ? CFDataGetLength(xor) : 0;
3379     uint8_t *serial = (serialData) ? (uint8_t*)CFDataGetBytePtr(serialData) : NULL;
3380     CFIndex serialLen = (serial) ? CFDataGetLength(serialData) : 0;
3381
3382     require(hash && serial && params, errOut);
3383
3384     const uint32_t FNV_OFFSET_BASIS = 2166136261;
3385     const uint32_t FNV_PRIME = 16777619;
3386     bool notInHash = false;
3387     CFIndex ix, count = CFArrayGetCount(params);
3388     for (ix = 0; ix < count; ix++) {
3389         int32_t param;
3390         CFNumberRef cfnum = (CFNumberRef)CFArrayGetValueAtIndex(params, ix);
3391         if (!isNumber(cfnum) ||
3392             !CFNumberGetValue(cfnum, kCFNumberSInt32Type, &param)) {
3393             secinfo("validupdate", "error processing filter params at index %ld", (long)ix);
3394             continue;
3395         }
3396         /* process one param */
3397         uint32_t hval = FNV_OFFSET_BASIS ^ param;
3398         CFIndex i = serialLen;
3399         while (i > 0) {
3400             hval = ((hval ^ (serial[--i])) * FNV_PRIME) & 0xFFFFFFFF;
3401         }
3402         hval = hval % (hashLen * 8);
3403         if ((hash[hval/8] & (1 << (hval % 8))) == 0) {
3404             notInHash = true; /* definitely not in hash */
3405             break;
3406         }
3407     }
3408     if (!notInHash) {
3409         /* probabilistically might be in hash if we get here. */
3410         result = true;
3411     }
3412
3413 errOut:
3414     CFReleaseSafe(nto1);
3415     CFReleaseSafe(propListData);
3416     return result;
3417 }
3418
3419 static SecValidInfoRef _SecRevocationDbValidInfoForCertificate(SecRevocationDbConnectionRef dbc,
3420                                                                SecCertificateRef certificate,
3421                                                                CFDataRef issuerHash,
3422                                                                CFErrorRef *error) {
3423     __block CFErrorRef localError = NULL;
3424     __block SecValidInfoFlags flags = 0;
3425     __block SecValidInfoFormat format = kSecValidInfoFormatUnknown;
3426     __block CFDataRef data = NULL;
3427
3428     bool matched = false;
3429     bool isOnList = false;
3430     int64_t groupId = 0;
3431     CFDataRef serial = NULL;
3432     CFDataRef certHash = NULL;
3433     CFDateRef notBeforeDate = NULL;
3434     CFDateRef notAfterDate = NULL;
3435     CFDataRef names = NULL;
3436     CFDataRef policies = NULL;
3437     SecValidInfoRef result = NULL;
3438
3439     require((serial = SecCertificateCopySerialNumberData(certificate, NULL)) != NULL, errOut);
3440     require((certHash = SecCertificateCopySHA256Digest(certificate)) != NULL, errOut);
3441     require_quiet((groupId = _SecRevocationDbGroupIdForIssuerHash(dbc, issuerHash, &localError)) > 0, errOut);
3442
3443     /* Look up the group record to determine flags and format. */
3444     format = _SecRevocationDbGetGroupFormat(dbc, groupId, &flags, &data, &policies, &localError);
3445
3446     if (format == kSecValidInfoFormatUnknown) {
3447         /* No group record found for this issuer. Don't return a SecValidInfoRef */
3448         goto errOut;
3449     }
3450     else if (format == kSecValidInfoFormatSerial) {
3451         /* Look up certificate's serial number in the serials table. */
3452         matched = _SecRevocationDbSerialInGroup(dbc, serial, groupId, &localError);
3453     }
3454     else if (format == kSecValidInfoFormatSHA256) {
3455         /* Look up certificate's SHA-256 hash in the hashes table. */
3456         matched = _SecRevocationDbCertHashInGroup(dbc, certHash, groupId, &localError);
3457     }
3458     else if (format == kSecValidInfoFormatNto1) {
3459         /* Perform a Bloom filter match against the serial. If matched is false,
3460            then the cert is definitely not in the list. But if matched is true,
3461            we don't know for certain, so we would need to check OCSP. */
3462         matched = _SecRevocationDbSerialInFilter(dbc, serial, data);
3463     }
3464
3465     if (matched) {
3466         /* Found a specific match for this certificate. */
3467         secdebug("validupdate", "Valid db matched certificate: %@, format=%d, flags=0x%lx",
3468                  certHash, format, flags);
3469         isOnList = true;
3470     }
3471
3472     /* If supplemental constraints are present for this issuer, then we always match. */
3473     if ((flags & kSecValidInfoDateConstraints) &&
3474         (_SecRevocationDbCopyDateConstraints(dbc, groupId, &notBeforeDate, &notAfterDate, &localError))) {
3475         secdebug("validupdate", "Valid db matched supplemental date constraints for groupId %lld: nb=%@, na=%@",
3476                  (long long)groupId, notBeforeDate, notAfterDate);
3477     }
3478
3479
3480     /* Return SecValidInfo for certificates for which an issuer entry is found. */
3481     result = SecValidInfoCreate(format, flags, isOnList,
3482                                 certHash, issuerHash, /*anchorHash*/ NULL,
3483                                 notBeforeDate, notAfterDate,
3484                                 names, policies);
3485
3486     if (result && SecIsAppleTrustAnchor(certificate, 0)) {
3487         /* Prevent a catch-22. */
3488         secdebug("validupdate", "Valid db match for Apple trust anchor: %@, format=%d, flags=0x%lx",
3489                  certHash, format, flags);
3490         CFReleaseNull(result);
3491     }
3492
3493 errOut:
3494     (void) CFErrorPropagate(localError, error);
3495     CFReleaseSafe(data);
3496     CFReleaseSafe(certHash);
3497     CFReleaseSafe(serial);
3498     CFReleaseSafe(notBeforeDate);
3499     CFReleaseSafe(notAfterDate);
3500     CFReleaseSafe(names);
3501     CFReleaseSafe(policies);
3502     return result;
3503 }
3504
3505 static SecValidInfoRef _SecRevocationDbCopyMatching(SecRevocationDbConnectionRef dbc,
3506                                                     SecCertificateRef certificate,
3507                                                     SecCertificateRef issuer) {
3508     SecValidInfoRef result = NULL;
3509     CFErrorRef error = NULL;
3510     CFDataRef issuerHash = NULL;
3511
3512     require(dbc && certificate && issuer, errOut);
3513     require(issuerHash = SecCertificateCopySHA256Digest(issuer), errOut);
3514
3515     /* Check for the result in the cache. */
3516     result = SecRevocationDbCacheRead(dbc->db, certificate, issuerHash);
3517
3518     /* Upon cache miss, get the result from the database and add it to the cache. */
3519     if (!result) {
3520         result = _SecRevocationDbValidInfoForCertificate(dbc, certificate, issuerHash, &error);
3521         SecRevocationDbCacheWrite(dbc->db, result);
3522     }
3523
3524 errOut:
3525     CFReleaseSafe(issuerHash);
3526     CFReleaseSafe(error);
3527     return result;
3528 }
3529
3530 /* Return the update source as a retained CFStringRef.
3531    If the value cannot be obtained, NULL is returned.
3532 */
3533 CFStringRef SecRevocationDbCopyUpdateSource(void) {
3534     __block CFStringRef result = NULL;
3535     SecRevocationDbWith(^(SecRevocationDbRef db) {
3536         (void) SecRevocationDbPerformRead(db, NULL, ^bool(SecRevocationDbConnectionRef dbc, CFErrorRef *blockError) {
3537             result = _SecRevocationDbCopyUpdateSource(dbc, blockError);
3538             return (bool)result;
3539         });
3540     });
3541     return result;
3542 }
3543
3544 /* Set the next update value for the revocation database.
3545    (This function is expected to be called only by the database
3546    maintainer, normally the system instance of trustd. If the
3547    caller does not have write access, this is a no-op.)
3548 */
3549 bool SecRevocationDbSetNextUpdateTime(CFAbsoluteTime nextUpdate, CFErrorRef *error) {
3550     __block bool ok = true;
3551     __block CFErrorRef localError = NULL;
3552     SecRevocationDbWith(^(SecRevocationDbRef rdb) {
3553         ok &= SecRevocationDbPerformWrite(rdb, &localError, ^bool(SecRevocationDbConnectionRef dbc, CFErrorRef *blockError) {
3554             return _SecRevocationDbSetNextUpdateTime(dbc, nextUpdate, blockError);
3555         });
3556     });
3557     (void) CFErrorPropagate(localError, error);
3558     return ok;
3559 }
3560
3561 /* Return the next update value as a CFAbsoluteTime.
3562    If the value cannot be obtained, -1 is returned.
3563 */
3564 CFAbsoluteTime SecRevocationDbGetNextUpdateTime(void) {
3565     __block CFAbsoluteTime result = -1;
3566     SecRevocationDbWith(^(SecRevocationDbRef db) {
3567         (void) SecRevocationDbPerformRead(db, NULL, ^bool(SecRevocationDbConnectionRef dbc, CFErrorRef *blockError) {
3568             result = _SecRevocationDbGetNextUpdateTime(dbc, blockError);
3569             return true;
3570         });
3571     });
3572     return result;
3573 }
3574
3575 /* Return the serial background queue for database updates.
3576    If the queue cannot be obtained, NULL is returned.
3577 */
3578 dispatch_queue_t SecRevocationDbGetUpdateQueue(void) {
3579     __block dispatch_queue_t result = NULL;
3580     SecRevocationDbWith(^(SecRevocationDbRef db) {
3581         result = (db) ? db->update_queue : NULL;
3582     });
3583     return result;
3584 }
3585
3586 /* Release all connections to the revocation database.
3587 */
3588 void SecRevocationDbReleaseAllConnections(void) {
3589     SecRevocationDbWith(^(SecRevocationDbRef db) {
3590         SecDbReleaseAllConnections((db) ? db->db : NULL);
3591     });
3592 }
3593
3594 /* === SecRevocationDb API === */
3595
3596 /* Given a certificate and its issuer, returns a SecValidInfoRef if the
3597    valid database contains matching info; otherwise returns NULL.
3598    Caller must release the returned SecValidInfoRef when finished.
3599 */
3600 SecValidInfoRef SecRevocationDbCopyMatching(SecCertificateRef certificate,
3601                                             SecCertificateRef issuer) {
3602     __block SecValidInfoRef result = NULL;
3603     SecRevocationDbWith(^(SecRevocationDbRef db) {
3604         (void) SecRevocationDbPerformRead(db, NULL, ^bool(SecRevocationDbConnectionRef dbc, CFErrorRef *blockError) {
3605             result = _SecRevocationDbCopyMatching(dbc, certificate, issuer);
3606             return (bool)result;
3607         });
3608     });
3609     return result;
3610 }
3611
3612 /* Given an issuer, returns true if an entry for this issuer exists in
3613    the database (i.e. a known CA). If the provided certificate is NULL,
3614    or its entry is not found, the function returns false.
3615 */
3616 bool SecRevocationDbContainsIssuer(SecCertificateRef issuer) {
3617     if (!issuer) {
3618         return false;
3619     }
3620     __block bool result = false;
3621     SecRevocationDbWith(^(SecRevocationDbRef db) {
3622         (void) SecRevocationDbPerformRead(db, NULL, ^bool(SecRevocationDbConnectionRef dbc, CFErrorRef *blockError) {
3623             CFDataRef issuerHash = SecCertificateCopySHA256Digest(issuer);
3624             int64_t groupId = _SecRevocationDbGroupIdForIssuerHash(dbc, issuerHash, blockError);
3625             CFReleaseSafe(issuerHash);
3626             result = (groupId > 0);
3627             return result;
3628         });
3629     });
3630     return result;
3631 }
3632
3633 /* Return the current version of the revocation database.
3634    A version of 0 indicates an empty database which must be populated.
3635    If the version cannot be obtained, -1 is returned.
3636 */
3637 CFIndex SecRevocationDbGetVersion(void) {
3638     __block CFIndex result = -1;
3639     SecRevocationDbWith(^(SecRevocationDbRef db) {
3640         (void) SecRevocationDbPerformRead(db, NULL, ^bool(SecRevocationDbConnectionRef dbc, CFErrorRef *blockError) {
3641             result = (CFIndex)_SecRevocationDbGetVersion(dbc, blockError);
3642             return (result >= 0);
3643         });
3644     });
3645     return result;
3646 }
3647
3648 /* Return the current schema version of the revocation database.
3649  A version of 0 indicates an empty database which must be populated.
3650  If the schema version cannot be obtained, -1 is returned.
3651  */
3652 CFIndex SecRevocationDbGetSchemaVersion(void) {
3653     __block CFIndex result = -1;
3654     SecRevocationDbWith(^(SecRevocationDbRef db) {
3655         result = (CFIndex)_SecRevocationDbGetSchemaVersion(db, NULL, NULL);
3656     });
3657     return result;
3658 }
3659
3660 /* Return the current update format of the revocation database.
3661  A version of 0 indicates the format was unknown.
3662  If the update format cannot be obtained, -1 is returned.
3663  */
3664 CFIndex SecRevocationDbGetUpdateFormat(void) {
3665     __block CFIndex result = -1;
3666     SecRevocationDbWith(^(SecRevocationDbRef db) {
3667         (void) SecRevocationDbPerformRead(db, NULL, ^bool(SecRevocationDbConnectionRef dbc, CFErrorRef *blockError) {
3668             result = (CFIndex)_SecRevocationDbGetUpdateFormat(dbc, blockError);
3669             return (result >= 0);
3670         });
3671     });
3672     return result;
3673 }
3674
3675 // MARK: -
3676 // MARK: Digests
3677 /*
3678  ==============================================================================
3679    Digest computation
3680  ==============================================================================
3681 */
3682
3683 /* Returns array of SHA-256 hashes computed over the contents of a valid.sqlite3
3684    database, in the order specified by the valid-server-api documentation. The
3685    resulting hashes can be compared against those in the update's 'hash' array.
3686
3687    Hash 0: full database (all fields in initial Valid specification)
3688    Hash 1: all issuer_hash arrays, plus not-after and not-before dates for each
3689    Hash 2: subset of issuer_hash arrays where the no-ca-v2 flag is set
3690 */
3691 static CF_RETURNS_RETAINED CFArrayRef SecRevocationDbComputeFullContentDigests(SecRevocationDbConnectionRef dbc, CFErrorRef *error) {
3692     if (!dbc) { return NULL; }
3693     __block bool ok = true;
3694     __block CFErrorRef localError = NULL;
3695     __block uint32_t N[4]={0,0,0,0};
3696     __block CC_SHA256_CTX hash0_ctx, hash1_ctx, hash2_ctx;
3697     CC_SHA256_Init(&hash0_ctx);
3698     CC_SHA256_Init(&hash1_ctx);
3699     CC_SHA256_Init(&hash2_ctx);
3700
3701     // Add version, check-again, and update (array count) fields as array of N.
3702     // (Note: 'N' is defined as "unsigned 32-bit integer in network byte order")
3703     int64_t version = _SecRevocationDbGetVersion(dbc, NULL);
3704     N[0] = OSSwapInt32(version & 0xffffffff);
3705     int64_t interval =  _SecRevocationDbGetUpdateInterval(dbc, NULL);
3706     if (interval < 0) {
3707         interval = kSecStdUpdateInterval; // if we didn't store it, assume default
3708     }
3709     N[1] = OSSwapInt32(interval & 0xffffffff);
3710     __block int64_t count = 0;
3711     ok = ok && SecDbWithSQL(dbc->dbconn, CFSTR("SELECT count(*) FROM groups"), &localError, ^bool(sqlite3_stmt *selectGroupsCount) {
3712         ok = ok && SecDbStep(dbc->dbconn, selectGroupsCount, &localError, ^void(bool *stop) {
3713             count = sqlite3_column_int64(selectGroupsCount, 0);
3714             *stop = true;
3715         });
3716         return ok;
3717     });
3718     N[2] = OSSwapInt32(count & 0xffffffff);
3719     CC_SHA256_Update(&hash0_ctx, N, sizeof(uint32_t) * 3);
3720
3721     // Sort the update array in order of minimum 'issuer-hash' entry.
3722     // The issuer-hash array is first sorted to determine the lowest issuer-hash,
3723     // and that value is used to sort the update entries.
3724     //
3725     // For our sqlite database, recreating the update array order means fetching
3726     // the groupid column from the issuers table after sorting on issuer_hash,
3727     // using DISTINCT to remove duplicates. Then, for each returned groupid, we
3728     // obtain its list of issuers, its list of serials or hashes, and other data.
3729
3730     ok = ok && SecDbWithSQL(dbc->dbconn, CFSTR("SELECT DISTINCT groupid FROM issuers ORDER BY issuer_hash ASC"), &localError, ^bool(sqlite3_stmt *selectGroups) {
3731         ok = ok && SecDbForEach(dbc->dbconn, selectGroups, &localError, ^bool(int row_index) {
3732             __block int64_t groupId = sqlite3_column_int64(selectGroups, 0);
3733             ok = ok && SecDbWithSQL(dbc->dbconn, CFSTR("SELECT flags,format,data FROM groups WHERE groupid=?"), &localError, ^bool(sqlite3_stmt *selectGroup) {
3734                 ok = ok && SecDbBindInt64(selectGroup, 1, groupId, &localError);
3735                 ok = ok && SecDbStep(dbc->dbconn, selectGroup, &localError, ^(bool *stop) {
3736                     // per-group info is hashed in the following order:
3737                     // - issuer_hash array data (sorted)
3738                     // - flag bytes, in order listed below
3739                     // - format string [serial|sha256|nto1]
3740                     // - add array data (sorted), if [serial|sha256]
3741                     // - params (if present)
3742                     // - xor data (if present)
3743
3744                     int64_t flags = sqlite3_column_int64(selectGroup, 0);
3745                     bool noCAv2 = (flags & kSecValidInfoNoCAv2Check);
3746
3747                     // instead of recreating the issuer_hash array in memory,
3748                     // hash its length (item count) followed by the data of each issuer_hash.
3749                     ok = ok && SecDbWithSQL(dbc->dbconn, CFSTR("SELECT count(*) FROM issuers WHERE groupid=?"), &localError, ^bool(sqlite3_stmt *selectIssuersCount) {
3750                         ok = ok && SecDbBindInt64(selectIssuersCount, 1, groupId, &localError);
3751                         ok = ok && SecDbStep(dbc->dbconn, selectIssuersCount, &localError, ^void(bool *stop) {
3752                             count = sqlite3_column_int64(selectIssuersCount, 0);
3753                             *stop = true;
3754                         });
3755                         return ok;
3756                     });
3757                     uint32_t n = OSSwapInt32(count & 0xffffffff);
3758                     CC_SHA256_Update(&hash0_ctx, &n, sizeof(uint32_t));
3759                     CC_SHA256_Update(&hash1_ctx, &n, sizeof(uint32_t));
3760                     if (noCAv2) {
3761                         CC_SHA256_Update(&hash2_ctx, &n, sizeof(uint32_t));
3762                     }
3763
3764                     // process issuer_hash entries for this group
3765                     ok = ok && SecDbWithSQL(dbc->dbconn, CFSTR("SELECT issuer_hash FROM issuers WHERE groupid=? ORDER BY issuer_hash ASC"), &localError, ^bool(sqlite3_stmt *selectIssuerHash) {
3766                         ok = ok && SecDbBindInt64(selectIssuerHash, 1, groupId, &localError);
3767                         ok = ok && SecDbForEach(dbc->dbconn, selectIssuerHash, &localError, ^bool(int row_index) {
3768                             uint8_t *p = (uint8_t *)sqlite3_column_blob(selectIssuerHash, 0);
3769                             CFDataRef data = NULL;
3770                             if (p != NULL) {
3771                                 CFIndex length = (CFIndex)sqlite3_column_bytes(selectIssuerHash, 0);
3772                                 data = CFDataCreate(kCFAllocatorDefault, p, length);
3773                             }
3774                             if (data != NULL) {
3775                                 hashData(data, &hash0_ctx);
3776                                 hashData(data, &hash1_ctx);
3777                                 if (noCAv2) {
3778                                     hashData(data, &hash2_ctx);
3779                                 }
3780                                 CFRelease(data);
3781                             } else {
3782                                 ok = false;
3783                             }
3784                             return ok;
3785                         });
3786                         return ok;
3787                     });
3788
3789                     // process flags, converting to array of unsigned 8-bit values, either 0 or 1:
3790                     // [ complete, check-ocsp, known-intermediates-only, no-ca, overridable, require-ct, valid ]
3791                     uint8_t C[8]={0,0,0,0,0,0,0,0};
3792                     C[0] = (flags & kSecValidInfoComplete) ? 1 : 0;
3793                     C[1] = (flags & kSecValidInfoCheckOCSP) ? 1 : 0;
3794                     C[2] = (flags & kSecValidInfoKnownOnly) ? 1 : 0;
3795                     C[3] = (flags & kSecValidInfoNoCACheck) ? 1 : 0;
3796                     C[4] = (flags & kSecValidInfoOverridable) ? 1 : 0;
3797                     C[5] = (flags & kSecValidInfoRequireCT) ? 1 : 0;
3798                     C[6] = (flags & kSecValidInfoAllowlist) ? 1 : 0;
3799                     CC_SHA256_Update(&hash0_ctx, C, sizeof(uint8_t) * 7);
3800
3801                     // process format, converting integer to string value [serial|sha256|nto1]
3802                     SecValidInfoFormat format = (SecValidInfoFormat)sqlite3_column_int(selectGroup, 1);
3803                     switch (format) {
3804                         case kSecValidInfoFormatSerial:
3805                             hashString(CFSTR("serial"), &hash0_ctx);
3806                             break;
3807                         case kSecValidInfoFormatSHA256:
3808                             hashString(CFSTR("sha256"), &hash0_ctx);
3809                             break;
3810                         case kSecValidInfoFormatNto1:
3811                             hashString(CFSTR("nto1"), &hash0_ctx);
3812                             break;
3813                         case kSecValidInfoFormatUnknown:
3814                         default:
3815                             ok = false; // unexpected format values are not allowed
3816                             break;
3817                     }
3818                     // process 'add' array (serial or sha256 format).
3819                     // instead of recreating the 'add' array in memory,
3820                     // hash its length (item count) followed by the data of each entry.
3821                     CFStringRef arrayCountSql = NULL;
3822                     if (format == kSecValidInfoFormatSerial) {
3823                         arrayCountSql = CFSTR("SELECT count(*) FROM serials WHERE groupid=?");
3824                     } else if (format == kSecValidInfoFormatSHA256) {
3825                         arrayCountSql = CFSTR("SELECT count(*) FROM hashes WHERE groupid=?");
3826                     }
3827                     if (arrayCountSql) {
3828                         ok = ok && SecDbWithSQL(dbc->dbconn, arrayCountSql, &localError, ^bool(sqlite3_stmt *selectAddCount) {
3829                             ok = ok && SecDbBindInt64(selectAddCount, 1, groupId, &localError);
3830                             ok = ok && SecDbStep(dbc->dbconn, selectAddCount, &localError, ^void(bool *stop) {
3831                                 count = sqlite3_column_int64(selectAddCount, 0);
3832                                 *stop = true;
3833                             });
3834                             return ok;
3835                         });
3836                         n = OSSwapInt32(count & 0xffffffff);
3837                         CC_SHA256_Update(&hash0_ctx, &n, sizeof(uint32_t));
3838                     }
3839                     // process data entries for this group
3840                     CFStringRef arrayDataSql = NULL;
3841                     if (format == kSecValidInfoFormatSerial) {
3842                         arrayDataSql = CFSTR("SELECT serial FROM serials WHERE groupid=? ORDER BY serial ASC");
3843                     } else if (format == kSecValidInfoFormatSHA256) {
3844                         arrayDataSql = CFSTR("SELECT sha256 FROM hashes WHERE groupid=? ORDER by sha256 ASC");
3845                     }
3846                     if (arrayDataSql) {
3847                         ok = ok && SecDbWithSQL(dbc->dbconn, arrayDataSql, &localError, ^bool(sqlite3_stmt *selectAddData) {
3848                             ok = ok && SecDbBindInt64(selectAddData, 1, groupId, &localError);
3849                             ok = ok && SecDbForEach(dbc->dbconn, selectAddData, &localError, ^bool(int row_index) {
3850                                 uint8_t *p = (uint8_t *)sqlite3_column_blob(selectAddData, 0);
3851                                 CFDataRef data = NULL;
3852                                 if (p != NULL) {
3853                                     CFIndex length = (CFIndex)sqlite3_column_bytes(selectAddData, 0);
3854                                     data = CFDataCreate(kCFAllocatorDefault, p, length);
3855                                 }
3856                                 if (data != NULL) {
3857                                     hashData(data, &hash0_ctx);
3858                                     CFRelease(data);
3859                                 } else {
3860                                     ok = false;
3861                                 }
3862                                 return ok;
3863                             });
3864                             return ok;
3865                         });
3866                     }
3867
3868                     // process params and xor data, if format is nto1
3869                     if (format == kSecValidInfoFormatNto1) {
3870                         uint8_t *p = (uint8_t *)sqlite3_column_blob(selectGroup, 2);
3871                         CFDataRef data = NULL;
3872                         if (p != NULL) {
3873                             CFIndex length = (CFIndex)sqlite3_column_bytes(selectGroup, 2);
3874                             data = CFDataCreate(kCFAllocatorDefault, p, length);
3875                         }
3876                         if (data != NULL) {
3877                             // unpack params and xor data
3878                             CFDataRef xor = NULL;
3879                             CFArrayRef params = NULL;
3880                             if (copyFilterComponents(data, &xor, &params)) {
3881                                 hashArray(params, &hash0_ctx);
3882                                 hashData(xor, &hash0_ctx);
3883                             } else {
3884                                 ok = false;
3885                             }
3886                             CFReleaseSafe(xor);
3887                             CFReleaseSafe(params);
3888                         }
3889                         CFReleaseSafe(data);
3890                     }
3891
3892                     // process date constraints [not-after, not-before]
3893                     CFAbsoluteTime notBefore = -3155760000.0; /* default: 1901-01-01 00:00:00-0000 */
3894                     CFAbsoluteTime notAfter = 31556908800.0;  /* default: 3001-01-01 00:00:00-0000 */
3895                     CFDateRef notBeforeDate = NULL;
3896                     CFDateRef notAfterDate = NULL;
3897                     if (_SecRevocationDbCopyDateConstraints(dbc, groupId, &notBeforeDate, &notAfterDate, &localError)) {
3898                         if (notBeforeDate) {
3899                             notBefore = CFDateGetAbsoluteTime(notBeforeDate);
3900                             CFReleaseNull(notBeforeDate);
3901                         }
3902                         if (notAfterDate) {
3903                             notAfter = CFDateGetAbsoluteTime(notAfterDate);
3904                             CFReleaseNull(notAfterDate);
3905                         }
3906                     }
3907                     double nb = htond(notBefore);
3908                     double na = htond(notAfter);
3909                     CC_SHA256_Update(&hash1_ctx, &na, sizeof(double));
3910                     CC_SHA256_Update(&hash1_ctx, &nb, sizeof(double));
3911
3912                     *stop = true;
3913                 }); // per-group step
3914                 return ok;
3915             }); // per-group select
3916             return ok;
3917         }); // for each group in list
3918         return ok;
3919     }); // select full group list
3920
3921     CFMutableArrayRef result = CFArrayCreateMutable(NULL, 0, &kCFTypeArrayCallBacks);
3922     if (result) {
3923         uint8_t digest[CC_SHA256_DIGEST_LENGTH];
3924         CFDataRef data = NULL;
3925         CC_SHA256_Final(digest, &hash0_ctx);
3926         if ((data = CFDataCreate(NULL, (const UInt8 *)digest, CC_SHA256_DIGEST_LENGTH)) != NULL) {
3927             CFArrayAppendValue(result, data);
3928             CFReleaseNull(data);
3929         }
3930         CC_SHA256_Final(digest, &hash1_ctx);
3931         if ((data = CFDataCreate(NULL, (const UInt8 *)digest, CC_SHA256_DIGEST_LENGTH)) != NULL) {
3932             CFArrayAppendValue(result, data);
3933             CFReleaseNull(data);
3934         }
3935         CC_SHA256_Final(digest, &hash2_ctx);
3936         if ((data = CFDataCreate(NULL, (const UInt8 *)digest, CC_SHA256_DIGEST_LENGTH)) != NULL) {
3937             CFArrayAppendValue(result, data);
3938             CFReleaseNull(data);
3939         }
3940     }
3941     (void) CFErrorPropagate(localError, error);
3942     return result;
3943 }
3944
3945 static bool SecRevocationDbComputeDigests(SecRevocationDbConnectionRef dbc, CFErrorRef *error) {
3946     secinfo("validupdate", "Started verifying db content");
3947     bool result = true;
3948     CFArrayRef expectedList = _SecRevocationDbCopyHashes(dbc, error);
3949     CFIndex expectedCount = (expectedList) ? CFArrayGetCount(expectedList) : 0;
3950     if (expectedCount < 1) {
3951         secinfo("validupdate", "Unable to read db_hash values");
3952         CFReleaseNull(expectedList);
3953         return result; // %%%% this will happen on first update, when db_hash isn't there
3954     }
3955     CFArrayRef computedList = SecRevocationDbComputeFullContentDigests(dbc, error);
3956     CFIndex computedCount = (computedList) ? CFArrayGetCount(computedList) : 0;
3957     for (CFIndex idx = 0; idx < expectedCount; idx++) {
3958         if (idx >= computedCount) {
3959             continue; // server provided additional hash value that we don't yet compute
3960         }
3961         CFDataRef expectedHash = (CFDataRef)CFArrayGetValueAtIndex(expectedList, idx);
3962         CFDataRef computedHash = (CFDataRef)CFArrayGetValueAtIndex(computedList, idx);
3963         if (!CFEqualSafe(expectedHash, computedHash)) {
3964             result = false;
3965             break;
3966         }
3967     }
3968     if (!result) {
3969         secinfo("validupdate", "Expected: %@", expectedList);
3970         secinfo("validupdate", "Computed: %@", computedList);
3971     }
3972     secinfo("validupdate", "Finished verifying db content; result=%s",
3973             (result) ? "SUCCESS" : "FAIL");
3974     CFReleaseSafe(expectedList);
3975     CFReleaseSafe(computedList);
3976     return result;
3977 }
3978