dyld3/shared-cache/dyld_shared_cache_builder.mm

   1 /* -*- mode: C++; c-basic-offset: 4; indent-tabs-mode: nil -*-
   2  *
   3  * Copyright (c) 2016 Apple Inc. All rights reserved.
   4  *
   5  * @APPLE_LICENSE_HEADER_START@
   6  *
   7  * This file contains Original Code and/or Modifications of Original Code
   8  * as defined in and that are subject to the Apple Public Source License
   9  * Version 2.0 (the 'License'). You may not use this file except in
  10  * compliance with the License. Please obtain a copy of the License at
  11  * http://www.opensource.apple.com/apsl/ and read it before using this
  12  * file.
  13  *
  14  * The Original Code and all software distributed under the License are
  15  * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  16  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  17  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
  18  * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
  19  * Please see the License for the specific language governing rights and
  20  * limitations under the License.
  21  *
  22  * @APPLE_LICENSE_HEADER_END@
  23  */
  24
  25 #include <sys/types.h>
  26 #include <sys/stat.h>
  27 #include <sys/mman.h>
  28 #include <sys/resource.h>
  29 #include <mach/mach.h>
  30 #include <mach/mach_time.h>
  31 #include <limits.h>
  32 #include <stdarg.h>
  33 #include <stdio.h>
  34 #include <stdlib.h>
  35 #include <math.h>
  36 #include <fcntl.h>
  37 #include <dlfcn.h>
  38 #include <signal.h>
  39 #include <errno.h>
  40 #include <sys/uio.h>
  41 #include <unistd.h>
  42 #include <sys/param.h>
  43 #include <sys/sysctl.h>
  44 #include <sys/resource.h>
  45 #include <dirent.h>
  46 #include <libgen.h>
  47 #include <pthread.h>
  48 #include <fts.h>
  49
  50 #include <vector>
  51 #include <array>
  52 #include <set>
  53 #include <map>
  54 #include <unordered_set>
  55 #include <algorithm>
  56 #include <fstream>
  57 #include <regex>
  58
  59 #include <spawn.h>
  60
  61 #include <Bom/Bom.h>
  62
  63 #include "Manifest.h"
  64 #include "Diagnostics.h"
  65 #include "DyldSharedCache.h"
  66 #include "BuilderUtils.h"
  67 #include "FileUtils.h"
  68 #include "JSONReader.h"
  69 #include "JSONWriter.h"
  70 #include "StringUtils.h"
  71 #include "mrm_shared_cache_builder.h"
  72
  73 #if !__has_feature(objc_arc)
  74 #error The use of libdispatch in this files requires it to be compiled with ARC in order to avoid leaks
  75 #endif
  76
  77 extern char** environ;
  78
  79 static dispatch_queue_t build_queue;
  80
  81 int runCommandAndWait(Diagnostics& diags, const char* args[])
  82 {
  83     pid_t pid;
  84     int   status;
  85     int   res = posix_spawn(&pid, args[0], nullptr, nullptr, (char**)args, environ);
  86     if (res != 0)
  87         diags.error("Failed to spawn %s: %s (%d)", args[0], strerror(res), res);
  88
  89     do {
  90         res = waitpid(pid, &status, 0);
  91     } while (res == -1 && errno == EINTR);
  92     if (res != -1) {
  93         if (WIFEXITED(status)) {
  94             res = WEXITSTATUS(status);
  95         } else {
  96             res = -1;
  97         }
  98     }
  99
 100     return res;
 101 }
 102
 103 void processRoots(Diagnostics& diags, std::set<std::string>& roots, const char *tempRootsDir)
 104 {
 105     std::set<std::string> processedRoots;
 106     struct stat           sb;
 107     int                   res = 0;
 108     const char*           args[8];
 109
 110     for (const auto& root : roots) {
 111         res = stat(root.c_str(), &sb);
 112
 113         if (res == 0 && S_ISDIR(sb.st_mode)) {
 114             processedRoots.insert(root);
 115             continue;
 116         }
 117
 118         char tempRootDir[MAXPATHLEN];
 119         strlcpy(tempRootDir, tempRootsDir, MAXPATHLEN);
 120         strlcat(tempRootDir, "/XXXXXXXX", MAXPATHLEN);
 121         mkdtemp(tempRootDir);
 122
 123         if (endsWith(root, ".cpio") || endsWith(root, ".cpio.gz") || endsWith(root, ".cpgz") || endsWith(root, ".cpio.bz2") || endsWith(root, ".cpbz2") || endsWith(root, ".pax") || endsWith(root, ".pax.gz") || endsWith(root, ".pgz") || endsWith(root, ".pax.bz2") || endsWith(root, ".pbz2")) {
 124             args[0] = (char*)"/usr/bin/ditto";
 125             args[1] = (char*)"-x";
 126             args[2] = (char*)root.c_str();
 127             args[3] = tempRootDir;
 128             args[4] = nullptr;
 129         } else if (endsWith(root, ".tar")) {
 130             args[0] = (char*)"/usr/bin/tar";
 131             args[1] = (char*)"xf";
 132             args[2] = (char*)root.c_str();
 133             args[3] = (char*)"-C";
 134             args[4] = tempRootDir;
 135             args[5] = nullptr;
 136         } else if (endsWith(root, ".tar.gz") || endsWith(root, ".tgz")) {
 137             args[0] = (char*)"/usr/bin/tar";
 138             args[1] = (char*)"xzf";
 139             args[2] = (char*)root.c_str();
 140             args[3] = (char*)"-C";
 141             args[4] = tempRootDir;
 142             args[5] = nullptr;
 143         } else if (endsWith(root, ".tar.bz2")
 144             || endsWith(root, ".tbz2")
 145             || endsWith(root, ".tbz")) {
 146             args[0] = (char*)"/usr/bin/tar";
 147             args[1] = (char*)"xjf";
 148             args[2] = (char*)root.c_str();
 149             args[3] = (char*)"-C";
 150             args[4] = tempRootDir;
 151             args[5] = nullptr;
 152         } else if (endsWith(root, ".xar")) {
 153             args[0] = (char*)"/usr/bin/xar";
 154             args[1] = (char*)"-xf";
 155             args[2] = (char*)root.c_str();
 156             args[3] = (char*)"-C";
 157             args[4] = tempRootDir;
 158             args[5] = nullptr;
 159         } else if (endsWith(root, ".zip")) {
 160             args[0] = (char*)"/usr/bin/ditto";
 161             args[1] = (char*)"-xk";
 162             args[2] = (char*)root.c_str();
 163             args[3] = tempRootDir;
 164             args[4] = nullptr;
 165         } else {
 166             diags.error("unknown archive type: %s", root.c_str());
 167             continue;
 168         }
 169
 170         if (res != runCommandAndWait(diags, args)) {
 171             fprintf(stderr, "Could not expand archive %s: %s (%d)", root.c_str(), strerror(res), res);
 172             exit(-1);
 173         }
 174         for (auto& existingRoot : processedRoots) {
 175             if (existingRoot == tempRootDir)
 176                 return;
 177         }
 178
 179         processedRoots.insert(tempRootDir);
 180     }
 181
 182     roots = processedRoots;
 183 }
 184
 185 void writeRootList(const std::string& dstRoot, const std::set<std::string>& roots)
 186 {
 187     if (roots.size() == 0)
 188         return;
 189
 190     std::string rootFile = dstRoot + "/roots.txt";
 191     FILE*       froots = ::fopen(rootFile.c_str(), "w");
 192     if (froots == NULL)
 193         return;
 194
 195     for (auto& root : roots) {
 196         fprintf(froots, "%s\n", root.c_str());
 197     }
 198
 199     ::fclose(froots);
 200 }
 201
 202 BOMCopierCopyOperation filteredCopyExcludingPaths(BOMCopier copier, const char* path, BOMFSObjType type, off_t size)
 203 {
 204     std::string absolutePath = &path[1];
 205     void *userData = BOMCopierUserData(copier);
 206     std::set<std::string> *cachePaths = (std::set<std::string>*)userData;
 207     if (cachePaths->count(absolutePath)) {
 208         return BOMCopierSkipFile;
 209     }
 210     return BOMCopierContinue;
 211 }
 212
 213 BOMCopierCopyOperation filteredCopyIncludingPaths(BOMCopier copier, const char* path, BOMFSObjType type, off_t size)
 214 {
 215     std::string absolutePath = &path[1];
 216     void *userData = BOMCopierUserData(copier);
 217     std::set<std::string> *cachePaths = (std::set<std::string>*)userData;
 218     for (const std::string& cachePath : *cachePaths) {
 219         if (startsWith(cachePath, absolutePath))
 220             return BOMCopierContinue;
 221     }
 222     if (cachePaths->count(absolutePath)) {
 223         return BOMCopierContinue;
 224     }
 225     return BOMCopierSkipFile;
 226 }
 227
 228 static std::string dispositionToString(Disposition disposition) {
 229     switch (disposition) {
 230         case Unknown:
 231             return "Unknown";
 232         case InternalDevelopment:
 233             return "InternalDevelopment";
 234         case Customer:
 235             return "Customer";
 236         case InternalMinDevelopment:
 237             return "InternalMinDevelopment";
 238     }
 239 }
 240
 241 static Disposition stringToDisposition(Diagnostics& diags, const std::string& str) {
 242     if (diags.hasError())
 243         return Unknown;
 244     if (str == "Unknown")
 245         return Unknown;
 246     if (str == "InternalDevelopment")
 247         return InternalDevelopment;
 248     if (str == "Customer")
 249         return Customer;
 250     if (str == "InternalMinDevelopment")
 251         return InternalMinDevelopment;
 252     return Unknown;
 253 }
 254
 255 static std::string platformToString(Platform platform) {
 256     switch (platform) {
 257         case unknown:
 258             return "unknown";
 259         case macOS:
 260             return "macOS";
 261         case iOS:
 262             return "iOS";
 263         case tvOS:
 264             return "tvOS";
 265         case watchOS:
 266             return "watchOS";
 267         case bridgeOS:
 268             return "bridgeOS";
 269         case iOSMac:
 270             return "UIKitForMac";
 271         case iOS_simulator:
 272             return "iOS_simulator";
 273         case tvOS_simulator:
 274             return "tvOS_simulator";
 275         case watchOS_simulator:
 276             return "watchOS_simulator";
 277     }
 278 }
 279
 280 static Platform stringToPlatform(Diagnostics& diags, const std::string& str) {
 281     if (diags.hasError())
 282         return unknown;
 283     if (str == "unknown")
 284         return unknown;
 285     if (str == "macOS")
 286         return macOS;
 287     if (str == "iOS")
 288         return iOS;
 289     if (str == "tvOS")
 290         return tvOS;
 291     if (str == "watchOS")
 292         return watchOS;
 293     if (str == "bridgeOS")
 294         return bridgeOS;
 295     if (str == "iOSMac")
 296         return iOSMac;
 297     if (str == "UIKitForMac")
 298         return iOSMac;
 299     if (str == "iOS_simulator")
 300         return iOS_simulator;
 301     if (str == "tvOS_simulator")
 302         return tvOS_simulator;
 303     if (str == "watchOS_simulator")
 304         return watchOS_simulator;
 305     return unknown;
 306 }
 307
 308 static std::string fileFlagsToString(FileFlags fileFlags) {
 309     switch (fileFlags) {
 310         case NoFlags:
 311             return "NoFlags";
 312         case MustBeInCache:
 313             return "MustBeInCache";
 314         case ShouldBeExcludedFromCacheIfUnusedLeaf:
 315             return "ShouldBeExcludedFromCacheIfUnusedLeaf";
 316         case RequiredClosure:
 317             return "RequiredClosure";
 318         case DylibOrderFile:
 319             return "DylibOrderFile";
 320         case DirtyDataOrderFile:
 321             return "DirtyDataOrderFile";
 322     }
 323 }
 324
 325 static FileFlags stringToFileFlags(Diagnostics& diags, const std::string& str) {
 326     if (diags.hasError())
 327         return NoFlags;
 328     if (str == "NoFlags")
 329         return NoFlags;
 330     if (str == "MustBeInCache")
 331         return MustBeInCache;
 332     if (str == "ShouldBeExcludedFromCacheIfUnusedLeaf")
 333         return ShouldBeExcludedFromCacheIfUnusedLeaf;
 334     if (str == "RequiredClosure")
 335         return RequiredClosure;
 336     if (str == "DylibOrderFile")
 337         return DylibOrderFile;
 338     if (str == "DirtyDataOrderFile")
 339         return DirtyDataOrderFile;
 340     return NoFlags;
 341 }
 342
 343 static dyld3::json::Node getBuildOptionsNode(BuildOptions_v1 buildOptions) {
 344     dyld3::json::Node buildOptionsNode;
 345     buildOptionsNode.map["version"].value             = dyld3::json::decimal(buildOptions.version);
 346     buildOptionsNode.map["updateName"].value          = buildOptions.updateName;
 347     buildOptionsNode.map["deviceName"].value          = buildOptions.deviceName;
 348     buildOptionsNode.map["disposition"].value         = dispositionToString(buildOptions.disposition);
 349     buildOptionsNode.map["platform"].value            = platformToString(buildOptions.platform);
 350     for (unsigned i = 0; i != buildOptions.numArchs; ++i) {
 351         dyld3::json::Node archNode;
 352         archNode.value = buildOptions.archs[i];
 353         buildOptionsNode.map["archs"].array.push_back(archNode);
 354     }
 355     return buildOptionsNode;
 356 }
 357
 358 struct SharedCacheBuilderOptions {
 359     Diagnostics           diags;
 360     std::set<std::string> roots;
 361     std::string           dylibCacheDir;
 362     std::string           artifactDir;
 363     std::string           release;
 364     bool                  emitDevCaches = true;
 365     bool                  emitElidedDylibs = true;
 366     bool                  listConfigs = false;
 367     bool                  copyRoots = false;
 368     bool                  debug = false;
 369     bool                  useMRM = false;
 370     std::string           dstRoot;
 371     std::string           emitJSONPath;
 372     std::string           buildAllPath;
 373     std::string           configuration;
 374     std::string           resultPath;
 375     std::string           baselineDifferenceResultPath;
 376     std::string           baselineCacheMapPath;
 377     bool                  baselineCopyRoots = false;
 378 };
 379
 380 static void loadMRMFiles(Diagnostics& diags,
 381                          SharedCacheBuilder* sharedCacheBuilder,
 382                          const std::vector<std::tuple<std::string, std::string, FileFlags>>& inputFiles,
 383                          std::vector<std::pair<const void*, size_t>>& mappedFiles,
 384                          const std::set<std::string>& baselineCacheFiles) {
 385
 386     for (const std::tuple<std::string, std::string, FileFlags>& inputFile : inputFiles) {
 387         const std::string& buildPath   = std::get<0>(inputFile);
 388         const std::string& runtimePath = std::get<1>(inputFile);
 389         FileFlags   fileFlags   = std::get<2>(inputFile);
 390
 391         struct stat stat_buf;
 392         int fd = ::open(buildPath.c_str(), O_RDONLY, 0);
 393         if (fd == -1) {
 394             if (baselineCacheFiles.count(runtimePath)) {
 395                 diags.error("can't open file '%s', errno=%d\n", buildPath.c_str(), errno);
 396                 return;
 397             } else {
 398                 diags.verbose("can't open file '%s', errno=%d\n", buildPath.c_str(), errno);
 399                 continue;
 400             }
 401         }
 402
 403         if (fstat(fd, &stat_buf) == -1) {
 404             if (baselineCacheFiles.count(runtimePath)) {
 405                 diags.error("can't stat open file '%s', errno=%d\n", buildPath.c_str(), errno);
 406                 ::close(fd);
 407                 return;
 408             } else {
 409                 diags.verbose("can't stat open file '%s', errno=%d\n", buildPath.c_str(), errno);
 410                 ::close(fd);
 411                 continue;
 412             }
 413         }
 414
 415         const void* buffer = mmap(NULL, (size_t)stat_buf.st_size, PROT_READ, MAP_PRIVATE, fd, 0);
 416         if (buffer == MAP_FAILED) {
 417             diags.error("mmap() for file at %s failed, errno=%d\n", buildPath.c_str(), errno);
 418             ::close(fd);
 419         }
 420         ::close(fd);
 421
 422         mappedFiles.emplace_back(buffer, (size_t)stat_buf.st_size);
 423
 424         addFile(sharedCacheBuilder, runtimePath.c_str(), (uint8_t*)buffer, (size_t)stat_buf.st_size, fileFlags);
 425     }
 426 }
 427
 428 static void unloadMRMFiles(std::vector<std::pair<const void*, size_t>>& mappedFiles) {
 429     for (auto mappedFile : mappedFiles)
 430         ::munmap((void*)mappedFile.first, mappedFile.second);
 431 }
 432
 433 static void writeMRMResults(bool cacheBuildSuccess, SharedCacheBuilder* sharedCacheBuilder, const SharedCacheBuilderOptions& options) {
 434     if (!cacheBuildSuccess) {
 435         uint64_t errorCount = 0;
 436         if (const char* const* errors = getErrors(sharedCacheBuilder, &errorCount)) {
 437             for (uint64 i = 0, e = errorCount; i != e; ++i) {
 438                 const char* errorMessage = errors[i];
 439                 fprintf(stderr, "ERROR: %s\n", errorMessage);
 440             }
 441         }
 442     }
 443
 444     // Now emit each cache we generated, or the errors for them.
 445     uint64_t cacheResultCount = 0;
 446     if (const CacheResult* const* cacheResults = getCacheResults(sharedCacheBuilder, &cacheResultCount)) {
 447         for (uint64 i = 0, e = cacheResultCount; i != e; ++i) {
 448             const CacheResult& result = *(cacheResults[i]);
 449             // Always print the warnings if we have roots, even if there are errors
 450             if ( (result.numErrors == 0) || !options.roots.empty() ) {
 451                 for (uint64_t warningIndex = 0; warningIndex != result.numWarnings; ++warningIndex) {
 452                     fprintf(stderr, "[%s] WARNING: %s\n", result.loggingPrefix, result.warnings[warningIndex]);
 453                 }
 454             }
 455             if (result.numErrors) {
 456                 for (uint64_t errorIndex = 0; errorIndex != result.numErrors; ++errorIndex) {
 457                     fprintf(stderr, "[%s] ERROR: %s\n", result.loggingPrefix, result.errors[errorIndex]);
 458                 }
 459                 cacheBuildSuccess = false;
 460             }
 461         }
 462     }
 463
 464     if (!cacheBuildSuccess) {
 465         exit(-1);
 466     }
 467
 468     // If we built caches, then write everything out.
 469     // TODO: Decide if we should we write any good caches anyway?
 470     if (cacheBuildSuccess && !options.dstRoot.empty()) {
 471         uint64_t fileResultCount = 0;
 472         if (const FileResult* const* fileResults = getFileResults(sharedCacheBuilder, &fileResultCount)) {
 473             for (uint64 i = 0, e = fileResultCount; i != e; ++i) {
 474                 const FileResult& result = *(fileResults[i]);
 475
 476                 switch (result.behavior) {
 477                     case AddFile:
 478                         break;
 479                     case ChangeFile:
 480                         continue;
 481                 }
 482
 483                 if (!result.data)
 484                     continue;
 485
 486                 const std::string path = options.dstRoot + result.path;
 487                 std::string pathTemplate = path + "-XXXXXX";
 488                 size_t templateLen = strlen(pathTemplate.c_str())+2;
 489                 char pathTemplateSpace[templateLen];
 490                 strlcpy(pathTemplateSpace, pathTemplate.c_str(), templateLen);
 491                 int fd = mkstemp(pathTemplateSpace);
 492                 if ( fd != -1 ) {
 493                     ::ftruncate(fd, result.size);
 494                     uint64_t writtenSize = pwrite(fd, result.data, result.size, 0);
 495                     if ( writtenSize == result.size ) {
 496                         ::fchmod(fd, S_IRUSR|S_IWUSR|S_IRGRP|S_IROTH); // mkstemp() makes file "rw-------", switch it to "rw-r--r--"
 497                         if ( ::rename(pathTemplateSpace, path.c_str()) == 0) {
 498                             ::close(fd);
 499                             continue; // success
 500                         }
 501                     }
 502                     else {
 503                         fprintf(stderr, "ERROR: could not write file %s\n", pathTemplateSpace);
 504                         cacheBuildSuccess = false;
 505                     }
 506                     ::close(fd);
 507                     ::unlink(pathTemplateSpace);
 508                 }
 509                 else {
 510                     fprintf(stderr, "ERROR: could not open file %s\n", pathTemplateSpace);
 511                     cacheBuildSuccess = false;
 512                 }
 513             }
 514         }
 515     }
 516 }
 517
 518 static void reportUnknownConfiguration(const std::string& configuration, dyld3::Manifest& manifest) {
 519     fprintf(stderr, "** Unknown config '%s' for build %s.\n",
 520             configuration.c_str(), manifest.build().c_str());
 521
 522     // Look for available configurations that the user might have meant.
 523     // Substring match: print configs that contain the user's string.
 524     // Regex match: if user wants "N61OS" then match .*N.*6.*1.*O.*S.*
 525
 526     std::string patternString = ".*";
 527     for (auto c : configuration) {
 528         if (isalnum(c)) {  // filter regex special characters
 529             patternString += c;
 530             patternString += ".*";
 531         }
 532     }
 533     std::regex pattern(patternString);
 534
 535     std::vector<std::string> likelyConfigs{};
 536     std::vector<std::string> allConfigs{};
 537     manifest.forEachConfiguration([&](const std::string& configName) {
 538         allConfigs.push_back(configName);
 539         if (!configuration.empty()) {
 540             if (configName.find(configuration) != std::string::npos || std::regex_match(configName, pattern)) {
 541                 likelyConfigs.push_back(configName);
 542             }
 543         }
 544     });
 545
 546     if (!likelyConfigs.empty()) {
 547         fprintf(stderr, "\nDid you mean:\n");
 548         for (auto configName: likelyConfigs) {
 549             fprintf(stderr, "%s\n", configName.c_str());
 550         }
 551     }
 552
 553     fprintf(stderr, "\nAvailable configurations:\n");
 554     for (auto configName : allConfigs) {
 555         fprintf(stderr, "%s\n", configName.c_str());
 556     }
 557 }
 558
 559 static void buildCacheFromPListManifest(Diagnostics& diags, const SharedCacheBuilderOptions& options) {
 560     // Get the list of configurations, without fetching all of the files.
 561     auto manifest = dyld3::Manifest(diags, options.dylibCacheDir + "/Manifest.plist", false);
 562
 563     if (manifest.build().empty()) {
 564         fprintf(stderr, "No manifest found at '%s/Manifest.plist'\n", options.dylibCacheDir.c_str());
 565         exit(-1);
 566     }
 567
 568     // List configurations if requested.
 569     if (options.listConfigs) {
 570         manifest.forEachConfiguration([](const std::string& configName) {
 571             printf("%s\n", configName.c_str());
 572         });
 573         // If we weren't passed a configuration then exit
 574         if (options.configuration.empty())
 575             exit(0);
 576     }
 577
 578     // Stop if the requested configuration is unavailable.
 579     if (!manifest.filterForConfig(options.configuration)) {
 580         reportUnknownConfiguration(options.configuration, manifest);
 581         exit(-1);
 582     }
 583
 584     // Now finish initializing the manifest.
 585     // This step is slow so we defer it until after the checks above.
 586     fprintf(stderr, "Building Caches for %s\n", manifest.build().c_str());
 587     manifest.populate(options.roots);
 588
 589     (void)mkpath_np((options.dstRoot + "/System/Library/Caches/com.apple.dyld/").c_str(), 0755);
 590     bool cacheBuildSuccess = false;
 591     if (options.useMRM) {
 592
 593         std::ofstream jsonFile;
 594         if (!options.emitJSONPath.empty()) {
 595             jsonFile.open(options.emitJSONPath, std::ofstream::out);
 596             if (!jsonFile.is_open()) {
 597                 diags.verbose("can't open file '%s'\n", options.emitJSONPath.c_str());
 598                 return;
 599             }
 600         }
 601         dyld3::json::Node buildInvocationNode;
 602
 603         buildInvocationNode.map["version"].value = "1";
 604
 605         // Find the archs for the configuration we want.
 606         __block std::set<std::string> validArchs;
 607         manifest.configuration(options.configuration).forEachArchitecture(^(const std::string& path) {
 608             validArchs.insert(path);
 609         });
 610
 611         if (validArchs.size() != 1) {
 612             fprintf(stderr, "MRM doesn't support more than one arch per configuration: %s\n",
 613                     options.configuration.c_str());
 614             exit(-1);
 615         }
 616
 617         const char* archs[validArchs.size()];
 618         uint64_t archIndex = 0;
 619         for (const std::string& arch : validArchs) {
 620             archs[archIndex++] = arch.c_str();
 621         }
 622
 623         BuildOptions_v1 buildOptions;
 624         buildOptions.version                            = 1;
 625         buildOptions.updateName                         = manifest.build().c_str();
 626         buildOptions.deviceName                         = options.configuration.c_str();
 627         buildOptions.disposition                        = Disposition::Unknown;
 628         buildOptions.platform                           = (Platform)manifest.platform();
 629         buildOptions.archs                              = archs;
 630         buildOptions.numArchs                           = validArchs.size();
 631         buildOptions.verboseDiagnostics                 = options.debug;
 632         buildOptions.isLocallyBuiltCache                = true;
 633
 634         __block struct SharedCacheBuilder* sharedCacheBuilder = createSharedCacheBuilder(&buildOptions);
 635         buildInvocationNode.map["buildOptions"] = getBuildOptionsNode(buildOptions);
 636
 637         std::set<std::string> requiredBinaries =  {
 638             "/usr/lib/libSystem.B.dylib"
 639         };
 640
 641         // Get the file data for every MachO in the BOM.
 642         __block dyld3::json::Node filesNode;
 643         __block std::vector<std::pair<const void*, size_t>> mappedFiles;
 644         manifest.forEachMachO(options.configuration, ^(const std::string &buildPath, const std::string &runtimePath, const std::string &arch, bool shouldBeExcludedIfLeaf) {
 645
 646             // Filter based on arch as the Manifest adds the file once for each UUID.
 647             if (!validArchs.count(arch))
 648                 return;
 649
 650             struct stat stat_buf;
 651             int fd = ::open(buildPath.c_str(), O_RDONLY, 0);
 652             if (fd == -1) {
 653                 diags.verbose("can't open file '%s', errno=%d\n", buildPath.c_str(), errno);
 654                 return;
 655             }
 656
 657             if (fstat(fd, &stat_buf) == -1) {
 658                 diags.verbose("can't stat open file '%s', errno=%d\n", buildPath.c_str(), errno);
 659                 ::close(fd);
 660                 return;
 661             }
 662
 663             const void* buffer = mmap(NULL, (size_t)stat_buf.st_size, PROT_READ, MAP_PRIVATE, fd, 0);
 664             if (buffer == MAP_FAILED) {
 665                 diags.verbose("mmap() for file at %s failed, errno=%d\n", buildPath.c_str(), errno);
 666                 ::close(fd);
 667             }
 668             ::close(fd);
 669
 670             mappedFiles.emplace_back(buffer, (size_t)stat_buf.st_size);
 671             FileFlags fileFlags = FileFlags::NoFlags;
 672             if (requiredBinaries.count(runtimePath))
 673                 fileFlags = FileFlags::RequiredClosure;
 674             addFile(sharedCacheBuilder, runtimePath.c_str(), (uint8_t*)buffer, (size_t)stat_buf.st_size, fileFlags);
 675
 676             dyld3::json::Node fileNode;
 677             fileNode.map["path"].value  = runtimePath;
 678             fileNode.map["flags"].value = fileFlagsToString(fileFlags);
 679             filesNode.array.push_back(fileNode);
 680         });
 681
 682         __block dyld3::json::Node symlinksNode;
 683         manifest.forEachSymlink(options.configuration, ^(const std::string &fromPath, const std::string &toPath) {
 684             addSymlink(sharedCacheBuilder, fromPath.c_str(), toPath.c_str());
 685
 686             dyld3::json::Node symlinkNode;
 687             symlinkNode.map["from-path"].value  = fromPath;
 688             symlinkNode.map["to-path"].value    = toPath;
 689             symlinksNode.array.push_back(symlinkNode);
 690         });
 691
 692         buildInvocationNode.map["symlinks"] = symlinksNode;
 693
 694         std::string orderFileData;
 695         if (!manifest.dylibOrderFile().empty()) {
 696             orderFileData = loadOrderFile(manifest.dylibOrderFile());
 697             if (!orderFileData.empty()) {
 698                 addFile(sharedCacheBuilder, "*order file data*", (uint8_t*)orderFileData.data(), orderFileData.size(), FileFlags::DylibOrderFile);
 699                 dyld3::json::Node fileNode;
 700                 fileNode.map["path"].value  = manifest.dylibOrderFile();
 701                 fileNode.map["flags"].value = fileFlagsToString(FileFlags::DylibOrderFile);
 702                 filesNode.array.push_back(fileNode);
 703             }
 704         }
 705
 706         std::string dirtyDataOrderFileData;
 707         if (!manifest.dirtyDataOrderFile().empty()) {
 708             dirtyDataOrderFileData = loadOrderFile(manifest.dirtyDataOrderFile());
 709             if (!dirtyDataOrderFileData.empty()) {
 710                 addFile(sharedCacheBuilder, "*dirty data order file data*", (uint8_t*)dirtyDataOrderFileData.data(), dirtyDataOrderFileData.size(), FileFlags::DirtyDataOrderFile);
 711                 dyld3::json::Node fileNode;
 712                 fileNode.map["path"].value  = manifest.dirtyDataOrderFile();
 713                 fileNode.map["flags"].value = fileFlagsToString(FileFlags::DirtyDataOrderFile);
 714                 filesNode.array.push_back(fileNode);
 715             }
 716         }
 717
 718         buildInvocationNode.map["files"] = filesNode;
 719
 720         if (jsonFile.is_open()) {
 721             dyld3::json::printJSON(buildInvocationNode, 0, jsonFile);
 722             jsonFile.close();
 723         }
 724
 725         cacheBuildSuccess = runSharedCacheBuilder(sharedCacheBuilder);
 726
 727         writeMRMResults(cacheBuildSuccess, sharedCacheBuilder, options);
 728
 729         destroySharedCacheBuilder(sharedCacheBuilder);
 730
 731         for (auto mappedFile : mappedFiles)
 732             ::munmap((void*)mappedFile.first, mappedFile.second);
 733     } else {
 734         manifest.calculateClosure();
 735
 736         cacheBuildSuccess = build(diags, manifest, options.dstRoot, false, options.debug, false, false, options.emitDevCaches, true);
 737     }
 738
 739     if (!cacheBuildSuccess) {
 740         exit(-1);
 741     }
 742
 743     // Compare this cache to the baseline cache and see if we have any roots to copy over
 744     if (!options.baselineDifferenceResultPath.empty() || options.baselineCopyRoots) {
 745         std::set<std::string> baselineDylibs = manifest.resultsForConfiguration(options.configuration);
 746
 747         std::set<std::string> newDylibs;
 748         std::map<std::string, std::string> missingDylibReasons;
 749         manifest.forEachConfiguration([&manifest, &newDylibs, &missingDylibReasons](const std::string& configName) {
 750             for (auto& arch : manifest.configuration(configName).architectures) {
 751                 for (auto& dylib : arch.second.results.dylibs) {
 752                     if (dylib.second.included) {
 753                         newDylibs.insert(manifest.installNameForUUID(dylib.first));
 754                     } else {
 755                         missingDylibReasons[manifest.installNameForUUID(dylib.first)] = dylib.second.exclusionInfo;
 756                     }
 757                 }
 758             }
 759         });
 760
 761         // Work out the set of dylibs in the old cache but not the new one
 762         std::map<std::string, std::string> dylibsMissingFromNewCache;
 763         if (options.baselineCopyRoots || !options.baselineDifferenceResultPath.empty()) {
 764             for (const std::string& baselineDylib : baselineDylibs) {
 765                 if (!newDylibs.count(baselineDylib)) {
 766                     auto reasonIt = missingDylibReasons.find(baselineDylib);
 767                     if (reasonIt != missingDylibReasons.end())
 768                         dylibsMissingFromNewCache[baselineDylib] = reasonIt->second;
 769                     else
 770                         dylibsMissingFromNewCache[baselineDylib] = "";
 771                 }
 772             }
 773
 774             if (!dylibsMissingFromNewCache.empty()) {
 775                 // Work out which dylibs are missing from the new cache, but are not
 776                 // coming from the -root which already has them on disk
 777                 std::set<std::string> pathsNotInRoots;
 778                 for (std::pair<std::string, std::string> dylibMissingFromNewCache : dylibsMissingFromNewCache) {
 779                     const std::string& dylibInstallName = dylibMissingFromNewCache.first;
 780                     bool foundInRoot = false;
 781                     for (auto& root : options.roots) {
 782                         struct stat sb;
 783                         std::string filePath = root + "/" + dylibInstallName;
 784                         if (!stat(filePath.c_str(), &sb)) {
 785                             foundInRoot = true;
 786                         }
 787                     }
 788                     if (!foundInRoot)
 789                         pathsNotInRoots.insert(dylibInstallName);
 790                 }
 791
 792                 BOMCopier copier = BOMCopierNewWithSys(BomSys_default());
 793                 BOMCopierSetUserData(copier, (void*)&pathsNotInRoots);
 794                 BOMCopierSetCopyFileStartedHandler(copier, filteredCopyIncludingPaths);
 795                 std::string dylibCacheRootDir = realFilePath(options.dylibCacheDir + "/Root");
 796                 if (dylibCacheRootDir == "") {
 797                     fprintf(stderr, "Could not find dylib Root directory to copy baseline roots from\n");
 798                     exit(1);
 799                 }
 800                 BOMCopierCopy(copier, dylibCacheRootDir.c_str(), options.dstRoot.c_str());
 801                 BOMCopierFree(copier);
 802
 803                 for (std::pair<std::string, std::string> dylibMissingFromNewCache : dylibsMissingFromNewCache) {
 804                     if (dylibMissingFromNewCache.second.empty())
 805                         diags.verbose("Dylib missing from new cache: '%s'\n", dylibMissingFromNewCache.first.c_str());
 806                     else
 807                         diags.verbose("Dylib missing from new cache: '%s' because '%s'\n",
 808                                       dylibMissingFromNewCache.first.c_str(), dylibMissingFromNewCache.second.c_str());
 809                 }
 810             }
 811         }
 812
 813         if (!options.baselineDifferenceResultPath.empty()) {
 814             auto cppToObjStr = [](const std::string& str) {
 815                 return [NSString stringWithUTF8String:str.c_str()];
 816             };
 817
 818             // Work out the set of dylibs in the cache and taken from the -root
 819             NSMutableArray<NSString*>* dylibsFromRoots = [NSMutableArray array];
 820             for (auto& root : options.roots) {
 821                 for (const std::string& dylibInstallName : newDylibs) {
 822                     struct stat sb;
 823                     std::string filePath = root + "/" + dylibInstallName;
 824                     if (!stat(filePath.c_str(), &sb)) {
 825                         [dylibsFromRoots addObject:cppToObjStr(dylibInstallName)];
 826                     }
 827                 }
 828             }
 829
 830             // Work out the set of dylibs in the new cache but not in the baseline cache.
 831             NSMutableArray<NSString*>* dylibsMissingFromBaselineCache = [NSMutableArray array];
 832             for (const std::string& newDylib : newDylibs) {
 833                 if (!baselineDylibs.count(newDylib))
 834                     [dylibsMissingFromBaselineCache addObject:cppToObjStr(newDylib)];
 835             }
 836
 837             // If a dylib which was cached is no longer eligible, say why
 838             NSMutableArray<NSDictionary*>* dylibsReasonsMissingFromNewCache = [NSMutableArray array];
 839             for (std::pair<std::string, std::string> dylibMissingFromNewCache : dylibsMissingFromNewCache) {
 840                 NSMutableDictionary* reasonDict = [[NSMutableDictionary alloc] init];
 841                 reasonDict[@"path"] = cppToObjStr(dylibMissingFromNewCache.first);
 842                 reasonDict[@"reason"] = cppToObjStr(dylibMissingFromNewCache.second);
 843                 [dylibsReasonsMissingFromNewCache addObject:reasonDict];
 844             }
 845
 846             NSMutableDictionary* cacheDict = [[NSMutableDictionary alloc] init];
 847             cacheDict[@"root-paths-in-cache"] = dylibsFromRoots;
 848             cacheDict[@"device-paths-to-delete"] = dylibsMissingFromBaselineCache;
 849             cacheDict[@"baseline-paths-evicted-from-cache"] = dylibsReasonsMissingFromNewCache;
 850
 851             NSError* error = nil;
 852             NSData*  outData = [NSPropertyListSerialization dataWithPropertyList:cacheDict
 853                                                                           format:NSPropertyListBinaryFormat_v1_0
 854                                                                          options:0
 855                                                                            error:&error];
 856             (void)[outData writeToFile:cppToObjStr(options.baselineDifferenceResultPath) atomically:YES];
 857         }
 858     }
 859
 860     if (options.copyRoots) {
 861         std::set<std::string> cachePaths;
 862         manifest.forEachConfiguration([&manifest, &cachePaths](const std::string& configName) {
 863             for (auto& arch : manifest.configuration(configName).architectures) {
 864                 for (auto& dylib : arch.second.results.dylibs) {
 865                     if (dylib.second.included) {
 866                         cachePaths.insert(manifest.installNameForUUID(dylib.first));
 867                     }
 868                 }
 869             }
 870         });
 871
 872         BOMCopier copier = BOMCopierNewWithSys(BomSys_default());
 873         BOMCopierSetUserData(copier, (void*)&cachePaths);
 874         BOMCopierSetCopyFileStartedHandler(copier, filteredCopyExcludingPaths);
 875         for (auto& root : options.roots) {
 876             BOMCopierCopy(copier, root.c_str(), options.dstRoot.c_str());
 877         }
 878         BOMCopierFree(copier);
 879     }
 880
 881     int err = sync_volume_np(options.dstRoot.c_str(), SYNC_VOLUME_FULLSYNC | SYNC_VOLUME_WAIT);
 882     if (err) {
 883         fprintf(stderr, "Volume sync failed errnor=%d (%s)\n", err, strerror(err));
 884     }
 885
 886     // Now that all the build commands have been issued lets put a barrier in after then which can tear down the app after
 887     // everything is written.
 888
 889     if (!options.resultPath.empty()) {
 890         manifest.write(options.resultPath);
 891     }
 892 }
 893
 894 static void buildCacheFromJSONManifest(Diagnostics& diags, const SharedCacheBuilderOptions& options,
 895                                        const std::string& jsonManifestPath) {
 896     dyld3::json::Node manifestNode = dyld3::json::readJSON(diags, jsonManifestPath.c_str());
 897     if (diags.hasError())
 898         return;
 899
 900     // Top level node should be a map of the options, files, and symlinks.
 901     if (manifestNode.map.empty()) {
 902         diags.error("Expected map for JSON manifest node\n");
 903         return;
 904     }
 905
 906     // Parse the nodes in the top level manifest node
 907     const dyld3::json::Node& versionNode          = dyld3::json::getRequiredValue(diags, manifestNode, "version");
 908     uint64_t manifestVersion                      = dyld3::json::parseRequiredInt(diags, versionNode);
 909     if (diags.hasError())
 910         return;
 911
 912     const uint64_t supportedManifestVersion = 1;
 913     if (manifestVersion != supportedManifestVersion) {
 914         diags.error("JSON manfiest version of %lld is unsupported.  Supported version is %lld\n",
 915                     manifestVersion, supportedManifestVersion);
 916         return;
 917     }
 918     const dyld3::json::Node& buildOptionsNode     = dyld3::json::getRequiredValue(diags, manifestNode, "buildOptions");
 919     const dyld3::json::Node& filesNode            = dyld3::json::getRequiredValue(diags, manifestNode, "files");
 920     const dyld3::json::Node* symlinksNode         = dyld3::json::getOptionalValue(diags, manifestNode, "symlinks");
 921
 922     // Parse the archs
 923     const dyld3::json::Node& archsNode = dyld3::json::getRequiredValue(diags, buildOptionsNode, "archs");
 924     if (diags.hasError())
 925         return;
 926     if (archsNode.array.empty()) {
 927         diags.error("Build options archs node is not an array\n");
 928         return;
 929     }
 930     const char* archs[archsNode.array.size()];
 931     uint64_t archIndex = 0;
 932     for (const dyld3::json::Node& archNode : archsNode.array) {
 933         archs[archIndex++] = dyld3::json::parseRequiredString(diags, archNode).c_str();
 934     }
 935
 936     // Parse the rest of the options node.
 937     BuildOptions_v1 buildOptions;
 938     buildOptions.version                            = dyld3::json::parseRequiredInt(diags, dyld3::json::getRequiredValue(diags, buildOptionsNode, "version"));
 939     buildOptions.updateName                         = dyld3::json::parseRequiredString(diags, dyld3::json::getRequiredValue(diags, buildOptionsNode, "updateName")).c_str();
 940     buildOptions.deviceName                         = dyld3::json::parseRequiredString(diags, dyld3::json::getRequiredValue(diags, buildOptionsNode, "deviceName")).c_str();
 941     buildOptions.disposition                        = stringToDisposition(diags, dyld3::json::parseRequiredString(diags, dyld3::json::getRequiredValue(diags, buildOptionsNode, "disposition")));
 942     buildOptions.platform                           = stringToPlatform(diags, dyld3::json::parseRequiredString(diags, dyld3::json::getRequiredValue(diags, buildOptionsNode, "platform")));
 943     buildOptions.archs                              = archs;
 944     buildOptions.numArchs                           = archsNode.array.size();
 945     buildOptions.verboseDiagnostics                 = options.debug;
 946     buildOptions.isLocallyBuiltCache                = true;
 947
 948     if (diags.hasError())
 949         return;
 950
 951     // Override the disposition if we don't want certaion caches.
 952     if (!options.emitDevCaches) {
 953         switch (buildOptions.disposition) {
 954             case Unknown:
 955                 // Nothing we can do here as we can't assume what caches are built here.
 956                 break;
 957             case InternalDevelopment:
 958                 // This builds both caches, but we don't want dev
 959                 buildOptions.disposition = Customer;
 960                 break;
 961             case Customer:
 962                 // This is already only the customer cache
 963                 break;
 964             case InternalMinDevelopment:
 965                 diags.error("Cannot request no dev cache for InternalMinDevelopment as that is already only a dev cache\n");
 966                 break;
 967         }
 968     }
 969
 970     if (diags.hasError())
 971         return;
 972
 973     struct SharedCacheBuilder* sharedCacheBuilder = createSharedCacheBuilder(&buildOptions);
 974
 975     // Parse the files
 976     if (filesNode.array.empty()) {
 977         diags.error("Build options files node is not an array\n");
 978         return;
 979     }
 980
 981     std::vector<std::tuple<std::string, std::string, FileFlags>> inputFiles;
 982     for (const dyld3::json::Node& fileNode : filesNode.array) {
 983         const std::string& path = dyld3::json::parseRequiredString(diags, dyld3::json::getRequiredValue(diags, fileNode, "path")).c_str();
 984         FileFlags fileFlags     = stringToFileFlags(diags, dyld3::json::parseRequiredString(diags, dyld3::json::getRequiredValue(diags, fileNode, "flags")));
 985
 986         // We can optionally have a sourcePath entry which is the path to get the source content from instead of the install path
 987         std::string sourcePath;
 988         const dyld3::json::Node* sourcePathNode = dyld3::json::getOptionalValue(diags, fileNode, "sourcePath");
 989         if ( sourcePathNode != nullptr ) {
 990             if (!sourcePathNode->array.empty()) {
 991                 diags.error("sourcePath node cannot be an array\n");
 992                 return;
 993             }
 994             if (!sourcePathNode->map.empty()) {
 995                 diags.error("sourcePath node cannot be a map\n");
 996                 return;
 997             }
 998             sourcePath = sourcePathNode->value;
 999         } else {
1000             sourcePath = path;
1001         }
1002
1003         // Check if one of the -root's has this path
1004         bool foundInOverlay = false;
1005         for (const std::string& overlay : options.roots) {
1006             struct stat sb;
1007             std::string filePath = overlay + path;
1008             if (!stat(filePath.c_str(), &sb)) {
1009                 foundInOverlay = true;
1010                 diags.verbose("Taking '%s' from overlay instead of dylib cache\n", path.c_str());
1011                 inputFiles.push_back({ filePath, path, fileFlags });
1012                 break;
1013             }
1014         }
1015
1016         if (foundInOverlay)
1017             continue;
1018
1019         // Build paths are relative to the build artifact root directory.
1020         std::string buildPath;
1021         switch (fileFlags) {
1022             case NoFlags:
1023             case MustBeInCache:
1024             case ShouldBeExcludedFromCacheIfUnusedLeaf:
1025             case RequiredClosure:
1026                 buildPath = "." + sourcePath;
1027                 break;
1028             case DylibOrderFile:
1029             case DirtyDataOrderFile:
1030                 buildPath = "." + sourcePath;
1031                 break;
1032         }
1033         inputFiles.push_back({ buildPath, path, fileFlags });
1034     }
1035
1036     if (diags.hasError())
1037         return;
1038
1039     // Parse the baseline from the map if we have it
1040     std::set<std::string> baselineDylibs;
1041     if ( !options.baselineCacheMapPath.empty() ) {
1042         dyld3::json::Node mapNode = dyld3::json::readJSON(diags, options.baselineCacheMapPath.c_str());
1043         if (diags.hasError())
1044             return;
1045
1046         // Top level node should be a map of the version and files
1047         if (mapNode.map.empty()) {
1048             diags.error("Expected map for JSON cache map node\n");
1049             return;
1050         }
1051
1052         // Parse the nodes in the top level manifest node
1053         const dyld3::json::Node& versionNode     = dyld3::json::getRequiredValue(diags, mapNode, "version");
1054         uint64_t mapVersion                      = dyld3::json::parseRequiredInt(diags, versionNode);
1055         if (diags.hasError())
1056             return;
1057
1058         const uint64_t supportedMapVersion = 1;
1059         if (mapVersion != supportedMapVersion) {
1060             diags.error("JSON map version of %lld is unsupported.  Supported version is %lld\n",
1061                         mapVersion, supportedMapVersion);
1062             return;
1063         }
1064
1065         // Parse the images
1066         const dyld3::json::Node& imagesNode = dyld3::json::getRequiredValue(diags, mapNode, "images");
1067         if (diags.hasError())
1068             return;
1069         if (imagesNode.array.empty()) {
1070             diags.error("Images node is not an array\n");
1071             return;
1072         }
1073
1074         for (const dyld3::json::Node& imageNode : imagesNode.array) {
1075             const dyld3::json::Node& pathNode = dyld3::json::getRequiredValue(diags, imageNode, "path");
1076             if (pathNode.value.empty()) {
1077                 diags.error("Image path node is not a string\n");
1078                 return;
1079             }
1080             baselineDylibs.insert(pathNode.value);
1081         }
1082     }
1083
1084     std::vector<std::pair<const void*, size_t>> mappedFiles;
1085     loadMRMFiles(diags, sharedCacheBuilder, inputFiles, mappedFiles, baselineDylibs);
1086
1087     if (diags.hasError())
1088         return;
1089
1090     // Parse the symlinks if we have them
1091     if (symlinksNode) {
1092         if (symlinksNode->array.empty()) {
1093             diags.error("Build options symlinks node is not an array\n");
1094             return;
1095         }
1096         for (const dyld3::json::Node& symlinkNode : symlinksNode->array) {
1097             const std::string& fromPath = dyld3::json::parseRequiredString(diags, dyld3::json::getRequiredValue(diags, symlinkNode, "path")).c_str();
1098             const std::string& toPath   = dyld3::json::parseRequiredString(diags, dyld3::json::getRequiredValue(diags, symlinkNode, "target")).c_str();
1099             addSymlink(sharedCacheBuilder, fromPath.c_str(), toPath.c_str());
1100         }
1101     }
1102
1103     if (diags.hasError())
1104         return;
1105
1106     // Don't create a directory if we are skipping writes, which means we have no dstRoot set
1107     if (!options.dstRoot.empty())
1108         (void)mkpath_np((options.dstRoot + "/System/Library/Caches/com.apple.dyld/").c_str(), 0755);
1109
1110     // Actually build the cache.
1111     bool cacheBuildSuccess = runSharedCacheBuilder(sharedCacheBuilder);
1112
1113     // Compare this cache to the baseline cache and see if we have any roots to copy over
1114     if (!options.baselineDifferenceResultPath.empty() || options.baselineCopyRoots) {
1115         std::set<std::string> newDylibs;
1116         if (cacheBuildSuccess) {
1117             uint64_t fileResultCount = 0;
1118             if (const char* const* fileResults = getFilesToRemove(sharedCacheBuilder, &fileResultCount)) {
1119                 for (uint64_t i = 0; i != fileResultCount; ++i)
1120                     newDylibs.insert(fileResults[i]);
1121             }
1122         }
1123
1124         if (options.baselineCopyRoots) {
1125             // Work out the set of dylibs in the old cache but not the new one
1126             std::set<std::string> dylibsMissingFromNewCache;
1127             for (const std::string& baselineDylib : baselineDylibs) {
1128                 if (!newDylibs.count(baselineDylib))
1129                     dylibsMissingFromNewCache.insert(baselineDylib);
1130             }
1131
1132             if (!dylibsMissingFromNewCache.empty()) {
1133                 BOMCopier copier = BOMCopierNewWithSys(BomSys_default());
1134                 BOMCopierSetUserData(copier, (void*)&dylibsMissingFromNewCache);
1135                 BOMCopierSetCopyFileStartedHandler(copier, filteredCopyIncludingPaths);
1136                 std::string dylibCacheRootDir = realFilePath(options.dylibCacheDir);
1137                 if (dylibCacheRootDir == "") {
1138                     fprintf(stderr, "Could not find dylib Root directory to copy baseline roots from\n");
1139                     exit(1);
1140                 }
1141                 BOMCopierCopy(copier, dylibCacheRootDir.c_str(), options.dstRoot.c_str());
1142                 BOMCopierFree(copier);
1143
1144                 for (const std::string& dylibMissingFromNewCache : dylibsMissingFromNewCache) {
1145                     diags.verbose("Dylib missing from new cache: '%s'\n", dylibMissingFromNewCache.c_str());
1146                 }
1147             }
1148         }
1149
1150         if (!options.baselineDifferenceResultPath.empty()) {
1151             auto cppToObjStr = [](const std::string& str) {
1152                 return [NSString stringWithUTF8String:str.c_str()];
1153             };
1154
1155             // Work out the set of dylibs in the cache and taken from the -root
1156             NSMutableArray<NSString*>* dylibsFromRoots = [NSMutableArray array];
1157             for (auto& root : options.roots) {
1158                 for (const std::string& dylibInstallName : newDylibs) {
1159                     struct stat sb;
1160                     std::string filePath = root + "/" + dylibInstallName;
1161                     if (!stat(filePath.c_str(), &sb)) {
1162                         [dylibsFromRoots addObject:cppToObjStr(dylibInstallName)];
1163                     }
1164                 }
1165             }
1166
1167             // Work out the set of dylibs in the new cache but not in the baseline cache.
1168             NSMutableArray<NSString*>* dylibsMissingFromBaselineCache = [NSMutableArray array];
1169             for (const std::string& newDylib : newDylibs) {
1170                 if (!baselineDylibs.count(newDylib))
1171                     [dylibsMissingFromBaselineCache addObject:cppToObjStr(newDylib)];
1172             }
1173
1174             NSMutableDictionary* cacheDict = [[NSMutableDictionary alloc] init];
1175             cacheDict[@"root-paths-in-cache"] = dylibsFromRoots;
1176             cacheDict[@"device-paths-to-delete"] = dylibsMissingFromBaselineCache;
1177
1178             NSError* error = nil;
1179             NSData*  outData = [NSPropertyListSerialization dataWithPropertyList:cacheDict
1180                                                                           format:NSPropertyListBinaryFormat_v1_0
1181                                                                          options:0
1182                                                                            error:&error];
1183             (void)[outData writeToFile:cppToObjStr(options.baselineDifferenceResultPath) atomically:YES];
1184         }
1185     }
1186
1187     writeMRMResults(cacheBuildSuccess, sharedCacheBuilder, options);
1188
1189     destroySharedCacheBuilder(sharedCacheBuilder);
1190
1191     unloadMRMFiles(mappedFiles);
1192 }
1193
1194 int main(int argc, const char* argv[])
1195 {
1196     @autoreleasepool {
1197         __block Diagnostics diags;
1198         SharedCacheBuilderOptions options;
1199         std::string jsonManifestPath;
1200         char* tempRootsDir = strdup("/tmp/dyld_shared_cache_builder.XXXXXX");
1201
1202         mkdtemp(tempRootsDir);
1203
1204         for (int i = 1; i < argc; ++i) {
1205             const char* arg = argv[i];
1206             if (arg[0] == '-') {
1207                 if (strcmp(arg, "-debug") == 0) {
1208                     diags = Diagnostics(true);
1209                     options.debug = true;
1210                 } else if (strcmp(arg, "-list_configs") == 0) {
1211                     options.listConfigs = true;
1212                 } else if (strcmp(arg, "-root") == 0) {
1213                     std::string realpath = realPath(argv[++i]);
1214                     if ( realpath.empty() || !fileExists(realpath) ) {
1215                         fprintf(stderr, "-root path doesn't exist: %s\n", argv[i]);
1216                         exit(-1);
1217                     }
1218                     options.roots.insert(realpath);
1219                 } else if (strcmp(arg, "-copy_roots") == 0) {
1220                     options.copyRoots = true;
1221                 } else if (strcmp(arg, "-dylib_cache") == 0) {
1222                     options.dylibCacheDir = realPath(argv[++i]);
1223                 } else if (strcmp(arg, "-artifact") == 0) {
1224                     options.artifactDir = realPath(argv[++i]);
1225                 } else if (strcmp(arg, "-no_development_cache") == 0) {
1226                     options.emitDevCaches = false;
1227                 } else if (strcmp(arg, "-no_overflow_dylibs") == 0) {
1228                     options.emitElidedDylibs = false;
1229                 } else if (strcmp(arg, "-development_cache") == 0) {
1230                     options.emitDevCaches = true;
1231                 } else if (strcmp(arg, "-overflow_dylibs") == 0) {
1232                     options.emitElidedDylibs = true;
1233                 } else if (strcmp(arg, "-mrm") == 0) {
1234                     options.useMRM = true;
1235                 } else if (strcmp(arg, "-emit_json") == 0) {
1236                     options.emitJSONPath = realPath(argv[++i]);
1237                 } else if (strcmp(arg, "-json_manifest") == 0) {
1238                     jsonManifestPath = realPath(argv[++i]);
1239                 } else if (strcmp(arg, "-build_all") == 0) {
1240                     options.buildAllPath = realPath(argv[++i]);
1241                 } else if (strcmp(arg, "-dst_root") == 0) {
1242                     options.dstRoot = realPath(argv[++i]);
1243                 } else if (strcmp(arg, "-release") == 0) {
1244                     options.release = argv[++i];
1245                 } else if (strcmp(arg, "-results") == 0) {
1246                     options.resultPath = realPath(argv[++i]);
1247                 } else if (strcmp(arg, "-baseline_diff_results") == 0) {
1248                     options.baselineDifferenceResultPath = realPath(argv[++i]);
1249                 } else if (strcmp(arg, "-baseline_copy_roots") == 0) {
1250                     options.baselineCopyRoots = true;
1251                 } else if (strcmp(arg, "-baseline_cache_map") == 0) {
1252                     options.baselineCacheMapPath = realPath(argv[++i]);
1253                 } else {
1254                     //usage();
1255                     fprintf(stderr, "unknown option: %s\n", arg);
1256                     exit(-1);
1257                 }
1258             } else {
1259                 if (!options.configuration.empty()) {
1260                     fprintf(stderr, "You may only specify one configuration\n");
1261                     exit(-1);
1262                 }
1263                 options.configuration = argv[i];
1264             }
1265         }
1266         (void)options.emitElidedDylibs; // not implemented yet
1267
1268         time_t mytime = time(0);
1269         fprintf(stderr, "Started: %s", asctime(localtime(&mytime)));
1270         processRoots(diags, options.roots, tempRootsDir);
1271
1272         struct rlimit rl = { OPEN_MAX, OPEN_MAX };
1273         (void)setrlimit(RLIMIT_NOFILE, &rl);
1274
1275         if (options.dylibCacheDir.empty() && options.artifactDir.empty() && options.release.empty()) {
1276             fprintf(stderr, "you must specify either -dylib_cache, -artifact or -release\n");
1277             exit(-1);
1278         } else if (!options.dylibCacheDir.empty() && !options.release.empty()) {
1279             fprintf(stderr, "you may not use -dylib_cache and -release at the same time\n");
1280             exit(-1);
1281         } else if (!options.dylibCacheDir.empty() && !options.artifactDir.empty()) {
1282             fprintf(stderr, "you may not use -dylib_cache and -artifact at the same time\n");
1283             exit(-1);
1284         }
1285
1286         if (options.configuration.empty() && jsonManifestPath.empty() && options.buildAllPath.empty()) {
1287             fprintf(stderr, "Must specify a configuration OR a -json_manifest path OR a -build_all path\n");
1288             exit(-1);
1289         }
1290
1291         if (!options.buildAllPath.empty()) {
1292             if (!options.dstRoot.empty()) {
1293                 fprintf(stderr, "Cannot combine -dst_root and -build_all\n");
1294                 exit(-1);
1295             }
1296             if (!options.configuration.empty()) {
1297                 fprintf(stderr, "Cannot combine configuration and -build_all\n");
1298                 exit(-1);
1299             }
1300             if (!jsonManifestPath.empty()) {
1301                 fprintf(stderr, "Cannot combine -json_manifest and -build_all\n");
1302                 exit(-1);
1303             }
1304             if (!options.baselineDifferenceResultPath.empty()) {
1305                 fprintf(stderr, "Cannot combine -baseline_diff_results and -build_all\n");
1306                 exit(-1);
1307             }
1308             if (options.baselineCopyRoots) {
1309                 fprintf(stderr, "Cannot combine -baseline_copy_roots and -build_all\n");
1310                 exit(-1);
1311             }
1312             if (!options.baselineCacheMapPath.empty()) {
1313                 fprintf(stderr, "Cannot combine -baseline_cache_map and -build_all\n");
1314                 exit(-1);
1315             }
1316         } else if (!options.listConfigs) {
1317             if (options.dstRoot.empty()) {
1318                 fprintf(stderr, "Must specify a valid -dst_root OR -list_configs\n");
1319                 exit(-1);
1320             }
1321
1322             if (options.configuration.empty() && jsonManifestPath.empty()) {
1323                 fprintf(stderr, "Must specify a configuration OR -json_manifest path OR -list_configs\n");
1324                 exit(-1);
1325             }
1326         }
1327
1328         if (!options.baselineDifferenceResultPath.empty() && (options.roots.size() > 1)) {
1329             fprintf(stderr, "Cannot use -baseline_diff_results with more that one -root\n");
1330             exit(-1);
1331         }
1332
1333         // Some options don't work with a JSON manifest
1334         if (!jsonManifestPath.empty()) {
1335             if (!options.resultPath.empty()) {
1336                 fprintf(stderr, "Cannot use -results with -json_manifest\n");
1337                 exit(-1);
1338             }
1339             if (!options.baselineDifferenceResultPath.empty() && options.baselineCacheMapPath.empty()) {
1340                 fprintf(stderr, "Must use -baseline_cache_map with -baseline_diff_results when using -json_manifest\n");
1341                 exit(-1);
1342             }
1343             if (options.baselineCopyRoots && options.baselineCacheMapPath.empty()) {
1344                 fprintf(stderr, "Must use -baseline_cache_map with -baseline_copy_roots when using -json_manifest\n");
1345                 exit(-1);
1346             }
1347         } else {
1348             if (!options.baselineCacheMapPath.empty()) {
1349                 fprintf(stderr, "Cannot use -baseline_cache_map without -json_manifest\n");
1350                 exit(-1);
1351             }
1352         }
1353
1354         if (!options.baselineCacheMapPath.empty()) {
1355             if (options.baselineDifferenceResultPath.empty() && options.baselineCopyRoots) {
1356                 fprintf(stderr, "Must use -baseline_cache_map with -baseline_diff_results or -baseline_copy_roots\n");
1357                 exit(-1);
1358             }
1359         }
1360
1361         // Find all the JSON files if we use -build_all
1362         __block std::vector<std::string> jsonPaths;
1363         if (!options.buildAllPath.empty()) {
1364             struct stat stat_buf;
1365             if (stat(options.buildAllPath.c_str(), &stat_buf) != 0) {
1366                 fprintf(stderr, "Could not find -build_all path '%s'\n", options.buildAllPath.c_str());
1367                 exit(-1);
1368             }
1369
1370             if ( (stat_buf.st_mode & S_IFMT) != S_IFDIR ) {
1371                 fprintf(stderr, "-build_all path is not a directory '%s'\n", options.buildAllPath.c_str());
1372                 exit(-1);
1373             }
1374
1375             auto processFile = ^(const std::string& path, const struct stat& statBuf) {
1376                 if ( !endsWith(path, ".json") )
1377                     return;
1378
1379                 jsonPaths.push_back(path);
1380             };
1381
1382             iterateDirectoryTree("", options.buildAllPath,
1383                                  ^(const std::string& dirPath) { return false; },
1384                                  processFile, true /* process files */, false /* recurse */);
1385
1386             if (jsonPaths.empty()) {
1387                 fprintf(stderr, "Didn't find any .json files inside -build_all path: %s\n", options.buildAllPath.c_str());
1388                 exit(-1);
1389             }
1390
1391             if (options.listConfigs) {
1392                 for (const std::string& path : jsonPaths) {
1393                     fprintf(stderr, "Found config: %s\n", path.c_str());
1394                 }
1395                 exit(-1);
1396             }
1397         }
1398
1399         if (!options.artifactDir.empty()) {
1400             // Find the dylib cache dir from inside the artifact dir
1401             struct stat stat_buf;
1402             if (stat(options.artifactDir.c_str(), &stat_buf) != 0) {
1403                 fprintf(stderr, "Could not find artifact path '%s'\n", options.artifactDir.c_str());
1404                 exit(-1);
1405             }
1406             std::string dir = options.artifactDir + "/AppleInternal/Developer/DylibCaches";
1407             if (stat(dir.c_str(), &stat_buf) != 0) {
1408                 fprintf(stderr, "Could not find artifact path '%s'\n", dir.c_str());
1409                 exit(-1);
1410             }
1411
1412             if (!options.release.empty()) {
1413                 // Use the given release
1414                 options.dylibCacheDir = dir + "/" + options.release + ".dlc";
1415             } else {
1416                 // Find a release directory
1417                 __block std::vector<std::string> subDirectories;
1418                 iterateDirectoryTree("", dir, ^(const std::string& dirPath) {
1419                     subDirectories.push_back(dirPath);
1420                     return false;
1421                 }, nullptr, false, false);
1422
1423                 if (subDirectories.empty()) {
1424                     fprintf(stderr, "Could not find dlc subdirectories inside '%s'\n", dir.c_str());
1425                     exit(-1);
1426                 }
1427
1428                 if (subDirectories.size() > 1) {
1429                     fprintf(stderr, "Found too many subdirectories inside artifact path '%s'.  Use -release to select one\n", dir.c_str());
1430                     exit(-1);
1431                 }
1432
1433                 options.dylibCacheDir = subDirectories.front();
1434             }
1435         }
1436
1437         if (options.dylibCacheDir.empty()) {
1438             options.dylibCacheDir = std::string("/AppleInternal/Developer/DylibCaches/") + options.release + ".dlc";
1439         }
1440
1441         //Move into the dir so we can use relative path manifests
1442         chdir(options.dylibCacheDir.c_str());
1443
1444         dispatch_async(dispatch_get_main_queue(), ^{
1445             if (!options.buildAllPath.empty()) {
1446                 bool requiresConcurrencyLimit = false;
1447                 dispatch_semaphore_t concurrencyLimit = NULL;
1448                 // Try build 1 cache per 8GB of RAM
1449                 uint64_t memSize = 0;
1450                 size_t sz = sizeof(memSize);
1451                 if ( sysctlbyname("hw.memsize", &memSize, &sz, NULL, 0) == 0 ) {
1452                     uint64_t maxThreads = std::max(memSize / 0x200000000ULL, 1ULL);
1453                     fprintf(stderr, "Detected %lldGb or less of memory, limiting concurrency to %lld threads\n",
1454                             memSize / (1 << 30), maxThreads);
1455                     requiresConcurrencyLimit = true;
1456                     concurrencyLimit = dispatch_semaphore_create(maxThreads);
1457                 }
1458
1459                 dispatch_apply(jsonPaths.size(), DISPATCH_APPLY_AUTO, ^(size_t index) {
1460                     // Horrible hack to limit concurrency in low spec build machines.
1461                     if (requiresConcurrencyLimit) { dispatch_semaphore_wait(concurrencyLimit, DISPATCH_TIME_FOREVER); }
1462
1463                     const std::string& jsonPath = jsonPaths[index];
1464                     buildCacheFromJSONManifest(diags, options, jsonPath);
1465
1466                     if (requiresConcurrencyLimit) { dispatch_semaphore_signal(concurrencyLimit); }
1467                 });
1468             } else if (!jsonManifestPath.empty()) {
1469                 buildCacheFromJSONManifest(diags, options, jsonManifestPath);
1470             } else {
1471                 buildCacheFromPListManifest(diags, options);
1472             }
1473
1474             const char* args[8];
1475             args[0] = (char*)"/bin/rm";
1476             args[1] = (char*)"-rf";
1477             args[2] = (char*)tempRootsDir;
1478             args[3] = nullptr;
1479             (void)runCommandAndWait(diags, args);
1480
1481             if (diags.hasError()) {
1482                 fprintf(stderr, "dyld_shared_cache_builder: error: %s", diags.errorMessage().c_str());
1483                 exit(-1);
1484             }
1485
1486             for (const std::string& warn : diags.warnings()) {
1487                 fprintf(stderr, "dyld_shared_cache_builder: warning: %s\n", warn.c_str());
1488             }
1489
1490             // Finally, write the roots.txt to tell us which roots we pulled in
1491             if (!options.dstRoot.empty())
1492                 writeRootList(options.dstRoot + "/System/Library/Caches/com.apple.dyld", options.roots);
1493             exit(0);
1494         });
1495     }
1496
1497     dispatch_main();
1498
1499     return 0;
1500 }