dyld3/shared-cache/dyld_shared_cache_builder.mm

   1 /* -*- mode: C++; c-basic-offset: 4; indent-tabs-mode: nil -*-
   2  *
   3  * Copyright (c) 2016 Apple Inc. All rights reserved.
   4  *
   5  * @APPLE_LICENSE_HEADER_START@
   6  *
   7  * This file contains Original Code and/or Modifications of Original Code
   8  * as defined in and that are subject to the Apple Public Source License
   9  * Version 2.0 (the 'License'). You may not use this file except in
  10  * compliance with the License. Please obtain a copy of the License at
  11  * http://www.opensource.apple.com/apsl/ and read it before using this
  12  * file.
  13  *
  14  * The Original Code and all software distributed under the License are
  15  * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  16  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  17  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
  18  * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
  19  * Please see the License for the specific language governing rights and
  20  * limitations under the License.
  21  *
  22  * @APPLE_LICENSE_HEADER_END@
  23  */
  24
  25 #include <sys/types.h>
  26 #include <sys/stat.h>
  27 #include <sys/mman.h>
  28 #include <sys/resource.h>
  29 #include <mach/mach.h>
  30 #include <mach/mach_time.h>
  31 #include <limits.h>
  32 #include <stdarg.h>
  33 #include <stdio.h>
  34 #include <stdlib.h>
  35 #include <math.h>
  36 #include <fcntl.h>
  37 #include <dlfcn.h>
  38 #include <signal.h>
  39 #include <errno.h>
  40 #include <sys/uio.h>
  41 #include <unistd.h>
  42 #include <sys/param.h>
  43 #include <sys/sysctl.h>
  44 #include <sys/resource.h>
  45 #include <dirent.h>
  46 #include <libgen.h>
  47 #include <pthread.h>
  48 #include <fts.h>
  49
  50 #include <vector>
  51 #include <array>
  52 #include <set>
  53 #include <map>
  54 #include <unordered_set>
  55 #include <algorithm>
  56
  57 #include <spawn.h>
  58
  59 #include <Bom/Bom.h>
  60
  61 #include "Manifest.h"
  62 #include "Diagnostics.h"
  63 #include "DyldSharedCache.h"
  64 #include "BuilderUtils.h"
  65 #include "FileUtils.h"
  66 #include "StringUtils.h"
  67 #include "MachOParser.h"
  68
  69 #if !__has_feature(objc_arc)
  70 #error The use of libdispatch in this files requires it to be compiled with ARC in order to avoid leaks
  71 #endif
  72
  73 extern char** environ;
  74
  75 static dispatch_queue_t build_queue;
  76
  77 static const char* tempRootDirTemplate = "/tmp/dyld_shared_cache_builder.XXXXXX";
  78 static char*       tempRootDir = nullptr;
  79
  80 int runCommandAndWait(Diagnostics& diags, const char* args[])
  81 {
  82     pid_t pid;
  83     int   status;
  84     int   res = posix_spawn(&pid, args[0], nullptr, nullptr, (char**)args, environ);
  85     if (res != 0)
  86         diags.error("Failed to spawn %s: %s (%d)", args[0], strerror(res), res);
  87
  88     do {
  89         res = waitpid(pid, &status, 0);
  90     } while (res == -1 && errno == EINTR);
  91     if (res != -1) {
  92         if (WIFEXITED(status)) {
  93             res = WEXITSTATUS(status);
  94         } else {
  95             res = -1;
  96         }
  97     }
  98
  99     return res;
 100 }
 101
 102 void processRoots(Diagnostics& diags, std::set<std::string>& roots)
 103 {
 104     std::set<std::string> processedRoots;
 105     struct stat           sb;
 106     int                   res = 0;
 107     const char*           args[8];
 108
 109     for (const auto& root : roots) {
 110         res = stat(root.c_str(), &sb);
 111
 112         if (res == 0 && S_ISDIR(sb.st_mode)) {
 113             roots.insert(root);
 114             return;
 115         } else if (endsWith(root, ".cpio") || endsWith(root, ".cpio.gz") || endsWith(root, ".cpgz") || endsWith(root, ".cpio.bz2") || endsWith(root, ".cpbz2") || endsWith(root, ".pax") || endsWith(root, ".pax.gz") || endsWith(root, ".pgz") || endsWith(root, ".pax.bz2") || endsWith(root, ".pbz2")) {
 116             args[0] = (char*)"/usr/bin/ditto";
 117             args[1] = (char*)"-x";
 118             args[2] = (char*)root.c_str();
 119             args[3] = tempRootDir;
 120             args[4] = nullptr;
 121         } else if (endsWith(root, ".tar")) {
 122             args[0] = (char*)"/usr/bin/tar";
 123             args[1] = (char*)"xf";
 124             args[2] = (char*)root.c_str();
 125             args[3] = (char*)"-C";
 126             args[4] = tempRootDir;
 127             args[5] = nullptr;
 128         } else if (endsWith(root, ".tar.gz") || endsWith(root, ".tgz")) {
 129             args[0] = (char*)"/usr/bin/tar";
 130             args[1] = (char*)"xzf";
 131             args[2] = (char*)root.c_str();
 132             args[3] = (char*)"-C";
 133             args[4] = tempRootDir;
 134             args[5] = nullptr;
 135         } else if (endsWith(root, ".tar.bz2")
 136             || endsWith(root, ".tbz2")
 137             || endsWith(root, ".tbz")) {
 138             args[0] = (char*)"/usr/bin/tar";
 139             args[1] = (char*)"xjf";
 140             args[2] = (char*)root.c_str();
 141             args[3] = (char*)"-C";
 142             args[4] = tempRootDir;
 143             args[5] = nullptr;
 144         } else if (endsWith(root, ".xar")) {
 145             args[0] = (char*)"/usr/bin/xar";
 146             args[1] = (char*)"-xf";
 147             args[2] = (char*)root.c_str();
 148             args[3] = (char*)"-C";
 149             args[4] = tempRootDir;
 150             args[5] = nullptr;
 151         } else if (endsWith(root, ".zip")) {
 152             args[0] = (char*)"/usr/bin/ditto";
 153             args[1] = (char*)"-xk";
 154             args[2] = (char*)root.c_str();
 155             args[3] = tempRootDir;
 156             args[4] = nullptr;
 157         } else {
 158             diags.error("unknown archive type: %s", root.c_str());
 159             continue;
 160         }
 161
 162         if (res != runCommandAndWait(diags, args)) {
 163             fprintf(stderr, "Could not expand archive %s: %s (%d)", root.c_str(), strerror(res), res);
 164             exit(-1);
 165         }
 166         for (auto& existingRoot : processedRoots) {
 167             if (existingRoot == tempRootDir)
 168                 return;
 169         }
 170
 171         processedRoots.insert(tempRootDir);
 172     }
 173
 174     roots = processedRoots;
 175 }
 176
 177 bool writeRootList(const std::string& dstRoot, const std::set<std::string>& roots)
 178 {
 179     if (roots.size() == 0)
 180         return false;
 181
 182     std::string rootFile = dstRoot + "/roots.txt";
 183     FILE*       froots = ::fopen(rootFile.c_str(), "w");
 184     if (froots == NULL)
 185         return false;
 186
 187     for (auto& root : roots) {
 188         fprintf(froots, "%s\n", root.c_str());
 189     }
 190
 191     ::fclose(froots);
 192     return true;
 193 }
 194
 195 std::set<std::string> cachePaths;
 196
 197 BOMCopierCopyOperation filteredCopy(BOMCopier copier, const char* path, BOMFSObjType type, off_t size)
 198 {
 199     std::string absolutePath = &path[1];
 200     if (cachePaths.count(absolutePath)) {
 201         return BOMCopierSkipFile;
 202     }
 203     return BOMCopierContinue;
 204 }
 205
 206 int main(int argc, const char* argv[])
 207 {
 208     @autoreleasepool {
 209         __block Diagnostics   diags;
 210         std::set<std::string> roots;
 211         std::string           dylibCacheDir;
 212         std::string           release;
 213         bool                  emitDevCaches = true;
 214         bool                  emitElidedDylibs = true;
 215         bool                  listConfigs = false;
 216         bool                  copyRoots = false;
 217         bool                  debug = false;
 218         std::string           dstRoot;
 219         std::string           configuration;
 220         std::string           resultPath;
 221
 222         tempRootDir = strdup(tempRootDirTemplate);
 223         mkdtemp(tempRootDir);
 224
 225         for (int i = 1; i < argc; ++i) {
 226             const char* arg = argv[i];
 227             if (arg[0] == '-') {
 228                 if (strcmp(arg, "-debug") == 0) {
 229                     diags = Diagnostics(true);
 230                     debug = true;
 231                 } else if (strcmp(arg, "-list_configs") == 0) {
 232                     listConfigs = true;
 233                 } else if (strcmp(arg, "-root") == 0) {
 234                     roots.insert(realPath(argv[++i]));
 235                 } else if (strcmp(arg, "-copy_roots") == 0) {
 236                     copyRoots = true;
 237                 } else if (strcmp(arg, "-dylib_cache") == 0) {
 238                     dylibCacheDir = realPath(argv[++i]);
 239                 } else if (strcmp(arg, "-no_development_cache") == 0) {
 240                     emitDevCaches = false;
 241                 } else if (strcmp(arg, "-no_overflow_dylibs") == 0) {
 242                     emitElidedDylibs = false;
 243                 } else if (strcmp(arg, "-development_cache") == 0) {
 244                     emitDevCaches = true;
 245                 } else if (strcmp(arg, "-overflow_dylibs") == 0) {
 246                     emitElidedDylibs = true;
 247                 } else if (strcmp(arg, "-dst_root") == 0) {
 248                     dstRoot = realPath(argv[++i]);
 249                 } else if (strcmp(arg, "-release") == 0) {
 250                     release = argv[++i];
 251                 } else if (strcmp(arg, "-results") == 0) {
 252                     resultPath = realPath(argv[++i]);
 253                 } else {
 254                     //usage();
 255                     diags.error("unknown option: %s\n", arg);
 256                 }
 257             } else {
 258                 if (!configuration.empty()) {
 259                     diags.error("You may only specify one configuration");
 260                 }
 261                 configuration = argv[i];
 262             }
 263         }
 264
 265         time_t mytime = time(0);
 266         fprintf(stderr, "Started: %s", asctime(localtime(&mytime)));
 267         processRoots(diags, roots);
 268
 269         struct rlimit rl = { OPEN_MAX, OPEN_MAX };
 270         (void)setrlimit(RLIMIT_NOFILE, &rl);
 271
 272         if (dylibCacheDir.empty() && release.empty()) {
 273             fprintf(stderr, "you must specify either -dylib_cache or -release");
 274             exit(-1);
 275         } else if (!dylibCacheDir.empty() && !release.empty()) {
 276             fprintf(stderr, "you may not use -dylib_cache and -release at the same time");
 277             exit(-1);
 278         }
 279
 280         if ((configuration.empty() || dstRoot.empty()) && !listConfigs) {
 281             fprintf(stderr, "Must specify a configuration and a valid -dst_root OR -list_configs\n");
 282             exit(-1);
 283         }
 284
 285         if (dylibCacheDir.empty()) {
 286             dylibCacheDir = std::string("/AppleInternal/Developer/DylibCaches/") + release + ".dlc";
 287         }
 288
 289         //Move into the dir so we can use relative path manifests
 290         chdir(dylibCacheDir.c_str());
 291
 292         dispatch_async(dispatch_get_main_queue(), ^{
 293             auto manifest = dyld3::Manifest(diags, dylibCacheDir + "/Manifest.plist", roots);
 294
 295             if (manifest.build().empty()) {
 296                 fprintf(stderr, "No manifest found at '%s/Manifest.plist'\n", dylibCacheDir.c_str());
 297                 exit(-1);
 298             }
 299             fprintf(stderr, "Building Caches for %s\n", manifest.build().c_str());
 300
 301             if (listConfigs) {
 302                 manifest.forEachConfiguration([](const std::string& configName) {
 303                     printf("%s\n", configName.c_str());
 304                 });
 305             }
 306
 307             if (!manifest.filterForConfig(configuration)) {
 308                 fprintf(stderr, "No config %s. Please run with -list_configs to see configurations available for this %s.\n",
 309                     configuration.c_str(), manifest.build().c_str());
 310                 exit(-1);
 311             }
 312             manifest.calculateClosure();
 313
 314             std::vector<dyld3::BuildQueueEntry> buildQueue;
 315
 316             bool cacheBuildSuccess = build(diags, manifest, dstRoot, false, debug, false, false);
 317
 318             if (!cacheBuildSuccess) {
 319                 exit(-1);
 320             }
 321
 322             writeRootList(dstRoot, roots);
 323
 324             if (copyRoots) {
 325                 manifest.forEachConfiguration([&manifest](const std::string& configName) {
 326                     for (auto& arch : manifest.configuration(configName).architectures) {
 327                         for (auto& dylib : arch.second.results.dylibs) {
 328                             if (dylib.second.included) {
 329                                 dyld3::MachOParser parser = manifest.parserForUUID(dylib.first);
 330                                 cachePaths.insert(parser.installName());
 331                             }
 332                         }
 333                     }
 334                 });
 335
 336                 BOMCopier copier = BOMCopierNewWithSys(BomSys_default());
 337                 BOMCopierSetCopyFileStartedHandler(copier, filteredCopy);
 338                 for (auto& root : roots) {
 339                     BOMCopierCopy(copier, root.c_str(), dstRoot.c_str());
 340                 }
 341                 BOMCopierFree(copier);
 342             }
 343
 344
 345
 346
 347             int err = sync_volume_np(dstRoot.c_str(), SYNC_VOLUME_FULLSYNC | SYNC_VOLUME_WAIT);
 348             if (err) {
 349                 fprintf(stderr, "Volume sync failed errnor=%d (%s)\n", err, strerror(err));
 350             }
 351
 352             // Create an empty FIPS data in the root
 353             (void)mkpath_np((dstRoot + "/private/var/db/FIPS/").c_str(), 0755);
 354             int fd = open((dstRoot + "/private/var/db/FIPS/fips_data").c_str(), O_CREAT | O_TRUNC, 0644);
 355             close(fd);
 356
 357             // Now that all the build commands have been issued lets put a barrier in after then which can tear down the app after
 358             // everything is written.
 359
 360             if (!resultPath.empty()) {
 361                 manifest.write(resultPath);
 362             }
 363
 364             const char* args[8];
 365             args[0] = (char*)"/bin/rm";
 366             args[1] = (char*)"-rf";
 367             args[2] = (char*)tempRootDir;
 368             args[3] = nullptr;
 369             (void)runCommandAndWait(diags, args);
 370
 371             for (const std::string& warn : diags.warnings()) {
 372                 fprintf(stderr, "dyld_shared_cache_builder: warning: %s\n", warn.c_str());
 373             }
 374             exit(0);
 375         });
 376     }
 377
 378     dispatch_main();
 379
 380     return 0;
 381 }