dyld3/shared-cache/multi_dyld_shared_cache_builder.mm

   1 /* -*- mode: C++; c-basic-offset: 4; indent-tabs-mode: nil -*-
   2  *
   3  * Copyright (c) 2016 Apple Inc. All rights reserved.
   4  *
   5  * @APPLE_LICENSE_HEADER_START@
   6  *
   7  * This file contains Original Code and/or Modifications of Original Code
   8  * as defined in and that are subject to the Apple Public Source License
   9  * Version 2.0 (the 'License'). You may not use this file except in
  10  * compliance with the License. Please obtain a copy of the License at
  11  * http://www.opensource.apple.com/apsl/ and read it before using this
  12  * file.
  13  *
  14  * The Original Code and all software distributed under the License are
  15  * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  16  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  17  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
  18  * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
  19  * Please see the License for the specific language governing rights and
  20  * limitations under the License.
  21  *
  22  * @APPLE_LICENSE_HEADER_END@
  23  */
  24
  25 #include <Bom/Bom.h>
  26 #include <dispatch/dispatch.h>
  27 #include <copyfile.h>
  28
  29 #include <sys/types.h>
  30 #include <sys/stat.h>
  31 #include <sys/mman.h>
  32 #include <sys/resource.h>
  33 #include <mach/mach.h>
  34 #include <mach/mach_time.h>
  35 #include <limits.h>
  36 #include <stdarg.h>
  37 #include <stdio.h>
  38 #include <stdlib.h>
  39 #include <math.h>
  40 #include <fcntl.h>
  41 #include <dlfcn.h>
  42 #include <signal.h>
  43 #include <errno.h>
  44 #include <sys/uio.h>
  45 #include <unistd.h>
  46 #include <sys/param.h>
  47 #include <sys/sysctl.h>
  48 #include <sys/resource.h>
  49 #include <dirent.h>
  50 #include <libgen.h>
  51 #include <pthread.h>
  52
  53 #include <vector>
  54 #include <array>
  55 #include <set>
  56 #include <map>
  57 #include <algorithm>
  58
  59 #include "Manifest.h"
  60 #include "FileUtils.h"
  61 #include "BuilderUtils.h"
  62
  63 #define CACHE_BUILDER_COPY_FILE_MODE COPYFILE_ALL
  64
  65 #if !__has_feature(objc_arc)
  66 #error The use of libdispatch in this files requires it to be compiled with ARC in order to avoid leaks
  67 #endif
  68
  69 static dispatch_queue_t build_queue = dispatch_queue_create("com.apple.dyld.cache-builder.build", DISPATCH_QUEUE_CONCURRENT);
  70
  71 #define kDylibCachePrefix "/AppleInternal/Developer/DylibCaches/"
  72
  73 void createArtifact(Diagnostics& diags, dyld3::Manifest& manifest, const std::string& dylibCachePath, bool includeExecutables)
  74 {
  75     auto copy_state = copyfile_state_alloc();
  76     mkpath_np((dylibCachePath + "/Metadata").c_str(), 0755);
  77     (void)copyfile(manifest.metabomFile().c_str(), (dylibCachePath + "/Metadata/metabom.bom").c_str(), copy_state, CACHE_BUILDER_COPY_FILE_MODE);
  78
  79     if (!manifest.dylibOrderFile().empty()) {
  80         (void)copyfile(realPath(manifest.dylibOrderFile()).c_str(), (dylibCachePath + "/Metadata/dylibOrderFile.txt").c_str(), copy_state, CACHE_BUILDER_COPY_FILE_MODE);
  81     }
  82
  83     if (!manifest.dirtyDataOrderFile().empty()) {
  84         (void)copyfile(realPath(manifest.dirtyDataOrderFile()).c_str(), (dylibCachePath + "/Metadata/dirtyDataOrderFile.txt").c_str(), copy_state, CACHE_BUILDER_COPY_FILE_MODE);
  85     }
  86
  87     std::set<dyld3::UUID> uuids;
  88     std::map<std::string, std::string> copy_pairs;
  89
  90     manifest.forEachConfiguration([&manifest, &uuids](const std::string& config) {
  91         manifest.configuration(config).forEachArchitecture([&manifest, &config, &uuids](const std::string& arch) {
  92             auto results = manifest.configuration(config).architecture(arch).results;
  93             for (const auto& image : results.dylibs) {
  94                 uuids.insert(image.first);
  95             }
  96             for (const auto& image : results.bundles) {
  97                 uuids.insert(image.first);
  98             }
  99             for (const auto& image : results.executables) {
 100                 uuids.insert(image.first);
 101             }
 102         });
 103     });
 104
 105     for (auto& uuid : uuids) {
 106         auto buildPath = manifest.buildPathForUUID(uuid);
 107         auto installPath = manifest.runtimePathForUUID(uuid);
 108         assert(!buildPath.empty() && !installPath.empty());
 109         copy_pairs.insert(std::make_pair(installPath, buildPath));
 110     }
 111
 112     for (const auto& copy_pair : copy_pairs) {
 113         std::string from = realPath(copy_pair.second);
 114         std::string to = dylibCachePath + "/Root/" + copy_pair.first;
 115         mkpath_np(dirPath(to).c_str(), 0755);
 116         int err = copyfile(from.c_str(), to.c_str(), copy_state, CACHE_BUILDER_COPY_FILE_MODE);
 117         diags.verbose("COPYING (%d) %s -> %s\n", err, from.c_str(), to.c_str());
 118
 119     }
 120     copyfile_state_free(copy_state);
 121
 122     fprintf(stderr, "[Artifact] dylibs copied\n");
 123 }
 124
 125 void addArtifactPaths(Diagnostics& diags, dyld3::Manifest& manifest)
 126 {
 127     manifest.setDylibOrderFile("./Metadata/dylibOrderFile.txt");
 128     manifest.setDirtyDataOrderFile("./Metadata/dirtyDataOrderFile.txt");
 129     manifest.setMetabomFile("./Metadata/metabom.bom");
 130
 131     for (auto& projects : manifest.projects()) {
 132         manifest.addProjectSource(projects.first, "./Root", true);
 133     }
 134 }
 135
 136 int main(int argc, const char* argv[])
 137 {
 138     @autoreleasepool {
 139         __block Diagnostics diags;
 140         bool                verbose = false;
 141         std::string         masterDstRoot;
 142         std::string         dylibCacheDir;
 143         std::string         resultPath;
 144         std::string         manifestPath;
 145         bool                preflight = false;
 146         __block bool        allBuildsSucceeded = true;
 147         bool                skipWrites = false;
 148         bool                skipBuilds = false;
 149         bool                agileChooseSHA256CdHash = false;
 150         time_t              mytime = time(0);
 151         fprintf(stderr, "Started: %s", asctime(localtime(&mytime)));
 152
 153         // parse command line options
 154         for (int i = 1; i < argc; ++i) {
 155             const char* arg = argv[i];
 156             if (arg[0] == '-') {
 157                 if (strcmp(arg, "-debug") == 0) {
 158                     verbose = true;
 159                     diags = Diagnostics(true);
 160                 } else if (strcmp(arg, "-skip_writes") == 0) {
 161                     skipWrites = true;
 162                 } else if (strcmp(arg, "-skip_builds") == 0) {
 163                     skipBuilds = true;
 164                 } else if (strcmp(arg, "-delete_writes") == 0) {
 165                     skipWrites = true;
 166                 } else if (strcmp(arg, "-dylib_cache") == 0) {
 167                     dylibCacheDir = argv[++i];
 168                 } else if (strcmp(arg, "-preflight") == 0) {
 169                     preflight = true;
 170                     skipWrites = true;
 171                 } else if (strcmp(arg, "-master_dst_root") == 0) {
 172                     masterDstRoot = argv[++i];
 173                     if (masterDstRoot.empty()) {
 174                         diags.error("-master_dst_root missing path argument");
 175                     }
 176                 } else if (strcmp(arg, "-results") == 0) {
 177                     resultPath = argv[++i];
 178                 } else if (strcmp(arg, "-plist") == 0) {
 179                     manifestPath = argv[++i];
 180                 } else if (strcmp(arg, "-agile_choose_sha256_cdhash") == 0) {
 181                     agileChooseSHA256CdHash = true;
 182                 } else {
 183                     // usage();
 184                     diags.error("unknown option: %s", arg);
 185                 }
 186             } else {
 187                 manifestPath = argv[i];
 188             }
 189         }
 190
 191         if (getenv("LGG_SKIP_CACHE_FUN") != nullptr) {
 192             skipBuilds = true;
 193         }
 194
 195         if (diags.hasError()) {
 196             printf("%s\n", diags.errorMessage().c_str());
 197             exit(-1);
 198         }
 199
 200         dispatch_async(dispatch_get_main_queue(), ^{
 201             if (manifestPath.empty()) {
 202                 fprintf(stderr, "mainfest path argument is required\n");
 203                 exit(-1);
 204             }
 205
 206             (void)chdir(dirname(strdup(manifestPath.c_str())));
 207             __block auto manifest = dyld3::Manifest(diags, manifestPath);
 208
 209             if (manifest.build().empty()) {
 210                 fprintf(stderr, "No version found in manifest\n");
 211                 exit(-1);
 212             }
 213
 214             fprintf(stderr, "Building Caches for %s\n", manifest.build().c_str());
 215
 216             if (masterDstRoot.empty()) {
 217                 fprintf(stderr, "-master_dst_root required path argument\n");
 218                 exit(-1);
 219             }
 220
 221             if (manifest.version() < 4) {
 222                 fprintf(stderr, "must specify valid manifest file\n");
 223                 exit(-1);
 224             }
 225
 226             struct rlimit rl = { OPEN_MAX, OPEN_MAX };
 227             (void)setrlimit(RLIMIT_NOFILE, &rl);
 228
 229             manifest.calculateClosure();
 230
 231             if (!skipWrites && !skipBuilds) {
 232                 (void)mkpath_np((masterDstRoot + "/Boms/").c_str(), 0755);
 233                 dispatch_group_async(buildGroup(), build_queue, ^{
 234                     createArtifact(diags, manifest, masterDstRoot + "/Artifact.dlc/", true);
 235                 });
 236             }
 237
 238             if (!dylibCacheDir.empty()) {
 239                 dispatch_group_async(buildGroup(), build_queue, ^{
 240                     createArtifact(diags, manifest, dylibCacheDir + "/AppleInternal/Developer/DylibCaches/" + manifest.build() + ".dlc/", false);
 241                 });
 242             }
 243
 244             if (!skipBuilds) {
 245                 dispatch_group_async(buildGroup(), build_queue, ^{
 246                     makeBoms(manifest, masterDstRoot);
 247                 });
 248                 allBuildsSucceeded = build(diags, manifest, masterDstRoot, true, verbose, skipWrites,
 249                                            agileChooseSHA256CdHash, true, false);
 250             }
 251
 252             manifest.write(resultPath);
 253
 254             addArtifactPaths(diags, manifest);
 255             if (!dylibCacheDir.empty()) {
 256                 manifest.write(dylibCacheDir + "/AppleInternal/Developer/DylibCaches/" + manifest.build() + ".dlc/Manifest.plist");
 257                 manifest.writeJSON(dylibCacheDir + "/AppleInternal/Developer/DylibCaches/" + manifest.build() + ".dlc/Manifest.json");
 258             }
 259
 260             if (!skipWrites) {
 261                 mkpath_np((masterDstRoot + "/Artifact.dlc").c_str(), 0755);
 262                 auto copy_state = copyfile_state_alloc();
 263                 (void)copyfile(realPath(manifestPath).c_str(), (masterDstRoot + "/Artifact.dlc/BNIManifest.plist").c_str(), copy_state, COPYFILE_ALL);
 264                 copyfile_state_free(copy_state);
 265                 manifest.write(masterDstRoot + "/Artifact.dlc/Manifest.plist");
 266                 manifest.writeJSON(masterDstRoot + "/Artifact.dlc/Manifest.json");
 267             }
 268
 269             dispatch_group_wait(buildGroup(), DISPATCH_TIME_FOREVER);
 270             time_t mytime = time(0);
 271             fprintf(stderr, "Finished: %s", asctime(localtime(&mytime)));
 272
 273             if (preflight && !allBuildsSucceeded) {
 274                 exit(-1);
 275             }
 276
 277             exit(0);
 278         });
 279     }
 280     dispatch_main();
 281     return 0;
 282 }