dyld3/shared-cache/make_ios_dyld_cache.cpp

   1 /* -*- mode: C++; c-basic-offset: 4; indent-tabs-mode: nil -*-
   2  *
   3  * Copyright (c) 2016 Apple Inc. All rights reserved.
   4  *
   5  * @APPLE_LICENSE_HEADER_START@
   6  *
   7  * This file contains Original Code and/or Modifications of Original Code
   8  * as defined in and that are subject to the Apple Public Source License
   9  * Version 2.0 (the 'License'). You may not use this file except in
  10  * compliance with the License. Please obtain a copy of the License at
  11  * http://www.opensource.apple.com/apsl/ and read it before using this
  12  * file.
  13  *
  14  * The Original Code and all software distributed under the License are
  15  * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  16  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  17  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
  18  * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
  19  * Please see the License for the specific language governing rights and
  20  * limitations under the License.
  21  *
  22  * @APPLE_LICENSE_HEADER_END@
  23  */
  24
  25 #include <sys/types.h>
  26 #include <sys/stat.h>
  27 #include <sys/mman.h>
  28 #include <mach/mach.h>
  29 #include <mach/mach_time.h>
  30 #include <limits.h>
  31 #include <stdarg.h>
  32 #include <stdio.h>
  33 #include <stdlib.h>
  34 #include <math.h>
  35 #include <fcntl.h>
  36 #include <dlfcn.h>
  37 #include <signal.h>
  38 #include <errno.h>
  39 #include <assert.h>
  40 #include <sys/uio.h>
  41 #include <unistd.h>
  42 #include <sys/param.h>
  43 #include <sys/sysctl.h>
  44 #include <sys/resource.h>
  45 #include <dirent.h>
  46 #include <rootless.h>
  47 #include <dscsym.h>
  48 #include <dispatch/dispatch.h>
  49 #include <pthread/pthread.h>
  50
  51 #include <algorithm>
  52 #include <vector>
  53 #include <unordered_set>
  54 #include <unordered_set>
  55 #include <iostream>
  56 #include <fstream>
  57
  58 #include "MachOFile.h"
  59 #include "FileUtils.h"
  60 #include "StringUtils.h"
  61 #include "DyldSharedCache.h"
  62
  63
  64
  65 struct MappedMachOsByCategory
  66 {
  67     std::string                                 archName;
  68     std::vector<DyldSharedCache::MappedMachO>   dylibsForCache;
  69     std::vector<DyldSharedCache::MappedMachO>   otherDylibsAndBundles;
  70     std::vector<DyldSharedCache::MappedMachO>   mainExecutables;
  71 };
  72
  73 static bool verbose = false;
  74
  75
  76 static bool addIfMachO(const std::string& buildRootPath, const std::string& runtimePath, const struct stat& statBuf, dyld3::Platform platform, std::vector<MappedMachOsByCategory>& files)
  77 {
  78     // read start of file to determine if it is mach-o or a fat file
  79     std::string fullPath = buildRootPath + runtimePath;
  80     int fd = ::open(fullPath.c_str(), O_RDONLY);
  81     if ( fd < 0 )
  82         return false;
  83     bool result = false;
  84     const void* wholeFile = ::mmap(NULL, (size_t)statBuf.st_size, PROT_READ, MAP_PRIVATE, fd, 0);
  85     if ( wholeFile != MAP_FAILED ) {
  86         Diagnostics diag;
  87         bool usedWholeFile = false;
  88         for (MappedMachOsByCategory& file : files) {
  89             uint64_t sliceOffset = 0;
  90             uint64_t sliceLength = statBuf.st_size;
  91             bool fatButMissingSlice;
  92             const void* slice = MAP_FAILED;
  93             const dyld3::FatFile* fh = (dyld3::FatFile*)wholeFile;
  94             const dyld3::MachOFile* mh = (dyld3::MachOFile*)wholeFile;
  95             if ( fh->isFatFileWithSlice(diag, statBuf.st_size, file.archName.c_str(), sliceOffset, sliceLength, fatButMissingSlice) ) {
  96                 slice = ::mmap(NULL, sliceLength, PROT_READ, MAP_PRIVATE, fd, sliceOffset);
  97                 if ( slice != MAP_FAILED ) {
  98                     //fprintf(stderr, "mapped slice at %p size=0x%0lX, offset=0x%0lX for %s\n", p, len, offset, fullPath.c_str());
  99                     mh = (dyld3::MachOFile*)slice;
 100                     if ( !mh->isMachO(diag, sliceLength) ) {
 101                         ::munmap((void*)slice, sliceLength);
 102                         slice = MAP_FAILED;
 103                     }
 104                 }
 105             }
 106             else if ( !fatButMissingSlice && mh->isMachO(diag, sliceLength) ) {
 107                 slice           = wholeFile;
 108                 sliceLength     = statBuf.st_size;
 109                 sliceOffset     = 0;
 110                 usedWholeFile   = true;
 111                 //fprintf(stderr, "mapped whole file at %p size=0x%0lX for %s\n", p, len, inputPath.c_str());
 112             }
 113             if ( slice != MAP_FAILED ) {
 114                 mh = (dyld3::MachOFile*)slice;
 115                 if ( mh->platform() != platform ) {
 116                     fprintf(stderr, "skipped wrong platform binary: %s\n", fullPath.c_str());
 117                     result = false;
 118                 }
 119                 else {
 120                     bool sip = true; // assume anything found in the simulator runtime is a platform binary
 121                     if ( mh->isDynamicExecutable() ) {
 122                         bool issetuid = (statBuf.st_mode & (S_ISUID|S_ISGID));
 123                         file.mainExecutables.emplace_back(runtimePath, mh, sliceLength, issetuid, sip, sliceOffset, statBuf.st_mtime, statBuf.st_ino);
 124                     }
 125                     else {
 126                         if ( parser.canBePlacedInDyldCache(runtimePath) ) {
 127                             file.dylibsForCache.emplace_back(runtimePath, mh, sliceLength, false, sip, sliceOffset, statBuf.st_mtime, statBuf.st_ino);
 128                         }
 129                     }
 130                     result = true;
 131                 }
 132             }
 133         }
 134         if ( !usedWholeFile )
 135             ::munmap((void*)wholeFile, statBuf.st_size);
 136     }
 137     ::close(fd);
 138     return result;
 139 }
 140
 141
 142 static bool parsePathsFile(const std::string& filePath, std::vector<std::string>& paths) {
 143     std::ifstream myfile( filePath );
 144     if ( myfile.is_open() ) {
 145     std::string line;
 146         while ( std::getline(myfile, line) ) {
 147             size_t pos = line.find('#');
 148             if ( pos != std::string::npos )
 149                 line.resize(pos);
 150             while ( line.size() != 0 && isspace(line.back()) ) {
 151                 line.pop_back();
 152             }
 153             if ( !line.empty() )
 154                 paths.push_back(line);
 155         }
 156         myfile.close();
 157         return true;
 158     }
 159     return false;
 160 }
 161
 162
 163 static void mapAllFiles(const std::string& dylibsRootDir, const std::vector<std::string>& paths, dyld3::Platform platform, std::vector<MappedMachOsByCategory>& files)
 164 {
 165     for (const std::string& runtimePath : paths) {
 166         std::string fullPath = dylibsRootDir + runtimePath;
 167         struct stat statBuf;
 168         if ( (stat(fullPath.c_str(), &statBuf) != 0) || !addIfMachO(dylibsRootDir, runtimePath, statBuf, platform, files) )
 169              fprintf(stderr, "could not load: %s\n", fullPath.c_str());
 170     }
 171 }
 172
 173
 174
 175 inline uint32_t absolutetime_to_milliseconds(uint64_t abstime)
 176 {
 177     return (uint32_t)(abstime/1000/1000);
 178 }
 179
 180
 181 #define TERMINATE_IF_LAST_ARG( s )      \
 182     do {                                \
 183         if ( i == argc - 1 ) {          \
 184             fprintf(stderr, s );        \
 185             return 1;                   \
 186         }                               \
 187     } while ( 0 )
 188
 189 int main(int argc, const char* argv[])
 190 {
 191     std::string                     rootPath;
 192     std::string                     dylibListFile;
 193     bool                            force = false;
 194     std::string                     cacheDir;
 195     std::string                     dylibsList;
 196     std::unordered_set<std::string> archStrs;
 197
 198     dyld3::Platform platform = dyld3::Platform::iOS;
 199
 200     // parse command line options
 201     for (int i = 1; i < argc; ++i) {
 202         const char* arg = argv[i];
 203         if (strcmp(arg, "-debug") == 0) {
 204             verbose = true;
 205         }
 206         else if (strcmp(arg, "-verbose") == 0) {
 207             verbose = true;
 208         }
 209         else if (strcmp(arg, "-tvOS") == 0) {
 210             platform = dyld3::Platform::tvOS;
 211         }
 212         else if (strcmp(arg, "-iOS") == 0) {
 213             platform = dyld3::Platform::iOS;
 214         }
 215         else if (strcmp(arg, "-watchOS") == 0) {
 216             platform = dyld3::Platform::watchOS;
 217         }
 218         else if ( strcmp(arg, "-root") == 0 ) {
 219             TERMINATE_IF_LAST_ARG("-root missing path argument\n");
 220             rootPath = argv[++i];
 221         }
 222         else if ( strcmp(arg, "-dylibs_list") == 0 ) {
 223             TERMINATE_IF_LAST_ARG("-dylibs_list missing path argument\n");
 224             dylibsList = argv[++i];
 225         }
 226         else if (strcmp(arg, "-cache_dir") == 0) {
 227             TERMINATE_IF_LAST_ARG("-cache_dir missing path argument\n");
 228             cacheDir = argv[++i];
 229         }
 230         else if (strcmp(arg, "-arch") == 0) {
 231             TERMINATE_IF_LAST_ARG("-arch missing argument\n");
 232             archStrs.insert(argv[++i]);
 233         }
 234         else if (strcmp(arg, "-force") == 0) {
 235             force = true;
 236         }
 237         else {
 238             //usage();
 239             fprintf(stderr, "update_dyld_sim_shared_cache: unknown option: %s\n", arg);
 240             return 1;
 241         }
 242     }
 243
 244     if ( cacheDir.empty() ) {
 245         fprintf(stderr, "missing -cache_dir <path> option to specify directory in which to write cache file(s)\n");
 246         return 1;
 247     }
 248
 249     if ( rootPath.empty() ) {
 250         fprintf(stderr, "missing -runtime_dir <path> option to specify directory which is root of simulator runtime)\n");
 251         return 1;
 252     }
 253     else {
 254         // canonicalize rootPath
 255         char resolvedPath[PATH_MAX];
 256         if ( realpath(rootPath.c_str(), resolvedPath) != NULL ) {
 257             rootPath = resolvedPath;
 258         }
 259         if ( rootPath.back() != '/' )
 260             rootPath = rootPath + "/";
 261     }
 262
 263     int err = mkpath_np(cacheDir.c_str(), S_IRWXU | S_IRGRP|S_IXGRP | S_IROTH|S_IXOTH);
 264     if ( (err != 0) && (err != EEXIST) ) {
 265         fprintf(stderr, "mkpath_np fail: %d", err);
 266         return 1;
 267     }
 268
 269     if ( archStrs.empty() ) {
 270         switch ( platform ) {
 271             case dyld3::Platform::iOS:
 272             case dyld3::Platform::tvOS:
 273                 archStrs.insert("arm64");
 274                 break;
 275             case dyld3::Platform::watchOS:
 276                 archStrs.insert("armv7k");
 277                 archStrs.insert("arm64_32");
 278                 break;
 279              case dyld3::Platform::unknown:
 280              case dyld3::Platform::macOS:
 281                 assert(0 && "macOS not support with this tool");
 282                 break;
 283         }
 284     }
 285
 286     uint64_t t1 = mach_absolute_time();
 287
 288     // find all mach-o files for requested architectures
 289     std::vector<MappedMachOsByCategory> allFileSets;
 290     if ( archStrs.count("arm64") )
 291         allFileSets.push_back({"arm64"});
 292     if ( archStrs.count("arm64_32") )
 293         allFileSets.push_back({"arm64_32"});
 294     if ( archStrs.count("armv7k") )
 295         allFileSets.push_back({"armv7k"});
 296     std::vector<std::string> paths;
 297     parsePathsFile(dylibsList, paths);
 298     mapAllFiles(rootPath, paths, platform, allFileSets);
 299
 300     uint64_t t2 = mach_absolute_time();
 301
 302     fprintf(stderr, "time to scan file system and construct lists of mach-o files: %ums\n", absolutetime_to_milliseconds(t2-t1));
 303
 304     // build all caches in parallel
 305     __block bool cacheBuildFailure = false;
 306     dispatch_apply(allFileSets.size(), dispatch_get_global_queue(DISPATCH_QUEUE_PRIORITY_DEFAULT, 0), ^(size_t index) {
 307         const MappedMachOsByCategory& fileSet = allFileSets[index];
 308         const std::string outFile = cacheDir + "/dyld_shared_cache_" + fileSet.archName;
 309
 310         fprintf(stderr, "make %s cache with %lu dylibs, %lu other dylibs, %lu programs\n", fileSet.archName.c_str(), fileSet.dylibsForCache.size(), fileSet.otherDylibsAndBundles.size(), fileSet.mainExecutables.size());
 311
 312         // build cache new cache file
 313         DyldSharedCache::CreateOptions options;
 314         options.archName                     = fileSet.archName;
 315         options.platform                     = platform;
 316         options.excludeLocalSymbols          = true;
 317         options.optimizeStubs                = false;
 318         options.optimizeObjC                 = true;
 319         options.codeSigningDigestMode = (platform() == dyld3::Platform::watchOS) ?
 320                                         DyldSharedCache::Agile : DyldSharedCache::SHA256only;
 321         options.dylibsRemovedDuringMastering = true;
 322         options.inodesAreSameAsRuntime       = false;
 323         options.cacheSupportsASLR            = true;
 324         options.forSimulator                 = false;
 325         options.isLocallyBuiltCache          = true;
 326         options.verbose                      = verbose;
 327         options.evictLeafDylibsOnOverflow    = false;
 328         options.pathPrefixes                 = { rootPath };
 329         DyldSharedCache::CreateResults results = DyldSharedCache::create(options, fileSet.dylibsForCache, fileSet.otherDylibsAndBundles, fileSet.mainExecutables);
 330
 331         // print any warnings
 332         for (const std::string& warn : results.warnings) {
 333             fprintf(stderr, "update_dyld_sim_shared_cache: warning: %s %s\n", fileSet.archName.c_str(), warn.c_str());
 334         }
 335         if ( !results.errorMessage.empty() ) {
 336             // print error (if one)
 337             fprintf(stderr, "update_dyld_sim_shared_cache: %s\n", results.errorMessage.c_str());
 338             cacheBuildFailure = true;
 339         }
 340         else {
 341             // save new cache file to disk and write new .map file
 342             assert(results.cacheContent != nullptr);
 343             if ( !safeSave(results.cacheContent, results.cacheLength, outFile) )
 344                 cacheBuildFailure = true;
 345             if ( !cacheBuildFailure ) {
 346                 std::string mapStr = results.cacheContent->mapFile();
 347                 std::string outFileMap = cacheDir + "/dyld_shared_cache_" + fileSet.archName + ".map";
 348                 safeSave(mapStr.c_str(), mapStr.size(), outFileMap);
 349             }
 350             // free created cache buffer
 351             vm_deallocate(mach_task_self(), (vm_address_t)results.cacheContent, results.cacheLength);
 352         }
 353     });
 354
 355     // we could unmap all input files, but tool is about to quit
 356
 357     return (cacheBuildFailure ? 1 : 0);
 358 }
 359