interlinked-dylibs/update_dyld_shared_cache_compat.cpp

   1 /* -*- mode: C++; c-basic-offset: 4; indent-tabs-mode: nil -*-
   2  *
   3  * Copyright (c) 2014 Apple Inc. All rights reserved.
   4  *
   5  * @APPLE_LICENSE_HEADER_START@
   6  *
   7  * This file contains Original Code and/or Modifications of Original Code
   8  * as defined in and that are subject to the Apple Public Source License
   9  * Version 2.0 (the 'License'). You may not use this file except in
  10  * compliance with the License. Please obtain a copy of the License at
  11  * http://www.opensource.apple.com/apsl/ and read it before using this
  12  * file.
  13  *
  14  * The Original Code and all software distributed under the License are
  15  * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  16  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  17  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
  18  * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
  19  * Please see the License for the specific language governing rights and
  20  * limitations under the License.
  21  *
  22  * @APPLE_LICENSE_HEADER_END@
  23  */
  24
  25 #include <sys/types.h>
  26 #include <sys/stat.h>
  27 #include <sys/mman.h>
  28 #include <mach/mach.h>
  29 #include <mach/mach_time.h>
  30 #include <limits.h>
  31 #include <stdarg.h>
  32 #include <stdio.h>
  33 #include <stdlib.h>
  34 #include <math.h>
  35 #include <fcntl.h>
  36 #include <dlfcn.h>
  37 #include <signal.h>
  38 #include <errno.h>
  39 #include <sys/uio.h>
  40 #include <unistd.h>
  41 #include <sys/param.h>
  42 #include <sys/sysctl.h>
  43 #include <sys/resource.h>
  44 #include <dirent.h>
  45
  46 #include <vector>
  47 #include <set>
  48 #include <map>
  49
  50 #include "mega-dylib-utils.h"
  51
  52 /*
  53  This is a compatibility driver to allow us to support migrating to the new cache codebase and format without forcing B&I to alter gencaches.
  54  This tool only supports flags used by B&I
  55  This intention is for this tool to be removed in the near future and replaced with a tool that is not commandline compatible,
  56    but allows shared caches to be built directly out of BuildRecords.
  57  */
  58
  59 /*
  60    Example commandline:
  61    [60042] INFO - Executing Command: /SWE/Teams/DT-SWB/iOS/Binaries/Binaries5/update_dyld_shared_cache/update_dyld_shared_cache-357.0.91~2/Root/Applications/Xcode.app/Contents/Developer/Platforms/iPhoneOS.platform/usr/local/bin/update_dyld_shared_cache
  62         -debug
  63         -root /BuildRoot//Applications/Xcode.app/Contents/Developer/Platforms/iPhoneOS.platform/Developer/SDKs/iPhoneOS9.0.Internal.sdk
  64     -dylib_list /tmp/dylibs.K93aInternalOS.60042
  65         -cache_dir /tmp/K93aInternalOS.60042
  66     -arch armv7
  67     -iPhone
  68         -dont_map_local_symbols
  69         -overlay /SWE/Teams/DT-SWB/iOS/Binaries/Binaries5/XPCService_caches/XPCService_caches-119~94/Root/K93aInternalOS/
  70         -overlay /SWE/Teams/DT-SWB/iOS/Binaries/Binaries5/libxpc_caches/libxpc_caches-649~18/Root/K93aInternalOS/
  71  */
  72
  73 // record warnings and add to .map file
  74 static std::vector<std::unique_ptr<const char>> sWarnings;
  75
  76
  77 static void write_cache(const char* cachePath, SharedCache& cache) {
  78         // create temp file for cache
  79         int fd = ::open(cachePath, O_CREAT | O_RDWR | O_TRUNC, 0644);
  80         if ( fd == -1 )
  81                 terminate("can't create temp file %s, errnor=%d", cachePath, errno);
  82
  83         // try to allocate whole cache file contiguously
  84         fstore_t fcntlSpec = { F_ALLOCATECONTIG|F_ALLOCATEALL, F_PEOFPOSMODE, 0, static_cast<off_t>(cache.fileSize()), 0 };
  85         ::fcntl(fd, F_PREALLOCATE, &fcntlSpec);
  86
  87         // write out cache file
  88         if ( ::pwrite(fd, cache.buffer().get(), cache.fileSize(), 0) != cache.fileSize() )
  89         terminate("write() failure creating cache file, errno=%d", errno);
  90
  91         // flush to disk and close
  92         int result = ::fcntl(fd, F_FULLFSYNC, NULL);
  93         if ( result == -1 )
  94         warning("fcntl(F_FULLFSYNC) failed with errno=%d for %s\n", errno, cachePath);
  95         ::close(fd);
  96
  97         // write .map file
  98         char mapPath[MAXPATHLEN];
  99         strlcpy(mapPath, cachePath, MAXPATHLEN);
 100         strlcat(mapPath, ".map", MAXPATHLEN);
 101         cache.writeCacheMapFile(mapPath);
 102
 103         // append any warnings encountered
 104         if ( !sWarnings.empty() ) {
 105                 FILE* fmap = ::fopen(mapPath, "a");
 106                 if ( fmap != NULL ) {
 107                         fprintf(fmap, "\n\n");
 108                         for ( std::unique_ptr<const char>& msg : sWarnings ) {
 109                                 fprintf(fmap, "# %s", &*msg);
 110                         }
 111                         ::fclose(fmap);
 112                 }
 113         }
 114 }
 115
 116 int main(int argc, const char* argv[])
 117 {
 118         std::set<ArchPair> onlyArchs;
 119         const char *rootPath = NULL;
 120         std::vector<const char*> searchPaths;
 121         std::vector<std::unique_ptr<DylibProxy>> dylibs;
 122         const char* dylibListFile = NULL;
 123 //      bool force = false;
 124 //      bool keepSignatures = false;
 125         bool explicitCacheDir = false;
 126         bool dontMapLocalSymbols = false;
 127         bool verbose = false;
 128         bool iPhoneOS = false;
 129         const char* cacheDir = NULL;
 130         const char* archStr = NULL;
 131         ArchPair archPair(CPU_TYPE_ARM64, CPU_SUBTYPE_ARM64_ALL);
 132
 133         // parse command line options
 134         for(int i=1; i < argc; ++i) {
 135                 const char* arg = argv[i];
 136                 if ( arg[0] == '-' ) {
 137                         if ( strcmp(arg, "-debug") == 0 ) {
 138                                 verbose = true;
 139                         }
 140                         else if ( strcmp(arg, "-dont_map_local_symbols") == 0 ) {
 141                                 dontMapLocalSymbols = true;
 142                         }
 143                         else if ( strcmp(arg, "-iPhone") == 0 ) {
 144                                 iPhoneOS = true;
 145                         }
 146                         else if ( strcmp(arg, "-dylib_list") == 0 ) {
 147                                 dylibListFile = argv[++i];
 148                                 if ( dylibListFile == NULL )
 149                                         terminate("-dylib_list missing path argument\n");
 150                         }
 151                         else if ( (strcmp(arg, "-root") == 0) || (strcmp(arg, "--root") == 0) ) {
 152                                 rootPath = argv[++i];
 153                         }
 154                         else if ( strcmp(arg, "-overlay") == 0 ) {
 155                                 const char* path = argv[++i];
 156                                 if ( path == NULL )
 157                                         terminate("-overlay missing path argument\n");
 158                                 searchPaths.push_back(path);
 159                         }
 160                         else if ( strcmp(arg, "-cache_dir") == 0 ) {
 161                                 cacheDir = argv[++i];
 162                                 if ( cacheDir == NULL )
 163                                         terminate("-cache_dir missing path argument\n");
 164                                 explicitCacheDir = true;
 165                         }
 166                         else if ( strcmp(arg, "-arch") == 0 ) {
 167                                 archStr = argv[++i];
 168                                 archPair = archForString(archStr);  // terminates if unknown
 169                         }
 170                         else if ( strcmp(arg, "-force") == 0 ) {
 171                                 // ignore.  always forced for iOS
 172                         }
 173                         else {
 174                                 //usage();
 175                                 terminate("unknown option: %s\n", arg);
 176                         }
 177                 }
 178                 else {
 179                         //usage();
 180                         terminate("unknown option: %s\n", arg);
 181                 }
 182         }
 183
 184         if (!rootPath) {
 185                 terminate("-root is a required option\n");
 186         }
 187         if (!iPhoneOS) {
 188                 terminate("-iPhone is a required option\n");
 189         }
 190         if (!cacheDir) {
 191                 terminate("-cache_dir is a required option\n");
 192         }
 193         if (!dylibListFile) {
 194                 terminate("-dylib_list is a required option\n");
 195         }
 196         if (!archStr) {
 197                 terminate("-arch is a required option\n");
 198         }
 199
 200         char prodCachePath[MAXPATHLEN];
 201         char devCachePath[MAXPATHLEN];
 202         strcpy(prodCachePath, cacheDir);
 203         if ( prodCachePath[strlen(prodCachePath)-1] != '/' )
 204                 strcat(prodCachePath, "/");
 205         strcat(prodCachePath, "dyld_shared_cache_");
 206         strcat(prodCachePath, archStr);
 207         strcpy(devCachePath, prodCachePath);
 208         strcat(devCachePath, ".development");
 209
 210         verboseLog("developement cache path = %s\n", devCachePath);
 211         verboseLog("cache path = %s\n", prodCachePath);
 212
 213         // create cache dirs if needed
 214         char neededDirs[1024];
 215         strcpy(neededDirs, prodCachePath);
 216         char* lastSlash = strrchr(neededDirs, '/');
 217         if ( lastSlash != NULL )
 218                 lastSlash[1] = '\0';
 219         struct stat stat_buf;
 220         if ( stat(neededDirs, &stat_buf) != 0 ) {
 221                 const char* afterSlash = &neededDirs[1];
 222                 char* slash;
 223                 while ( (slash = strchr(afterSlash, '/')) != NULL ) {
 224                         *slash = '\0';
 225                         ::mkdir(neededDirs, S_IRWXU | S_IRGRP|S_IXGRP | S_IROTH|S_IXOTH);
 226                         *slash = '/';
 227                         afterSlash = slash+1;
 228                 }
 229         }
 230
 231         std::string dylibOrderFile = toolDir() + "/dylib-order.txt";
 232         std::string dirtyDataOrderFile = toolDir() + "/dirty-data-segments-order.txt";
 233
 234         std::unordered_map<std::string, std::unordered_set<std::string>> dependents;
 235         SharedCache devCache(rootPath, searchPaths, dylibListFile, archPair, dylibOrderFile, dirtyDataOrderFile);
 236
 237         if ( devCache.fileSize() == 0 )
 238                 terminate("Could not find all necessary dylibs\n");
 239
 240         devCache.buildUnoptimizedCache();
 241
 242         SharedCache prodCache = devCache;
 243
 244         prodCache.optimizeForProduction();
 245         devCache.optimizeForDevelopment();
 246
 247         verboseLog("developement cache size = %llu", devCache.fileSize());
 248         verboseLog("developement cache vm size = %llu", devCache.vmSize());
 249
 250         // optimize cache
 251         write_cache(devCachePath, devCache);
 252         if ( devCache.vmSize()+align(devCache.vmSize()/200, sharedRegionRegionAlignment(archPair)) > sharedRegionRegionSize(archPair)) {
 253                 terminate("update_dyld_shared_cache[%u] for arch=%s, shared cache will not fit in shared regions address space.  Overflow amount: %llu\n",
 254                                         getpid(), archStr, devCache.vmSize()+align(devCache.vmSize()/200, sharedRegionRegionAlignment(archPair)) - sharedRegionRegionSize(archPair));
 255         }
 256
 257         write_cache(prodCachePath, prodCache);
 258
 259         return 0;
 260 }
 261
 262
 263 void uniqueWarning(const char* format, ...)
 264 {
 265         va_list list;
 266         va_start(list, format);
 267         vfprintf(stderr, format, list);
 268         va_end(list);
 269         fprintf(stderr, "\n");
 270 }
 271
 272 void log(const char * __restrict format, ...)
 273 {
 274         va_list list;
 275         va_start(list, format);
 276         vfprintf(stderr, format, list);
 277         va_end(list);
 278         fprintf(stderr, "\n");
 279 }
 280 void verboseLog(const char* format, ...)
 281 {
 282         va_list list;
 283         va_start(list, format);
 284         vfprintf(stderr, format, list);
 285         va_end(list);
 286         fprintf(stderr, "\n");
 287 }
 288
 289 void warning(const char* format, ...)
 290 {
 291         fprintf(stderr, "update_dyld_shared_cache warning: ");
 292         va_list list;
 293         va_start(list, format);
 294         char* msg;
 295         vasprintf(&msg, format, list);
 296         va_end(list);
 297         fprintf(stderr, "%s\n", msg);
 298         sWarnings.push_back(std::unique_ptr<const char>(msg));
 299 }
 300
 301 void terminate(const char* format, ...)
 302 {
 303         fprintf(stderr, "update_dyld_shared_cache error: ");
 304         va_list list;
 305         va_start(list, format);
 306         vfprintf(stderr, format, list);
 307         va_end(list);
 308         exit(1);
 309 }