interlinked-dylibs/dyld_shared_cache_builder.mm

   1 /* -*- mode: C++; c-basic-offset: 4; indent-tabs-mode: nil -*-
   2  *
   3  * Copyright (c) 2016 Apple Inc. All rights reserved.
   4  *
   5  * @APPLE_LICENSE_HEADER_START@
   6  *
   7  * This file contains Original Code and/or Modifications of Original Code
   8  * as defined in and that are subject to the Apple Public Source License
   9  * Version 2.0 (the 'License'). You may not use this file except in
  10  * compliance with the License. Please obtain a copy of the License at
  11  * http://www.opensource.apple.com/apsl/ and read it before using this
  12  * file.
  13  *
  14  * The Original Code and all software distributed under the License are
  15  * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  16  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  17  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
  18  * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
  19  * Please see the License for the specific language governing rights and
  20  * limitations under the License.
  21  *
  22  * @APPLE_LICENSE_HEADER_END@
  23  */
  24
  25 #include <dispatch/dispatch.h>
  26 #include <Security/Security.h>
  27 #include <Security/SecCodeSigner.h>
  28
  29 #include <sys/types.h>
  30 #include <sys/stat.h>
  31 #include <sys/mman.h>
  32 #include <sys/resource.h>
  33 #include <mach/mach.h>
  34 #include <mach/mach_time.h>
  35 #include <limits.h>
  36 #include <stdarg.h>
  37 #include <stdio.h>
  38 #include <stdlib.h>
  39 #include <math.h>
  40 #include <fcntl.h>
  41 #include <dlfcn.h>
  42 #include <signal.h>
  43 #include <errno.h>
  44 #include <sys/uio.h>
  45 #include <unistd.h>
  46 #include <sys/param.h>
  47 #include <sys/sysctl.h>
  48 #include <sys/resource.h>
  49 #include <dirent.h>
  50 #include <libgen.h>
  51 #include <pthread.h>
  52 #include <fts.h>
  53
  54 #include <vector>
  55 #include <array>
  56 #include <set>
  57 #include <map>
  58 #include <algorithm>
  59
  60 #include <spawn.h>
  61
  62 #include <Bom/Bom.h>
  63
  64 #include "Manifest.h"
  65 #include "MultiCacheBuilder.h"
  66
  67 #include "mega-dylib-utils.h"
  68 #include "Logging.h"
  69
  70 #if !__has_feature(objc_arc)
  71 #error The use of libdispatch in this files requires it to be compiled with ARC in order to avoid leaks
  72 #endif
  73
  74 extern char** environ;
  75
  76 static dispatch_queue_t build_queue;
  77
  78 inline bool has_suffix(std::string const & value, std::string const & ending)
  79 {
  80         if (ending.size() > value.size()) return false;
  81         return std::equal(ending.rbegin(), ending.rend(), value.rbegin());
  82 }
  83
  84 static const char *tempRootDirTemplate = "/tmp/dyld_shared_cache_builder.XXXXXX";
  85 static char *tempRootDir = nullptr;
  86
  87 void processRoot( const std::string &root, std::set<std::string> &roots ) {
  88     struct stat sb;
  89     int res = 0;
  90         pid_t pid;
  91         int status;
  92         const char* args[8];
  93
  94         res = stat(root.c_str(), &sb);
  95
  96         if (res == 0 && S_ISDIR(sb.st_mode)) {
  97             roots.insert( root );
  98             return;
  99         } else if ( has_suffix( root, ".cpio" ) || has_suffix( root, ".cpio.gz" ) || has_suffix( root, ".cpgz" ) ||
 100                     has_suffix( root, ".cpio.bz2" ) || has_suffix( root, ".cpbz2" ) || has_suffix( root, ".pax" ) ||
 101                     has_suffix( root, ".pax.gz" ) || has_suffix( root, ".pgz" ) || has_suffix( root, ".pax.bz2" ) ||
 102                     has_suffix( root, ".pbz2" ) ) {
 103                 args[0] = (char *)"/usr/bin/ditto";
 104                 args[1] = (char *)"-x";
 105                 args[2] = (char *)root.c_str();
 106                 args[3] = tempRootDir;
 107                 args[4] = nullptr;
 108         } else if (has_suffix(root, ".tar")) {
 109                 args[0] = (char *)"/usr/bin/tar";
 110                 args[1] = (char *)"xf";
 111                 args[2] = (char *)root.c_str();
 112                 args[3] = (char *)"-C";
 113                 args[4] = tempRootDir;
 114                 args[5] = nullptr;
 115         } else if (has_suffix(root, ".tar.gz") || has_suffix(root, ".tgz")) {
 116                 args[0] = (char *)"/usr/bin/tar";
 117                 args[1] = (char *)"xzf";
 118                 args[2] = (char *)root.c_str();
 119                 args[3] = (char *)"-C";
 120                 args[4] = tempRootDir;
 121                 args[5] = nullptr;
 122         } else if (has_suffix(root, ".tar.bz2")
 123                            || has_suffix(root, ".tbz2")
 124                            || has_suffix(root, ".tbz")) {
 125                 args[0] = (char *)"/usr/bin/tar";
 126                 args[1] = (char *)"xjf";
 127                 args[2] = (char *)root.c_str();
 128                 args[3] = (char *)"-C";
 129                 args[4] = tempRootDir;
 130                 args[5] = nullptr;
 131         } else if (has_suffix(root, ".xar")) {
 132                 args[0] = (char *)"/usr/bin/xar";
 133                 args[1] = (char *)"-xf";
 134                 args[2] = (char *)root.c_str();
 135                 args[3] = (char *)"-C";
 136                 args[4] = tempRootDir;
 137                 args[5] = nullptr;
 138         } else if (has_suffix(root, ".zip")) {
 139                 args[0] = (char *)"/usr/bin/ditto";
 140                 args[1] = (char *)"-xk";
 141                 args[2] = (char *)root.c_str();
 142                 args[3] = tempRootDir;
 143                 args[4] = nullptr;
 144         } else {
 145                 terminate("unknown archive type: %s", root.c_str());
 146         }
 147
 148         res = posix_spawn(&pid, args[0], nullptr, nullptr, (char**)args, environ);
 149         if (res != 0) terminate("Failed to spawn %s: %s (%d)", args[0], strerror(res), res);
 150
 151         do {
 152                 res = waitpid(pid, &status, 0);
 153         } while (res == -1 && errno == EINTR);
 154         if (res != -1) {
 155                 if (WIFEXITED(status)) {
 156                         res = WEXITSTATUS(status);
 157                 } else {
 158                         res = -1;
 159                 }
 160         }
 161
 162         if (res != 0) terminate("Could not expand archive %s: %s (%d)", root.c_str(), strerror(res), res);
 163
 164         for (auto &existingRoot : roots) {
 165                 if (existingRoot == tempRootDir)
 166                         return;
 167         }
 168
 169         roots.insert( tempRootDir );
 170 }
 171
 172 bool writeRootList( const std::string &dstRoot, const std::set<std::string> &roots ) {
 173     if ( roots.size() == 0 ) return false;
 174
 175     std::string rootFile = dstRoot + "/roots.txt";
 176         FILE* froots = ::fopen(rootFile.c_str(), "w");
 177         if ( froots == NULL )
 178                 return false;
 179
 180         for (auto &root : roots) {
 181                         fprintf(froots, "%s\n", root.c_str());
 182         }
 183
 184         ::fclose(froots);
 185         return true;
 186 }
 187
 188 std::set<std::string> cachePaths;
 189
 190 BOMCopierCopyOperation filteredCopy(BOMCopier copier, const char *path, BOMFSObjType type, off_t size) {
 191         std::string absolutePath = &path[1];
 192         if (cachePaths.count(absolutePath)) {
 193                 return BOMCopierSkipFile;
 194         }
 195         return BOMCopierContinue;
 196 }
 197
 198 int main (int argc, const char * argv[]) {
 199         @autoreleasepool {
 200                 std::set<std::string>    roots;
 201                 std::vector<std::string> inputRoots;
 202                 std::string              dylibCacheDir;
 203                 std::string              release;
 204                 bool                     emitDevCaches = true;
 205                 bool                     emitElidedDylibs = true;
 206                 bool                     listConfigs = false;
 207                 bool                     copyRoots = false;
 208                 std::string              dstRoot;
 209                 std::string              configuration;
 210                 std::string              resultPath;
 211
 212                 time_t mytime = time(0);
 213                 log("Started: %s", asctime(localtime(&mytime)));
 214
 215                 tempRootDir = strdup(tempRootDirTemplate);
 216                 mkdtemp(tempRootDir);
 217
 218                 for (int i = 1; i < argc; ++i) {
 219                         const char* arg = argv[i];
 220                         if (arg[0] == '-') {
 221                                 if (strcmp(arg, "-debug") == 0) {
 222                                         setVerbose(true);
 223                                 } else if (strcmp(arg, "-list_configs") == 0) {
 224                                         listConfigs = true;
 225                                 } else if (strcmp(arg, "-root") == 0) {
 226                                         std::string root = argv[++i];
 227                                         inputRoots.push_back(root);
 228                                         processRoot(root, roots);
 229                                 } else if (strcmp(arg, "-copy_roots") == 0) {
 230                                         copyRoots = true;
 231                                 } else if (strcmp(arg, "-dylib_cache") == 0) {
 232                                         dylibCacheDir = argv[++i];
 233                                 } else if (strcmp(arg, "-no_development_cache") == 0) {
 234                                         emitDevCaches = false;
 235                                 } else if (strcmp(arg, "-no_overflow_dylibs") == 0) {
 236                                         emitElidedDylibs = false;
 237                                 } else if (strcmp(arg, "-development_cache") == 0) {
 238                                         emitDevCaches = true;
 239                                 } else if (strcmp(arg, "-overflow_dylibs") == 0) {
 240                                         emitElidedDylibs = true;
 241                                 } else if (strcmp(arg, "-dst_root") == 0) {
 242                                         dstRoot = argv[++i];
 243                                 } else if (strcmp(arg, "-release") == 0) {
 244                                         release = argv[++i];
 245                                 } else if (strcmp(arg, "-results") == 0) {
 246                                         resultPath = argv[++i];
 247                                 } else {
 248                                         //usage();
 249                                         terminate("unknown option: %s\n", arg);
 250                                 }
 251                         } else {
 252                                 if (!configuration.empty()) {
 253                                         terminate("You may only specify one configruation");
 254                                 }
 255                                 configuration = argv[i];
 256                         }
 257                 }
 258
 259                 struct rlimit rl = {OPEN_MAX, OPEN_MAX};
 260                 (void)setrlimit(RLIMIT_NOFILE, &rl);
 261
 262                 if (dylibCacheDir.empty() && release.empty()) {
 263                         terminate("you must specify either -dylib_cache or -release");
 264                 } else if (!dylibCacheDir.empty() && !release.empty()) {
 265                         terminate("you may not use -dylib_cache and -release at the same time");
 266                 }
 267
 268                 if ((configuration.empty() || dstRoot.empty()) && !listConfigs) {
 269                         terminate("Must specify a configuration and a -dst_root OR -list_configs\n");
 270                 }
 271
 272
 273                 if (dylibCacheDir.empty()) {
 274                         dylibCacheDir = std::string("/AppleInternal/Developer/DylibCaches/") + release + ".dlc";
 275                 }
 276
 277                 //Move into the dir so we can use relative path manifests
 278                 chdir(dylibCacheDir.c_str());
 279
 280                 auto manifest = Manifest(dylibCacheDir + "/Manifest.plist", roots);
 281
 282                 if (manifest.build.empty()) {
 283                         terminate("No manifest found at '%s/Manifest.plist'\n", dylibCacheDir.c_str());
 284                 }
 285                 log("Building Caches for %s", manifest.build.c_str());
 286
 287                 if (listConfigs) {
 288                         for (auto& config : manifest.configurations) {
 289                                 printf("%s\n", config.first.c_str());
 290                         }
 291                         exit(0);
 292                 }
 293
 294                 std::map<std::string, Manifest::Configuration> filteredConfigs;
 295
 296                 for (auto& config : manifest.configurations) {
 297                         if (config.first == configuration) {
 298                                 filteredConfigs[config.first] = config.second;
 299
 300                                 for (auto& arch : filteredConfigs[config.first].architectures) {
 301                                         arch.second.results = Manifest::Results();
 302                                 }
 303                         }
 304                 }
 305
 306                 if ( filteredConfigs.empty() ) {
 307                         terminate( "No config %s. Please run with -list_configs to see configurations available for this %s.\n",
 308                                    configuration.c_str(), manifest.build.c_str() );
 309                 }
 310
 311                 manifest.configurations = filteredConfigs;
 312                 manifest.calculateClosure(false);
 313
 314                 // FIXME: Plumb through no_development
 315
 316                 std::shared_ptr<MultiCacheBuilder> builder = std::make_shared<MultiCacheBuilder>( manifest, false, false, true );
 317                 builder->buildCaches(dstRoot);
 318                 writeRootList(dstRoot, roots);
 319
 320                 if (copyRoots) {
 321                         for (auto& config : manifest.configurations) {
 322                                 for (auto& arch : config.second.architectures) {
 323                                         for (auto& dylib : arch.second.results.dylibs) {
 324                                                 if (dylib.second.included) {
 325                                                         MachOProxy *proxy = manifest.dylibProxy( dylib.first, arch.first );
 326                                                         cachePaths.insert( proxy->installName );
 327                                                         for ( auto &alias : proxy->installNameAliases ) {
 328                                                                                 cachePaths.insert(alias);
 329                                                         }
 330                                                 }
 331                                         }
 332                                 }
 333                         }
 334
 335                         BOMCopier copier = BOMCopierNewWithSys(BomSys_default());
 336                         BOMCopierSetCopyFileStartedHandler(copier, filteredCopy);
 337                         for (auto& root : roots) {
 338                                 BOMCopierCopy(copier, root.c_str(), dstRoot.c_str());
 339                         }
 340                         BOMCopierFree(copier);
 341                 }
 342
 343         // Create an empty FIPS data in the root
 344         (void)mkpath_np((dstRoot + "/private/var/db/FIPS/").c_str(), 0755);
 345         int fd = open((dstRoot + "/private/var/db/FIPS/fips_data").c_str(), O_CREAT | O_TRUNC, 0644);
 346         close(fd);
 347
 348                 // Now that all the build commands have been issued lets put a barrier in after then which can tear down the app after
 349                 // everything is written.
 350
 351                 builder->logStats();
 352                 if (!resultPath.empty()) {
 353                         manifest.write(resultPath);
 354                 }
 355
 356                 pid_t       pid;
 357                 int         res = 0;
 358                 int         status;
 359                 const char* args[8];
 360
 361                 args[0] = (char*)"/bin/rm";
 362                 args[1] = (char*)"-rf";
 363                 args[2] = (char*)tempRootDir;
 364         args[3] = nullptr;
 365
 366         res = posix_spawn(&pid, args[0], nullptr, nullptr, (char**)args, environ);
 367                 if (res != 0)
 368                         terminate("Failed to spawn %s: %s (%d)", args[0], strerror(res), res);
 369
 370                 do {
 371                         res = waitpid(pid, &status, 0);
 372                 } while (res == -1 && errno == EINTR);
 373                 if (res != -1) {
 374                         if (WIFEXITED(status)) {
 375                                 res = WEXITSTATUS(status);
 376                         } else {
 377                                 res = -1;
 378                         }
 379                 }
 380
 381                 dumpLogAndExit();
 382         }
 383
 384         dispatch_main();
 385
 386         return 0;
 387 }