dyld3/shared-cache/BuilderUtils.mm

   1 /*
   2  * Copyright (c) 2017 Apple Inc. All rights reserved.
   3  *
   4  * @APPLE_LICENSE_HEADER_START@
   5  *
   6  * This file contains Original Code and/or Modifications of Original Code
   7  * as defined in and that are subject to the Apple Public Source License
   8  * Version 2.0 (the 'License'). You may not use this file except in
   9  * compliance with the License. Please obtain a copy of the License at
  10  * http://www.opensource.apple.com/apsl/ and read it before using this
  11  * file.
  12  *
  13  * The Original Code and all software distributed under the License are
  14  * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  15  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  16  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
  17  * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
  18  * Please see the License for the specific language governing rights and
  19  * limitations under the License.
  20  *
  21  * @APPLE_LICENSE_HEADER_END@
  22  */
  23
  24
  25 #include <set>
  26 #include <array>
  27 #include <string>
  28 #include <sstream>
  29 #include <iomanip> // std::setfill, std::setw
  30 #include <pthread.h>
  31 #include <mach/mach.h>
  32 #include <sys/types.h>
  33 #include <sys/sysctl.h>
  34 #include <dispatch/dispatch.h>
  35
  36 #include <Bom/Bom.h>
  37 #include <Security/Security.h>
  38 #include <Security/SecCodeSigner.h>
  39 #include <CommonCrypto/CommonCrypto.h>
  40
  41 #include "Manifest.h"
  42 #include "Diagnostics.h"
  43 #include "FileUtils.h"
  44
  45 #include "BuilderUtils.h"
  46
  47 static dispatch_queue_t write_queue = dispatch_queue_create("com.apple.dyld.cache-builder.write", DISPATCH_QUEUE_CONCURRENT);
  48 static dispatch_group_t build_group = dispatch_group_create();
  49
  50 dispatch_group_t buildGroup() {
  51     return build_group;
  52 }
  53
  54 void insertFileInBom(const std::string& path, BOMBom bom)
  55 {
  56     std::vector<std::string> components;
  57     std::vector<std::string> processed_components;
  58     std::stringstream ss(path);
  59     std::string item;
  60
  61     while (std::getline(ss, item, '/')) {
  62         if (!item.empty()) {
  63             components.push_back(item);
  64         }
  65     }
  66
  67     std::string partialPath = ".";
  68     std::string lastComponent = components.back();
  69     components.pop_back();
  70     BOMFSObject fso = BOMFSObjectNew(BOMDirectoryType);
  71     BOMFSObjectSetFlags(fso, B_PATHONLY);
  72     BOMFSObjectSetPathName(fso, ".", true);
  73     BOMFSObjectSetShortName(fso, ".", true);
  74     (void)BOMBomInsertFSObject(bom, fso, false);
  75     BOMFSObjectFree(fso);
  76
  77     for (const auto& component : components) {
  78         partialPath = partialPath + "/" + component;
  79         fso = BOMFSObjectNew(BOMDirectoryType);
  80         BOMFSObjectSetFlags(fso, B_PATHONLY);
  81         BOMFSObjectSetPathName(fso, partialPath.c_str(), true);
  82         BOMFSObjectSetShortName(fso, component.c_str(), true);
  83         (void)BOMBomInsertFSObject(bom, fso, false);
  84         BOMFSObjectFree(fso);
  85     }
  86
  87     partialPath = partialPath + "/" + lastComponent;
  88     fso = BOMFSObjectNew(BOMFileType);
  89     BOMFSObjectSetFlags(fso, B_PATHONLY);
  90     BOMFSObjectSetPathName(fso, partialPath.c_str(), true);
  91     BOMFSObjectSetShortName(fso, lastComponent.c_str(), true);
  92     (void)BOMBomInsertFSObject(bom, fso, false);
  93     BOMFSObjectFree(fso);
  94 }
  95
  96 void makeBoms(dyld3::Manifest& manifest, const std::string& masterDstRoot)
  97 {
  98     mkpath_np((masterDstRoot + "/Boms/").c_str(), 0755);
  99
 100     manifest.forEachConfiguration([&manifest, &masterDstRoot](const std::string& configName) {
 101         auto                     config = manifest.configuration(configName);
 102         std::vector<std::string> prodBomPaths;
 103         std::vector<std::string> devBomPaths;
 104
 105         std::string runtimePath =  "/System/Library/Caches/com.apple.dyld/";
 106         if (manifest.platform() == dyld3::Platform::macOS) {
 107             runtimePath =  "/private/var/db/dyld/";
 108         }
 109
 110         for (auto& arch : config.architectures) {
 111             std::string cachePath = "dyld_shared_cache_" + arch.first;
 112             prodBomPaths.push_back(cachePath);
 113             if (manifest.platform() != dyld3::Platform::macOS) {
 114                 cachePath += ".development";
 115             }
 116             devBomPaths.push_back(cachePath);
 117             char buffer[MAXPATHLEN];
 118             sprintf(buffer, "%s/Boms/%s.prod.bom", masterDstRoot.c_str(), configName.c_str());
 119             BOMBom bom = BOMBomNew(buffer);
 120             for (auto& path : prodBomPaths) {
 121                 insertFileInBom(runtimePath + path, bom);
 122             }
 123             BOMBomFree(bom);
 124
 125             sprintf(buffer, "%s/Boms/%s.dev.bom", masterDstRoot.c_str(), configName.c_str());
 126             bom = BOMBomNew(buffer);
 127             for (auto& path : devBomPaths) {
 128                 insertFileInBom(runtimePath + path, bom);
 129             }
 130             BOMBomFree(bom);
 131
 132             sprintf(buffer, "%s/Boms/%s.full.bom", masterDstRoot.c_str(), configName.c_str());
 133             bom = BOMBomNew(buffer);
 134             for (auto& path : prodBomPaths) {
 135                 insertFileInBom(runtimePath + path, bom);
 136             }
 137             for (auto& path : devBomPaths) {
 138                 insertFileInBom(runtimePath + path, bom);
 139             }
 140             BOMBomFree(bom);
 141         }
 142     });
 143 }
 144
 145 bool build(Diagnostics& diags, dyld3::Manifest& manifest, const std::string& masterDstRoot, bool dedupe, bool verbose,
 146            bool skipWrites, bool agileChooseSHA256CdHash, bool emitDevCaches, bool isLocallyBuiltCache)
 147 {
 148     dispatch_queue_t                   queue = dispatch_get_global_queue(DISPATCH_QUEUE_PRIORITY_DEFAULT, 0);
 149     dispatch_queue_t                   warningQueue = dispatch_queue_create("com.apple.dyld.cache-builder.warnings", DISPATCH_QUEUE_SERIAL);
 150     std::vector<std::set<std::string>> dedupedCacheSets;
 151     if (dedupe) {
 152         manifest.forEachConfiguration([&manifest, &dedupedCacheSets](const std::string& configName) {
 153             auto config = manifest.configuration(configName);
 154             bool dupeFound = false;
 155
 156             for (auto& cacheSet : dedupedCacheSets) {
 157                 if (config == manifest.configuration(*cacheSet.begin())) {
 158                     cacheSet.insert(configName);
 159                     dupeFound = true;
 160                     break;
 161                 }
 162             }
 163
 164             if (!dupeFound) {
 165                 std::set<std::string> temp;
 166                 temp.insert(configName);
 167                 dedupedCacheSets.push_back(temp);
 168             }
 169         });
 170     } else {
 171         manifest.forEachConfiguration([&manifest, &dedupedCacheSets](const std::string& configName) {
 172             std::set<std::string> temp;
 173             temp.insert(configName);
 174             dedupedCacheSets.push_back(temp);
 175         });
 176     }
 177
 178     std::vector<dyld3::BuildQueueEntry> buildQueue;
 179
 180     for (auto& cacheSet : dedupedCacheSets) {
 181         //FIXME we may want to consider moving to hashes of UUID sets
 182         std::string setName;
 183
 184         for (auto& archName : cacheSet) {
 185             if (!setName.empty()) {
 186                 setName += "|";
 187             }
 188             setName += archName;
 189         }
 190
 191         std::stringstream fileNameStream;
 192 #pragma clang diagnostic push
 193 #pragma clang diagnostic ignored "-Wdeprecated-declarations"
 194         std::array<uint8_t, CC_SHA1_DIGEST_LENGTH> digest = { 0 };
 195         CC_SHA1(setName.c_str(), (unsigned int)setName.length(), &digest[0]);
 196 #pragma clang diagnostic pop
 197
 198         fileNameStream << std::hex << std::uppercase << std::setfill('0');
 199         for (int c : digest) {
 200             fileNameStream << std::setw(2) << c;
 201         }
 202
 203         std::string fileName(fileNameStream.str());
 204
 205         if (dedupe) {
 206             for (auto& config : cacheSet) {
 207                 if (!skipWrites) {
 208                     int err = symlink(("DedupedConfigs/" + fileName).c_str(), (masterDstRoot + "/" + config).c_str());
 209                     if (err) {
 210                         diags.warning("Could not create symlink '%s' -> 'DedupedConfigs/%s' (%d)", config.c_str(), fileName.c_str(), err);
 211                     }
 212                 }
 213             }
 214         }
 215
 216         manifest.configuration(*cacheSet.begin()).forEachArchitecture([&masterDstRoot, &dedupe, &fileName, &setName, &manifest,
 217                                                                        &buildQueue, &cacheSet, skipWrites, verbose, emitDevCaches, isLocallyBuiltCache](const std::string& arch) {
 218             std::string configPath;
 219             std::string runtimePath =  "/System/Library/Caches/com.apple.dyld/";
 220             if (manifest.platform() == dyld3::Platform::macOS) {
 221                 runtimePath =  "/private/var/db/dyld/";
 222             }
 223             if (dedupe) {
 224                 configPath = masterDstRoot + "/DedupedConfigs/" + fileName + runtimePath;
 225             } else {
 226                 configPath = masterDstRoot + runtimePath;
 227             }
 228
 229             mkpath_np(configPath.c_str(), 0755);
 230             if (manifest.platform() == dyld3::Platform::macOS) {
 231                 buildQueue.push_back(manifest.makeQueueEntry(configPath + "dyld_shared_cache_" + arch, cacheSet, arch, false, setName + "/" + arch,
 232                                                              isLocallyBuiltCache, skipWrites, verbose));
 233             } else {
 234                 if (emitDevCaches)
 235                     buildQueue.push_back(manifest.makeQueueEntry(configPath + "dyld_shared_cache_" + arch + ".development", cacheSet, arch, false, setName + "/" + arch,
 236                                                                  isLocallyBuiltCache, skipWrites, verbose));
 237                 buildQueue.push_back(manifest.makeQueueEntry(configPath + "dyld_shared_cache_" + arch, cacheSet, arch, true, setName + "/" + arch,
 238                                                              isLocallyBuiltCache, skipWrites, verbose));
 239             }
 240         });
 241     }
 242
 243     __block bool cacheBuildFailure = false;
 244     __block std::set<std::string> warnings;
 245     __block std::set<std::string> errors;
 246
 247     dispatch_sync(warningQueue, ^{
 248         auto manifestWarnings = diags.warnings();
 249         //warnings.insert(manifestWarnings.begin(), manifestWarnings.end());
 250     });
 251
 252     bool requuiresConcurrencyLimit = false;
 253     dispatch_semaphore_t concurrencyLimit = NULL;
 254     // Limit cuncurrency to 8 threads for machines with 32GB of RAM and to 1 thread if we have 4GB or less of memory
 255     uint64_t memSize = 0;
 256     size_t sz = sizeof(memSize);;
 257     if ( sysctlbyname("hw.memsize", &memSize, &sz, NULL, 0) == 0 ) {
 258         if ( memSize <= 0x100000000ULL ) {
 259             fprintf(stderr, "Detected 4Gb or less of memory, limiting concurrency to 1 thread\n");
 260             requuiresConcurrencyLimit = true;
 261             concurrencyLimit = dispatch_semaphore_create(1);
 262         } else if ( memSize <= 0x800000000ULL ) {
 263             fprintf(stderr, "Detected 32Gb or less of memory, limiting concurrency to 8 threads\n");
 264             requuiresConcurrencyLimit = true;
 265             concurrencyLimit = dispatch_semaphore_create(8);
 266         }
 267     }
 268
 269     dispatch_apply(buildQueue.size(), queue, ^(size_t index) {
 270         auto queueEntry = buildQueue[index];
 271         pthread_setname_np(queueEntry.options.loggingPrefix.substr(0, MAXTHREADNAMESIZE - 1).c_str());
 272
 273         // Horrible hack to limit concurrency in low spec build machines.
 274         if (requuiresConcurrencyLimit) { dispatch_semaphore_wait(concurrencyLimit, DISPATCH_TIME_FOREVER); }
 275         DyldSharedCache::CreateResults results = DyldSharedCache::create(queueEntry.options, queueEntry.fileSystem, queueEntry.dylibsForCache, queueEntry.otherDylibsAndBundles, queueEntry.mainExecutables);
 276         if (requuiresConcurrencyLimit) { dispatch_semaphore_signal(concurrencyLimit); }
 277
 278         dispatch_sync(warningQueue, ^{
 279             warnings.insert(results.warnings.begin(), results.warnings.end());
 280             bool chooseSecondCdHash = agileChooseSHA256CdHash;
 281             if (agileChooseSHA256CdHash && !results.agileSignature) {
 282                 // Ignore this option for caches that are not signed agile (which is the majority).
 283                 chooseSecondCdHash = false;
 284             }
 285             for (const auto& configName : queueEntry.configNames) {
 286                 auto& configResults = manifest.configuration(configName).architecture(queueEntry.options.archs->name()).results;
 287                 for (const auto& mh : results.evictions) {
 288                     configResults.exclude(mh, "VM overflow, evicting");
 289                 }
 290                 configResults.warnings = results.warnings;
 291                 if (queueEntry.options.optimizeStubs) {
 292                     configResults.productionCache.cdHash = chooseSecondCdHash ? results.cdHashSecond : results.cdHashFirst;
 293                 } else {
 294                     configResults.developmentCache.cdHash =  chooseSecondCdHash ? results.cdHashSecond : results.cdHashFirst;
 295                 }
 296             }
 297         });
 298         if (!results.errorMessage.empty()) {
 299             fprintf(stderr, "[%s] ERROR: %s\n", queueEntry.options.loggingPrefix.c_str(), results.errorMessage.c_str());
 300             cacheBuildFailure = true;
 301         } else if (skipWrites) {
 302             fprintf(stderr, "[%s] Skipped writing cache to: %s\n", queueEntry.options.loggingPrefix.c_str(), queueEntry.outputPath.c_str());
 303         }
 304     });
 305
 306     // print any warnings
 307     for (const std::string& warn : warnings) {
 308         fprintf(stderr, "[WARNING] %s\n", warn.c_str());
 309     }
 310
 311     int err = sync_volume_np(masterDstRoot.c_str(), SYNC_VOLUME_FULLSYNC | SYNC_VOLUME_WAIT);
 312     if (err) {
 313         fprintf(stderr, "Volume sync failed errnor=%d (%s)\n", err, strerror(err));
 314     }
 315
 316     return !cacheBuildFailure;
 317 }