dyld3/shared-cache/BuilderUtils.mm

   1 /*
   2  * Copyright (c) 2017 Apple Inc. All rights reserved.
   3  *
   4  * @APPLE_LICENSE_HEADER_START@
   5  *
   6  * This file contains Original Code and/or Modifications of Original Code
   7  * as defined in and that are subject to the Apple Public Source License
   8  * Version 2.0 (the 'License'). You may not use this file except in
   9  * compliance with the License. Please obtain a copy of the License at
  10  * http://www.opensource.apple.com/apsl/ and read it before using this
  11  * file.
  12  *
  13  * The Original Code and all software distributed under the License are
  14  * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  15  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  16  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
  17  * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
  18  * Please see the License for the specific language governing rights and
  19  * limitations under the License.
  20  *
  21  * @APPLE_LICENSE_HEADER_END@
  22  */
  23
  24
  25 #include <set>
  26 #include <array>
  27 #include <string>
  28 #include <sstream>
  29 #include <iomanip> // std::setfill, std::setw
  30 #include <pthread.h>
  31 #include <mach/mach.h>
  32 #include <dispatch/dispatch.h>
  33
  34 #include <Bom/Bom.h>
  35 #include <Security/Security.h>
  36 #include <Security/SecCodeSigner.h>
  37 #include <CommonCrypto/CommonCrypto.h>
  38
  39 #include "Manifest.h"
  40 #include "Diagnostics.h"
  41 #include "FileUtils.h"
  42
  43 #include "BuilderUtils.h"
  44
  45 static dispatch_queue_t write_queue = dispatch_queue_create("com.apple.dyld.cache-builder.write", DISPATCH_QUEUE_CONCURRENT);
  46 static dispatch_group_t build_group = dispatch_group_create();
  47
  48 dispatch_group_t buildGroup() {
  49     return build_group;
  50 }
  51
  52 void insertFileInBom(const std::string& path, BOMBom bom)
  53 {
  54     std::vector<std::string> components;
  55     std::vector<std::string> processed_components;
  56     std::stringstream ss(path);
  57     std::string item;
  58
  59     while (std::getline(ss, item, '/')) {
  60         if (!item.empty()) {
  61             components.push_back(item);
  62         }
  63     }
  64
  65     std::string partialPath = ".";
  66     std::string lastComponent = components.back();
  67     components.pop_back();
  68     BOMFSObject fso = BOMFSObjectNew(BOMDirectoryType);
  69     BOMFSObjectSetFlags(fso, B_PATHONLY);
  70     BOMFSObjectSetPathName(fso, ".", true);
  71     BOMFSObjectSetShortName(fso, ".", true);
  72     (void)BOMBomInsertFSObject(bom, fso, false);
  73     BOMFSObjectFree(fso);
  74
  75     for (const auto& component : components) {
  76         partialPath = partialPath + "/" + component;
  77         fso = BOMFSObjectNew(BOMDirectoryType);
  78         BOMFSObjectSetFlags(fso, B_PATHONLY);
  79         BOMFSObjectSetPathName(fso, partialPath.c_str(), true);
  80         BOMFSObjectSetShortName(fso, component.c_str(), true);
  81         (void)BOMBomInsertFSObject(bom, fso, false);
  82         BOMFSObjectFree(fso);
  83     }
  84
  85     partialPath = partialPath + "/" + lastComponent;
  86     fso = BOMFSObjectNew(BOMFileType);
  87     BOMFSObjectSetFlags(fso, B_PATHONLY);
  88     BOMFSObjectSetPathName(fso, partialPath.c_str(), true);
  89     BOMFSObjectSetShortName(fso, lastComponent.c_str(), true);
  90     (void)BOMBomInsertFSObject(bom, fso, false);
  91     BOMFSObjectFree(fso);
  92 }
  93
  94 void makeBoms(dyld3::Manifest& manifest, const std::string& masterDstRoot)
  95 {
  96     mkpath_np((masterDstRoot + "/Boms/").c_str(), 0755);
  97
  98     manifest.forEachConfiguration([&manifest, &masterDstRoot](const std::string& configName) {
  99         auto                     config = manifest.configuration(configName);
 100         std::vector<std::string> prodBomPaths;
 101         std::vector<std::string> devBomPaths;
 102
 103         std::string runtimePath =  "/System/Library/Caches/com.apple.dyld/";
 104         if (manifest.platform() == dyld3::Platform::macOS) {
 105             runtimePath =  "/private/var/db/dyld/";
 106         }
 107
 108         for (auto& arch : config.architectures) {
 109             std::string cachePath = "dyld_shared_cache_" + arch.first;
 110             prodBomPaths.push_back(cachePath);
 111             if (manifest.platform() != dyld3::Platform::macOS) {
 112                 cachePath += ".development";
 113             }
 114             devBomPaths.push_back(cachePath);
 115             char buffer[MAXPATHLEN];
 116             sprintf(buffer, "%s/Boms/%s.prod.bom", masterDstRoot.c_str(), configName.c_str());
 117             BOMBom bom = BOMBomNew(buffer);
 118             for (auto& path : prodBomPaths) {
 119                 insertFileInBom(runtimePath + path, bom);
 120             }
 121             BOMBomFree(bom);
 122
 123             sprintf(buffer, "%s/Boms/%s.dev.bom", masterDstRoot.c_str(), configName.c_str());
 124             bom = BOMBomNew(buffer);
 125             for (auto& path : devBomPaths) {
 126                 insertFileInBom(runtimePath + path, bom);
 127             }
 128             BOMBomFree(bom);
 129
 130             sprintf(buffer, "%s/Boms/%s.full.bom", masterDstRoot.c_str(), configName.c_str());
 131             bom = BOMBomNew(buffer);
 132             for (auto& path : prodBomPaths) {
 133                 insertFileInBom(runtimePath + path, bom);
 134             }
 135             for (auto& path : devBomPaths) {
 136                 insertFileInBom(runtimePath + path, bom);
 137             }
 138             BOMBomFree(bom);
 139         }
 140     });
 141 }
 142
 143 bool build(Diagnostics& diags, dyld3::Manifest& manifest, const std::string& masterDstRoot, bool dedupe, bool verbose,
 144            bool skipWrites, bool agileChooseSHA256CdHash, bool emitDevCaches, bool isLocallyBuiltCache)
 145 {
 146     dispatch_queue_t                   queue = dispatch_get_global_queue(DISPATCH_QUEUE_PRIORITY_DEFAULT, 0);
 147     dispatch_queue_t                   warningQueue = dispatch_queue_create("com.apple.dyld.cache-builder.warnings", DISPATCH_QUEUE_SERIAL);
 148     std::vector<std::set<std::string>> dedupedCacheSets;
 149     if (dedupe) {
 150         manifest.forEachConfiguration([&manifest, &dedupedCacheSets](const std::string& configName) {
 151             auto config = manifest.configuration(configName);
 152             bool dupeFound = false;
 153
 154             for (auto& cacheSet : dedupedCacheSets) {
 155                 if (config == manifest.configuration(*cacheSet.begin())) {
 156                     cacheSet.insert(configName);
 157                     dupeFound = true;
 158                     break;
 159                 }
 160             }
 161
 162             if (!dupeFound) {
 163                 std::set<std::string> temp;
 164                 temp.insert(configName);
 165                 dedupedCacheSets.push_back(temp);
 166             }
 167         });
 168     } else {
 169         manifest.forEachConfiguration([&manifest, &dedupedCacheSets](const std::string& configName) {
 170             std::set<std::string> temp;
 171             temp.insert(configName);
 172             dedupedCacheSets.push_back(temp);
 173         });
 174     }
 175
 176     std::vector<dyld3::BuildQueueEntry> buildQueue;
 177
 178     for (auto& cacheSet : dedupedCacheSets) {
 179         //FIXME we may want to consider moving to hashes of UUID sets
 180         std::string setName;
 181
 182         for (auto& archName : cacheSet) {
 183             if (!setName.empty()) {
 184                 setName += "|";
 185             }
 186             setName += archName;
 187         }
 188
 189         std::stringstream fileNameStream;
 190         std::array<uint8_t, CC_SHA1_DIGEST_LENGTH> digest = { 0 };
 191         CC_SHA1(setName.c_str(), (unsigned int)setName.length(), &digest[0]);
 192
 193         fileNameStream << std::hex << std::uppercase << std::setfill('0');
 194         for (int c : digest) {
 195             fileNameStream << std::setw(2) << c;
 196         }
 197
 198         std::string fileName(fileNameStream.str());
 199
 200         if (dedupe) {
 201             for (auto& config : cacheSet) {
 202                 if (!skipWrites) {
 203                     int err = symlink(("DedupedConfigs/" + fileName).c_str(), (masterDstRoot + "/" + config).c_str());
 204                     if (err) {
 205                         diags.warning("Could not create symlink '%s' -> 'DedupedConfigs/%s' (%d)", config.c_str(), fileName.c_str(), err);
 206                     }
 207                 }
 208             }
 209         }
 210
 211         manifest.configuration(*cacheSet.begin()).forEachArchitecture([&masterDstRoot, &dedupe, &fileName, &setName, &manifest,
 212                                                                        &buildQueue, &cacheSet, skipWrites, verbose, emitDevCaches, isLocallyBuiltCache](const std::string& arch) {
 213             std::string configPath;
 214             std::string runtimePath =  "/System/Library/Caches/com.apple.dyld/";
 215             if (manifest.platform() == dyld3::Platform::macOS) {
 216                 runtimePath =  "/private/var/db/dyld/";
 217             }
 218             if (dedupe) {
 219                 configPath = masterDstRoot + "/DedupedConfigs/" + fileName + runtimePath;
 220             } else {
 221                 configPath = masterDstRoot + runtimePath;
 222             }
 223
 224             mkpath_np(configPath.c_str(), 0755);
 225             if (manifest.platform() == dyld3::Platform::macOS) {
 226                 buildQueue.push_back(manifest.makeQueueEntry(configPath + "dyld_shared_cache_" + arch, cacheSet, arch, false, setName + "/" + arch,
 227                                                              isLocallyBuiltCache, skipWrites, verbose));
 228             } else {
 229                 if (emitDevCaches)
 230                     buildQueue.push_back(manifest.makeQueueEntry(configPath + "dyld_shared_cache_" + arch + ".development", cacheSet, arch, false, setName + "/" + arch,
 231                                                                  isLocallyBuiltCache, skipWrites, verbose));
 232                 buildQueue.push_back(manifest.makeQueueEntry(configPath + "dyld_shared_cache_" + arch, cacheSet, arch, true, setName + "/" + arch,
 233                                                              isLocallyBuiltCache, skipWrites, verbose));
 234             }
 235         });
 236     }
 237
 238     __block bool cacheBuildFailure = false;
 239     __block std::set<std::string> warnings;
 240     __block std::set<std::string> errors;
 241
 242     dispatch_sync(warningQueue, ^{
 243         auto manifestWarnings = diags.warnings();
 244         //warnings.insert(manifestWarnings.begin(), manifestWarnings.end());
 245     });
 246
 247     dispatch_apply(buildQueue.size(), queue, ^(size_t index) {
 248         auto queueEntry = buildQueue[index];
 249         pthread_setname_np(queueEntry.options.loggingPrefix.substr(0, MAXTHREADNAMESIZE - 1).c_str());
 250
 251         DyldSharedCache::CreateResults results = DyldSharedCache::create(queueEntry.options, queueEntry.dylibsForCache, queueEntry.otherDylibsAndBundles, queueEntry.mainExecutables);
 252         dispatch_sync(warningQueue, ^{
 253             warnings.insert(results.warnings.begin(), results.warnings.end());
 254             bool chooseSecondCdHash = agileChooseSHA256CdHash;
 255             if (agileChooseSHA256CdHash && !results.agileSignature) {
 256                 // Ignore this option for caches that are not signed agile (which is the majority).
 257                 chooseSecondCdHash = false;
 258             }
 259             for (const auto& configName : queueEntry.configNames) {
 260                 auto& configResults = manifest.configuration(configName).architecture(queueEntry.options.archName).results;
 261                 for (const auto& mh : results.evictions) {
 262                     configResults.exclude(mh, "VM overflow, evicting");
 263                 }
 264                 configResults.warnings = results.warnings;
 265                 if (queueEntry.options.optimizeStubs) {
 266                     configResults.productionCache.cdHash = chooseSecondCdHash ? results.cdHashSecond : results.cdHashFirst;
 267                 } else {
 268                     configResults.developmentCache.cdHash =  chooseSecondCdHash ? results.cdHashSecond : results.cdHashFirst;
 269                 }
 270             }
 271         });
 272         if (!results.errorMessage.empty()) {
 273             fprintf(stderr, "[%s] ERROR: %s\n", queueEntry.options.loggingPrefix.c_str(), results.errorMessage.c_str());
 274             cacheBuildFailure = true;
 275         } else if (skipWrites) {
 276             fprintf(stderr, "[%s] Skipped writing cache to: %s\n", queueEntry.options.loggingPrefix.c_str(), queueEntry.outputPath.c_str());
 277         }
 278     });
 279
 280     // print any warnings
 281     for (const std::string& warn : warnings) {
 282         fprintf(stderr, "[WARNING] %s\n", warn.c_str());
 283     }
 284
 285     int err = sync_volume_np(masterDstRoot.c_str(), SYNC_VOLUME_FULLSYNC | SYNC_VOLUME_WAIT);
 286     if (err) {
 287         fprintf(stderr, "Volume sync failed errnor=%d (%s)\n", err, strerror(err));
 288     }
 289
 290     return !cacheBuildFailure;
 291 }