dyld3/shared-cache/BuilderUtils.mm

   1 /*
   2  * Copyright (c) 2017 Apple Inc. All rights reserved.
   3  *
   4  * @APPLE_LICENSE_HEADER_START@
   5  *
   6  * This file contains Original Code and/or Modifications of Original Code
   7  * as defined in and that are subject to the Apple Public Source License
   8  * Version 2.0 (the 'License'). You may not use this file except in
   9  * compliance with the License. Please obtain a copy of the License at
  10  * http://www.opensource.apple.com/apsl/ and read it before using this
  11  * file.
  12  *
  13  * The Original Code and all software distributed under the License are
  14  * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  15  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  16  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
  17  * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
  18  * Please see the License for the specific language governing rights and
  19  * limitations under the License.
  20  *
  21  * @APPLE_LICENSE_HEADER_END@
  22  */
  23
  24
  25 #include <set>
  26 #include <string>
  27 #include <sstream>
  28 #include <iomanip> // std::setfill, std::setw
  29 #include <pthread.h>
  30 #include <mach/mach.h>
  31 #include <dispatch/dispatch.h>
  32
  33 #include <Bom/Bom.h>
  34 #include <Security/Security.h>
  35 #include <Security/SecCodeSigner.h>
  36 #include <CommonCrypto/CommonCrypto.h>
  37
  38 #include "Manifest.h"
  39 #include "Diagnostics.h"
  40 #include "FileUtils.h"
  41
  42 #include "BuilderUtils.h"
  43
  44 static dispatch_queue_t write_queue = dispatch_queue_create("com.apple.dyld.cache-builder.write", DISPATCH_QUEUE_CONCURRENT);
  45 static dispatch_group_t build_group = dispatch_group_create();
  46
  47 dispatch_group_t buildGroup() {
  48     return build_group;
  49 }
  50
  51 void insertFileInBom(const std::string& path, BOMBom bom)
  52 {
  53     std::vector<std::string> components;
  54     std::vector<std::string> processed_components;
  55     std::stringstream ss(path);
  56     std::string item;
  57
  58     while (std::getline(ss, item, '/')) {
  59         if (!item.empty()) {
  60             components.push_back(item);
  61         }
  62     }
  63
  64     std::string partialPath = ".";
  65     std::string lastComponent = components.back();
  66     components.pop_back();
  67     BOMFSObject fso = BOMFSObjectNew(BOMDirectoryType);
  68     BOMFSObjectSetFlags(fso, B_PATHONLY);
  69     BOMFSObjectSetPathName(fso, ".", true);
  70     BOMFSObjectSetShortName(fso, ".", true);
  71     (void)BOMBomInsertFSObject(bom, fso, false);
  72     BOMFSObjectFree(fso);
  73
  74     for (const auto& component : components) {
  75         partialPath = partialPath + "/" + component;
  76         fso = BOMFSObjectNew(BOMDirectoryType);
  77         BOMFSObjectSetFlags(fso, B_PATHONLY);
  78         BOMFSObjectSetPathName(fso, partialPath.c_str(), true);
  79         BOMFSObjectSetShortName(fso, component.c_str(), true);
  80         (void)BOMBomInsertFSObject(bom, fso, false);
  81         BOMFSObjectFree(fso);
  82     }
  83
  84     partialPath = partialPath + "/" + lastComponent;
  85     fso = BOMFSObjectNew(BOMFileType);
  86     BOMFSObjectSetFlags(fso, B_PATHONLY);
  87     BOMFSObjectSetPathName(fso, partialPath.c_str(), true);
  88     BOMFSObjectSetShortName(fso, lastComponent.c_str(), true);
  89     (void)BOMBomInsertFSObject(bom, fso, false);
  90     BOMFSObjectFree(fso);
  91 }
  92
  93 void makeBoms(dyld3::Manifest& manifest, const std::string& masterDstRoot)
  94 {
  95     mkpath_np((masterDstRoot + "/Boms/").c_str(), 0755);
  96
  97     manifest.forEachConfiguration([&manifest, &masterDstRoot](const std::string& configName) {
  98         auto                     config = manifest.configuration(configName);
  99         std::vector<std::string> prodBomPaths;
 100         std::vector<std::string> devBomPaths;
 101
 102         std::string runtimePath =  "/System/Library/Caches/com.apple.dyld/";
 103         if (manifest.platform() == dyld3::Platform::macOS) {
 104             runtimePath =  "/private/var/db/dyld/";
 105         }
 106
 107         for (auto& arch : config.architectures) {
 108             std::string cachePath = "dyld_shared_cache_" + arch.first;
 109             prodBomPaths.push_back(cachePath);
 110             if (manifest.platform() != dyld3::Platform::macOS) {
 111                 cachePath += ".development";
 112             }
 113             devBomPaths.push_back(cachePath);
 114             char buffer[MAXPATHLEN];
 115             sprintf(buffer, "%s/Boms/%s.prod.bom", masterDstRoot.c_str(), configName.c_str());
 116             BOMBom bom = BOMBomNew(buffer);
 117             for (auto& path : prodBomPaths) {
 118                 insertFileInBom(runtimePath + path, bom);
 119             }
 120             BOMBomFree(bom);
 121
 122             sprintf(buffer, "%s/Boms/%s.dev.bom", masterDstRoot.c_str(), configName.c_str());
 123             bom = BOMBomNew(buffer);
 124             for (auto& path : devBomPaths) {
 125                 insertFileInBom(runtimePath + path, bom);
 126             }
 127             BOMBomFree(bom);
 128
 129             sprintf(buffer, "%s/Boms/%s.full.bom", masterDstRoot.c_str(), configName.c_str());
 130             bom = BOMBomNew(buffer);
 131             for (auto& path : prodBomPaths) {
 132                 insertFileInBom(runtimePath + path, bom);
 133             }
 134             for (auto& path : devBomPaths) {
 135                 insertFileInBom(runtimePath + path, bom);
 136             }
 137             BOMBomFree(bom);
 138         }
 139     });
 140 }
 141
 142 bool build(Diagnostics& diags, dyld3::Manifest& manifest, const std::string& masterDstRoot, bool dedupe, bool verbose,
 143            bool skipWrites, bool agileChooseSHA256CdHash)
 144 {
 145     dispatch_queue_t                   queue = dispatch_get_global_queue(DISPATCH_QUEUE_PRIORITY_DEFAULT, 0);
 146     dispatch_queue_t                   warningQueue = dispatch_queue_create("com.apple.dyld.cache-builder.warnings", DISPATCH_QUEUE_SERIAL);
 147     std::vector<std::set<std::string>> dedupedCacheSets;
 148     if (dedupe) {
 149         manifest.forEachConfiguration([&manifest, &dedupedCacheSets](const std::string& configName) {
 150             auto config = manifest.configuration(configName);
 151             bool dupeFound = false;
 152
 153             for (auto& cacheSet : dedupedCacheSets) {
 154                 if (config == manifest.configuration(*cacheSet.begin())) {
 155                     cacheSet.insert(configName);
 156                     dupeFound = true;
 157                     break;
 158                 }
 159             }
 160
 161             if (!dupeFound) {
 162                 std::set<std::string> temp;
 163                 temp.insert(configName);
 164                 dedupedCacheSets.push_back(temp);
 165             }
 166         });
 167     } else {
 168         manifest.forEachConfiguration([&manifest, &dedupedCacheSets](const std::string& configName) {
 169             std::set<std::string> temp;
 170             temp.insert(configName);
 171             dedupedCacheSets.push_back(temp);
 172         });
 173     }
 174
 175     std::vector<dyld3::BuildQueueEntry> buildQueue;
 176
 177     for (auto& cacheSet : dedupedCacheSets) {
 178         //FIXME we may want to consider moving to hashes of UUID sets
 179         std::string setName;
 180
 181         for (auto& archName : cacheSet) {
 182             if (!setName.empty()) {
 183                 setName += "|";
 184             }
 185             setName += archName;
 186         }
 187
 188         std::stringstream fileNameStream;
 189         std::array<uint8_t, CC_SHA1_DIGEST_LENGTH> digest = { 0 };
 190         CC_SHA1(setName.c_str(), (unsigned int)setName.length(), &digest[0]);
 191
 192         fileNameStream << std::hex << std::uppercase << std::setfill('0');
 193         for (int c : digest) {
 194             fileNameStream << std::setw(2) << c;
 195         }
 196
 197         std::string fileName(fileNameStream.str());
 198
 199         if (dedupe) {
 200             for (auto& config : cacheSet) {
 201                 if (!skipWrites) {
 202                     int err = symlink(("DedupedConfigs/" + fileName).c_str(), (masterDstRoot + "/" + config).c_str());
 203                     if (err) {
 204                         diags.warning("Could not create symlink '%s' -> 'DedupedConfigs/%s' (%d)", config.c_str(), fileName.c_str(), err);
 205                     }
 206                 }
 207             }
 208         }
 209
 210         manifest.configuration(*cacheSet.begin()).forEachArchitecture([&masterDstRoot, &dedupe, &fileName, &setName, &manifest, &buildQueue, &cacheSet, verbose](const std::string& arch) {
 211             std::string configPath;
 212             std::string runtimePath =  "/System/Library/Caches/com.apple.dyld/";
 213             if (manifest.platform() == dyld3::Platform::macOS) {
 214                 runtimePath =  "/private/var/db/dyld/";
 215             }
 216             if (dedupe) {
 217                 configPath = masterDstRoot + "/DedupedConfigs/" + fileName + runtimePath;
 218             } else {
 219                 configPath = masterDstRoot + runtimePath;
 220             }
 221
 222             if (manifest.platform() == dyld3::Platform::macOS) {
 223                 buildQueue.push_back(manifest.makeQueueEntry(configPath + "dyld_shared_cache_" + arch, cacheSet, arch, false, setName + "/" + arch, verbose));
 224             } else {
 225                 buildQueue.push_back(manifest.makeQueueEntry(configPath + "dyld_shared_cache_" + arch + ".development", cacheSet, arch, false, setName + "/" + arch, verbose));
 226                 buildQueue.push_back(manifest.makeQueueEntry(configPath + "dyld_shared_cache_" + arch, cacheSet, arch, true, setName + "/" + arch, verbose));
 227             }
 228         });
 229     }
 230
 231     __block bool cacheBuildFailure = false;
 232     __block std::set<std::string> warnings;
 233     __block std::set<std::string> errors;
 234
 235     dispatch_sync(warningQueue, ^{
 236         auto manifestWarnings = diags.warnings();
 237         warnings.insert(manifestWarnings.begin(), manifestWarnings.end());
 238     });
 239
 240     dispatch_apply(buildQueue.size(), queue, ^(size_t index) {
 241         auto queueEntry = buildQueue[index];
 242         pthread_setname_np(queueEntry.options.loggingPrefix.substr(0, MAXTHREADNAMESIZE - 1).c_str());
 243
 244         DyldSharedCache::CreateResults results = DyldSharedCache::create(queueEntry.options, queueEntry.dylibsForCache, queueEntry.otherDylibsAndBundles, queueEntry.mainExecutables);
 245         dispatch_sync(warningQueue, ^{
 246             warnings.insert(results.warnings.begin(), results.warnings.end());
 247             bool chooseSecondCdHash = agileChooseSHA256CdHash;
 248             if (agileChooseSHA256CdHash && !results.agileSignature) {
 249                 // Ignore this option for caches that are not signed agile (which is the majority).
 250                 chooseSecondCdHash = false;
 251             }
 252             for (const auto& configName : queueEntry.configNames) {
 253                 auto& configResults = manifest.configuration(configName).architecture(queueEntry.options.archName).results;
 254                 for (const auto& mh : results.evictions) {
 255                     auto parser = dyld3::MachOParser(mh);
 256                     configResults.exclude(&parser, "VM overflow, evicting");
 257                 }
 258                 configResults.warnings = results.warnings;
 259                 if (queueEntry.options.optimizeStubs) {
 260                     configResults.developmentCache.cdHash = chooseSecondCdHash ? results.cdHashSecond : results.cdHashFirst;
 261                 } else {
 262                     configResults.productionCache.cdHash =  chooseSecondCdHash ? results.cdHashSecond : results.cdHashFirst;
 263                 }
 264             }
 265         });
 266         if (!results.errorMessage.empty()) {
 267             fprintf(stderr, "[%s] ERROR: %s\n", queueEntry.options.loggingPrefix.c_str(), results.errorMessage.c_str());
 268         } else if (!skipWrites) {
 269             dispatch_sync(write_queue, ^{
 270                 // save new cache file to disk and write new .map file
 271                 assert(results.cacheContent != nullptr);
 272                 mkpath_np(dirPath(queueEntry.outputPath).c_str(), 0755);
 273                 if (!safeSave(results.cacheContent, results.cacheLength, queueEntry.outputPath)) {
 274                     cacheBuildFailure = true;
 275                     fprintf(stderr, "[%s] ERROR: Could not write cache to: %s\n", queueEntry.options.loggingPrefix.c_str(), queueEntry.outputPath.c_str());
 276                 } else {
 277                     fprintf(stderr, "[%s] Wrote cache to: %s\n", queueEntry.options.loggingPrefix.c_str(), queueEntry.outputPath.c_str());
 278                     std::string mapStr = results.cacheContent->mapFile();
 279                     std::string outFileMap = queueEntry.outputPath + ".map";
 280                     safeSave(mapStr.c_str(), mapStr.size(), outFileMap);
 281                 }
 282                 // free created cache buffer
 283                 vm_deallocate(mach_task_self(), (vm_address_t)results.cacheContent, results.cacheLength);
 284             });
 285         } else {
 286             fprintf(stderr, "[%s] Skipped writing cache to: %s\n", queueEntry.options.loggingPrefix.c_str(), queueEntry.outputPath.c_str());
 287             vm_deallocate(mach_task_self(), (vm_address_t)results.cacheContent, results.cacheLength);
 288         }
 289     });
 290
 291     // print any warnings
 292     for (const std::string& warn : warnings) {
 293         fprintf(stderr, "[WARNING] %s\n", warn.c_str());
 294     }
 295
 296     int err = sync_volume_np(masterDstRoot.c_str(), SYNC_VOLUME_FULLSYNC | SYNC_VOLUME_WAIT);
 297     if (err) {
 298         fprintf(stderr, "Volume sync failed errnor=%d (%s)\n", err, strerror(err));
 299     }
 300
 301     return !cacheBuildFailure;
 302 }