2 * Copyright (c) 2017 Apple Inc. All rights reserved.
4 * @APPLE_LICENSE_HEADER_START@
6 * This file contains Original Code and/or Modifications of Original Code
7 * as defined in and that are subject to the Apple Public Source License
8 * Version 2.0 (the 'License'). You may not use this file except in
9 * compliance with the License. Please obtain a copy of the License at
10 * http://www.opensource.apple.com/apsl/ and read it before using this
13 * The Original Code and all software distributed under the License are
14 * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
15 * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
16 * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
17 * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
18 * Please see the License for the specific language governing rights and
19 * limitations under the License.
21 * @APPLE_LICENSE_HEADER_END@
27 #include <Metabom/MBTypes.h>
28 #include <Metabom/MBEntry.h>
29 #include <Metabom/MBMetabom.h>
30 #include <Metabom/MBIterator.h>
34 #include <CommonCrypto/CommonDigest.h>
35 #include <CommonCrypto/CommonDigestSPI.h>
36 #include <Foundation/Foundation.h>
38 #include "MachOFileAbstraction.hpp"
39 #include "FileAbstraction.hpp"
41 #include "FileUtils.h"
42 #include "StringUtils.h"
44 #include <mach-o/loader.h>
45 #include <mach-o/fat.h>
53 //FIXME this should be in a class
54 static inline NSString* cppToObjStr(const std::string& str) { return [NSString stringWithUTF8String:str.c_str()]; }
56 template <class Set1, class Set2>
57 inline bool is_disjoint(const Set1& set1, const Set2& set2)
59 if (set1.empty() || set2.empty())
62 typename Set1::const_iterator it1 = set1.begin(), it1End = set1.end();
63 typename Set2::const_iterator it2 = set2.begin(), it2End = set2.end();
65 if (*it1 > *set2.rbegin() || *it2 > *set1.rbegin())
68 while (it1 != it1End && it2 != it2End) {
81 //hACK: If we declare this in manifest
82 static NSDictionary* gManifestDict;
84 } /* Anonymous namespace */
87 void Manifest::Results::exclude(MachOParser* parser, const std::string& reason)
89 auto dylibUUID = parser->uuid();
90 dylibs[dylibUUID].uuid = dylibUUID;
91 dylibs[dylibUUID].installname = parser->installName();
92 dylibs[dylibUUID].included = false;
93 dylibs[dylibUUID].exclusionInfo = reason;
96 void Manifest::Results::exclude(Manifest& manifest, const UUID& uuid, const std::string& reason)
98 auto parser = manifest.parserForUUID(uuid);
99 dylibs[uuid].uuid = uuid;
100 dylibs[uuid].installname = parser.installName();
101 dylibs[uuid].included = false;
102 dylibs[uuid].exclusionInfo = reason;
105 Manifest::CacheImageInfo& Manifest::Results::dylibForInstallname(const std::string& installname)
107 auto i = find_if(dylibs.begin(), dylibs.end(), [&installname](std::pair<UUID, CacheImageInfo> d) { return d.second.installname == installname; });
108 assert(i != dylibs.end());
112 bool Manifest::Architecture::operator==(const Architecture& O) const
114 for (auto& dylib : results.dylibs) {
115 if (dylib.second.included) {
116 auto Odylib = O.results.dylibs.find(dylib.first);
117 if (Odylib == O.results.dylibs.end()
118 || Odylib->second.included == false
119 || Odylib->second.uuid != dylib.second.uuid)
124 for (const auto& Odylib : O.results.dylibs) {
125 if (Odylib.second.included) {
126 auto dylib = results.dylibs.find(Odylib.first);
127 if (dylib == results.dylibs.end()
128 || dylib->second.included == false
129 || dylib->second.uuid != Odylib.second.uuid)
134 for (auto& bundle : results.bundles) {
135 if (bundle.second.included) {
136 auto Obundle = O.results.bundles.find(bundle.first);
137 if (Obundle == O.results.bundles.end()
138 || Obundle->second.included == false
139 || Obundle->second.uuid != bundle.second.uuid)
144 for (const auto& Obundle : O.results.bundles) {
145 if (Obundle.second.included) {
146 auto bundle = results.bundles.find(Obundle.first);
147 if (bundle == results.bundles.end()
148 || bundle->second.included == false
149 || bundle->second.uuid != Obundle.second.uuid)
154 for (auto& executable : results.executables) {
155 if (executable.second.included) {
156 auto Oexecutable = O.results.executables.find(executable.first);
157 if (Oexecutable == O.results.executables.end()
158 || Oexecutable->second.included == false
159 || Oexecutable->second.uuid != executable.second.uuid)
164 for (const auto& Oexecutable : O.results.executables) {
165 if (Oexecutable.second.included) {
166 auto executable = results.executables.find(Oexecutable.first);
167 if (executable == results.executables.end()
168 || executable->second.included == false
169 || executable->second.uuid != Oexecutable.second.uuid)
177 bool Manifest::Configuration::operator==(const Configuration& O) const
179 return architectures == O.architectures;
182 bool Manifest::Configuration::operator!=(const Configuration& other) const { return !(*this == other); }
184 const Manifest::Architecture& Manifest::Configuration::architecture(const std::string& architecture) const
186 assert(architectures.find(architecture) != architectures.end());
187 return architectures.find(architecture)->second;
190 void Manifest::Configuration::forEachArchitecture(std::function<void(const std::string& archName)> lambda) const
192 for (const auto& architecutre : architectures) {
193 lambda(architecutre.first);
197 bool Manifest::Architecture::operator!=(const Architecture& other) const { return !(*this == other); }
199 const std::map<std::string, Manifest::Project>& Manifest::projects()
204 const Manifest::Configuration& Manifest::configuration(const std::string& configuration) const
206 assert(_configurations.find(configuration) != _configurations.end());
207 return _configurations.find(configuration)->second;
210 void Manifest::forEachConfiguration(std::function<void(const std::string& configName)> lambda) const
212 for (const auto& configuration : _configurations) {
213 lambda(configuration.first);
217 void Manifest::addProjectSource(const std::string& project, const std::string& source, bool first)
219 auto& sources = _projects[project].sources;
220 if (std::find(sources.begin(), sources.end(), source) == sources.end()) {
222 sources.insert(sources.begin(), source);
224 sources.push_back(source);
229 const std::string Manifest::projectPath(const std::string& projectName)
231 auto project = _projects.find(projectName);
232 if (project == _projects.end())
234 if (project->second.sources.size() == 0)
236 return project->second.sources[0];
239 const bool Manifest::empty(void)
241 for (const auto& configuration : _configurations) {
242 if (configuration.second.architectures.size() != 0)
248 const std::string Manifest::dylibOrderFile() const { return _dylibOrderFile; };
249 void Manifest::setDylibOrderFile(const std::string& dylibOrderFile) { _dylibOrderFile = dylibOrderFile; };
251 const std::string Manifest::dirtyDataOrderFile() const { return _dirtyDataOrderFile; };
252 void Manifest::setDirtyDataOrderFile(const std::string& dirtyDataOrderFile) { _dirtyDataOrderFile = dirtyDataOrderFile; };
254 const std::string Manifest::metabomFile() const { return _metabomFile; };
255 void Manifest::setMetabomFile(const std::string& metabomFile) { _metabomFile = metabomFile; };
257 const Platform Manifest::platform() const { return _platform; };
258 void Manifest::setPlatform(const Platform platform) { _platform = platform; };
260 const std::string& Manifest::build() const { return _build; };
261 void Manifest::setBuild(const std::string& build) { _build = build; };
262 const uint32_t Manifest::version() const { return _manifestVersion; };
263 void Manifest::setVersion(const uint32_t manifestVersion) { _manifestVersion = manifestVersion; };
265 BuildQueueEntry Manifest::makeQueueEntry(const std::string& outputPath, const std::set<std::string>& configs, const std::string& arch, bool optimizeStubs, const std::string& prefix, bool verbose)
267 dyld3::BuildQueueEntry retval;
269 DyldSharedCache::CreateOptions options;
270 options.archName = arch;
271 options.platform = platform();
272 options.excludeLocalSymbols = true;
273 options.optimizeStubs = optimizeStubs;
274 options.optimizeObjC = true;
275 options.codeSigningDigestMode = (platform() == dyld3::Platform::watchOS) ?
276 DyldSharedCache::Agile : DyldSharedCache::SHA256only;
277 options.dylibsRemovedDuringMastering = true;
278 options.inodesAreSameAsRuntime = false;
279 options.cacheSupportsASLR = true;
280 options.forSimulator = false;
281 options.verbose = verbose;
282 options.evictLeafDylibsOnOverflow = false;
283 options.loggingPrefix = prefix;
284 options.pathPrefixes = { "" };
285 options.dylibOrdering = loadOrderFile(_dylibOrderFile);
286 options.dirtyDataSegmentOrdering = loadOrderFile(_dirtyDataOrderFile);
288 dyld3::BuildQueueEntry queueEntry;
289 retval.configNames = configs;
290 retval.options = options;
291 retval.outputPath = outputPath;
292 retval.dylibsForCache = dylibsForCache(*configs.begin(), arch);
293 retval.otherDylibsAndBundles = otherDylibsAndBundles(*configs.begin(), arch);
294 retval.mainExecutables = mainExecutables(*configs.begin(), arch);
299 bool Manifest::loadParser(const void* p, size_t size, uint64_t sliceOffset, const std::string& runtimePath, const std::string& buildPath, const std::set<std::string>& architectures)
301 const mach_header* mh = reinterpret_cast<const mach_header*>(p);
302 if (!MachOParser::isValidMachO(_diags, "", _platform, p, size, runtimePath.c_str(), false)) {
306 auto parser = MachOParser(mh);
307 if (_diags.hasError()) {
308 // Clear the error and punt
309 _diags.verbose("MachoParser error: %s\n", _diags.errorMessage().c_str());
314 auto uuid = parser.uuid();
315 auto archName = parser.archName();
317 if (parser.fileType() == MH_DYLIB && architectures.count(parser.archName()) != 0) {
318 std::string installName = parser.installName();
319 auto index = std::make_pair(installName, parser.archName());
320 auto i = _installNameMap.find(index);
322 if ( installName == "/System/Library/Caches/com.apple.xpc/sdk.dylib"
323 || installName == "/System/Library/Caches/com.apple.xpcd/xpcd_cache.dylib" ) {
324 // HACK to deal with device specific dylibs. These must not be inseted into the installNameMap
325 _uuidMap.insert(std::make_pair(uuid, UUIDInfo(mh, size, sliceOffset, uuid, parser.archName(), runtimePath, buildPath, installName)));
326 } else if (i == _installNameMap.end()) {
327 _installNameMap.insert(std::make_pair(index, uuid));
328 _uuidMap.insert(std::make_pair(uuid, UUIDInfo(mh, size, sliceOffset, uuid, parser.archName(), runtimePath, buildPath, installName)));
329 if (installName[0] != '@' && installName != runtimePath) {
330 _diags.warning("Dylib located at '%s' has installname '%s'", runtimePath.c_str(), installName.c_str());
333 auto info = infoForUUID(i->second);
334 _diags.warning("Multiple dylibs claim installname '%s' ('%s' and '%s')", installName.c_str(), runtimePath.c_str(), info.runtimePath.c_str());
336 // This is the "Good" one, overwrite
337 if (runtimePath == installName) {
338 _uuidMap.erase(uuid);
339 _uuidMap.insert(std::make_pair(uuid, UUIDInfo(mh, size, sliceOffset, uuid, parser.archName(), runtimePath, buildPath, installName)));
343 _uuidMap.insert(std::make_pair(uuid, UUIDInfo(mh, size, sliceOffset, uuid, parser.archName(), runtimePath, buildPath, "")));
348 //FIXME: assert we have not errored first
349 bool Manifest::loadParsers(const std::string& buildPath, const std::string& runtimePath, const std::set<std::string>& architectures)
351 __block bool retval = false;
352 const void* p = (uint8_t*)(-1);
353 struct stat stat_buf;
355 std::tie(p, stat_buf) = fileCache.cacheLoad(_diags, buildPath);
357 if (p == (uint8_t*)(-1)) {
361 if (FatUtil::isFatFile(p)) {
362 FatUtil::forEachSlice(_diags, p, stat_buf.st_size, ^(uint32_t sliceCpuType, uint32_t sliceCpuSubType, const void* sliceStart, size_t sliceSize, bool& stop) {
363 if (loadParser(sliceStart, sliceSize, (uintptr_t)sliceStart-(uintptr_t)p, runtimePath, buildPath, architectures))
367 return loadParser(p, stat_buf.st_size, 0, runtimePath, buildPath, architectures);
372 const Manifest::UUIDInfo& Manifest::infoForUUID(const UUID& uuid) const {
373 auto i = _uuidMap.find(uuid);
374 assert(i != _uuidMap.end());
378 const Manifest::UUIDInfo Manifest::infoForInstallNameAndarch(const std::string& installName, const std::string arch) const {
380 auto uuidI = _installNameMap.find(std::make_pair(installName, arch));
381 if (uuidI == _installNameMap.end())
384 auto i = _uuidMap.find(uuidI->second);
385 if (i == _uuidMap.end())
390 MachOParser Manifest::parserForUUID(const UUID& uuid) const {
391 return MachOParser(infoForUUID(uuid).mh);
394 const std::string Manifest::buildPathForUUID(const UUID& uuid) {
395 return infoForUUID(uuid).buildPath;
398 const std::string Manifest::runtimePathForUUID(const UUID& uuid) {
399 return infoForUUID(uuid).runtimePath;
402 Manifest::Manifest(Diagnostics& D, const std::string& path) : Manifest(D, path, std::set<std::string>())
406 Manifest::Manifest(Diagnostics& D, const std::string& path, const std::set<std::string>& overlays) :
409 NSMutableDictionary* manifestDict = [NSMutableDictionary dictionaryWithContentsOfFile:cppToObjStr(path)];
410 NSString* platStr = manifestDict[@"platform"];
411 std::set<std::string> architectures;
413 if (platStr == nullptr)
415 std::string platformString = [platStr UTF8String];
416 setMetabomFile([manifestDict[@"metabomFile"] UTF8String]);
418 if (platformString == "ios") {
419 setPlatform(dyld3::Platform::iOS);
420 } else if ( (platformString == "tvos") || (platformString == "atv") ) {
421 setPlatform(dyld3::Platform::tvOS);
422 } else if ( (platformString == "watchos") || (platformString == "watch") ) {
423 setPlatform(dyld3::Platform::watchOS);
424 } else if ( (platformString == "bridgeos") || (platformString == "bridge") ) {
425 setPlatform(dyld3::Platform::bridgeOS);
426 } else if ( (platformString == "macos") || (platformString == "osx") ) {
427 setPlatform(dyld3::Platform::macOS);
429 //Fixme should we error?
430 setPlatform(dyld3::Platform::iOS);
433 for (NSString* project in manifestDict[@"projects"]) {
434 for (NSString* source in manifestDict[@"projects"][project]) {
435 addProjectSource([project UTF8String], [source UTF8String]);
439 for (NSString* configuration in manifestDict[@"configurations"]) {
440 std::string configStr = [configuration UTF8String];
441 std::string configTag = [manifestDict[@"configurations"][configuration][@"metabomTag"] UTF8String];
443 if (manifestDict[@"configurations"][configuration][@"metabomExcludeTags"]) {
444 for (NSString* excludeTag in manifestDict[@"configurations"][configuration][@"metabomExcludeTags"]) {
445 _metabomExcludeTagMap[configStr].insert([excludeTag UTF8String]);
446 _configurations[configStr].metabomExcludeTags.insert([excludeTag UTF8String]);
450 if (manifestDict[@"configurations"][configuration][@"metabomRestrictTags"]) {
451 for (NSString* restrictTag in manifestDict[@"configurations"][configuration][@"metabomRestrictTags"]) {
452 _metabomRestrictedTagMap[configStr].insert([restrictTag UTF8String]);
453 _configurations[configStr].metabomRestrictTags.insert([restrictTag UTF8String]);
457 _configurations[configStr].metabomTag = configTag;
458 _configurations[configStr].metabomTags.insert(configTag);
459 _configurations[configStr].platformName =
460 [manifestDict[@"configurations"][configuration][@"platformName"] UTF8String];
462 if (endsWith(configStr, "InternalOS")) {
463 _configurations[configStr].disposition = "internal";
464 _configurations[configStr].device = configStr.substr(0, configStr.length()-strlen("InternalOS"));
465 } else if (endsWith(configStr, "VendorOS")) {
466 _configurations[configStr].disposition = "internal";
467 _configurations[configStr].device = configStr.substr(0, configStr.length()-strlen("VendorOS"));
468 } else if (endsWith(configStr, "VendorUIOS")) {
469 _configurations[configStr].disposition = "internal";
470 _configurations[configStr].device = configStr.substr(0, configStr.length()-strlen("VendorUIOS"));
471 } else if (endsWith(configStr, "CarrierOS")) {
472 _configurations[configStr].disposition = "internal";
473 _configurations[configStr].device = configStr.substr(0, configStr.length()-strlen("CarrierOS"));
474 } else if (endsWith(configStr, "FactoryOS")) {
475 _configurations[configStr].disposition = "internal";
476 _configurations[configStr].device = configStr.substr(0, configStr.length()-strlen("FactoryOS"));
477 } else if (endsWith(configStr, "DesenseOS")) {
478 _configurations[configStr].disposition = "internal";
479 _configurations[configStr].device = configStr.substr(0, configStr.length()-strlen("DesenseOS"));
480 } else if (endsWith(configStr, "MinosOS")) {
481 _configurations[configStr].disposition = "minos";
482 _configurations[configStr].device = configStr.substr(0, configStr.length()-strlen("MinosOS"));
483 } else if (endsWith(configStr, "DemoOS")) {
484 _configurations[configStr].disposition = "demo";
485 _configurations[configStr].device = configStr.substr(0, configStr.length()-strlen("DemoOS"));
486 } else if (endsWith(configStr, "MinosOS")) {
487 _configurations[configStr].disposition = "minos";
488 _configurations[configStr].device = configStr.substr(0, configStr.length()-strlen("MinosOS"));
489 } else if (endsWith(configStr, "DeveloperOS")) {
490 _configurations[configStr].disposition = "user";
491 _configurations[configStr].device = configStr.substr(0, configStr.length()-strlen("DeveloperOS"));
492 } else if (endsWith(configStr, "OS")) {
493 _configurations[configStr].disposition = "user";
494 _configurations[configStr].device = configStr.substr(0, configStr.length()-strlen("OS"));
497 for (NSString* architecutre in manifestDict[@"configurations"][configuration][@"architectures"]) {
498 //HACK until B&I stops mastering armv7s
499 if ([architecutre isEqual:@"armv7s"]) break;
500 _configurations[configStr].architectures[[architecutre UTF8String]] = Architecture();
501 architectures.insert([architecutre UTF8String]);
505 setVersion([manifestDict[@"manifest-version"] unsignedIntValue]);
506 setBuild([manifestDict[@"build"] UTF8String]);
507 if (manifestDict[@"dylibOrderFile"]) {
508 setDylibOrderFile([manifestDict[@"dylibOrderFile"] UTF8String]);
510 if (manifestDict[@"dirtyDataOrderFile"]) {
511 setDirtyDataOrderFile([manifestDict[@"dirtyDataOrderFile"] UTF8String]);
514 auto metabom = MBMetabomOpen(metabomFile().c_str(), false);
515 auto metabomEnumerator = MBIteratorNewWithPath(metabom, ".", "");
518 // FIXME error handling (NULL metabom)
520 //First we iterate through the bom and build our objects
522 while ((entry = MBIteratorNext(metabomEnumerator))) {
523 BOMFSObject fsObject = MBEntryGetFSObject(entry);
524 BOMFSObjType entryType = BOMFSObjectType(fsObject);
525 std::string entryPath = BOMFSObjectPathName(fsObject);
526 if (entryPath[0] == '.') {
527 entryPath.erase(0, 1);
530 // Skip artifacts that happen to be in the build chain
531 if ( startsWith(entryPath, "/Applications/Xcode.app") ) {
535 // Skip variants we can't deal with
536 if ( endsWith(entryPath, "_profile.dylib") || endsWith(entryPath, "_debug.dylib") || endsWith(entryPath, "_profile") || endsWith(entryPath, "_debug") || endsWith(entryPath, "/CoreADI") ) {
540 // Skip images that are only used in InternalOS variants
541 if ( startsWith(entryPath, "/AppleInternal/") || startsWith(entryPath, "/usr/local/") || startsWith(entryPath, "/Developer/")) {
545 // Skip genCache generated dylibs
546 if ( endsWith(entryPath, "/System/Library/Caches/com.apple.xpc/sdk.dylib") || endsWith(entryPath, "/System/Library/Caches/com.apple.xpcd/xpcd_cache.dylib")) {
551 auto tagCount = MBEntryGetNumberOfProjectTags(entry);
552 if (entryType == BOMFileType && BOMFSObjectIsBinaryObject(fsObject) && MBEntryGetNumberOfProjectTags(entry) != 0 && tagCount != 0) {
554 MBEntryGetProjectTags(entry, &tag);
556 MBTag* tags = (MBTag*)malloc(sizeof(MBTag) * tagCount);
557 MBEntryGetProjectTags(entry, tags);
559 //Sigh, we can have duplicate entries for the same tag, so build a set to work with
560 std::set<std::string> tagStrs;
561 std::map<std::string, MBTag> tagStrMap;
562 for (auto i = 0; i < tagCount; ++i) {
563 tagStrs.insert(MBMetabomGetProjectForTag(metabom, tags[i]));
564 tagStrMap.insert(std::make_pair(MBMetabomGetProjectForTag(metabom, tags[i]), tags[i]));
567 if (tagStrs.size() > 1) {
568 std::string projects;
569 for (const auto& tagStr : tagStrs) {
570 if (!projects.empty())
573 projects += "'" + tagStr + "'";
575 _diags.warning("Bom entry '%s' is claimed by multiple projects: %s, taking first entry", entryPath.c_str(), projects.c_str());
577 tag = tagStrMap[*tagStrs.begin()];
581 std::string projectName = MBMetabomGetProjectForTag(metabom, tag);
582 tagCount = MBEntryGetNumberOfPackageTags(entry);
583 MBTag* tags = (MBTag*)malloc(sizeof(MBTag) * tagCount);
584 MBEntryGetPackageTags(entry, tags);
585 std::set<std::string> tagStrs;
587 for (auto i = 0; i < tagCount; ++i) {
588 tagStrs.insert(MBMetabomGetPackageForTag(metabom, tags[i]));
591 _metabomTagMap.insert(std::make_pair(entryPath, tagStrs));
592 bool foundParser = false;
593 for (const auto& overlay : overlays) {
594 if (loadParsers(overlay + "/" + entryPath, entryPath, architectures)) {
601 (void)loadParsers(projectPath(projectName) + "/" + entryPath, entryPath, architectures);
606 MBIteratorFree(metabomEnumerator);
607 MBMetabomFree(metabom);
610 void Manifest::insert(std::vector<DyldSharedCache::MappedMachO>& mappedMachOs, const CacheImageInfo& imageInfo) {
611 auto info = infoForUUID(imageInfo.uuid);
612 auto runtimePath = info.runtimePath;
613 mappedMachOs.emplace_back(runtimePath, info.mh, info.size, false, false, info.sliceFileOffset, 0, 0);
616 std::vector<DyldSharedCache::MappedMachO> Manifest::dylibsForCache(const std::string& configuration, const std::string& architecture)
618 std::vector<DyldSharedCache::MappedMachO> retval;
619 const auto& dylibs = _configurations[configuration].architectures[architecture].results.dylibs;
620 for (const auto& dylib : dylibs) {
621 if (dylib.second.included) {
622 insert(retval, dylib.second);
628 std::vector<DyldSharedCache::MappedMachO> Manifest::otherDylibsAndBundles(const std::string& configuration, const std::string& architecture)
630 std::vector<DyldSharedCache::MappedMachO> retval;
631 const auto& dylibs = _configurations[configuration].architectures[architecture].results.dylibs;
632 for (const auto& dylib : dylibs) {
633 if (!dylib.second.included) {
634 insert(retval, dylib.second);
638 const auto& bundles = _configurations[configuration].architectures[architecture].results.bundles;
639 for (const auto& bundle : bundles) {
640 insert(retval, bundle.second);
646 std::vector<DyldSharedCache::MappedMachO> Manifest::mainExecutables(const std::string& configuration, const std::string& architecture)
648 std::vector<DyldSharedCache::MappedMachO> retval;
649 const auto& executables = _configurations[configuration].architectures[architecture].results.executables;
650 for (const auto& executable : executables) {
651 insert(retval, executable.second);
657 bool Manifest::filterForConfig(const std::string& configName)
659 for (const auto configuration : _configurations) {
660 if (configName == configuration.first) {
661 std::map<std::string, Configuration> filteredConfigs;
662 filteredConfigs[configName] = configuration.second;
664 _configurations = filteredConfigs;
666 for (auto& arch : configuration.second.architectures) {
667 arch.second.results = Manifest::Results();
675 void Manifest::dedupeDispositions(void) {
676 // Since this is all hacky and inference based for now only do it for iOS until XBS
677 // is reved to give us real info. All the other platforms are way smaller anyway.
678 if (_platform != Platform::iOS)
681 std::map<std::pair<std::string, std::string>, std::set<std::string>> dispositionSets;
683 for (const auto& configuration : _configurations) {
684 dispositionSets[std::make_pair(configuration.second.device, configuration.second.disposition)].insert(configuration.first);
687 for (const auto& dSet : dispositionSets) {
688 for (const auto &c1 : dSet.second) {
689 for (const auto &c2 : dSet.second) {
690 _configurations[c1].metabomTags.insert(_configurations[c2].metabomTag);
696 void Manifest::calculateClosure()
698 auto closureSemaphore = dispatch_semaphore_create(32);
699 auto closureGroup = dispatch_group_create();
700 auto closureQueue = dispatch_queue_create("com.apple.dyld.cache.closure", dispatch_queue_attr_make_with_qos_class(DISPATCH_QUEUE_CONCURRENT, QOS_CLASS_USER_INITIATED, 0));
702 dedupeDispositions();
703 for (auto& config : _configurations) {
704 for (auto& arch : config.second.architectures) {
705 dispatch_semaphore_wait(closureSemaphore, DISPATCH_TIME_FOREVER);
706 dispatch_group_async(closureGroup, closureQueue, [&] {
707 calculateClosure(config.first, arch.first);
708 dispatch_semaphore_signal(closureSemaphore);
713 dispatch_group_wait(closureGroup, DISPATCH_TIME_FOREVER);
716 void Manifest::remove(const std::string& config, const std::string& arch)
718 if (_configurations.count(config))
719 _configurations[config].architectures.erase(arch);
722 void Manifest::removeDylib(MachOParser parser, const std::string& reason, const std::string& configuration,
723 const std::string& architecture, std::unordered_set<UUID>& processedIdentifiers)
726 auto configIter = _configurations.find(configuration);
727 if (configIter == _configurations.end())
729 auto archIter = configIter->second.architectures.find( architecture );
730 if ( archIter == configIter->second.architectures.end() ) return;
731 auto& archManifest = archIter->second;
733 if (archManifest.results.dylibs.count(parser->uuid()) == 0) {
734 archManifest.results.dylibs[parser->uuid()].uuid = parser->uuid();
735 archManifest.results.dylibs[parser->uuid()].installname = parser->installName();
736 processedIdentifiers.insert(parser->uuid());
738 archManifest.results.exclude(MachOProxy::forIdentifier(parser->uuid(), architecture), reason);
740 processedIdentifiers.insert(parser->uuid());
742 for (const auto& dependent : proxy->dependentIdentifiers) {
743 auto dependentProxy = MachOProxy::forIdentifier(dependent, architecture);
744 auto dependentResultIter = archManifest.results.dylibs.find(dependentProxy->identifier);
745 if ( dependentProxy &&
746 ( dependentResultIter == archManifest.results.dylibs.end() || dependentResultIter->second.included == true ) ) {
747 removeDylib(dependentProxy, "Missing dependency: " + proxy->installName, configuration, architecture,
748 processedIdentifiers);
754 const std::string Manifest::removeLargestLeafDylib(const std::set<std::string>& configurations, const std::string& architecture)
756 // Find the leaf nodes
757 __block std::map<std::string, uint64_t> dependentCounts;
758 for (const auto& dylib : _configurations[*configurations.begin()].architectures[architecture].results.dylibs) {
759 if (!dylib.second.included)
761 std::string installName;
762 auto info = infoForUUID(dylib.first);
763 auto parser = MachOParser(info.mh);
764 dependentCounts[parser.installName()] = 0;
767 for (const auto& dylib : _configurations[*configurations.begin()].architectures[architecture].results.dylibs) {
768 if (!dylib.second.included)
770 auto info = infoForUUID(dylib.first);
771 auto parser = MachOParser(info.mh);
772 parser.forEachDependentDylib(^(const char *loadPath, bool isWeak, bool isReExport, bool isUpward, uint32_t compatVersion, uint32_t curVersion, bool &stop) {
774 dependentCounts[loadPath]++;
779 // Figure out which leaf is largest
780 UUIDInfo largestLeaf;
782 for (const auto& dependentCount : dependentCounts) {
783 if (dependentCount.second == 0) {
784 auto info = infoForInstallNameAndarch(dependentCount.first, architecture);
785 assert(info.mh != nullptr);
786 if (info.size > largestLeaf.size) {
792 if (largestLeaf.mh == nullptr) {
793 _diags.error("Fatal overflow, could not evict more dylibs");
797 // Remove it ferom all configs
798 for (const auto& config : configurations) {
799 configuration(config).architecture(architecture).results.exclude(*this, largestLeaf.uuid, "Cache Overflow");
802 return largestLeaf.installName;
805 void Manifest::calculateClosure(const std::string& configuration, const std::string& architecture)
807 __block auto& configManifest = _configurations[configuration];
808 __block auto& archManifest = _configurations[configuration].architectures[architecture];
809 __block std::set<UUID> newUuids;
810 std::set<UUID> processedUuids;
811 std::set<UUID> cachedUUIDs;
814 for (auto& uuidInfo : _uuidMap) {
815 auto info = uuidInfo.second;
816 if (info.arch != architecture) {
820 auto i = _metabomTagMap.find(info.runtimePath);
821 assert(i != _metabomTagMap.end());
822 auto tags = i->second;
823 if (!is_disjoint(tags, configManifest.metabomTags)) {
824 newUuids.insert(info.uuid);
829 // Pull in all dependencies
830 while (!newUuids.empty()) {
831 std::set<UUID> uuidsToProcess = newUuids;
834 for (const auto& uuid : uuidsToProcess) {
835 if (processedUuids.count(uuid) > 0) {
838 processedUuids.insert(uuid);
840 auto parser = parserForUUID(uuid);
841 auto runtimePath = runtimePathForUUID(uuid);
842 assert(parser.header() != 0);
844 parser.forEachDependentDylib(^(const char* loadPath, bool isWeak, bool isReExport, bool isUpward, uint32_t compatVersion, uint32_t curVersion, bool& stop) {
845 auto i = _installNameMap.find(std::make_pair(loadPath, architecture));
846 if (i != _installNameMap.end())
847 newUuids.insert(i->second);
850 if (parser.fileType() == MH_DYLIB) {
851 // Add the dylib to the results
852 if (archManifest.results.dylibs.count(uuid) == 0 ) {
853 archManifest.results.dylibs[uuid].uuid = uuid;
854 archManifest.results.dylibs[uuid].installname = parser.installName();
857 // HACK to insert device specific dylib closures into all caches
858 if ( parser.installName() == std::string("/System/Library/Caches/com.apple.xpc/sdk.dylib")
859 || parser.installName() == std::string("/System/Library/Caches/com.apple.xpcd/xpcd_cache.dylib") ) {
860 archManifest.results.exclude(&parser, "Device specific dylib");
864 std::set<std::string> reasons;
865 if (parser.canBePlacedInDyldCache(runtimePath, reasons)) {
866 auto i = _metabomTagMap.find(runtimePath);
867 assert(i != _metabomTagMap.end());
868 auto restrictions = _metabomRestrictedTagMap.find(configuration);
869 if (restrictions != _metabomRestrictedTagMap.end() && !is_disjoint(restrictions->second, i->second)) {
870 archManifest.results.exclude(&parser, "Dylib '" + runtimePath + "' removed due to explict restriction");
873 // It can be placed in the cache, grab its dependents and queue them for inclusion
874 cachedUUIDs.insert(parser.uuid());
876 // It can't be placed in the cache, print out the reasons why
877 std::string reasonString = "Rejected from cached dylibs: " + runtimePath + " " + architecture + " (\"";
878 for (auto i = reasons.begin(); i != reasons.end(); ++i) {
880 if (i != --reasons.end()) {
881 reasonString += "\", \"";
884 reasonString += "\")";
885 archManifest.results.exclude(&parser, reasonString);
887 } else if (parser.fileType() == MH_BUNDLE) {
888 if (archManifest.results.bundles.count(uuid) == 0) {
889 archManifest.results.bundles[uuid].uuid = uuid;
891 } else if (parser.fileType() == MH_EXECUTE) {
892 //HACK exclude all launchd and installd variants until we can do something about xpcd_cache.dylib and friends
893 if (runtimePath == "/sbin/launchd"
894 || runtimePath == "/usr/local/sbin/launchd.debug"
895 || runtimePath == "/usr/local/sbin/launchd.development"
896 || runtimePath == "/usr/libexec/installd") {
899 if (archManifest.results.executables.count(uuid) == 0) {
900 archManifest.results.executables[uuid].uuid = uuid;
906 __block std::set<UUID> removedUUIDs;
907 __block bool doAgain = true;
909 //Trim out dylibs that are missing dependencies
912 for (const auto& uuid : cachedUUIDs) {
913 __block std::set<std::string> badDependencies;
914 __block auto parser = parserForUUID(uuid);
915 parser.forEachDependentDylib(^(const char* loadPath, bool isWeak, bool isReExport, bool isUpward, uint32_t compatVersion, uint32_t curVersion, bool& stop) {
919 auto i = _installNameMap.find(std::make_pair(loadPath, architecture));
920 if (i == _installNameMap.end() || removedUUIDs.count(i->second)) {
921 removedUUIDs.insert(uuid);
922 badDependencies.insert(loadPath);
926 if (badDependencies.size()) {
927 std::string reasonString = "Rejected from cached dylibs: " + std::string(parser.installName()) + " " + architecture + " (\"";
928 for (auto i = badDependencies.begin(); i != badDependencies.end(); ++i) {
930 if (i != --badDependencies.end()) {
931 reasonString += "\", \"";
934 reasonString += "\")";
935 archManifest.results.exclude(&parser, reasonString);
940 for (const auto& removedUUID : removedUUIDs) {
941 cachedUUIDs.erase(removedUUID);
945 //Trim out excluded leaf dylibs
946 __block std::set<std::string> linkedDylibs;
948 for(const auto& uuid : cachedUUIDs) {
949 auto parser = parserForUUID(uuid);
950 parser.forEachDependentDylib(^(const char* loadPath, bool isWeak, bool isReExport, bool isUpward, uint32_t compatVersion, uint32_t curVersion, bool& stop) {
951 linkedDylibs.insert(loadPath);
955 for(const auto& uuid : cachedUUIDs) {
956 auto info = infoForUUID(uuid);
957 auto i = _metabomTagMap.find(info.runtimePath);
958 assert(i != _metabomTagMap.end());
959 auto exclusions = _metabomExcludeTagMap.find(configuration);
960 if (exclusions == _metabomExcludeTagMap.end() || is_disjoint(exclusions->second, i->second))
963 if (linkedDylibs.count(info.installName) != 0)
966 archManifest.results.exclude(*this, info.uuid, "Dylib '" + info.runtimePath + "' excluded leaf node");
970 void Manifest::writeJSON(const std::string& path) {
971 NSMutableDictionary* jsonDict = [[NSMutableDictionary alloc] init];
972 for (auto& configuration : _configurations) {
973 jsonDict[cppToObjStr(configuration.first)] = [[NSMutableDictionary alloc] init];
975 for (auto& arch : configuration.second.architectures) {
976 NSMutableOrderedSet* includedDylibsSet = [[NSMutableOrderedSet alloc] init];
977 NSMutableOrderedSet* executablesSet = [[NSMutableOrderedSet alloc] init];
978 NSMutableOrderedSet* otherSet = [[NSMutableOrderedSet alloc] init];
979 for (auto& dylib : arch.second.results.dylibs) {
980 NSString *runtimePath = cppToObjStr(runtimePathForUUID(dylib.second.uuid));
981 if (dylib.second.included) {
982 [includedDylibsSet addObject:runtimePath];
984 [otherSet addObject:runtimePath];
988 for (auto& executable : arch.second.results.executables) {
989 NSString *runtimePath = cppToObjStr(runtimePathForUUID(executable.second.uuid));
990 [executablesSet addObject:runtimePath];
993 for (auto& bundle : arch.second.results.bundles) {
994 NSString *runtimePath = cppToObjStr(runtimePathForUUID(bundle.second.uuid));
995 [otherSet addObject:runtimePath];
998 [includedDylibsSet sortUsingComparator:^NSComparisonResult(id _Nonnull obj1, id _Nonnull obj2) {
999 return [obj1 compare:obj2];
1002 [executablesSet sortUsingComparator:^NSComparisonResult(id _Nonnull obj1, id _Nonnull obj2) {
1003 return [obj1 compare:obj2];
1006 [otherSet sortUsingComparator:^NSComparisonResult(id _Nonnull obj1, id _Nonnull obj2) {
1007 return [obj1 compare:obj2];
1010 jsonDict[cppToObjStr(configuration.first)][cppToObjStr(arch.first)] = @{ @"cachedDylibs" : [includedDylibsSet array], @"mainExecutables" : [executablesSet array], @"other" : [otherSet array]};;
1014 NSError* error = nil;
1015 NSData *jsonData = [NSJSONSerialization dataWithJSONObject:jsonDict options:0x0 error:&error];
1016 (void)[jsonData writeToFile:cppToObjStr(path) atomically:YES];
1019 void Manifest::write(const std::string& path)
1024 NSMutableDictionary* cacheDict = [[NSMutableDictionary alloc] init];
1025 NSMutableDictionary* projectDict = [[NSMutableDictionary alloc] init];
1026 NSMutableDictionary* configurationsDict = [[NSMutableDictionary alloc] init];
1027 NSMutableDictionary* resultsDict = [[NSMutableDictionary alloc] init];
1029 cacheDict[@"manifest-version"] = @(version());
1030 cacheDict[@"build"] = cppToObjStr(build());
1031 cacheDict[@"dylibOrderFile"] = cppToObjStr(dylibOrderFile());
1032 cacheDict[@"dirtyDataOrderFile"] = cppToObjStr(dirtyDataOrderFile());
1033 cacheDict[@"metabomFile"] = cppToObjStr(metabomFile());
1035 cacheDict[@"projects"] = projectDict;
1036 cacheDict[@"results"] = resultsDict;
1037 cacheDict[@"configurations"] = configurationsDict;
1039 for (const auto& project : projects()) {
1040 NSMutableArray* sources = [[NSMutableArray alloc] init];
1042 for (const auto& source : project.second.sources) {
1043 [sources addObject:cppToObjStr(source)];
1046 projectDict[cppToObjStr(project.first)] = sources;
1049 for (auto& configuration : _configurations) {
1050 NSMutableArray* archArray = [[NSMutableArray alloc] init];
1051 for (auto& arch : configuration.second.architectures) {
1052 [archArray addObject:cppToObjStr(arch.first)];
1055 NSMutableArray* excludeTags = [[NSMutableArray alloc] init];
1056 for (const auto& excludeTag : configuration.second.metabomExcludeTags) {
1057 [excludeTags addObject:cppToObjStr(excludeTag)];
1060 configurationsDict[cppToObjStr(configuration.first)] = @{
1061 @"platformName" : cppToObjStr(configuration.second.platformName),
1062 @"metabomTag" : cppToObjStr(configuration.second.metabomTag),
1063 @"metabomExcludeTags" : excludeTags,
1064 @"architectures" : archArray
1068 for (auto& configuration : _configurations) {
1069 NSMutableDictionary* archResultsDict = [[NSMutableDictionary alloc] init];
1070 for (auto& arch : configuration.second.architectures) {
1071 NSMutableDictionary* dylibsDict = [[NSMutableDictionary alloc] init];
1072 NSMutableArray* warningsArray = [[NSMutableArray alloc] init];
1073 NSMutableDictionary* devRegionsDict = [[NSMutableDictionary alloc] init];
1074 NSMutableDictionary* prodRegionsDict = [[NSMutableDictionary alloc] init];
1075 NSString* prodCDHash = cppToObjStr(arch.second.results.productionCache.cdHash);
1076 NSString* devCDHash = cppToObjStr(arch.second.results.developmentCache.cdHash);
1078 for (auto& dylib : arch.second.results.dylibs) {
1079 NSMutableDictionary* dylibDict = [[NSMutableDictionary alloc] init];
1080 if (dylib.second.included) {
1081 dylibDict[@"included"] = @YES;
1083 dylibDict[@"included"] = @NO;
1084 dylibDict[@"exclusionInfo"] = cppToObjStr(dylib.second.exclusionInfo);
1086 dylibsDict[cppToObjStr(dylib.second.installname)] = dylibDict;
1089 for (auto& warning : arch.second.results.warnings) {
1090 [warningsArray addObject:cppToObjStr(warning)];
1093 BOOL built = arch.second.results.failure.empty();
1094 archResultsDict[cppToObjStr(arch.first)] = @{
1095 @"dylibs" : dylibsDict,
1096 @"built" : @(built),
1097 @"failure" : cppToObjStr(arch.second.results.failure),
1098 @"productionCache" : @{ @"cdhash" : prodCDHash, @"regions" : prodRegionsDict },
1099 @"developmentCache" : @{ @"cdhash" : devCDHash, @"regions" : devRegionsDict },
1100 @"warnings" : warningsArray
1103 resultsDict[cppToObjStr(configuration.first)] = archResultsDict;
1106 switch (platform()) {
1108 cacheDict[@"platform"] = @"ios";
1110 case Platform::tvOS:
1111 cacheDict[@"platform"] = @"tvos";
1113 case Platform::watchOS:
1114 cacheDict[@"platform"] = @"watchos";
1116 case Platform::bridgeOS:
1117 cacheDict[@"platform"] = @"bridgeos";
1119 case Platform::macOS:
1120 cacheDict[@"platform"] = @"macos";
1122 case Platform::unknown:
1123 cacheDict[@"platform"] = @"unknown";
1127 NSError* error = nil;
1128 NSData* outData = [NSPropertyListSerialization dataWithPropertyList:cacheDict
1129 format:NSPropertyListBinaryFormat_v1_0
1132 (void)[outData writeToFile:cppToObjStr(path) atomically:YES];