dyld-551.3.tar.gz
[apple/dyld.git] / dyld3 / shared-cache / Manifest.mm
1 /*
2 * Copyright (c) 2017 Apple Inc. All rights reserved.
3 *
4 * @APPLE_LICENSE_HEADER_START@
5 *
6 * This file contains Original Code and/or Modifications of Original Code
7 * as defined in and that are subject to the Apple Public Source License
8 * Version 2.0 (the 'License'). You may not use this file except in
9 * compliance with the License. Please obtain a copy of the License at
10 * http://www.opensource.apple.com/apsl/ and read it before using this
11 * file.
12 *
13 * The Original Code and all software distributed under the License are
14 * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
15 * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
16 * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
17 * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
18 * Please see the License for the specific language governing rights and
19 * limitations under the License.
20 *
21 * @APPLE_LICENSE_HEADER_END@
22 */
23
24
25 extern "C" {
26 #include <Bom/Bom.h>
27 #include <Metabom/MBTypes.h>
28 #include <Metabom/MBEntry.h>
29 #include <Metabom/MBMetabom.h>
30 #include <Metabom/MBIterator.h>
31 };
32
33 #include <algorithm>
34 #include <CommonCrypto/CommonDigest.h>
35 #include <CommonCrypto/CommonDigestSPI.h>
36 #include <Foundation/Foundation.h>
37
38 #include "MachOFileAbstraction.hpp"
39 #include "FileAbstraction.hpp"
40 #include "Trie.hpp"
41 #include "FileUtils.h"
42 #include "StringUtils.h"
43
44 #include <mach-o/loader.h>
45 #include <mach-o/fat.h>
46
47 #include <array>
48 #include <vector>
49
50 #include "Manifest.h"
51
52 namespace {
53 //FIXME this should be in a class
54 static inline NSString* cppToObjStr(const std::string& str) { return [NSString stringWithUTF8String:str.c_str()]; }
55
56 template <class Set1, class Set2>
57 inline bool is_disjoint(const Set1& set1, const Set2& set2)
58 {
59 if (set1.empty() || set2.empty())
60 return true;
61
62 typename Set1::const_iterator it1 = set1.begin(), it1End = set1.end();
63 typename Set2::const_iterator it2 = set2.begin(), it2End = set2.end();
64
65 if (*it1 > *set2.rbegin() || *it2 > *set1.rbegin())
66 return true;
67
68 while (it1 != it1End && it2 != it2End) {
69 if (*it1 == *it2)
70 return false;
71 if (*it1 < *it2) {
72 it1++;
73 } else {
74 it2++;
75 }
76 }
77
78 return true;
79 }
80
81 //hACK: If we declare this in manifest
82 static NSDictionary* gManifestDict;
83
84 } /* Anonymous namespace */
85
86 namespace dyld3 {
87 void Manifest::Results::exclude(MachOParser* parser, const std::string& reason)
88 {
89 auto dylibUUID = parser->uuid();
90 dylibs[dylibUUID].uuid = dylibUUID;
91 dylibs[dylibUUID].installname = parser->installName();
92 dylibs[dylibUUID].included = false;
93 dylibs[dylibUUID].exclusionInfo = reason;
94 }
95
96 void Manifest::Results::exclude(Manifest& manifest, const UUID& uuid, const std::string& reason)
97 {
98 auto parser = manifest.parserForUUID(uuid);
99 dylibs[uuid].uuid = uuid;
100 dylibs[uuid].installname = parser.installName();
101 dylibs[uuid].included = false;
102 dylibs[uuid].exclusionInfo = reason;
103 }
104
105 Manifest::CacheImageInfo& Manifest::Results::dylibForInstallname(const std::string& installname)
106 {
107 auto i = find_if(dylibs.begin(), dylibs.end(), [&installname](std::pair<UUID, CacheImageInfo> d) { return d.second.installname == installname; });
108 assert(i != dylibs.end());
109 return i->second;
110 }
111
112 bool Manifest::Architecture::operator==(const Architecture& O) const
113 {
114 for (auto& dylib : results.dylibs) {
115 if (dylib.second.included) {
116 auto Odylib = O.results.dylibs.find(dylib.first);
117 if (Odylib == O.results.dylibs.end()
118 || Odylib->second.included == false
119 || Odylib->second.uuid != dylib.second.uuid)
120 return false;
121 }
122 }
123
124 for (const auto& Odylib : O.results.dylibs) {
125 if (Odylib.second.included) {
126 auto dylib = results.dylibs.find(Odylib.first);
127 if (dylib == results.dylibs.end()
128 || dylib->second.included == false
129 || dylib->second.uuid != Odylib.second.uuid)
130 return false;
131 }
132 }
133
134 for (auto& bundle : results.bundles) {
135 if (bundle.second.included) {
136 auto Obundle = O.results.bundles.find(bundle.first);
137 if (Obundle == O.results.bundles.end()
138 || Obundle->second.included == false
139 || Obundle->second.uuid != bundle.second.uuid)
140 return false;
141 }
142 }
143
144 for (const auto& Obundle : O.results.bundles) {
145 if (Obundle.second.included) {
146 auto bundle = results.bundles.find(Obundle.first);
147 if (bundle == results.bundles.end()
148 || bundle->second.included == false
149 || bundle->second.uuid != Obundle.second.uuid)
150 return false;
151 }
152 }
153
154 for (auto& executable : results.executables) {
155 if (executable.second.included) {
156 auto Oexecutable = O.results.executables.find(executable.first);
157 if (Oexecutable == O.results.executables.end()
158 || Oexecutable->second.included == false
159 || Oexecutable->second.uuid != executable.second.uuid)
160 return false;
161 }
162 }
163
164 for (const auto& Oexecutable : O.results.executables) {
165 if (Oexecutable.second.included) {
166 auto executable = results.executables.find(Oexecutable.first);
167 if (executable == results.executables.end()
168 || executable->second.included == false
169 || executable->second.uuid != Oexecutable.second.uuid)
170 return false;
171 }
172 }
173
174 return true;
175 }
176
177 bool Manifest::Configuration::operator==(const Configuration& O) const
178 {
179 return architectures == O.architectures;
180 }
181
182 bool Manifest::Configuration::operator!=(const Configuration& other) const { return !(*this == other); }
183
184 const Manifest::Architecture& Manifest::Configuration::architecture(const std::string& architecture) const
185 {
186 assert(architectures.find(architecture) != architectures.end());
187 return architectures.find(architecture)->second;
188 }
189
190 void Manifest::Configuration::forEachArchitecture(std::function<void(const std::string& archName)> lambda) const
191 {
192 for (const auto& architecutre : architectures) {
193 lambda(architecutre.first);
194 }
195 }
196
197 bool Manifest::Architecture::operator!=(const Architecture& other) const { return !(*this == other); }
198
199 const std::map<std::string, Manifest::Project>& Manifest::projects()
200 {
201 return _projects;
202 }
203
204 const Manifest::Configuration& Manifest::configuration(const std::string& configuration) const
205 {
206 assert(_configurations.find(configuration) != _configurations.end());
207 return _configurations.find(configuration)->second;
208 }
209
210 void Manifest::forEachConfiguration(std::function<void(const std::string& configName)> lambda) const
211 {
212 for (const auto& configuration : _configurations) {
213 lambda(configuration.first);
214 }
215 }
216
217 void Manifest::addProjectSource(const std::string& project, const std::string& source, bool first)
218 {
219 auto& sources = _projects[project].sources;
220 if (std::find(sources.begin(), sources.end(), source) == sources.end()) {
221 if (first) {
222 sources.insert(sources.begin(), source);
223 } else {
224 sources.push_back(source);
225 }
226 }
227 }
228
229 const std::string Manifest::projectPath(const std::string& projectName)
230 {
231 auto project = _projects.find(projectName);
232 if (project == _projects.end())
233 return "";
234 if (project->second.sources.size() == 0)
235 return "";
236 return project->second.sources[0];
237 }
238
239 const bool Manifest::empty(void)
240 {
241 for (const auto& configuration : _configurations) {
242 if (configuration.second.architectures.size() != 0)
243 return false;
244 }
245 return true;
246 }
247
248 const std::string Manifest::dylibOrderFile() const { return _dylibOrderFile; };
249 void Manifest::setDylibOrderFile(const std::string& dylibOrderFile) { _dylibOrderFile = dylibOrderFile; };
250
251 const std::string Manifest::dirtyDataOrderFile() const { return _dirtyDataOrderFile; };
252 void Manifest::setDirtyDataOrderFile(const std::string& dirtyDataOrderFile) { _dirtyDataOrderFile = dirtyDataOrderFile; };
253
254 const std::string Manifest::metabomFile() const { return _metabomFile; };
255 void Manifest::setMetabomFile(const std::string& metabomFile) { _metabomFile = metabomFile; };
256
257 const Platform Manifest::platform() const { return _platform; };
258 void Manifest::setPlatform(const Platform platform) { _platform = platform; };
259
260 const std::string& Manifest::build() const { return _build; };
261 void Manifest::setBuild(const std::string& build) { _build = build; };
262 const uint32_t Manifest::version() const { return _manifestVersion; };
263 void Manifest::setVersion(const uint32_t manifestVersion) { _manifestVersion = manifestVersion; };
264
265 BuildQueueEntry Manifest::makeQueueEntry(const std::string& outputPath, const std::set<std::string>& configs, const std::string& arch, bool optimizeStubs, const std::string& prefix, bool verbose)
266 {
267 dyld3::BuildQueueEntry retval;
268
269 DyldSharedCache::CreateOptions options;
270 options.archName = arch;
271 options.platform = platform();
272 options.excludeLocalSymbols = true;
273 options.optimizeStubs = optimizeStubs;
274 options.optimizeObjC = true;
275 options.codeSigningDigestMode = (platform() == dyld3::Platform::watchOS) ?
276 DyldSharedCache::Agile : DyldSharedCache::SHA256only;
277 options.dylibsRemovedDuringMastering = true;
278 options.inodesAreSameAsRuntime = false;
279 options.cacheSupportsASLR = true;
280 options.forSimulator = false;
281 options.verbose = verbose;
282 options.evictLeafDylibsOnOverflow = true;
283 options.loggingPrefix = prefix;
284 options.pathPrefixes = { "" };
285 options.dylibOrdering = loadOrderFile(_dylibOrderFile);
286 options.dirtyDataSegmentOrdering = loadOrderFile(_dirtyDataOrderFile);
287
288 dyld3::BuildQueueEntry queueEntry;
289 retval.configNames = configs;
290 retval.options = options;
291 retval.outputPath = outputPath;
292 retval.dylibsForCache = dylibsForCache(*configs.begin(), arch);
293 retval.otherDylibsAndBundles = otherDylibsAndBundles(*configs.begin(), arch);
294 retval.mainExecutables = mainExecutables(*configs.begin(), arch);
295
296 return retval;
297 }
298
299 bool Manifest::loadParser(const void* p, size_t size, uint64_t sliceOffset, const std::string& runtimePath, const std::string& buildPath, const std::set<std::string>& architectures)
300 {
301 const mach_header* mh = reinterpret_cast<const mach_header*>(p);
302 if (!MachOParser::isValidMachO(_diags, "", _platform, p, size, runtimePath.c_str(), false)) {
303 return false;
304 }
305
306 auto parser = MachOParser(mh);
307 if (_diags.hasError()) {
308 // Clear the error and punt
309 _diags.verbose("MachoParser error: %s\n", _diags.errorMessage().c_str());
310 _diags.clearError();
311 return false;
312 }
313
314 auto uuid = parser.uuid();
315 auto archName = parser.archName();
316
317 if (parser.fileType() == MH_DYLIB && architectures.count(parser.archName()) != 0) {
318 std::string installName = parser.installName();
319 auto index = std::make_pair(installName, parser.archName());
320 auto i = _installNameMap.find(index);
321
322 if ( installName == "/System/Library/Caches/com.apple.xpc/sdk.dylib"
323 || installName == "/System/Library/Caches/com.apple.xpcd/xpcd_cache.dylib" ) {
324 // HACK to deal with device specific dylibs. These must not be inseted into the installNameMap
325 _uuidMap.insert(std::make_pair(uuid, UUIDInfo(mh, size, sliceOffset, uuid, parser.archName(), runtimePath, buildPath, installName)));
326 } else if (i == _installNameMap.end()) {
327 _installNameMap.insert(std::make_pair(index, uuid));
328 _uuidMap.insert(std::make_pair(uuid, UUIDInfo(mh, size, sliceOffset, uuid, parser.archName(), runtimePath, buildPath, installName)));
329 if (installName[0] != '@' && installName != runtimePath) {
330 _diags.warning("Dylib located at '%s' has installname '%s'", runtimePath.c_str(), installName.c_str());
331 }
332 } else {
333 auto info = infoForUUID(i->second);
334 _diags.warning("Multiple dylibs claim installname '%s' ('%s' and '%s')", installName.c_str(), runtimePath.c_str(), info.runtimePath.c_str());
335
336 // This is the "Good" one, overwrite
337 if (runtimePath == installName) {
338 _uuidMap.erase(uuid);
339 _uuidMap.insert(std::make_pair(uuid, UUIDInfo(mh, size, sliceOffset, uuid, parser.archName(), runtimePath, buildPath, installName)));
340 }
341 }
342 } else {
343 _uuidMap.insert(std::make_pair(uuid, UUIDInfo(mh, size, sliceOffset, uuid, parser.archName(), runtimePath, buildPath, "")));
344 }
345 return true;
346 }
347
348 //FIXME: assert we have not errored first
349 bool Manifest::loadParsers(const std::string& buildPath, const std::string& runtimePath, const std::set<std::string>& architectures)
350 {
351 __block bool retval = false;
352 const void* p = (uint8_t*)(-1);
353 struct stat stat_buf;
354
355 std::tie(p, stat_buf) = fileCache.cacheLoad(_diags, buildPath);
356
357 if (p == (uint8_t*)(-1)) {
358 return false;
359 }
360
361 if (FatUtil::isFatFile(p)) {
362 FatUtil::forEachSlice(_diags, p, stat_buf.st_size, ^(uint32_t sliceCpuType, uint32_t sliceCpuSubType, const void* sliceStart, size_t sliceSize, bool& stop) {
363 if (loadParser(sliceStart, sliceSize, (uintptr_t)sliceStart-(uintptr_t)p, runtimePath, buildPath, architectures))
364 retval = true;
365 });
366 } else {
367 return loadParser(p, stat_buf.st_size, 0, runtimePath, buildPath, architectures);
368 }
369 return retval;
370 }
371
372 const Manifest::UUIDInfo& Manifest::infoForUUID(const UUID& uuid) const {
373 auto i = _uuidMap.find(uuid);
374 assert(i != _uuidMap.end());
375 return i->second;
376 }
377
378 const Manifest::UUIDInfo Manifest::infoForInstallNameAndarch(const std::string& installName, const std::string arch) const {
379 UUIDInfo retval;
380 auto uuidI = _installNameMap.find(std::make_pair(installName, arch));
381 if (uuidI == _installNameMap.end())
382 return UUIDInfo();
383
384 auto i = _uuidMap.find(uuidI->second);
385 if (i == _uuidMap.end())
386 return UUIDInfo();
387 return i->second;
388 }
389
390 MachOParser Manifest::parserForUUID(const UUID& uuid) const {
391 return MachOParser(infoForUUID(uuid).mh);
392 }
393
394 const std::string Manifest::buildPathForUUID(const UUID& uuid) {
395 return infoForUUID(uuid).buildPath;
396 }
397
398 const std::string Manifest::runtimePathForUUID(const UUID& uuid) {
399 return infoForUUID(uuid).runtimePath;
400 }
401
402 Manifest::Manifest(Diagnostics& D, const std::string& path) : Manifest(D, path, std::set<std::string>())
403 {
404 }
405
406 Manifest::Manifest(Diagnostics& D, const std::string& path, const std::set<std::string>& overlays) :
407 _diags(D)
408 {
409 NSMutableDictionary* manifestDict = [NSMutableDictionary dictionaryWithContentsOfFile:cppToObjStr(path)];
410 NSString* platStr = manifestDict[@"platform"];
411 std::set<std::string> architectures;
412
413 if (platStr == nullptr)
414 platStr = @"ios";
415 std::string platformString = [platStr UTF8String];
416 setMetabomFile([manifestDict[@"metabomFile"] UTF8String]);
417
418 if (platformString == "ios") {
419 setPlatform(dyld3::Platform::iOS);
420 } else if ( (platformString == "tvos") || (platformString == "atv") ) {
421 setPlatform(dyld3::Platform::tvOS);
422 } else if ( (platformString == "watchos") || (platformString == "watch") ) {
423 setPlatform(dyld3::Platform::watchOS);
424 } else if ( (platformString == "bridgeos") || (platformString == "bridge") ) {
425 setPlatform(dyld3::Platform::bridgeOS);
426 } else if ( (platformString == "macos") || (platformString == "osx") ) {
427 setPlatform(dyld3::Platform::macOS);
428 } else {
429 //Fixme should we error?
430 setPlatform(dyld3::Platform::iOS);
431 }
432
433 for (NSString* project in manifestDict[@"projects"]) {
434 for (NSString* source in manifestDict[@"projects"][project]) {
435 addProjectSource([project UTF8String], [source UTF8String]);
436 }
437 }
438
439 for (NSString* configuration in manifestDict[@"configurations"]) {
440 std::string configStr = [configuration UTF8String];
441 std::string configTag = [manifestDict[@"configurations"][configuration][@"metabomTag"] UTF8String];
442
443 if (manifestDict[@"configurations"][configuration][@"metabomExcludeTags"]) {
444 for (NSString* excludeTag in manifestDict[@"configurations"][configuration][@"metabomExcludeTags"]) {
445 _metabomExcludeTagMap[configStr].insert([excludeTag UTF8String]);
446 _configurations[configStr].metabomExcludeTags.insert([excludeTag UTF8String]);
447 }
448 }
449
450 if (manifestDict[@"configurations"][configuration][@"metabomRestrictTags"]) {
451 for (NSString* restrictTag in manifestDict[@"configurations"][configuration][@"metabomRestrictTags"]) {
452 _metabomRestrictedTagMap[configStr].insert([restrictTag UTF8String]);
453 _configurations[configStr].metabomRestrictTags.insert([restrictTag UTF8String]);
454 }
455 }
456
457 _configurations[configStr].metabomTag = configTag;
458 _configurations[configStr].metabomTags.insert(configTag);
459 _configurations[configStr].platformName =
460 [manifestDict[@"configurations"][configuration][@"platformName"] UTF8String];
461
462 if (endsWith(configStr, "InternalOS")) {
463 _configurations[configStr].disposition = "internal";
464 _configurations[configStr].device = configStr.substr(0, configStr.length()-strlen("InternalOS"));
465 } else if (endsWith(configStr, "VendorOS")) {
466 _configurations[configStr].disposition = "internal";
467 _configurations[configStr].device = configStr.substr(0, configStr.length()-strlen("VendorOS"));
468 } else if (endsWith(configStr, "VendorUIOS")) {
469 _configurations[configStr].disposition = "internal";
470 _configurations[configStr].device = configStr.substr(0, configStr.length()-strlen("VendorUIOS"));
471 } else if (endsWith(configStr, "CarrierOS")) {
472 _configurations[configStr].disposition = "internal";
473 _configurations[configStr].device = configStr.substr(0, configStr.length()-strlen("CarrierOS"));
474 } else if (endsWith(configStr, "FactoryOS")) {
475 _configurations[configStr].disposition = "internal";
476 _configurations[configStr].device = configStr.substr(0, configStr.length()-strlen("FactoryOS"));
477 } else if (endsWith(configStr, "DesenseOS")) {
478 _configurations[configStr].disposition = "internal";
479 _configurations[configStr].device = configStr.substr(0, configStr.length()-strlen("DesenseOS"));
480 } else if (endsWith(configStr, "MinosOS")) {
481 _configurations[configStr].disposition = "minos";
482 _configurations[configStr].device = configStr.substr(0, configStr.length()-strlen("MinosOS"));
483 } else if (endsWith(configStr, "DemoOS")) {
484 _configurations[configStr].disposition = "demo";
485 _configurations[configStr].device = configStr.substr(0, configStr.length()-strlen("DemoOS"));
486 } else if (endsWith(configStr, "MinosOS")) {
487 _configurations[configStr].disposition = "minos";
488 _configurations[configStr].device = configStr.substr(0, configStr.length()-strlen("MinosOS"));
489 } else if (endsWith(configStr, "DeveloperOS")) {
490 _configurations[configStr].disposition = "user";
491 _configurations[configStr].device = configStr.substr(0, configStr.length()-strlen("DeveloperOS"));
492 } else if (endsWith(configStr, "OS")) {
493 _configurations[configStr].disposition = "user";
494 _configurations[configStr].device = configStr.substr(0, configStr.length()-strlen("OS"));
495 }
496
497 for (NSString* architecutre in manifestDict[@"configurations"][configuration][@"architectures"]) {
498 //HACK until B&I stops mastering armv7s
499 if ([architecutre isEqual:@"armv7s"]) break;
500 _configurations[configStr].architectures[[architecutre UTF8String]] = Architecture();
501 architectures.insert([architecutre UTF8String]);
502 }
503 }
504
505 setVersion([manifestDict[@"manifest-version"] unsignedIntValue]);
506 setBuild([manifestDict[@"build"] UTF8String]);
507 if (manifestDict[@"dylibOrderFile"]) {
508 setDylibOrderFile([manifestDict[@"dylibOrderFile"] UTF8String]);
509 }
510 if (manifestDict[@"dirtyDataOrderFile"]) {
511 setDirtyDataOrderFile([manifestDict[@"dirtyDataOrderFile"] UTF8String]);
512 }
513
514 auto metabom = MBMetabomOpen(metabomFile().c_str(), false);
515 auto metabomEnumerator = MBIteratorNewWithPath(metabom, ".", "");
516 MBEntry entry;
517
518 // FIXME error handling (NULL metabom)
519
520 //First we iterate through the bom and build our objects
521
522 while ((entry = MBIteratorNext(metabomEnumerator))) {
523 BOMFSObject fsObject = MBEntryGetFSObject(entry);
524 BOMFSObjType entryType = BOMFSObjectType(fsObject);
525 std::string entryPath = BOMFSObjectPathName(fsObject);
526 if (entryPath[0] == '.') {
527 entryPath.erase(0, 1);
528 }
529
530 // Skip artifacts that happen to be in the build chain
531 if ( startsWith(entryPath, "/Applications/Xcode.app") ) {
532 continue;
533 }
534
535 // Skip variants we can't deal with
536 if ( endsWith(entryPath, "_profile.dylib") || endsWith(entryPath, "_debug.dylib") || endsWith(entryPath, "_profile") || endsWith(entryPath, "_debug") || endsWith(entryPath, "/CoreADI") ) {
537 continue;
538 }
539
540 // Skip images that are only used in InternalOS variants
541 if ( startsWith(entryPath, "/AppleInternal/") || startsWith(entryPath, "/usr/local/") || startsWith(entryPath, "/Developer/")) {
542 continue;
543 }
544
545 // Skip genCache generated dylibs
546 if ( endsWith(entryPath, "/System/Library/Caches/com.apple.xpc/sdk.dylib") || endsWith(entryPath, "/System/Library/Caches/com.apple.xpcd/xpcd_cache.dylib")) {
547 continue;
548 }
549
550 MBTag tag;
551 auto tagCount = MBEntryGetNumberOfProjectTags(entry);
552 if (entryType == BOMFileType && BOMFSObjectIsBinaryObject(fsObject) && MBEntryGetNumberOfProjectTags(entry) != 0 && tagCount != 0) {
553 if (tagCount == 1) {
554 MBEntryGetProjectTags(entry, &tag);
555 } else {
556 MBTag* tags = (MBTag*)malloc(sizeof(MBTag) * tagCount);
557 MBEntryGetProjectTags(entry, tags);
558
559 //Sigh, we can have duplicate entries for the same tag, so build a set to work with
560 std::set<std::string> tagStrs;
561 std::map<std::string, MBTag> tagStrMap;
562 for (auto i = 0; i < tagCount; ++i) {
563 tagStrs.insert(MBMetabomGetProjectForTag(metabom, tags[i]));
564 tagStrMap.insert(std::make_pair(MBMetabomGetProjectForTag(metabom, tags[i]), tags[i]));
565 }
566
567 if (tagStrs.size() > 1) {
568 std::string projects;
569 for (const auto& tagStr : tagStrs) {
570 if (!projects.empty())
571 projects += ", ";
572
573 projects += "'" + tagStr + "'";
574 }
575 _diags.warning("Bom entry '%s' is claimed by multiple projects: %s, taking first entry", entryPath.c_str(), projects.c_str());
576 }
577 tag = tagStrMap[*tagStrs.begin()];
578 free(tags);
579 }
580
581 std::string projectName = MBMetabomGetProjectForTag(metabom, tag);
582 tagCount = MBEntryGetNumberOfPackageTags(entry);
583 MBTag* tags = (MBTag*)malloc(sizeof(MBTag) * tagCount);
584 MBEntryGetPackageTags(entry, tags);
585 std::set<std::string> tagStrs;
586
587 for (auto i = 0; i < tagCount; ++i) {
588 tagStrs.insert(MBMetabomGetPackageForTag(metabom, tags[i]));
589 }
590
591 _metabomTagMap.insert(std::make_pair(entryPath, tagStrs));
592 bool foundParser = false;
593 for (const auto& overlay : overlays) {
594 if (loadParsers(overlay + "/" + entryPath, entryPath, architectures)) {
595 foundParser = true;
596 break;
597 }
598 }
599
600 if (!foundParser) {
601 (void)loadParsers(projectPath(projectName) + "/" + entryPath, entryPath, architectures);
602 }
603 }
604 }
605
606 MBIteratorFree(metabomEnumerator);
607 MBMetabomFree(metabom);
608 }
609
610 void Manifest::insert(std::vector<DyldSharedCache::MappedMachO>& mappedMachOs, const CacheImageInfo& imageInfo) {
611 auto info = infoForUUID(imageInfo.uuid);
612 auto runtimePath = info.runtimePath;
613 mappedMachOs.emplace_back(runtimePath, info.mh, info.size, false, false, info.sliceFileOffset, 0, 0);
614 }
615
616 std::vector<DyldSharedCache::MappedMachO> Manifest::dylibsForCache(const std::string& configuration, const std::string& architecture)
617 {
618 std::vector<DyldSharedCache::MappedMachO> retval;
619 const auto& dylibs = _configurations[configuration].architectures[architecture].results.dylibs;
620 for (const auto& dylib : dylibs) {
621 if (dylib.second.included) {
622 insert(retval, dylib.second);
623 }
624 }
625 return retval;
626 }
627
628 std::vector<DyldSharedCache::MappedMachO> Manifest::otherDylibsAndBundles(const std::string& configuration, const std::string& architecture)
629 {
630 std::vector<DyldSharedCache::MappedMachO> retval;
631 const auto& dylibs = _configurations[configuration].architectures[architecture].results.dylibs;
632 for (const auto& dylib : dylibs) {
633 if (!dylib.second.included) {
634 insert(retval, dylib.second);
635 }
636 }
637
638 const auto& bundles = _configurations[configuration].architectures[architecture].results.bundles;
639 for (const auto& bundle : bundles) {
640 insert(retval, bundle.second);
641 }
642
643 return retval;
644 }
645
646 std::vector<DyldSharedCache::MappedMachO> Manifest::mainExecutables(const std::string& configuration, const std::string& architecture)
647 {
648 std::vector<DyldSharedCache::MappedMachO> retval;
649 const auto& executables = _configurations[configuration].architectures[architecture].results.executables;
650 for (const auto& executable : executables) {
651 insert(retval, executable.second);
652 }
653
654 return retval;
655 }
656
657 bool Manifest::filterForConfig(const std::string& configName)
658 {
659 for (const auto configuration : _configurations) {
660 if (configName == configuration.first) {
661 std::map<std::string, Configuration> filteredConfigs;
662 filteredConfigs[configName] = configuration.second;
663
664 _configurations = filteredConfigs;
665
666 for (auto& arch : configuration.second.architectures) {
667 arch.second.results = Manifest::Results();
668 }
669 return true;
670 }
671 }
672 return false;
673 }
674
675 void Manifest::dedupeDispositions(void) {
676 // Since this is all hacky and inference based for now only do it for iOS until XBS
677 // is reved to give us real info. All the other platforms are way smaller anyway.
678 if (_platform != Platform::iOS)
679 return;
680
681 std::map<std::pair<std::string, std::string>, std::set<std::string>> dispositionSets;
682
683 for (const auto& configuration : _configurations) {
684 dispositionSets[std::make_pair(configuration.second.device, configuration.second.disposition)].insert(configuration.first);
685 }
686
687 for (const auto& dSet : dispositionSets) {
688 for (const auto &c1 : dSet.second) {
689 for (const auto &c2 : dSet.second) {
690 _configurations[c1].metabomTags.insert(_configurations[c2].metabomTag);
691 }
692 }
693 }
694 }
695
696 void Manifest::calculateClosure()
697 {
698 auto closureSemaphore = dispatch_semaphore_create(32);
699 auto closureGroup = dispatch_group_create();
700 auto closureQueue = dispatch_queue_create("com.apple.dyld.cache.closure", dispatch_queue_attr_make_with_qos_class(DISPATCH_QUEUE_CONCURRENT, QOS_CLASS_USER_INITIATED, 0));
701
702 dedupeDispositions();
703 for (auto& config : _configurations) {
704 for (auto& arch : config.second.architectures) {
705 dispatch_semaphore_wait(closureSemaphore, DISPATCH_TIME_FOREVER);
706 dispatch_group_async(closureGroup, closureQueue, [&] {
707 calculateClosure(config.first, arch.first);
708 dispatch_semaphore_signal(closureSemaphore);
709 });
710 }
711 }
712
713 dispatch_group_wait(closureGroup, DISPATCH_TIME_FOREVER);
714 }
715
716 void Manifest::remove(const std::string& config, const std::string& arch)
717 {
718 if (_configurations.count(config))
719 _configurations[config].architectures.erase(arch);
720 }
721
722 void Manifest::removeDylib(MachOParser parser, const std::string& reason, const std::string& configuration,
723 const std::string& architecture, std::unordered_set<UUID>& processedIdentifiers)
724 {
725 #if 0
726 auto configIter = _configurations.find(configuration);
727 if (configIter == _configurations.end())
728 return;
729 auto archIter = configIter->second.architectures.find( architecture );
730 if ( archIter == configIter->second.architectures.end() ) return;
731 auto& archManifest = archIter->second;
732
733 if (archManifest.results.dylibs.count(parser->uuid()) == 0) {
734 archManifest.results.dylibs[parser->uuid()].uuid = parser->uuid();
735 archManifest.results.dylibs[parser->uuid()].installname = parser->installName();
736 processedIdentifiers.insert(parser->uuid());
737 }
738 archManifest.results.exclude(MachOProxy::forIdentifier(parser->uuid(), architecture), reason);
739
740 processedIdentifiers.insert(parser->uuid());
741
742 for (const auto& dependent : proxy->dependentIdentifiers) {
743 auto dependentProxy = MachOProxy::forIdentifier(dependent, architecture);
744 auto dependentResultIter = archManifest.results.dylibs.find(dependentProxy->identifier);
745 if ( dependentProxy &&
746 ( dependentResultIter == archManifest.results.dylibs.end() || dependentResultIter->second.included == true ) ) {
747 removeDylib(dependentProxy, "Missing dependency: " + proxy->installName, configuration, architecture,
748 processedIdentifiers);
749 }
750 }
751 #endif
752 }
753
754 const std::string Manifest::removeLargestLeafDylib(const std::set<std::string>& configurations, const std::string& architecture)
755 {
756 // Find the leaf nodes
757 __block std::map<std::string, uint64_t> dependentCounts;
758 for (const auto& dylib : _configurations[*configurations.begin()].architectures[architecture].results.dylibs) {
759 if (!dylib.second.included)
760 continue;
761 std::string installName;
762 auto info = infoForUUID(dylib.first);
763 auto parser = MachOParser(info.mh);
764 dependentCounts[parser.installName()] = 0;
765 }
766
767 for (const auto& dylib : _configurations[*configurations.begin()].architectures[architecture].results.dylibs) {
768 if (!dylib.second.included)
769 continue;
770 auto info = infoForUUID(dylib.first);
771 auto parser = MachOParser(info.mh);
772 parser.forEachDependentDylib(^(const char *loadPath, bool isWeak, bool isReExport, bool isUpward, uint32_t compatVersion, uint32_t curVersion, bool &stop) {
773 if (!isWeak) {
774 dependentCounts[loadPath]++;
775 }
776 });
777 }
778
779 // Figure out which leaf is largest
780 UUIDInfo largestLeaf;
781
782 for (const auto& dependentCount : dependentCounts) {
783 if (dependentCount.second == 0) {
784 auto info = infoForInstallNameAndarch(dependentCount.first, architecture);
785 assert(info.mh != nullptr);
786 if (info.size > largestLeaf.size) {
787 largestLeaf = info;
788 }
789 }
790 }
791
792 if (largestLeaf.mh == nullptr) {
793 _diags.error("Fatal overflow, could not evict more dylibs");
794 return "";
795 }
796
797 // Remove it ferom all configs
798 for (const auto& config : configurations) {
799 configuration(config).architecture(architecture).results.exclude(*this, largestLeaf.uuid, "Cache Overflow");
800 }
801
802 return largestLeaf.installName;
803 }
804
805 void Manifest::calculateClosure(const std::string& configuration, const std::string& architecture)
806 {
807 __block auto& configManifest = _configurations[configuration];
808 __block auto& archManifest = _configurations[configuration].architectures[architecture];
809 __block std::set<UUID> newUuids;
810 std::set<UUID> processedUuids;
811 std::set<UUID> cachedUUIDs;
812
813 // Seed anchors
814 for (auto& uuidInfo : _uuidMap) {
815 auto info = uuidInfo.second;
816 if (info.arch != architecture) {
817 continue;
818 }
819
820 auto i = _metabomTagMap.find(info.runtimePath);
821 assert(i != _metabomTagMap.end());
822 auto tags = i->second;
823 if (!is_disjoint(tags, configManifest.metabomTags)) {
824 newUuids.insert(info.uuid);
825
826 }
827 }
828
829 // Pull in all dependencies
830 while (!newUuids.empty()) {
831 std::set<UUID> uuidsToProcess = newUuids;
832 newUuids.clear();
833
834 for (const auto& uuid : uuidsToProcess) {
835 if (processedUuids.count(uuid) > 0) {
836 continue;
837 }
838 processedUuids.insert(uuid);
839
840 auto parser = parserForUUID(uuid);
841 auto runtimePath = runtimePathForUUID(uuid);
842 assert(parser.header() != 0);
843
844 parser.forEachDependentDylib(^(const char* loadPath, bool isWeak, bool isReExport, bool isUpward, uint32_t compatVersion, uint32_t curVersion, bool& stop) {
845 auto i = _installNameMap.find(std::make_pair(loadPath, architecture));
846 if (i != _installNameMap.end())
847 newUuids.insert(i->second);
848 });
849
850 if (parser.fileType() == MH_DYLIB) {
851 // Add the dylib to the results
852 if (archManifest.results.dylibs.count(uuid) == 0 ) {
853 archManifest.results.dylibs[uuid].uuid = uuid;
854 archManifest.results.dylibs[uuid].installname = parser.installName();
855 }
856
857 // HACK to insert device specific dylib closures into all caches
858 if ( parser.installName() == std::string("/System/Library/Caches/com.apple.xpc/sdk.dylib")
859 || parser.installName() == std::string("/System/Library/Caches/com.apple.xpcd/xpcd_cache.dylib") ) {
860 archManifest.results.exclude(&parser, "Device specific dylib");
861 continue;
862 }
863
864 std::set<std::string> reasons;
865 if (parser.canBePlacedInDyldCache(runtimePath, reasons)) {
866 auto i = _metabomTagMap.find(runtimePath);
867 assert(i != _metabomTagMap.end());
868 auto restrictions = _metabomRestrictedTagMap.find(configuration);
869 if (restrictions != _metabomRestrictedTagMap.end() && !is_disjoint(restrictions->second, i->second)) {
870 archManifest.results.exclude(&parser, "Dylib '" + runtimePath + "' removed due to explict restriction");
871 }
872
873 // It can be placed in the cache, grab its dependents and queue them for inclusion
874 cachedUUIDs.insert(parser.uuid());
875 } else {
876 // It can't be placed in the cache, print out the reasons why
877 std::string reasonString = "Rejected from cached dylibs: " + runtimePath + " " + architecture + " (\"";
878 for (auto i = reasons.begin(); i != reasons.end(); ++i) {
879 reasonString += *i;
880 if (i != --reasons.end()) {
881 reasonString += "\", \"";
882 }
883 }
884 reasonString += "\")";
885 archManifest.results.exclude(&parser, reasonString);
886 }
887 } else if (parser.fileType() == MH_BUNDLE) {
888 if (archManifest.results.bundles.count(uuid) == 0) {
889 archManifest.results.bundles[uuid].uuid = uuid;
890 }
891 } else if (parser.fileType() == MH_EXECUTE) {
892 //HACK exclude all launchd and installd variants until we can do something about xpcd_cache.dylib and friends
893 if (runtimePath == "/sbin/launchd"
894 || runtimePath == "/usr/local/sbin/launchd.debug"
895 || runtimePath == "/usr/local/sbin/launchd.development"
896 || runtimePath == "/usr/libexec/installd") {
897 continue;
898 }
899 if (archManifest.results.executables.count(uuid) == 0) {
900 archManifest.results.executables[uuid].uuid = uuid;
901 }
902 }
903 }
904 }
905
906 __block std::set<UUID> removedUUIDs;
907 __block bool doAgain = true;
908
909 //Trim out dylibs that are missing dependencies
910 while ( doAgain ) {
911 doAgain = false;
912 for (const auto& uuid : cachedUUIDs) {
913 __block std::set<std::string> badDependencies;
914 __block auto parser = parserForUUID(uuid);
915 parser.forEachDependentDylib(^(const char* loadPath, bool isWeak, bool isReExport, bool isUpward, uint32_t compatVersion, uint32_t curVersion, bool& stop) {
916 if (isWeak)
917 return;
918
919 auto i = _installNameMap.find(std::make_pair(loadPath, architecture));
920 if (i == _installNameMap.end() || removedUUIDs.count(i->second)) {
921 removedUUIDs.insert(uuid);
922 badDependencies.insert(loadPath);
923 doAgain = true;
924 }
925
926 if (badDependencies.size()) {
927 std::string reasonString = "Rejected from cached dylibs: " + std::string(parser.installName()) + " " + architecture + " (\"";
928 for (auto i = badDependencies.begin(); i != badDependencies.end(); ++i) {
929 reasonString += *i;
930 if (i != --badDependencies.end()) {
931 reasonString += "\", \"";
932 }
933 }
934 reasonString += "\")";
935 archManifest.results.exclude(&parser, reasonString);
936 }
937 });
938 }
939
940 for (const auto& removedUUID : removedUUIDs) {
941 cachedUUIDs.erase(removedUUID);
942 }
943 }
944
945 //Trim out excluded leaf dylibs
946 __block std::set<std::string> linkedDylibs;
947
948 for(const auto& uuid : cachedUUIDs) {
949 auto parser = parserForUUID(uuid);
950 parser.forEachDependentDylib(^(const char* loadPath, bool isWeak, bool isReExport, bool isUpward, uint32_t compatVersion, uint32_t curVersion, bool& stop) {
951 linkedDylibs.insert(loadPath);
952 });
953 }
954
955 for(const auto& uuid : cachedUUIDs) {
956 auto info = infoForUUID(uuid);
957 auto i = _metabomTagMap.find(info.runtimePath);
958 assert(i != _metabomTagMap.end());
959 auto exclusions = _metabomExcludeTagMap.find(configuration);
960 if (exclusions == _metabomExcludeTagMap.end() || is_disjoint(exclusions->second, i->second))
961 continue;
962
963 if (linkedDylibs.count(info.installName) != 0)
964 continue;
965
966 archManifest.results.exclude(*this, info.uuid, "Dylib '" + info.runtimePath + "' excluded leaf node");
967 }
968 }
969
970 void Manifest::writeJSON(const std::string& path) {
971 NSMutableDictionary* jsonDict = [[NSMutableDictionary alloc] init];
972 for (auto& configuration : _configurations) {
973 jsonDict[cppToObjStr(configuration.first)] = [[NSMutableDictionary alloc] init];
974
975 for (auto& arch : configuration.second.architectures) {
976 NSMutableOrderedSet* includedDylibsSet = [[NSMutableOrderedSet alloc] init];
977 NSMutableOrderedSet* executablesSet = [[NSMutableOrderedSet alloc] init];
978 NSMutableOrderedSet* otherSet = [[NSMutableOrderedSet alloc] init];
979 for (auto& dylib : arch.second.results.dylibs) {
980 NSString *runtimePath = cppToObjStr(runtimePathForUUID(dylib.second.uuid));
981 if (dylib.second.included) {
982 [includedDylibsSet addObject:runtimePath];
983 } else {
984 [otherSet addObject:runtimePath];
985 }
986 }
987
988 for (auto& executable : arch.second.results.executables) {
989 NSString *runtimePath = cppToObjStr(runtimePathForUUID(executable.second.uuid));
990 [executablesSet addObject:runtimePath];
991 }
992
993 for (auto& bundle : arch.second.results.bundles) {
994 NSString *runtimePath = cppToObjStr(runtimePathForUUID(bundle.second.uuid));
995 [otherSet addObject:runtimePath];
996 }
997
998 [includedDylibsSet sortUsingComparator:^NSComparisonResult(id _Nonnull obj1, id _Nonnull obj2) {
999 return [obj1 compare:obj2];
1000 }];
1001
1002 [executablesSet sortUsingComparator:^NSComparisonResult(id _Nonnull obj1, id _Nonnull obj2) {
1003 return [obj1 compare:obj2];
1004 }];
1005
1006 [otherSet sortUsingComparator:^NSComparisonResult(id _Nonnull obj1, id _Nonnull obj2) {
1007 return [obj1 compare:obj2];
1008 }];
1009
1010 jsonDict[cppToObjStr(configuration.first)][cppToObjStr(arch.first)] = @{ @"cachedDylibs" : [includedDylibsSet array], @"mainExecutables" : [executablesSet array], @"other" : [otherSet array]};;
1011 }
1012 }
1013
1014 NSError* error = nil;
1015 NSData *jsonData = [NSJSONSerialization dataWithJSONObject:jsonDict options:0x0 error:&error];
1016 (void)[jsonData writeToFile:cppToObjStr(path) atomically:YES];
1017 }
1018
1019 void Manifest::write(const std::string& path)
1020 {
1021 if (path.empty())
1022 return;
1023
1024 NSMutableDictionary* cacheDict = [[NSMutableDictionary alloc] init];
1025 NSMutableDictionary* projectDict = [[NSMutableDictionary alloc] init];
1026 NSMutableDictionary* configurationsDict = [[NSMutableDictionary alloc] init];
1027 NSMutableDictionary* resultsDict = [[NSMutableDictionary alloc] init];
1028
1029 cacheDict[@"manifest-version"] = @(version());
1030 cacheDict[@"build"] = cppToObjStr(build());
1031 cacheDict[@"dylibOrderFile"] = cppToObjStr(dylibOrderFile());
1032 cacheDict[@"dirtyDataOrderFile"] = cppToObjStr(dirtyDataOrderFile());
1033 cacheDict[@"metabomFile"] = cppToObjStr(metabomFile());
1034
1035 cacheDict[@"projects"] = projectDict;
1036 cacheDict[@"results"] = resultsDict;
1037 cacheDict[@"configurations"] = configurationsDict;
1038
1039 for (const auto& project : projects()) {
1040 NSMutableArray* sources = [[NSMutableArray alloc] init];
1041
1042 for (const auto& source : project.second.sources) {
1043 [sources addObject:cppToObjStr(source)];
1044 }
1045
1046 projectDict[cppToObjStr(project.first)] = sources;
1047 }
1048
1049 for (auto& configuration : _configurations) {
1050 NSMutableArray* archArray = [[NSMutableArray alloc] init];
1051 for (auto& arch : configuration.second.architectures) {
1052 [archArray addObject:cppToObjStr(arch.first)];
1053 }
1054
1055 NSMutableArray* excludeTags = [[NSMutableArray alloc] init];
1056 for (const auto& excludeTag : configuration.second.metabomExcludeTags) {
1057 [excludeTags addObject:cppToObjStr(excludeTag)];
1058 }
1059
1060 configurationsDict[cppToObjStr(configuration.first)] = @{
1061 @"platformName" : cppToObjStr(configuration.second.platformName),
1062 @"metabomTag" : cppToObjStr(configuration.second.metabomTag),
1063 @"metabomExcludeTags" : excludeTags,
1064 @"architectures" : archArray
1065 };
1066 }
1067
1068 for (auto& configuration : _configurations) {
1069 NSMutableDictionary* archResultsDict = [[NSMutableDictionary alloc] init];
1070 for (auto& arch : configuration.second.architectures) {
1071 NSMutableDictionary* dylibsDict = [[NSMutableDictionary alloc] init];
1072 NSMutableArray* warningsArray = [[NSMutableArray alloc] init];
1073 NSMutableDictionary* devRegionsDict = [[NSMutableDictionary alloc] init];
1074 NSMutableDictionary* prodRegionsDict = [[NSMutableDictionary alloc] init];
1075 NSString* prodCDHash = cppToObjStr(arch.second.results.productionCache.cdHash);
1076 NSString* devCDHash = cppToObjStr(arch.second.results.developmentCache.cdHash);
1077
1078 for (auto& dylib : arch.second.results.dylibs) {
1079 NSMutableDictionary* dylibDict = [[NSMutableDictionary alloc] init];
1080 if (dylib.second.included) {
1081 dylibDict[@"included"] = @YES;
1082 } else {
1083 dylibDict[@"included"] = @NO;
1084 dylibDict[@"exclusionInfo"] = cppToObjStr(dylib.second.exclusionInfo);
1085 }
1086 dylibsDict[cppToObjStr(dylib.second.installname)] = dylibDict;
1087 }
1088
1089 for (auto& warning : arch.second.results.warnings) {
1090 [warningsArray addObject:cppToObjStr(warning)];
1091 }
1092
1093 BOOL built = arch.second.results.failure.empty();
1094 archResultsDict[cppToObjStr(arch.first)] = @{
1095 @"dylibs" : dylibsDict,
1096 @"built" : @(built),
1097 @"failure" : cppToObjStr(arch.second.results.failure),
1098 @"productionCache" : @{ @"cdhash" : prodCDHash, @"regions" : prodRegionsDict },
1099 @"developmentCache" : @{ @"cdhash" : devCDHash, @"regions" : devRegionsDict },
1100 @"warnings" : warningsArray
1101 };
1102 }
1103 resultsDict[cppToObjStr(configuration.first)] = archResultsDict;
1104 }
1105
1106 switch (platform()) {
1107 case Platform::iOS:
1108 cacheDict[@"platform"] = @"ios";
1109 break;
1110 case Platform::tvOS:
1111 cacheDict[@"platform"] = @"tvos";
1112 break;
1113 case Platform::watchOS:
1114 cacheDict[@"platform"] = @"watchos";
1115 break;
1116 case Platform::bridgeOS:
1117 cacheDict[@"platform"] = @"bridgeos";
1118 break;
1119 case Platform::macOS:
1120 cacheDict[@"platform"] = @"macos";
1121 break;
1122 case Platform::unknown:
1123 cacheDict[@"platform"] = @"unknown";
1124 break;
1125 }
1126
1127 NSError* error = nil;
1128 NSData* outData = [NSPropertyListSerialization dataWithPropertyList:cacheDict
1129 format:NSPropertyListBinaryFormat_v1_0
1130 options:0
1131 error:&error];
1132 (void)[outData writeToFile:cppToObjStr(path) atomically:YES];
1133 }
1134 }