]> git.saurik.com Git - apple/dyld.git/blob - dyld3/shared-cache/mrm_shared_cache_builder.cpp
dyld-832.7.3.tar.gz
[apple/dyld.git] / dyld3 / shared-cache / mrm_shared_cache_builder.cpp
1 /* -*- mode: C++; c-basic-offset: 4; indent-tabs-mode: nil -*-
2 *
3 * Copyright (c) 2017 Apple Inc. All rights reserved.
4 *
5 * @APPLE_LICENSE_HEADER_START@
6 *
7 * This file contains Original Code and/or Modifications of Original Code
8 * as defined in and that are subject to the Apple Public Source License
9 * Version 2.0 (the 'License'). You may not use this file except in
10 * compliance with the License. Please obtain a copy of the License at
11 * http://www.opensource.apple.com/apsl/ and read it before using this
12 * file.
13 *
14 * The Original Code and all software distributed under the License are
15 * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
16 * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
17 * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
18 * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
19 * Please see the License for the specific language governing rights and
20 * limitations under the License.
21 *
22 * @APPLE_LICENSE_HEADER_END@
23 */
24
25 #include "mrm_shared_cache_builder.h"
26 #include "SharedCacheBuilder.h"
27 #include "ClosureFileSystem.h"
28 #include "FileUtils.h"
29 #include "JSONReader.h"
30 #include <pthread.h>
31 #include <memory>
32 #include <vector>
33 #include <map>
34 #include <sys/stat.h>
35
36 static const uint64_t kMinBuildVersion = 1; //The minimum version BuildOptions struct we can support
37 static const uint64_t kMaxBuildVersion = 2; //The maximum version BuildOptions struct we can support
38
39 static const uint32_t MajorVersion = 1;
40 static const uint32_t MinorVersion = 2;
41
42 namespace dyld3 {
43 namespace closure {
44
45 struct FileInfo {
46 const char* path;
47 const uint8_t* data;
48 const uint64_t length;
49 FileFlags flags;
50 uint64_t mtime;
51 uint64_t inode;
52 };
53
54 class FileSystemMRM : public FileSystem {
55 public:
56 FileSystemMRM() : FileSystem() { }
57
58 bool getRealPath(const char possiblePath[MAXPATHLEN], char realPath[MAXPATHLEN]) const override {
59 Diagnostics diag;
60 std::string resolvedPath = symlinkResolver.realPath(diag, possiblePath);
61 if (diag.hasError()) {
62 diag.verbose("MRM error: %s\n", diag.errorMessage().c_str());
63 diag.clearError();
64 return false;
65 }
66
67 // FIXME: Should we only return real paths of files which point to macho's? For now that is what we are doing
68 auto it = fileMap.find(resolvedPath);
69 if (it == fileMap.end())
70 return false;
71
72 memcpy(realPath, resolvedPath.c_str(), std::min((size_t)MAXPATHLEN, resolvedPath.size() + 1));
73 return true;
74 }
75
76 bool loadFile(const char* path, LoadedFileInfo& info, char realerPath[MAXPATHLEN], void (^error)(const char* format, ...)) const override {
77 Diagnostics diag;
78 std::string resolvedPath = symlinkResolver.realPath(diag, path);
79 if (diag.hasError()) {
80 diag.verbose("MRM error: %s\n", diag.errorMessage().c_str());
81 diag.clearError();
82 return false;
83 }
84
85 auto it = fileMap.find(resolvedPath);
86 if (it == fileMap.end())
87 return false;
88
89 if (resolvedPath == path)
90 realerPath[0] = '\0';
91 else
92 memcpy(realerPath, resolvedPath.c_str(), std::min((size_t)MAXPATHLEN, resolvedPath.size() + 1));
93
94 // The file exists at this exact path. Lets use it!
95 const FileInfo& fileInfo = files[it->second];
96
97 info.fileContent = fileInfo.data;
98 info.fileContentLen = fileInfo.length;
99 info.sliceOffset = 0;
100 info.sliceLen = fileInfo.length;
101 info.isOSBinary = true;
102 info.inode = fileInfo.inode;
103 info.mtime = fileInfo.mtime;
104 info.unload = nullptr;
105 info.path = path;
106 return true;
107 }
108
109 void unloadFile(const LoadedFileInfo& info) const override {
110 if (info.unload)
111 info.unload(info);
112 }
113
114 void unloadPartialFile(LoadedFileInfo& info, uint64_t keepStartOffset, uint64_t keepLength) const override {
115 // Note we don't actually unload the data here, but we do want to update the offsets for other data structures to track where we are
116 info.fileContent = (const void*)((char*)info.fileContent + keepStartOffset);
117 info.fileContentLen = keepLength;
118 }
119
120 bool fileExists(const char* path, uint64_t* inode=nullptr, uint64_t* mtime=nullptr,
121 bool* issetuid=nullptr, bool* inodesMatchRuntime = nullptr) const override {
122 Diagnostics diag;
123 std::string resolvedPath = symlinkResolver.realPath(diag, path);
124 if (diag.hasError()) {
125 diag.verbose("MRM error: %s\n", diag.errorMessage().c_str());
126 diag.clearError();
127 return false;
128 }
129
130 auto it = fileMap.find(resolvedPath);
131 if (it == fileMap.end())
132 return false;
133
134 // The file exists at this exact path. Lets use it!
135 const FileInfo& fileInfo = files[it->second];
136 if (inode)
137 *inode = fileInfo.inode;
138 if (mtime)
139 *mtime = fileInfo.mtime;
140 if (issetuid)
141 *issetuid = false;
142 if (inodesMatchRuntime)
143 *inodesMatchRuntime = false;
144 return true;
145 }
146
147 // MRM file APIs
148 bool addFile(const char* path, uint8_t* data, uint64_t size, Diagnostics& diag, FileFlags fileFlags) {
149 auto iteratorAndInserted = fileMap.insert(std::make_pair(path, files.size()));
150 if (!iteratorAndInserted.second) {
151 diag.error("Already have content for path: '%s'", path);
152 return false;
153 }
154
155 symlinkResolver.addFile(diag, path);
156 if (diag.hasError())
157 return false;
158
159 // on iOS, inode is just a placeholder
160 // Note its safe to just use the index here as we only compare it during closure building
161 // and never record it in the closures
162 uint64_t inode = files.size() + 1;
163 uint64_t mtime = 0;
164
165 files.push_back((FileInfo){ path, data, size, fileFlags, mtime, inode });
166 return true;
167 }
168
169 bool addSymlink(const char* fromPath, const char* toPath, Diagnostics& diag) {
170 symlinkResolver.addSymlink(diag, fromPath, toPath);
171 return !diag.hasError();
172 }
173
174 void forEachFileInfo(std::function<void(const char* path, FileFlags fileFlags)> lambda) {
175 for (const FileInfo& fileInfo : files)
176 lambda(fileInfo.path, fileInfo.flags);
177 }
178
179 size_t fileCount() const {
180 return files.size();
181 }
182
183 std::vector<DyldSharedCache::FileAlias> getResolvedSymlinks(Diagnostics& diag) {
184 return symlinkResolver.getResolvedSymlinks(diag);
185 }
186
187 private:
188 std::vector<FileInfo> files;
189 std::map<std::string, uint64_t> fileMap;
190 SymlinkResolver symlinkResolver;
191 };
192
193 } // namespace closure
194 } // namespace dyld3
195
196 struct BuildInstance {
197 std::unique_ptr<DyldSharedCache::CreateOptions> options;
198 std::unique_ptr<SharedCacheBuilder> builder;
199 std::vector<CacheBuilder::InputFile> inputFiles;
200 std::vector<const char*> errors;
201 std::vector<const char*> warnings;
202 std::vector<std::string> errorStrings; // Owns the data for the errors
203 std::vector<std::string> warningStrings; // Owns the data for the warnings
204 uint8_t* cacheData = nullptr;
205 uint64_t cacheSize = 0;
206 std::string jsonMap;
207 std::string macOSMap; // For compatibility with update_dyld_shared_cache's .map file
208 std::string macOSMapPath; // Owns the string for the path
209 std::string cdHash; // Owns the data for the cdHash
210 std::string cdHashType; // Owns the data for the cdHashType
211 std::string uuid; // Owns the data for the uuid
212 };
213
214 struct BuildFileResult {
215 std::string path;
216 const uint8_t* data;
217 uint64_t size;
218 };
219
220 struct TranslationResult {
221 const uint8_t* data;
222 size_t size;
223 std::string cdHash;
224 std::string path;
225 bool bufferWasMalloced;
226 };
227
228 struct MRMSharedCacheBuilder {
229 MRMSharedCacheBuilder(const BuildOptions_v1* options);
230 const BuildOptions_v1* options;
231 dyld3::closure::FileSystemMRM fileSystem;
232
233 std::string dylibOrderFileData;
234 std::string dirtyDataOrderFileData;
235 void* objcOptimizationsFileData;
236 size_t objcOptimizationsFileLength;
237
238 // An array of builders and their options as we may have more than one builder for a given device variant.
239 std::vector<BuildInstance> builders;
240
241 // The paths in all of the caches
242 // We keep this here to own the std::string path data
243 std::map<std::string, std::unordered_set<const BuildInstance*>> dylibsInCaches;
244
245 // The results from all of the builders
246 // We keep this in a vector to own the data.
247 std::vector<FileResult*> fileResults;
248 std::vector<FileResult> fileResultStorage;
249 std::vector<std::pair<uint64_t, bool>> fileResultBuffers;
250
251 // The results from all of the builders
252 // We keep this in a vector to own the data.
253 std::vector<CacheResult*> cacheResults;
254 std::vector<CacheResult> cacheResultStorage;
255
256 // The files to remove. These are in every copy of the caches we built
257 std::vector<const char*> filesToRemove;
258
259 std::vector<const char*> errors;
260 std::vector<std::string> errorStorage;
261 pthread_mutex_t lock;
262
263 enum State {
264 AcceptingFiles,
265 Building,
266 FinishedBuilding
267 };
268
269 State state = AcceptingFiles;
270
271 void runSync(void (^block)()) {
272 pthread_mutex_lock(&lock);
273 block();
274 pthread_mutex_unlock(&lock);
275 }
276
277 __attribute__((format(printf, 2, 3)))
278 void error(const char* format, ...) {
279 va_list list;
280 va_start(list, format);
281 Diagnostics diag;
282 diag.error(format, list);
283 va_end(list);
284
285 errorStorage.push_back(diag.errorMessage());
286 errors.push_back(errorStorage.back().data());
287 }
288 };
289
290 MRMSharedCacheBuilder::MRMSharedCacheBuilder(const BuildOptions_v1* options)
291 : options(options)
292 , lock(PTHREAD_MUTEX_INITIALIZER)
293 , objcOptimizationsFileData(nullptr)
294 , objcOptimizationsFileLength(0)
295 {
296
297 }
298
299 void validiateBuildOptions(const BuildOptions_v1* options, MRMSharedCacheBuilder& builder) {
300 if (options->version < kMinBuildVersion) {
301 builder.error("Builder version %llu is less than minimum supported version of %llu", options->version, kMinBuildVersion);
302 }
303 if (options->version > kMaxBuildVersion) {
304 builder.error("Builder version %llu is greater than maximum supported version of %llu", options->version, kMaxBuildVersion);
305 }
306 if (!options->updateName) {
307 builder.error("updateName must not be null");
308 }
309 if (!options->deviceName) {
310 builder.error("deviceName must not be null");
311 }
312 switch (options->disposition) {
313 case Disposition::Unknown:
314 case Disposition::InternalDevelopment:
315 case Disposition::Customer:
316 case Disposition::InternalMinDevelopment:
317 break;
318 default:
319 builder.error("unknown disposition value");
320 break;
321 }
322 switch (options->platform) {
323 case Platform::unknown:
324 builder.error("platform must not be unknown");
325 break;
326 case Platform::macOS:
327 case Platform::iOS:
328 case Platform::tvOS:
329 case Platform::watchOS:
330 case Platform::bridgeOS:
331 case Platform::iOSMac:
332 case Platform::iOS_simulator:
333 case Platform::tvOS_simulator:
334 case Platform::watchOS_simulator:
335 break;
336 default:
337 builder.error("unknown platform value");
338 break;
339 }
340 if (!options->archs) {
341 builder.error("archs must not be null");
342 }
343 if (!options->numArchs) {
344 builder.error("numArchs must not be 0");
345 }
346 }
347
348 void getVersion(uint32_t *major, uint32_t *minor) {
349 *major = MajorVersion;
350 *minor = MinorVersion;
351 }
352
353 struct MRMSharedCacheBuilder* createSharedCacheBuilder(const BuildOptions_v1* options) {
354 MRMSharedCacheBuilder* builder = new MRMSharedCacheBuilder(options);
355
356 // Check the option struct values are valid
357 validiateBuildOptions(options, *builder);
358
359 return builder;
360 }
361
362 bool addFile(struct MRMSharedCacheBuilder* builder, const char* path, uint8_t* data, uint64_t size, FileFlags fileFlags) {
363 __block bool success = false;
364 builder->runSync(^() {
365 if (builder->state != MRMSharedCacheBuilder::AcceptingFiles) {
366 builder->error("Cannot add file: '%s' as we have already started building", path);
367 return;
368 }
369 size_t pathLength = strlen(path);
370 if (pathLength == 0) {
371 builder->error("Empty path");
372 return;
373 }
374 if (pathLength >= MAXPATHLEN) {
375 builder->error("Path is too long: '%s'", path);
376 return;
377 }
378 if (data == nullptr) {
379 builder->error("Data cannot be null for file: '%s'", path);
380 return;
381 }
382 switch (fileFlags) {
383 case NoFlags:
384 case MustBeInCache:
385 case ShouldBeExcludedFromCacheIfUnusedLeaf:
386 case RequiredClosure:
387 break;
388 case DylibOrderFile:
389 builder->dylibOrderFileData = std::string((char*)data, size);
390 success = true;
391 return;
392 case DirtyDataOrderFile:
393 builder->dirtyDataOrderFileData = std::string((char*)data, size);
394 success = true;
395 return;
396 case ObjCOptimizationsFile:
397 builder->objcOptimizationsFileData = data;
398 builder->objcOptimizationsFileLength = size;
399 success = true;
400 return;
401 default:
402 builder->error("unknown file flags value");
403 break;
404 }
405 Diagnostics diag;
406 if (!builder->fileSystem.addFile(path, data, size, diag, fileFlags)) {
407 builder->errorStorage.push_back(diag.errorMessage());
408 builder->errors.push_back(builder->errorStorage.back().data());
409 return;
410 }
411 success = true;
412 });
413 return success;
414 }
415
416 bool addSymlink(struct MRMSharedCacheBuilder* builder, const char* fromPath, const char* toPath) {
417 __block bool success = false;
418 builder->runSync(^() {
419 if (builder->state != MRMSharedCacheBuilder::AcceptingFiles) {
420 builder->error("Cannot add file: '%s' as we have already started building", fromPath);
421 return;
422 }
423 size_t pathLength = strlen(fromPath);
424 if (pathLength == 0) {
425 builder->error("Empty path");
426 return;
427 }
428 if (pathLength >= MAXPATHLEN) {
429 builder->error("Path is too long: '%s'", fromPath);
430 return;
431 }
432 Diagnostics diag;
433 if (!builder->fileSystem.addSymlink(fromPath, toPath, diag)) {
434 builder->errorStorage.push_back(diag.errorMessage());
435 builder->errors.push_back(builder->errorStorage.back().data());
436 return;
437 }
438 success = true;
439 });
440 return success;
441 }
442
443 static DyldSharedCache::LocalSymbolsMode platformExcludeLocalSymbols(Platform platform) {
444 switch (platform) {
445 case Platform::unknown:
446 case Platform::macOS:
447 return DyldSharedCache::LocalSymbolsMode::keep;
448 case Platform::iOS:
449 case Platform::tvOS:
450 case Platform::watchOS:
451 case Platform::bridgeOS:
452 return DyldSharedCache::LocalSymbolsMode::unmap;
453 case Platform::iOSMac:
454 case Platform::iOS_simulator:
455 case Platform::tvOS_simulator:
456 case Platform::watchOS_simulator:
457 return DyldSharedCache::LocalSymbolsMode::keep;
458 }
459 }
460
461 static DyldSharedCache::LocalSymbolsMode excludeLocalSymbols(const BuildOptions_v1* options) {
462 if ( options->version >= 2 ) {
463 const BuildOptions_v2* v2 = (const BuildOptions_v2*)options;
464 if ( v2->optimizeForSize )
465 return DyldSharedCache::LocalSymbolsMode::strip;
466 }
467
468 // Old build options always use the platform default
469 return platformExcludeLocalSymbols(options->platform);
470 }
471
472 static bool optimizeDyldDlopens(const BuildOptions_v1* options) {
473 // Old builds always default to dyld3 optimisations
474 if ( options->version < 2 ) {
475 return true;
476 }
477
478 // If we want to optimize for size instead of speed, then disable dyld3 dlopen closures
479 const BuildOptions_v2* v2 = (const BuildOptions_v2*)options;
480 return !v2->optimizeForSize;
481 }
482
483 static DyldSharedCache::CodeSigningDigestMode platformCodeSigningDigestMode(Platform platform) {
484 switch (platform) {
485 case Platform::unknown:
486 case Platform::macOS:
487 case Platform::iOS:
488 case Platform::tvOS:
489 return DyldSharedCache::SHA256only;
490 case Platform::watchOS:
491 return DyldSharedCache::Agile;
492 case Platform::bridgeOS:
493 case Platform::iOSMac:
494 case Platform::iOS_simulator:
495 case Platform::tvOS_simulator:
496 case Platform::watchOS_simulator:
497 return DyldSharedCache::SHA256only;
498 }
499 }
500
501 static bool platformIsForSimulator(Platform platform) {
502 switch (platform) {
503 case Platform::unknown:
504 case Platform::macOS:
505 case Platform::iOS:
506 case Platform::tvOS:
507 case Platform::watchOS:
508 case Platform::bridgeOS:
509 case Platform::iOSMac:
510 return false;
511 case Platform::iOS_simulator:
512 case Platform::tvOS_simulator:
513 case Platform::watchOS_simulator:
514 return true;
515 }
516 }
517
518 static const char* dispositionName(Disposition disposition) {
519 switch (disposition) {
520 case Disposition::Unknown:
521 return "";
522 case Disposition::InternalDevelopment:
523 return "Internal";
524 case Disposition::Customer:
525 return "Customer";
526 case Disposition::InternalMinDevelopment:
527 return "InternalMinDevelopment";
528 }
529 }
530
531 // This is a JSON file containing the list of classes for which
532 // we should try to build IMP caches.
533 dyld3::json::Node parseObjcOptimizationsFile(Diagnostics& diags, const void* data, size_t length) {
534 return dyld3::json::readJSON(diags, data, length);
535 }
536
537 bool runSharedCacheBuilder(struct MRMSharedCacheBuilder* builder) {
538 __block bool success = false;
539 builder->runSync(^() {
540 if (builder->state != MRMSharedCacheBuilder::AcceptingFiles) {
541 builder->error("Builder has already been run");
542 return;
543 }
544 builder->state = MRMSharedCacheBuilder::Building;
545 if (builder->fileSystem.fileCount() == 0) {
546 builder->error("Cannot run builder with no files");
547 }
548
549 __block Diagnostics diag;
550 std::vector<DyldSharedCache::FileAlias> aliases = builder->fileSystem.getResolvedSymlinks(diag);
551 if (diag.hasError()) {
552 diag.verbose("Symlink resolver error: %s\n", diag.errorMessage().c_str());
553 }
554
555 if (!builder->errors.empty()) {
556 builder->error("Skipping running shared cache builder due to previous errors");
557 return;
558 }
559
560 __block std::vector<SharedCacheBuilder::InputFile> inputFiles;
561 builder->fileSystem.forEachFileInfo(^(const char* path, FileFlags fileFlags) {
562 SharedCacheBuilder::InputFile::State state = SharedCacheBuilder::InputFile::Unset;
563 switch (fileFlags) {
564 case FileFlags::NoFlags:
565 state = SharedCacheBuilder::InputFile::Unset;
566 break;
567 case FileFlags::MustBeInCache:
568 state = SharedCacheBuilder::InputFile::MustBeIncluded;
569 break;
570 case FileFlags::ShouldBeExcludedFromCacheIfUnusedLeaf:
571 state = SharedCacheBuilder::InputFile::MustBeExcludedIfUnused;
572 break;
573 case FileFlags::RequiredClosure:
574 state = SharedCacheBuilder::InputFile::MustBeIncluded;
575 break;
576 case FileFlags::DylibOrderFile:
577 case FileFlags::DirtyDataOrderFile:
578 case FileFlags::ObjCOptimizationsFile:
579 builder->error("Order files should not be in the file system");
580 return;
581 }
582 inputFiles.emplace_back((SharedCacheBuilder::InputFile){ path, state });
583 });
584
585 auto addCacheConfiguration = ^(bool isOptimized) {
586 for (uint64_t i = 0; i != builder->options->numArchs; ++i) {
587 // HACK: Skip i386 for macOS
588 if ( (builder->options->platform == Platform::macOS) && (strcmp(builder->options->archs[i], "i386") == 0 ) )
589 continue;
590 auto options = std::make_unique<DyldSharedCache::CreateOptions>((DyldSharedCache::CreateOptions){});
591 const char *cacheSuffix = (isOptimized ? "" : ".development");
592 if ( builder->options->platform == Platform::macOS )
593 cacheSuffix = "";
594 std::string runtimePath = (builder->options->platform == Platform::macOS) ? MACOSX_MRM_DYLD_SHARED_CACHE_DIR : IPHONE_DYLD_SHARED_CACHE_DIR;
595 options->outputFilePath = runtimePath + "dyld_shared_cache_" + builder->options->archs[i] + cacheSuffix;
596 options->outputMapFilePath = options->outputFilePath + ".json";
597 options->archs = &dyld3::GradedArchs::forName(builder->options->archs[i]);
598 options->platform = (dyld3::Platform)builder->options->platform;
599 options->localSymbolMode = excludeLocalSymbols(builder->options);
600 options->optimizeStubs = isOptimized;
601 options->optimizeDyldDlopens = optimizeDyldDlopens(builder->options);
602 options->optimizeDyldLaunches = true;
603 options->codeSigningDigestMode = platformCodeSigningDigestMode(builder->options->platform);
604 options->dylibsRemovedDuringMastering = true;
605 options->inodesAreSameAsRuntime = false;
606 options->cacheSupportsASLR = true;
607 options->forSimulator = platformIsForSimulator(builder->options->platform);
608 options->isLocallyBuiltCache = builder->options->isLocallyBuiltCache;
609 options->verbose = builder->options->verboseDiagnostics;
610 options->evictLeafDylibsOnOverflow = true;
611 options->loggingPrefix = std::string(builder->options->deviceName) + dispositionName(builder->options->disposition) + "." + builder->options->archs[i] + cacheSuffix;
612 options->dylibOrdering = parseOrderFile(builder->dylibOrderFileData);
613 options->dirtyDataSegmentOrdering = parseOrderFile(builder->dirtyDataOrderFileData);
614 options->objcOptimizations = parseObjcOptimizationsFile(diag, builder->objcOptimizationsFileData, builder->objcOptimizationsFileLength);
615
616 auto cacheBuilder = std::make_unique<SharedCacheBuilder>(*options.get(), builder->fileSystem);
617 builder->builders.emplace_back((BuildInstance) { std::move(options), std::move(cacheBuilder), inputFiles });
618 }
619 };
620
621 // Enqueue a cache for each configuration
622 switch (builder->options->disposition) {
623 case Disposition::Unknown:
624 case Disposition::InternalDevelopment:
625 // HACK: MRM for the mac should only get development, even if it requested both
626 if (builder->options->platform == Platform::macOS) {
627 addCacheConfiguration(false);
628 } else {
629 addCacheConfiguration(false);
630 addCacheConfiguration(true);
631 }
632 break;
633 case Disposition::Customer:
634 addCacheConfiguration(true);
635 break;
636 case Disposition::InternalMinDevelopment:
637 addCacheConfiguration(false);
638 break;
639 }
640
641 // FIXME: This step can run in parallel.
642 for (auto& buildInstance : builder->builders) {
643 SharedCacheBuilder* cacheBuilder = buildInstance.builder.get();
644 cacheBuilder->build(buildInstance.inputFiles, aliases);
645
646 // First put the warnings in to a vector to own them.
647 buildInstance.warningStrings.reserve(cacheBuilder->warnings().size());
648 for (const std::string& warning : cacheBuilder->warnings())
649 buildInstance.warningStrings.push_back(warning);
650
651 // Then copy to a vector to reference the owner
652 buildInstance.warnings.reserve(buildInstance.warningStrings.size());
653 for (const std::string& warning : buildInstance.warningStrings)
654 buildInstance.warnings.push_back(warning.c_str());
655
656 if (!cacheBuilder->errorMessage().empty()) {
657 // First put the errors in to a vector to own them.
658 buildInstance.errorStrings.push_back(cacheBuilder->errorMessage());
659
660 // Then copy to a vector to reference the owner
661 buildInstance.errors.reserve(buildInstance.errorStrings.size());
662 for (const std::string& error : buildInstance.errorStrings)
663 buildInstance.errors.push_back(error.c_str());
664 }
665
666 if (cacheBuilder->errorMessage().empty()) {
667 cacheBuilder->writeBuffer(buildInstance.cacheData, buildInstance.cacheSize);
668 buildInstance.jsonMap = cacheBuilder->getMapFileJSONBuffer(builder->options->deviceName);
669 if ( buildInstance.options->platform == dyld3::Platform::macOS ) {
670 // For compatibility with update_dyld_shared_cache, put a .map file next to the shared cache
671 buildInstance.macOSMap = cacheBuilder->getMapFileBuffer();
672 buildInstance.macOSMapPath = buildInstance.options->outputFilePath + ".map";
673 }
674 buildInstance.cdHash = cacheBuilder->cdHashFirst();
675 buildInstance.uuid = cacheBuilder->uuid();
676 switch (buildInstance.options->codeSigningDigestMode) {
677 case DyldSharedCache::SHA256only:
678 buildInstance.cdHashType = "sha256";
679 break;
680 case DyldSharedCache::SHA1only:
681 buildInstance.cdHashType = "sha1";
682 break;
683 case DyldSharedCache::Agile:
684 buildInstance.cdHashType = "sha1";
685 break;
686 }
687
688 // Track the dylibs which were included in this cache
689 cacheBuilder->forEachCacheDylib(^(const std::string &path) {
690 builder->dylibsInCaches[path.c_str()].insert(&buildInstance);
691 });
692 cacheBuilder->forEachCacheSymlink(^(const std::string &path) {
693 builder->dylibsInCaches[path.c_str()].insert(&buildInstance);
694 });
695 }
696 // Free the cache builder now so that we don't keep too much memory resident
697 cacheBuilder->deleteBuffer();
698 buildInstance.builder.reset();
699 }
700
701 // Now that we have run all of the builds, collect the results
702 // First push file results for each of the shared caches we built
703 for (auto& buildInstance : builder->builders) {
704 CacheResult cacheBuildResult;
705 cacheBuildResult.version = 1;
706 cacheBuildResult.loggingPrefix = buildInstance.options->loggingPrefix.c_str();
707 cacheBuildResult.deviceConfiguration = buildInstance.options->loggingPrefix.c_str();
708 cacheBuildResult.warnings = buildInstance.warnings.empty() ? nullptr : buildInstance.warnings.data();
709 cacheBuildResult.numWarnings = buildInstance.warnings.size();
710 cacheBuildResult.errors = buildInstance.errors.empty() ? nullptr : buildInstance.errors.data();
711 cacheBuildResult.numErrors = buildInstance.errors.size();
712 cacheBuildResult.uuidString = buildInstance.uuid.c_str();
713 cacheBuildResult.mapJSON = buildInstance.jsonMap.c_str();
714
715 builder->cacheResultStorage.emplace_back(cacheBuildResult);
716
717 if (!buildInstance.errors.empty())
718 continue;
719
720 FileResult cacheFileResult;
721 cacheFileResult.version = 1;
722 cacheFileResult.path = buildInstance.options->outputFilePath.c_str();
723 cacheFileResult.behavior = AddFile;
724 cacheFileResult.data = buildInstance.cacheData;
725 cacheFileResult.size = buildInstance.cacheSize;
726 cacheFileResult.hashArch = buildInstance.options->archs->name();
727 cacheFileResult.hashType = buildInstance.cdHashType.c_str();
728 cacheFileResult.hash = buildInstance.cdHash.c_str();
729
730 builder->fileResultBuffers.push_back({ builder->fileResultStorage.size(), true });
731 builder->fileResultStorage.emplace_back(cacheFileResult);
732
733 // Add a file result for the .map file
734 if ( !buildInstance.macOSMap.empty() ) {
735 FileResult cacheFileResult;
736 cacheFileResult.version = 1;
737 cacheFileResult.path = buildInstance.macOSMapPath.c_str();
738 cacheFileResult.behavior = AddFile;
739 cacheFileResult.data = (const uint8_t*)buildInstance.macOSMap.data();
740 cacheFileResult.size = buildInstance.macOSMap.size();
741 cacheFileResult.hashArch = buildInstance.options->archs->name();
742 cacheFileResult.hashType = buildInstance.cdHashType.c_str();
743 cacheFileResult.hash = buildInstance.cdHash.c_str();
744
745 builder->fileResultStorage.emplace_back(cacheFileResult);
746 }
747 }
748
749 // Copy from the storage to the vector we can return to the API.
750 for (auto &fileResult : builder->fileResultStorage)
751 builder->fileResults.push_back(&fileResult);
752 for (auto &cacheResult : builder->cacheResultStorage)
753 builder->cacheResults.push_back(&cacheResult);
754
755
756 // Add entries to tell us to remove all of the dylibs from disk which are in every cache.
757 const size_t numCaches = builder->builders.size();
758 for (const auto& dylibAndCount : builder->dylibsInCaches) {
759 const char* pathToRemove = dylibAndCount.first.c_str();
760
761 if ( builder->options->platform == Platform::macOS ) {
762 // macOS has to leave the simulator support binaries on disk
763 if ( strcmp(pathToRemove, "/usr/lib/system/libsystem_kernel.dylib") == 0 )
764 continue;
765 if ( strcmp(pathToRemove, "/usr/lib/system/libsystem_platform.dylib") == 0 )
766 continue;
767 if ( strcmp(pathToRemove, "/usr/lib/system/libsystem_pthread.dylib") == 0 )
768 continue;
769 }
770
771 if (dylibAndCount.second.size() == numCaches) {
772 builder->filesToRemove.push_back(pathToRemove);
773 } else {
774 // File is not in every cache, so likely has perhaps only x86_64h slice
775 // but we built both x86_64 and x86_64h caches.
776 // We may still delete it if its in all caches it's eligible for, ie, we
777 // assume the cache builder knows about all possible arch's on the system and
778 // can delete anything it knows can't run
779 bool canDeletePath = true;
780 for (auto& buildInstance : builder->builders) {
781 if ( dylibAndCount.second.count(&buildInstance) != 0 )
782 continue;
783 // This builder didn't get this image. See if the image was ineligible
784 // based on slide, ie, that dyld at runtime couldn't load this anyway, so
785 // so removing it from disk won't hurt
786 Diagnostics loaderDiag;
787 const dyld3::GradedArchs* archs = buildInstance.options->archs;
788 dyld3::Platform platform = buildInstance.options->platform;
789 char realerPath[MAXPATHLEN];
790 dyld3::closure::LoadedFileInfo fileInfo = dyld3::MachOAnalyzer::load(loaderDiag, builder->fileSystem,
791 pathToRemove, *archs, platform, realerPath);
792 if ( (platform == dyld3::Platform::macOS) && loaderDiag.hasError() ) {
793 // Try again with iOSMac
794 loaderDiag.clearError();
795 fileInfo = dyld3::MachOAnalyzer::load(loaderDiag, builder->fileSystem,
796 pathToRemove, *archs, dyld3::Platform::iOSMac, realerPath);
797 }
798
799 // We don't need the file content now, as we only needed to know if this file could be loaded
800 builder->fileSystem.unloadFile(fileInfo);
801
802 if ( loaderDiag.hasError() || (fileInfo.fileContent == nullptr) ) {
803 // This arch/platform combination couldn't load this path, so we can remove it
804 continue;
805 }
806
807 // This arch was compatible, so the dylib was rejected from this cache for some other reason, eg,
808 // cache overflow. We need to keep it on-disk
809 canDeletePath = false;
810 break;
811 }
812 if ( canDeletePath )
813 builder->filesToRemove.push_back(pathToRemove);
814 }
815 }
816
817 // Quit if we had any errors.
818 for (auto& buildInstance : builder->builders) {
819 if (!buildInstance.errors.empty())
820 return;
821 }
822
823 builder->state = MRMSharedCacheBuilder::FinishedBuilding;
824 success = true;
825 });
826 return success;
827 }
828
829 const char* const* getErrors(const struct MRMSharedCacheBuilder* builder, uint64_t* errorCount) {
830 if (builder->errors.empty())
831 return nullptr;
832 *errorCount = builder->errors.size();
833 return builder->errors.data();
834 }
835
836 const struct FileResult* const* getFileResults(struct MRMSharedCacheBuilder* builder, uint64_t* resultCount) {
837 if (builder->fileResults.empty())
838 return nullptr;
839 *resultCount = builder->fileResults.size();
840 return builder->fileResults.data();
841 }
842
843 const struct CacheResult* const* getCacheResults(struct MRMSharedCacheBuilder* builder, uint64_t* resultCount) {
844 if (builder->cacheResults.empty())
845 return nullptr;
846 *resultCount = builder->cacheResults.size();
847 return builder->cacheResults.data();
848 }
849
850 const char* const* getFilesToRemove(const struct MRMSharedCacheBuilder* builder, uint64_t* fileCount) {
851 if (builder->filesToRemove.empty())
852 return nullptr;
853 *fileCount = builder->filesToRemove.size();
854 return builder->filesToRemove.data();
855 }
856
857 void destroySharedCacheBuilder(struct MRMSharedCacheBuilder* builder) {
858 for (auto &indexAndIsDataMalloced : builder->fileResultBuffers) {
859 FileResult& fileResult = builder->fileResultStorage[indexAndIsDataMalloced.first];
860 if (indexAndIsDataMalloced.second) {
861 free((void*)fileResult.data);
862 } else {
863 vm_deallocate(mach_task_self(), (vm_address_t)fileResult.data, fileResult.size);
864 }
865 fileResult.data = nullptr;
866 }
867 delete builder;
868 }