1 /* -*- mode: C++; c-basic-offset: 4; indent-tabs-mode: nil -*-
3 * Copyright (c) 2017 Apple Inc. All rights reserved.
5 * @APPLE_LICENSE_HEADER_START@
7 * This file contains Original Code and/or Modifications of Original Code
8 * as defined in and that are subject to the Apple Public Source License
9 * Version 2.0 (the 'License'). You may not use this file except in
10 * compliance with the License. Please obtain a copy of the License at
11 * http://www.opensource.apple.com/apsl/ and read it before using this
14 * The Original Code and all software distributed under the License are
15 * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
16 * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
17 * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
18 * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
19 * Please see the License for the specific language governing rights and
20 * limitations under the License.
22 * @APPLE_LICENSE_HEADER_END@
25 #include "mrm_shared_cache_builder.h"
26 #include "SharedCacheBuilder.h"
27 #include "ClosureFileSystem.h"
28 #include "FileUtils.h"
29 #include "JSONReader.h"
36 static const uint64_t kMinBuildVersion
= 1; //The minimum version BuildOptions struct we can support
37 static const uint64_t kMaxBuildVersion
= 2; //The maximum version BuildOptions struct we can support
39 static const uint32_t MajorVersion
= 1;
40 static const uint32_t MinorVersion
= 2;
48 const uint64_t length
;
54 class FileSystemMRM
: public FileSystem
{
56 FileSystemMRM() : FileSystem() { }
58 bool getRealPath(const char possiblePath
[MAXPATHLEN
], char realPath
[MAXPATHLEN
]) const override
{
60 std::string resolvedPath
= symlinkResolver
.realPath(diag
, possiblePath
);
61 if (diag
.hasError()) {
62 diag
.verbose("MRM error: %s\n", diag
.errorMessage().c_str());
67 // FIXME: Should we only return real paths of files which point to macho's? For now that is what we are doing
68 auto it
= fileMap
.find(resolvedPath
);
69 if (it
== fileMap
.end())
72 memcpy(realPath
, resolvedPath
.c_str(), std::min((size_t)MAXPATHLEN
, resolvedPath
.size() + 1));
76 bool loadFile(const char* path
, LoadedFileInfo
& info
, char realerPath
[MAXPATHLEN
], void (^error
)(const char* format
, ...)) const override
{
78 std::string resolvedPath
= symlinkResolver
.realPath(diag
, path
);
79 if (diag
.hasError()) {
80 diag
.verbose("MRM error: %s\n", diag
.errorMessage().c_str());
85 auto it
= fileMap
.find(resolvedPath
);
86 if (it
== fileMap
.end())
89 if (resolvedPath
== path
)
92 memcpy(realerPath
, resolvedPath
.c_str(), std::min((size_t)MAXPATHLEN
, resolvedPath
.size() + 1));
94 // The file exists at this exact path. Lets use it!
95 const FileInfo
& fileInfo
= files
[it
->second
];
97 info
.fileContent
= fileInfo
.data
;
98 info
.fileContentLen
= fileInfo
.length
;
100 info
.sliceLen
= fileInfo
.length
;
101 info
.isOSBinary
= true;
102 info
.inode
= fileInfo
.inode
;
103 info
.mtime
= fileInfo
.mtime
;
104 info
.unload
= nullptr;
109 void unloadFile(const LoadedFileInfo
& info
) const override
{
114 void unloadPartialFile(LoadedFileInfo
& info
, uint64_t keepStartOffset
, uint64_t keepLength
) const override
{
115 // Note we don't actually unload the data here, but we do want to update the offsets for other data structures to track where we are
116 info
.fileContent
= (const void*)((char*)info
.fileContent
+ keepStartOffset
);
117 info
.fileContentLen
= keepLength
;
120 bool fileExists(const char* path
, uint64_t* inode
=nullptr, uint64_t* mtime
=nullptr,
121 bool* issetuid
=nullptr, bool* inodesMatchRuntime
= nullptr) const override
{
123 std::string resolvedPath
= symlinkResolver
.realPath(diag
, path
);
124 if (diag
.hasError()) {
125 diag
.verbose("MRM error: %s\n", diag
.errorMessage().c_str());
130 auto it
= fileMap
.find(resolvedPath
);
131 if (it
== fileMap
.end())
134 // The file exists at this exact path. Lets use it!
135 const FileInfo
& fileInfo
= files
[it
->second
];
137 *inode
= fileInfo
.inode
;
139 *mtime
= fileInfo
.mtime
;
142 if (inodesMatchRuntime
)
143 *inodesMatchRuntime
= false;
148 bool addFile(const char* path
, uint8_t* data
, uint64_t size
, Diagnostics
& diag
, FileFlags fileFlags
) {
149 auto iteratorAndInserted
= fileMap
.insert(std::make_pair(path
, files
.size()));
150 if (!iteratorAndInserted
.second
) {
151 diag
.error("Already have content for path: '%s'", path
);
155 symlinkResolver
.addFile(diag
, path
);
159 // on iOS, inode is just a placeholder
160 // Note its safe to just use the index here as we only compare it during closure building
161 // and never record it in the closures
162 uint64_t inode
= files
.size() + 1;
165 files
.push_back((FileInfo
){ path
, data
, size
, fileFlags
, mtime
, inode
});
169 bool addSymlink(const char* fromPath
, const char* toPath
, Diagnostics
& diag
) {
170 symlinkResolver
.addSymlink(diag
, fromPath
, toPath
);
171 return !diag
.hasError();
174 void forEachFileInfo(std::function
<void(const char* path
, FileFlags fileFlags
)> lambda
) {
175 for (const FileInfo
& fileInfo
: files
)
176 lambda(fileInfo
.path
, fileInfo
.flags
);
179 size_t fileCount() const {
183 std::vector
<DyldSharedCache::FileAlias
> getResolvedSymlinks(Diagnostics
& diag
) {
184 return symlinkResolver
.getResolvedSymlinks(diag
);
188 std::vector
<FileInfo
> files
;
189 std::map
<std::string
, uint64_t> fileMap
;
190 SymlinkResolver symlinkResolver
;
193 } // namespace closure
196 struct BuildInstance
{
197 std::unique_ptr
<DyldSharedCache::CreateOptions
> options
;
198 std::unique_ptr
<SharedCacheBuilder
> builder
;
199 std::vector
<CacheBuilder::InputFile
> inputFiles
;
200 std::vector
<const char*> errors
;
201 std::vector
<const char*> warnings
;
202 std::vector
<std::string
> errorStrings
; // Owns the data for the errors
203 std::vector
<std::string
> warningStrings
; // Owns the data for the warnings
204 uint8_t* cacheData
= nullptr;
205 uint64_t cacheSize
= 0;
207 std::string macOSMap
; // For compatibility with update_dyld_shared_cache's .map file
208 std::string macOSMapPath
; // Owns the string for the path
209 std::string cdHash
; // Owns the data for the cdHash
210 std::string cdHashType
; // Owns the data for the cdHashType
211 std::string uuid
; // Owns the data for the uuid
214 struct BuildFileResult
{
220 struct TranslationResult
{
225 bool bufferWasMalloced
;
228 struct MRMSharedCacheBuilder
{
229 MRMSharedCacheBuilder(const BuildOptions_v1
* options
);
230 const BuildOptions_v1
* options
;
231 dyld3::closure::FileSystemMRM fileSystem
;
233 std::string dylibOrderFileData
;
234 std::string dirtyDataOrderFileData
;
235 void* objcOptimizationsFileData
;
236 size_t objcOptimizationsFileLength
;
238 // An array of builders and their options as we may have more than one builder for a given device variant.
239 std::vector
<BuildInstance
> builders
;
241 // The paths in all of the caches
242 // We keep this here to own the std::string path data
243 std::map
<std::string
, std::unordered_set
<const BuildInstance
*>> dylibsInCaches
;
245 // The results from all of the builders
246 // We keep this in a vector to own the data.
247 std::vector
<FileResult
*> fileResults
;
248 std::vector
<FileResult
> fileResultStorage
;
249 std::vector
<std::pair
<uint64_t, bool>> fileResultBuffers
;
251 // The results from all of the builders
252 // We keep this in a vector to own the data.
253 std::vector
<CacheResult
*> cacheResults
;
254 std::vector
<CacheResult
> cacheResultStorage
;
256 // The files to remove. These are in every copy of the caches we built
257 std::vector
<const char*> filesToRemove
;
259 std::vector
<const char*> errors
;
260 std::vector
<std::string
> errorStorage
;
261 pthread_mutex_t lock
;
269 State state
= AcceptingFiles
;
271 void runSync(void (^block
)()) {
272 pthread_mutex_lock(&lock
);
274 pthread_mutex_unlock(&lock
);
277 __attribute__((format(printf
, 2, 3)))
278 void error(const char* format
, ...) {
280 va_start(list
, format
);
282 diag
.error(format
, list
);
285 errorStorage
.push_back(diag
.errorMessage());
286 errors
.push_back(errorStorage
.back().data());
290 MRMSharedCacheBuilder::MRMSharedCacheBuilder(const BuildOptions_v1
* options
)
292 , lock(PTHREAD_MUTEX_INITIALIZER
)
293 , objcOptimizationsFileData(nullptr)
294 , objcOptimizationsFileLength(0)
299 void validiateBuildOptions(const BuildOptions_v1
* options
, MRMSharedCacheBuilder
& builder
) {
300 if (options
->version
< kMinBuildVersion
) {
301 builder
.error("Builder version %llu is less than minimum supported version of %llu", options
->version
, kMinBuildVersion
);
303 if (options
->version
> kMaxBuildVersion
) {
304 builder
.error("Builder version %llu is greater than maximum supported version of %llu", options
->version
, kMaxBuildVersion
);
306 if (!options
->updateName
) {
307 builder
.error("updateName must not be null");
309 if (!options
->deviceName
) {
310 builder
.error("deviceName must not be null");
312 switch (options
->disposition
) {
313 case Disposition::Unknown
:
314 case Disposition::InternalDevelopment
:
315 case Disposition::Customer
:
316 case Disposition::InternalMinDevelopment
:
319 builder
.error("unknown disposition value");
322 switch (options
->platform
) {
323 case Platform::unknown
:
324 builder
.error("platform must not be unknown");
326 case Platform::macOS
:
329 case Platform::watchOS
:
330 case Platform::bridgeOS
:
331 case Platform::iOSMac
:
332 case Platform::iOS_simulator
:
333 case Platform::tvOS_simulator
:
334 case Platform::watchOS_simulator
:
337 builder
.error("unknown platform value");
340 if (!options
->archs
) {
341 builder
.error("archs must not be null");
343 if (!options
->numArchs
) {
344 builder
.error("numArchs must not be 0");
348 void getVersion(uint32_t *major
, uint32_t *minor
) {
349 *major
= MajorVersion
;
350 *minor
= MinorVersion
;
353 struct MRMSharedCacheBuilder
* createSharedCacheBuilder(const BuildOptions_v1
* options
) {
354 MRMSharedCacheBuilder
* builder
= new MRMSharedCacheBuilder(options
);
356 // Check the option struct values are valid
357 validiateBuildOptions(options
, *builder
);
362 bool addFile(struct MRMSharedCacheBuilder
* builder
, const char* path
, uint8_t* data
, uint64_t size
, FileFlags fileFlags
) {
363 __block
bool success
= false;
364 builder
->runSync(^() {
365 if (builder
->state
!= MRMSharedCacheBuilder::AcceptingFiles
) {
366 builder
->error("Cannot add file: '%s' as we have already started building", path
);
369 size_t pathLength
= strlen(path
);
370 if (pathLength
== 0) {
371 builder
->error("Empty path");
374 if (pathLength
>= MAXPATHLEN
) {
375 builder
->error("Path is too long: '%s'", path
);
378 if (data
== nullptr) {
379 builder
->error("Data cannot be null for file: '%s'", path
);
385 case ShouldBeExcludedFromCacheIfUnusedLeaf
:
386 case RequiredClosure
:
389 builder
->dylibOrderFileData
= std::string((char*)data
, size
);
392 case DirtyDataOrderFile
:
393 builder
->dirtyDataOrderFileData
= std::string((char*)data
, size
);
396 case ObjCOptimizationsFile
:
397 builder
->objcOptimizationsFileData
= data
;
398 builder
->objcOptimizationsFileLength
= size
;
402 builder
->error("unknown file flags value");
406 if (!builder
->fileSystem
.addFile(path
, data
, size
, diag
, fileFlags
)) {
407 builder
->errorStorage
.push_back(diag
.errorMessage());
408 builder
->errors
.push_back(builder
->errorStorage
.back().data());
416 bool addSymlink(struct MRMSharedCacheBuilder
* builder
, const char* fromPath
, const char* toPath
) {
417 __block
bool success
= false;
418 builder
->runSync(^() {
419 if (builder
->state
!= MRMSharedCacheBuilder::AcceptingFiles
) {
420 builder
->error("Cannot add file: '%s' as we have already started building", fromPath
);
423 size_t pathLength
= strlen(fromPath
);
424 if (pathLength
== 0) {
425 builder
->error("Empty path");
428 if (pathLength
>= MAXPATHLEN
) {
429 builder
->error("Path is too long: '%s'", fromPath
);
433 if (!builder
->fileSystem
.addSymlink(fromPath
, toPath
, diag
)) {
434 builder
->errorStorage
.push_back(diag
.errorMessage());
435 builder
->errors
.push_back(builder
->errorStorage
.back().data());
443 static DyldSharedCache::LocalSymbolsMode
platformExcludeLocalSymbols(Platform platform
) {
445 case Platform::unknown
:
446 case Platform::macOS
:
447 return DyldSharedCache::LocalSymbolsMode::keep
;
450 case Platform::watchOS
:
451 case Platform::bridgeOS
:
452 return DyldSharedCache::LocalSymbolsMode::unmap
;
453 case Platform::iOSMac
:
454 case Platform::iOS_simulator
:
455 case Platform::tvOS_simulator
:
456 case Platform::watchOS_simulator
:
457 return DyldSharedCache::LocalSymbolsMode::keep
;
461 static DyldSharedCache::LocalSymbolsMode
excludeLocalSymbols(const BuildOptions_v1
* options
) {
462 if ( options
->version
>= 2 ) {
463 const BuildOptions_v2
* v2
= (const BuildOptions_v2
*)options
;
464 if ( v2
->optimizeForSize
)
465 return DyldSharedCache::LocalSymbolsMode::strip
;
468 // Old build options always use the platform default
469 return platformExcludeLocalSymbols(options
->platform
);
472 static bool optimizeDyldDlopens(const BuildOptions_v1
* options
) {
473 // Old builds always default to dyld3 optimisations
474 if ( options
->version
< 2 ) {
478 // If we want to optimize for size instead of speed, then disable dyld3 dlopen closures
479 const BuildOptions_v2
* v2
= (const BuildOptions_v2
*)options
;
480 return !v2
->optimizeForSize
;
483 static DyldSharedCache::CodeSigningDigestMode
platformCodeSigningDigestMode(Platform platform
) {
485 case Platform::unknown
:
486 case Platform::macOS
:
489 return DyldSharedCache::SHA256only
;
490 case Platform::watchOS
:
491 return DyldSharedCache::Agile
;
492 case Platform::bridgeOS
:
493 case Platform::iOSMac
:
494 case Platform::iOS_simulator
:
495 case Platform::tvOS_simulator
:
496 case Platform::watchOS_simulator
:
497 return DyldSharedCache::SHA256only
;
501 static bool platformIsForSimulator(Platform platform
) {
503 case Platform::unknown
:
504 case Platform::macOS
:
507 case Platform::watchOS
:
508 case Platform::bridgeOS
:
509 case Platform::iOSMac
:
511 case Platform::iOS_simulator
:
512 case Platform::tvOS_simulator
:
513 case Platform::watchOS_simulator
:
518 static const char* dispositionName(Disposition disposition
) {
519 switch (disposition
) {
520 case Disposition::Unknown
:
522 case Disposition::InternalDevelopment
:
524 case Disposition::Customer
:
526 case Disposition::InternalMinDevelopment
:
527 return "InternalMinDevelopment";
531 // This is a JSON file containing the list of classes for which
532 // we should try to build IMP caches.
533 dyld3::json::Node
parseObjcOptimizationsFile(Diagnostics
& diags
, const void* data
, size_t length
) {
534 return dyld3::json::readJSON(diags
, data
, length
);
537 bool runSharedCacheBuilder(struct MRMSharedCacheBuilder
* builder
) {
538 __block
bool success
= false;
539 builder
->runSync(^() {
540 if (builder
->state
!= MRMSharedCacheBuilder::AcceptingFiles
) {
541 builder
->error("Builder has already been run");
544 builder
->state
= MRMSharedCacheBuilder::Building
;
545 if (builder
->fileSystem
.fileCount() == 0) {
546 builder
->error("Cannot run builder with no files");
549 __block Diagnostics diag
;
550 std::vector
<DyldSharedCache::FileAlias
> aliases
= builder
->fileSystem
.getResolvedSymlinks(diag
);
551 if (diag
.hasError()) {
552 diag
.verbose("Symlink resolver error: %s\n", diag
.errorMessage().c_str());
555 if (!builder
->errors
.empty()) {
556 builder
->error("Skipping running shared cache builder due to previous errors");
560 __block
std::vector
<SharedCacheBuilder::InputFile
> inputFiles
;
561 builder
->fileSystem
.forEachFileInfo(^(const char* path
, FileFlags fileFlags
) {
562 SharedCacheBuilder::InputFile::State state
= SharedCacheBuilder::InputFile::Unset
;
564 case FileFlags::NoFlags
:
565 state
= SharedCacheBuilder::InputFile::Unset
;
567 case FileFlags::MustBeInCache
:
568 state
= SharedCacheBuilder::InputFile::MustBeIncluded
;
570 case FileFlags::ShouldBeExcludedFromCacheIfUnusedLeaf
:
571 state
= SharedCacheBuilder::InputFile::MustBeExcludedIfUnused
;
573 case FileFlags::RequiredClosure
:
574 state
= SharedCacheBuilder::InputFile::MustBeIncluded
;
576 case FileFlags::DylibOrderFile
:
577 case FileFlags::DirtyDataOrderFile
:
578 case FileFlags::ObjCOptimizationsFile
:
579 builder
->error("Order files should not be in the file system");
582 inputFiles
.emplace_back((SharedCacheBuilder::InputFile
){ path
, state
});
585 auto addCacheConfiguration
= ^(bool isOptimized
) {
586 for (uint64_t i
= 0; i
!= builder
->options
->numArchs
; ++i
) {
587 // HACK: Skip i386 for macOS
588 if ( (builder
->options
->platform
== Platform::macOS
) && (strcmp(builder
->options
->archs
[i
], "i386") == 0 ) )
590 auto options
= std::make_unique
<DyldSharedCache::CreateOptions
>((DyldSharedCache::CreateOptions
){});
591 const char *cacheSuffix
= (isOptimized
? "" : ".development");
592 if ( builder
->options
->platform
== Platform::macOS
)
594 std::string runtimePath
= (builder
->options
->platform
== Platform::macOS
) ? MACOSX_MRM_DYLD_SHARED_CACHE_DIR
: IPHONE_DYLD_SHARED_CACHE_DIR
;
595 options
->outputFilePath
= runtimePath
+ "dyld_shared_cache_" + builder
->options
->archs
[i
] + cacheSuffix
;
596 options
->outputMapFilePath
= options
->outputFilePath
+ ".json";
597 options
->archs
= &dyld3::GradedArchs::forName(builder
->options
->archs
[i
]);
598 options
->platform
= (dyld3::Platform
)builder
->options
->platform
;
599 options
->localSymbolMode
= excludeLocalSymbols(builder
->options
);
600 options
->optimizeStubs
= isOptimized
;
601 options
->optimizeDyldDlopens
= optimizeDyldDlopens(builder
->options
);
602 options
->optimizeDyldLaunches
= true;
603 options
->codeSigningDigestMode
= platformCodeSigningDigestMode(builder
->options
->platform
);
604 options
->dylibsRemovedDuringMastering
= true;
605 options
->inodesAreSameAsRuntime
= false;
606 options
->cacheSupportsASLR
= true;
607 options
->forSimulator
= platformIsForSimulator(builder
->options
->platform
);
608 options
->isLocallyBuiltCache
= builder
->options
->isLocallyBuiltCache
;
609 options
->verbose
= builder
->options
->verboseDiagnostics
;
610 options
->evictLeafDylibsOnOverflow
= true;
611 options
->loggingPrefix
= std::string(builder
->options
->deviceName
) + dispositionName(builder
->options
->disposition
) + "." + builder
->options
->archs
[i
] + cacheSuffix
;
612 options
->dylibOrdering
= parseOrderFile(builder
->dylibOrderFileData
);
613 options
->dirtyDataSegmentOrdering
= parseOrderFile(builder
->dirtyDataOrderFileData
);
614 options
->objcOptimizations
= parseObjcOptimizationsFile(diag
, builder
->objcOptimizationsFileData
, builder
->objcOptimizationsFileLength
);
616 auto cacheBuilder
= std::make_unique
<SharedCacheBuilder
>(*options
.get(), builder
->fileSystem
);
617 builder
->builders
.emplace_back((BuildInstance
) { std::move(options
), std::move(cacheBuilder
), inputFiles
});
621 // Enqueue a cache for each configuration
622 switch (builder
->options
->disposition
) {
623 case Disposition::Unknown
:
624 case Disposition::InternalDevelopment
:
625 // HACK: MRM for the mac should only get development, even if it requested both
626 if (builder
->options
->platform
== Platform::macOS
) {
627 addCacheConfiguration(false);
629 addCacheConfiguration(false);
630 addCacheConfiguration(true);
633 case Disposition::Customer
:
634 addCacheConfiguration(true);
636 case Disposition::InternalMinDevelopment
:
637 addCacheConfiguration(false);
641 // FIXME: This step can run in parallel.
642 for (auto& buildInstance
: builder
->builders
) {
643 SharedCacheBuilder
* cacheBuilder
= buildInstance
.builder
.get();
644 cacheBuilder
->build(buildInstance
.inputFiles
, aliases
);
646 // First put the warnings in to a vector to own them.
647 buildInstance
.warningStrings
.reserve(cacheBuilder
->warnings().size());
648 for (const std::string
& warning
: cacheBuilder
->warnings())
649 buildInstance
.warningStrings
.push_back(warning
);
651 // Then copy to a vector to reference the owner
652 buildInstance
.warnings
.reserve(buildInstance
.warningStrings
.size());
653 for (const std::string
& warning
: buildInstance
.warningStrings
)
654 buildInstance
.warnings
.push_back(warning
.c_str());
656 if (!cacheBuilder
->errorMessage().empty()) {
657 // First put the errors in to a vector to own them.
658 buildInstance
.errorStrings
.push_back(cacheBuilder
->errorMessage());
660 // Then copy to a vector to reference the owner
661 buildInstance
.errors
.reserve(buildInstance
.errorStrings
.size());
662 for (const std::string
& error
: buildInstance
.errorStrings
)
663 buildInstance
.errors
.push_back(error
.c_str());
666 if (cacheBuilder
->errorMessage().empty()) {
667 cacheBuilder
->writeBuffer(buildInstance
.cacheData
, buildInstance
.cacheSize
);
668 buildInstance
.jsonMap
= cacheBuilder
->getMapFileJSONBuffer(builder
->options
->deviceName
);
669 if ( buildInstance
.options
->platform
== dyld3::Platform::macOS
) {
670 // For compatibility with update_dyld_shared_cache, put a .map file next to the shared cache
671 buildInstance
.macOSMap
= cacheBuilder
->getMapFileBuffer();
672 buildInstance
.macOSMapPath
= buildInstance
.options
->outputFilePath
+ ".map";
674 buildInstance
.cdHash
= cacheBuilder
->cdHashFirst();
675 buildInstance
.uuid
= cacheBuilder
->uuid();
676 switch (buildInstance
.options
->codeSigningDigestMode
) {
677 case DyldSharedCache::SHA256only
:
678 buildInstance
.cdHashType
= "sha256";
680 case DyldSharedCache::SHA1only
:
681 buildInstance
.cdHashType
= "sha1";
683 case DyldSharedCache::Agile
:
684 buildInstance
.cdHashType
= "sha1";
688 // Track the dylibs which were included in this cache
689 cacheBuilder
->forEachCacheDylib(^(const std::string
&path
) {
690 builder
->dylibsInCaches
[path
.c_str()].insert(&buildInstance
);
692 cacheBuilder
->forEachCacheSymlink(^(const std::string
&path
) {
693 builder
->dylibsInCaches
[path
.c_str()].insert(&buildInstance
);
696 // Free the cache builder now so that we don't keep too much memory resident
697 cacheBuilder
->deleteBuffer();
698 buildInstance
.builder
.reset();
701 // Now that we have run all of the builds, collect the results
702 // First push file results for each of the shared caches we built
703 for (auto& buildInstance
: builder
->builders
) {
704 CacheResult cacheBuildResult
;
705 cacheBuildResult
.version
= 1;
706 cacheBuildResult
.loggingPrefix
= buildInstance
.options
->loggingPrefix
.c_str();
707 cacheBuildResult
.deviceConfiguration
= buildInstance
.options
->loggingPrefix
.c_str();
708 cacheBuildResult
.warnings
= buildInstance
.warnings
.empty() ? nullptr : buildInstance
.warnings
.data();
709 cacheBuildResult
.numWarnings
= buildInstance
.warnings
.size();
710 cacheBuildResult
.errors
= buildInstance
.errors
.empty() ? nullptr : buildInstance
.errors
.data();
711 cacheBuildResult
.numErrors
= buildInstance
.errors
.size();
712 cacheBuildResult
.uuidString
= buildInstance
.uuid
.c_str();
713 cacheBuildResult
.mapJSON
= buildInstance
.jsonMap
.c_str();
715 builder
->cacheResultStorage
.emplace_back(cacheBuildResult
);
717 if (!buildInstance
.errors
.empty())
720 FileResult cacheFileResult
;
721 cacheFileResult
.version
= 1;
722 cacheFileResult
.path
= buildInstance
.options
->outputFilePath
.c_str();
723 cacheFileResult
.behavior
= AddFile
;
724 cacheFileResult
.data
= buildInstance
.cacheData
;
725 cacheFileResult
.size
= buildInstance
.cacheSize
;
726 cacheFileResult
.hashArch
= buildInstance
.options
->archs
->name();
727 cacheFileResult
.hashType
= buildInstance
.cdHashType
.c_str();
728 cacheFileResult
.hash
= buildInstance
.cdHash
.c_str();
730 builder
->fileResultBuffers
.push_back({ builder
->fileResultStorage
.size(), true });
731 builder
->fileResultStorage
.emplace_back(cacheFileResult
);
733 // Add a file result for the .map file
734 if ( !buildInstance
.macOSMap
.empty() ) {
735 FileResult cacheFileResult
;
736 cacheFileResult
.version
= 1;
737 cacheFileResult
.path
= buildInstance
.macOSMapPath
.c_str();
738 cacheFileResult
.behavior
= AddFile
;
739 cacheFileResult
.data
= (const uint8_t*)buildInstance
.macOSMap
.data();
740 cacheFileResult
.size
= buildInstance
.macOSMap
.size();
741 cacheFileResult
.hashArch
= buildInstance
.options
->archs
->name();
742 cacheFileResult
.hashType
= buildInstance
.cdHashType
.c_str();
743 cacheFileResult
.hash
= buildInstance
.cdHash
.c_str();
745 builder
->fileResultStorage
.emplace_back(cacheFileResult
);
749 // Copy from the storage to the vector we can return to the API.
750 for (auto &fileResult
: builder
->fileResultStorage
)
751 builder
->fileResults
.push_back(&fileResult
);
752 for (auto &cacheResult
: builder
->cacheResultStorage
)
753 builder
->cacheResults
.push_back(&cacheResult
);
756 // Add entries to tell us to remove all of the dylibs from disk which are in every cache.
757 const size_t numCaches
= builder
->builders
.size();
758 for (const auto& dylibAndCount
: builder
->dylibsInCaches
) {
759 const char* pathToRemove
= dylibAndCount
.first
.c_str();
761 if ( builder
->options
->platform
== Platform::macOS
) {
762 // macOS has to leave the simulator support binaries on disk
763 if ( strcmp(pathToRemove
, "/usr/lib/system/libsystem_kernel.dylib") == 0 )
765 if ( strcmp(pathToRemove
, "/usr/lib/system/libsystem_platform.dylib") == 0 )
767 if ( strcmp(pathToRemove
, "/usr/lib/system/libsystem_pthread.dylib") == 0 )
771 if (dylibAndCount
.second
.size() == numCaches
) {
772 builder
->filesToRemove
.push_back(pathToRemove
);
774 // File is not in every cache, so likely has perhaps only x86_64h slice
775 // but we built both x86_64 and x86_64h caches.
776 // We may still delete it if its in all caches it's eligible for, ie, we
777 // assume the cache builder knows about all possible arch's on the system and
778 // can delete anything it knows can't run
779 bool canDeletePath
= true;
780 for (auto& buildInstance
: builder
->builders
) {
781 if ( dylibAndCount
.second
.count(&buildInstance
) != 0 )
783 // This builder didn't get this image. See if the image was ineligible
784 // based on slide, ie, that dyld at runtime couldn't load this anyway, so
785 // so removing it from disk won't hurt
786 Diagnostics loaderDiag
;
787 const dyld3::GradedArchs
* archs
= buildInstance
.options
->archs
;
788 dyld3::Platform platform
= buildInstance
.options
->platform
;
789 char realerPath
[MAXPATHLEN
];
790 dyld3::closure::LoadedFileInfo fileInfo
= dyld3::MachOAnalyzer::load(loaderDiag
, builder
->fileSystem
,
791 pathToRemove
, *archs
, platform
, realerPath
);
792 if ( (platform
== dyld3::Platform::macOS
) && loaderDiag
.hasError() ) {
793 // Try again with iOSMac
794 loaderDiag
.clearError();
795 fileInfo
= dyld3::MachOAnalyzer::load(loaderDiag
, builder
->fileSystem
,
796 pathToRemove
, *archs
, dyld3::Platform::iOSMac
, realerPath
);
799 // We don't need the file content now, as we only needed to know if this file could be loaded
800 builder
->fileSystem
.unloadFile(fileInfo
);
802 if ( loaderDiag
.hasError() || (fileInfo
.fileContent
== nullptr) ) {
803 // This arch/platform combination couldn't load this path, so we can remove it
807 // This arch was compatible, so the dylib was rejected from this cache for some other reason, eg,
808 // cache overflow. We need to keep it on-disk
809 canDeletePath
= false;
813 builder
->filesToRemove
.push_back(pathToRemove
);
817 // Quit if we had any errors.
818 for (auto& buildInstance
: builder
->builders
) {
819 if (!buildInstance
.errors
.empty())
823 builder
->state
= MRMSharedCacheBuilder::FinishedBuilding
;
829 const char* const* getErrors(const struct MRMSharedCacheBuilder
* builder
, uint64_t* errorCount
) {
830 if (builder
->errors
.empty())
832 *errorCount
= builder
->errors
.size();
833 return builder
->errors
.data();
836 const struct FileResult
* const* getFileResults(struct MRMSharedCacheBuilder
* builder
, uint64_t* resultCount
) {
837 if (builder
->fileResults
.empty())
839 *resultCount
= builder
->fileResults
.size();
840 return builder
->fileResults
.data();
843 const struct CacheResult
* const* getCacheResults(struct MRMSharedCacheBuilder
* builder
, uint64_t* resultCount
) {
844 if (builder
->cacheResults
.empty())
846 *resultCount
= builder
->cacheResults
.size();
847 return builder
->cacheResults
.data();
850 const char* const* getFilesToRemove(const struct MRMSharedCacheBuilder
* builder
, uint64_t* fileCount
) {
851 if (builder
->filesToRemove
.empty())
853 *fileCount
= builder
->filesToRemove
.size();
854 return builder
->filesToRemove
.data();
857 void destroySharedCacheBuilder(struct MRMSharedCacheBuilder
* builder
) {
858 for (auto &indexAndIsDataMalloced
: builder
->fileResultBuffers
) {
859 FileResult
& fileResult
= builder
->fileResultStorage
[indexAndIsDataMalloced
.first
];
860 if (indexAndIsDataMalloced
.second
) {
861 free((void*)fileResult
.data
);
863 vm_deallocate(mach_task_self(), (vm_address_t
)fileResult
.data
, fileResult
.size
);
865 fileResult
.data
= nullptr;