1 /* -*- mode: C++; c-basic-offset: 4; indent-tabs-mode: nil -*-
3 * Copyright (c) 2017 Apple Inc. All rights reserved.
5 * @APPLE_LICENSE_HEADER_START@
7 * This file contains Original Code and/or Modifications of Original Code
8 * as defined in and that are subject to the Apple Public Source License
9 * Version 2.0 (the 'License'). You may not use this file except in
10 * compliance with the License. Please obtain a copy of the License at
11 * http://www.opensource.apple.com/apsl/ and read it before using this
14 * The Original Code and all software distributed under the License are
15 * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
16 * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
17 * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
18 * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
19 * Please see the License for the specific language governing rights and
20 * limitations under the License.
22 * @APPLE_LICENSE_HEADER_END@
25 #include "mrm_shared_cache_builder.h"
26 #include "CacheBuilder.h"
27 #include "ClosureFileSystem.h"
28 #include "FileUtils.h"
34 static const uint64_t kMinBuildVersion
= 1; //The minimum version BuildOptions struct we can support
35 static const uint64_t kMaxBuildVersion
= 1; //The maximum version BuildOptions struct we can support
43 const uint64_t length
;
49 class FileSystemMRM
: public FileSystem
{
51 FileSystemMRM() : FileSystem() { }
53 bool getRealPath(const char possiblePath
[MAXPATHLEN
], char realPath
[MAXPATHLEN
]) const override
{
55 std::string resolvedPath
= symlinkResolver
.realPath(diag
, possiblePath
);
56 if (diag
.hasError()) {
57 diag
.verbose("MRM error: %s\n", diag
.errorMessage().c_str());
62 // FIXME: Should we only return real paths of files which point to macho's? For now that is what we are doing
63 auto it
= fileMap
.find(resolvedPath
);
64 if (it
== fileMap
.end())
67 memcpy(realPath
, resolvedPath
.c_str(), std::min((size_t)MAXPATHLEN
, resolvedPath
.size() + 1));
71 bool loadFile(const char* path
, LoadedFileInfo
& info
, char realerPath
[MAXPATHLEN
], void (^error
)(const char* format
, ...)) const override
{
73 std::string resolvedPath
= symlinkResolver
.realPath(diag
, path
);
74 if (diag
.hasError()) {
75 diag
.verbose("MRM error: %s\n", diag
.errorMessage().c_str());
80 auto it
= fileMap
.find(resolvedPath
);
81 if (it
== fileMap
.end())
84 if (resolvedPath
== path
)
87 memcpy(realerPath
, resolvedPath
.c_str(), std::min((size_t)MAXPATHLEN
, resolvedPath
.size() + 1));
89 // The file exists at this exact path. Lets use it!
90 const FileInfo
& fileInfo
= files
[it
->second
];
92 info
.fileContent
= fileInfo
.data
;
93 info
.fileContentLen
= fileInfo
.length
;
95 info
.sliceLen
= fileInfo
.length
;
96 info
.inode
= fileInfo
.inode
;
97 info
.mtime
= fileInfo
.mtime
;
98 info
.unload
= nullptr;
103 void unloadFile(const LoadedFileInfo
& info
) const override
{
108 void unloadPartialFile(LoadedFileInfo
& info
, uint64_t keepStartOffset
, uint64_t keepLength
) const override
{
109 // Note we don't actually unload the data here, but we do want to update the offsets for other data structures to track where we are
110 info
.fileContent
= (const void*)((char*)info
.fileContent
+ keepStartOffset
);
111 info
.fileContentLen
= keepLength
;
114 bool fileExists(const char* path
, uint64_t* inode
=nullptr, uint64_t* mtime
=nullptr, bool* issetuid
=nullptr) const override
{
116 std::string resolvedPath
= symlinkResolver
.realPath(diag
, path
);
117 if (diag
.hasError()) {
118 diag
.verbose("MRM error: %s\n", diag
.errorMessage().c_str());
123 auto it
= fileMap
.find(resolvedPath
);
124 if (it
== fileMap
.end())
127 // The file exists at this exact path. Lets use it!
128 const FileInfo
& fileInfo
= files
[it
->second
];
130 *inode
= fileInfo
.inode
;
132 *mtime
= fileInfo
.mtime
;
139 bool addFile(const char* path
, uint8_t* data
, uint64_t size
, Diagnostics
& diag
, FileFlags fileFlags
) {
140 auto iteratorAndInserted
= fileMap
.insert(std::make_pair(path
, files
.size()));
141 if (!iteratorAndInserted
.second
) {
142 diag
.error("Already have content for path: '%s'", path
);
146 symlinkResolver
.addFile(diag
, path
);
150 // on iOS, inode is used to hold hash of path
152 for (const char* s
= path
; *s
!= '\0'; ++s
)
154 uint64_t inode
= hash
;
157 files
.push_back((FileInfo
){ path
, data
, size
, fileFlags
, mtime
, inode
});
161 bool addSymlink(const char* fromPath
, const char* toPath
, Diagnostics
& diag
) {
162 symlinkResolver
.addSymlink(diag
, fromPath
, toPath
);
163 return !diag
.hasError();
166 void forEachFileInfo(std::function
<void(const char* path
, FileFlags fileFlags
)> lambda
) {
167 for (const FileInfo
& fileInfo
: files
)
168 lambda(fileInfo
.path
, fileInfo
.flags
);
171 size_t fileCount() const {
175 std::vector
<DyldSharedCache::FileAlias
> getResolvedSymlinks(Diagnostics
& diag
) {
176 return symlinkResolver
.getResolvedSymlinks(diag
);
180 std::vector
<FileInfo
> files
;
181 std::map
<std::string
, uint64_t> fileMap
;
182 SymlinkResolver symlinkResolver
;
185 } // namespace closure
188 struct BuildInstance
{
189 std::unique_ptr
<DyldSharedCache::CreateOptions
> options
;
190 std::unique_ptr
<CacheBuilder
> builder
;
191 std::vector
<CacheBuilder::InputFile
> inputFiles
;
192 std::vector
<const char*> errors
;
193 std::vector
<const char*> warnings
;
194 std::vector
<std::string
> errorStrings
; // Owns the data for the errors
195 std::vector
<std::string
> warningStrings
; // Owns the data for the warnings
196 uint8_t* cacheData
= nullptr;
197 uint64_t cacheSize
= 0;
198 uint8_t* cacheMapData
= nullptr;
199 uint64_t cacheMapSize
= 0;
200 std::string cdHash
; // Owns the data for the cdHash
203 struct BuildFileResult
{
209 struct SharedCacheBuilder
{
210 SharedCacheBuilder(const BuildOptions_v1
* options
);
211 const BuildOptions_v1
* options
;
212 dyld3::closure::FileSystemMRM fileSystem
;
214 std::string dylibOrderFileData
;
215 std::string dirtyDataOrderFileData
;
217 // An array of builders and their options as we may have more than one builder for a given device variant.
218 std::vector
<BuildInstance
> builders
;
220 // The results from all of the builders
221 // We keep this in a vector to own the data.
222 std::vector
<BuildFileResult
> fileResults
;
224 std::vector
<std::string
> errors
;
225 pthread_mutex_t lock
;
233 State state
= AcceptingFiles
;
235 void runSync(void (^block
)()) {
236 pthread_mutex_lock(&lock
);
238 pthread_mutex_unlock(&lock
);
241 __attribute__((format(printf
, 2, 3)))
242 void error(const char* format
, ...) {
244 va_start(list
, format
);
246 diag
.error(format
, list
);
249 errors
.push_back(diag
.errorMessage());
253 SharedCacheBuilder::SharedCacheBuilder(const BuildOptions_v1
* options
) : options(options
), lock(PTHREAD_MUTEX_INITIALIZER
) {
257 void validiateBuildOptions(const BuildOptions_v1
* options
, SharedCacheBuilder
& builder
) {
258 if (options
->version
< kMinBuildVersion
) {
259 builder
.error("Builder version %llu is less than minimum supported version of %llu", options
->version
, kMinBuildVersion
);
261 if (options
->version
> kMaxBuildVersion
) {
262 builder
.error("Builder version %llu is greater than maximum supported version of %llu", options
->version
, kMaxBuildVersion
);
264 if (!options
->updateName
) {
265 builder
.error("updateName must not be null");
267 if (!options
->deviceName
) {
268 builder
.error("deviceName must not be null");
270 switch (options
->disposition
) {
271 case Disposition::Unknown
:
272 case Disposition::InternalDevelopment
:
273 case Disposition::Customer
:
276 builder
.error("unknown disposition value");
279 switch (options
->platform
) {
280 case Platform::unknown
:
281 builder
.error("platform must not be unknown");
283 case Platform::macOS
:
286 case Platform::watchOS
:
287 case Platform::bridgeOS
:
288 case Platform::iOSMac
:
289 case Platform::iOS_simulator
:
290 case Platform::tvOS_simulator
:
291 case Platform::watchOS_simulator
:
294 builder
.error("unknown platform value");
297 if (!options
->archs
) {
298 builder
.error("archs must not be null");
300 if (!options
->numArchs
) {
301 builder
.error("numArchs must not be 0");
305 struct SharedCacheBuilder
* createSharedCacheBuilder(const BuildOptions_v1
* options
) {
306 SharedCacheBuilder
* builder
= new SharedCacheBuilder(options
);
308 // Check the option struct values are valid
309 validiateBuildOptions(options
, *builder
);
314 bool addFile(struct SharedCacheBuilder
* builder
, const char* path
, uint8_t* data
, uint64_t size
, FileFlags fileFlags
) {
315 __block
bool success
= false;
316 builder
->runSync(^() {
317 if (builder
->state
!= SharedCacheBuilder::AcceptingFiles
) {
318 builder
->error("Cannot add file: '%s' as we have already started building", path
);
321 size_t pathLength
= strlen(path
);
322 if (pathLength
== 0) {
323 builder
->error("Empty path");
326 if (pathLength
>= MAXPATHLEN
) {
327 builder
->error("Path is too long: '%s'", path
);
330 if (data
== nullptr) {
331 builder
->error("Data cannot be null for file: '%s'", path
);
337 case ShouldBeExcludedFromCacheIfUnusedLeaf
:
338 case RequiredClosure
:
341 builder
->dylibOrderFileData
= std::string((char*)data
, size
);
344 case DirtyDataOrderFile
:
345 builder
->dirtyDataOrderFileData
= std::string((char*)data
, size
);
349 builder
->error("unknown file flags value");
353 if (!builder
->fileSystem
.addFile(path
, data
, size
, diag
, fileFlags
)) {
354 builder
->errors
.push_back(diag
.errorMessage());
362 bool addSymlink(struct SharedCacheBuilder
* builder
, const char* fromPath
, const char* toPath
) {
363 __block
bool success
= false;
364 builder
->runSync(^() {
365 if (builder
->state
!= SharedCacheBuilder::AcceptingFiles
) {
366 builder
->error("Cannot add file: '%s' as we have already started building", fromPath
);
369 size_t pathLength
= strlen(fromPath
);
370 if (pathLength
== 0) {
371 builder
->error("Empty path");
374 if (pathLength
>= MAXPATHLEN
) {
375 builder
->error("Path is too long: '%s'", fromPath
);
379 if (!builder
->fileSystem
.addSymlink(fromPath
, toPath
, diag
)) {
380 builder
->errors
.push_back(diag
.errorMessage());
388 static bool platformExcludeLocalSymbols(Platform platform
) {
390 case Platform::unknown
:
391 case Platform::macOS
:
395 case Platform::watchOS
:
396 case Platform::bridgeOS
:
398 case Platform::iOSMac
:
399 case Platform::iOS_simulator
:
400 case Platform::tvOS_simulator
:
401 case Platform::watchOS_simulator
:
406 static DyldSharedCache::CodeSigningDigestMode
platformCodeSigningDigestMode(Platform platform
) {
408 case Platform::unknown
:
409 case Platform::macOS
:
412 return DyldSharedCache::SHA256only
;
413 case Platform::watchOS
:
414 return DyldSharedCache::Agile
;
415 case Platform::bridgeOS
:
416 case Platform::iOSMac
:
417 case Platform::iOS_simulator
:
418 case Platform::tvOS_simulator
:
419 case Platform::watchOS_simulator
:
420 return DyldSharedCache::SHA256only
;
424 static bool platformIsForSimulator(Platform platform
) {
426 case Platform::unknown
:
427 case Platform::macOS
:
430 case Platform::watchOS
:
431 case Platform::bridgeOS
:
432 case Platform::iOSMac
:
434 case Platform::iOS_simulator
:
435 case Platform::tvOS_simulator
:
436 case Platform::watchOS_simulator
:
441 static const char* dispositionName(Disposition disposition
) {
442 switch (disposition
) {
443 case Disposition::Unknown
:
445 case Disposition::InternalDevelopment
:
447 case Disposition::Customer
:
449 case Disposition::InternalMinDevelopment
:
450 return "InternalMinDevelopment";
454 bool runSharedCacheBuilder(struct SharedCacheBuilder
* builder
) {
455 __block
bool success
= false;
456 builder
->runSync(^() {
457 if (builder
->state
!= SharedCacheBuilder::AcceptingFiles
) {
458 builder
->error("Builder has already been run");
461 builder
->state
= SharedCacheBuilder::Building
;
462 if (builder
->fileSystem
.fileCount() == 0) {
463 builder
->error("Cannot run builder with no files");
467 std::vector
<DyldSharedCache::FileAlias
> aliases
= builder
->fileSystem
.getResolvedSymlinks(diag
);
468 if (diag
.hasError()) {
469 diag
.verbose("Symlink resolver error: %s\n", diag
.errorMessage().c_str());
472 if (!builder
->errors
.empty()) {
473 builder
->error("Skipping running shared cache builder due to previous errors");
477 __block
std::vector
<CacheBuilder::InputFile
> inputFiles
;
478 builder
->fileSystem
.forEachFileInfo(^(const char* path
, FileFlags fileFlags
) {
479 CacheBuilder::InputFile::State state
= CacheBuilder::InputFile::Unset
;
481 case FileFlags::NoFlags
:
482 state
= CacheBuilder::InputFile::Unset
;
484 case FileFlags::MustBeInCache
:
485 state
= CacheBuilder::InputFile::MustBeIncluded
;
487 case FileFlags::ShouldBeExcludedFromCacheIfUnusedLeaf
:
488 state
= CacheBuilder::InputFile::MustBeExcludedIfUnused
;
490 case FileFlags::RequiredClosure
:
491 state
= CacheBuilder::InputFile::MustBeIncluded
;
493 case FileFlags::DylibOrderFile
:
494 case FileFlags::DirtyDataOrderFile
:
495 builder
->error("Order files should not be in the file system");
498 inputFiles
.emplace_back((CacheBuilder::InputFile
){ path
, state
});
501 auto addCacheConfiguration
= ^(bool isOptimized
) {
502 for (uint64_t i
= 0; i
!= builder
->options
->numArchs
; ++i
) {
503 auto options
= std::make_unique
<DyldSharedCache::CreateOptions
>((DyldSharedCache::CreateOptions
){});
504 const char *cacheSuffix
= (isOptimized
? "" : ".development");
505 std::string runtimePath
= (builder
->options
->platform
== Platform::macOS
) ? "/private/var/db/dyld/" : "/System/Library/Caches/com.apple.dyld/";
506 options
->outputFilePath
= runtimePath
+ "dyld_shared_cache_" + builder
->options
->archs
[i
] + cacheSuffix
;
507 options
->outputMapFilePath
= options
->outputFilePath
+ ".map";
508 options
->archName
= builder
->options
->archs
[i
];
509 options
->platform
= (dyld3::Platform
)builder
->options
->platform
;
510 options
->excludeLocalSymbols
= platformExcludeLocalSymbols(builder
->options
->platform
);
511 options
->optimizeStubs
= isOptimized
;
512 options
->optimizeObjC
= true;
513 options
->codeSigningDigestMode
= platformCodeSigningDigestMode(builder
->options
->platform
);
514 options
->dylibsRemovedDuringMastering
= true;
515 options
->inodesAreSameAsRuntime
= false;
516 options
->cacheSupportsASLR
= true;
517 options
->forSimulator
= platformIsForSimulator(builder
->options
->platform
);
518 options
->isLocallyBuiltCache
= builder
->options
->isLocallyBuiltCache
;
519 options
->verbose
= builder
->options
->verboseDiagnostics
;
520 options
->evictLeafDylibsOnOverflow
= true;
521 options
->loggingPrefix
= std::string(builder
->options
->deviceName
) + dispositionName(builder
->options
->disposition
) + "." + builder
->options
->archs
[i
] + cacheSuffix
;
522 options
->pathPrefixes
= { "" };
523 options
->dylibOrdering
= parseOrderFile(builder
->dylibOrderFileData
);
524 options
->dirtyDataSegmentOrdering
= parseOrderFile(builder
->dirtyDataOrderFileData
);
526 auto cacheBuilder
= std::make_unique
<CacheBuilder
>(*options
.get(), builder
->fileSystem
);
527 builder
->builders
.emplace_back((BuildInstance
) { std::move(options
), std::move(cacheBuilder
), inputFiles
});
531 // Enqueue a cache for each configuration
532 switch (builder
->options
->disposition
) {
533 case Disposition::Unknown
:
534 case Disposition::InternalDevelopment
:
535 addCacheConfiguration(false);
536 addCacheConfiguration(true);
538 case Disposition::Customer
:
539 addCacheConfiguration(true);
540 case Disposition::InternalMinDevelopment
:
541 addCacheConfiguration(false);
544 // FIXME: This step can run in parallel.
545 for (auto& buildInstance
: builder
->builders
) {
546 CacheBuilder
* builder
= buildInstance
.builder
.get();
547 builder
->build(buildInstance
.inputFiles
, aliases
);
549 // First put the warnings in to a vector to own them.
550 buildInstance
.warningStrings
.reserve(builder
->warnings().size());
551 for (const std::string
& warning
: builder
->warnings())
552 buildInstance
.warningStrings
.push_back(warning
);
554 // Then copy to a vector to reference the owner
555 buildInstance
.warnings
.reserve(buildInstance
.warningStrings
.size());
556 for (const std::string
& warning
: buildInstance
.warningStrings
)
557 buildInstance
.warnings
.push_back(warning
.c_str());
559 if (!builder
->errorMessage().empty()) {
560 // First put the errors in to a vector to own them.
561 buildInstance
.errorStrings
.push_back(builder
->errorMessage());
563 // Then copy to a vector to reference the owner
564 buildInstance
.errors
.reserve(buildInstance
.errorStrings
.size());
565 for (const std::string
& error
: buildInstance
.errorStrings
)
566 buildInstance
.errors
.push_back(error
.c_str());
569 if (builder
->errorMessage().empty()) {
570 builder
->writeBuffer(buildInstance
.cacheData
, buildInstance
.cacheSize
);
571 builder
->writeMapFileBuffer(buildInstance
.cacheMapData
, buildInstance
.cacheMapSize
);
572 buildInstance
.cdHash
= builder
->cdHashFirst();
576 // Now that we have run all of the builds, collect the results
577 // First push file results for each of the shared caches we built
578 __block
std::map
<std::string
, uint32_t> dylibsInCaches
;
579 for (auto& buildInstance
: builder
->builders
) {
580 CacheBuilder
* cacheBuilder
= buildInstance
.builder
.get();
581 if (!cacheBuilder
->errorMessage().empty())
583 builder
->fileResults
.push_back((BuildFileResult
) { buildInstance
.options
->outputFilePath
, buildInstance
.cacheData
, buildInstance
.cacheSize
});
584 builder
->fileResults
.push_back((BuildFileResult
) { buildInstance
.options
->outputMapFilePath
, buildInstance
.cacheMapData
, buildInstance
.cacheMapSize
});
586 cacheBuilder
->forEachCacheDylib(^(const std::string
&path
) {
587 ++dylibsInCaches
[path
];
591 // Add entries to tell us to remove all of the dylibs from disk which are in every cache.
592 const size_t numCaches
= builder
->builders
.size();
593 for (const auto& dylibAndCount
: dylibsInCaches
) {
594 if (dylibAndCount
.second
== numCaches
)
595 builder
->fileResults
.push_back((BuildFileResult
) { dylibAndCount
.first
, nullptr, 0 });
598 builder
->state
= SharedCacheBuilder::FinishedBuilding
;
604 uint64_t getErrorCount(const struct SharedCacheBuilder
* builder
) {
605 return builder
->errors
.size();
608 const char* getError(const struct SharedCacheBuilder
* builder
, uint64_t errorIndex
) {
609 if (errorIndex
>= builder
->errors
.size())
611 return builder
->errors
[errorIndex
].c_str();
614 uint64_t getCacheResultCount(const struct SharedCacheBuilder
* builder
) {
615 return builder
->builders
.size();
618 void getCacheResult(struct SharedCacheBuilder
* builder
, uint64_t cacheIndex
, BuildResult
* result
) {
619 if (cacheIndex
>= builder
->builders
.size())
622 BuildInstance
& buildInstance
= builder
->builders
[cacheIndex
];
625 result
->loggingPrefix
= buildInstance
.options
->loggingPrefix
.c_str();
626 result
->warnings
= buildInstance
.warnings
.empty() ? nullptr : buildInstance
.warnings
.data();
627 result
->numWarnings
= buildInstance
.warnings
.size();
628 result
->errors
= buildInstance
.errors
.empty() ? nullptr : buildInstance
.errors
.data();
629 result
->numErrors
= buildInstance
.errors
.size();
630 result
->sharedCachePath
= buildInstance
.options
->outputFilePath
.c_str();
631 result
->cdHash
= buildInstance
.cdHash
.c_str();
634 uint64_t getFileResultCount(const struct SharedCacheBuilder
* builder
) {
635 return builder
->fileResults
.size();
638 void getFileResult(struct SharedCacheBuilder
* builder
, uint64_t fileIndex
, FileResult
* result
) {
639 if (fileIndex
>= builder
->fileResults
.size())
641 const BuildFileResult
& buildFileResult
= builder
->fileResults
[fileIndex
];
642 *result
= (FileResult
) { buildFileResult
.path
.c_str(), buildFileResult
.data
, buildFileResult
.size
};
645 void destroySharedCacheBuilder(struct SharedCacheBuilder
* builder
) {