dyld-625.13.tar.gz
[apple/dyld.git] / dyld3 / shared-cache / mrm_shared_cache_builder.cpp
1 /* -*- mode: C++; c-basic-offset: 4; indent-tabs-mode: nil -*-
2 *
3 * Copyright (c) 2017 Apple Inc. All rights reserved.
4 *
5 * @APPLE_LICENSE_HEADER_START@
6 *
7 * This file contains Original Code and/or Modifications of Original Code
8 * as defined in and that are subject to the Apple Public Source License
9 * Version 2.0 (the 'License'). You may not use this file except in
10 * compliance with the License. Please obtain a copy of the License at
11 * http://www.opensource.apple.com/apsl/ and read it before using this
12 * file.
13 *
14 * The Original Code and all software distributed under the License are
15 * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
16 * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
17 * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
18 * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
19 * Please see the License for the specific language governing rights and
20 * limitations under the License.
21 *
22 * @APPLE_LICENSE_HEADER_END@
23 */
24
25 #include "mrm_shared_cache_builder.h"
26 #include "CacheBuilder.h"
27 #include "ClosureFileSystem.h"
28 #include "FileUtils.h"
29 #include <pthread.h>
30 #include <memory>
31 #include <vector>
32 #include <map>
33
34 static const uint64_t kMinBuildVersion = 1; //The minimum version BuildOptions struct we can support
35 static const uint64_t kMaxBuildVersion = 1; //The maximum version BuildOptions struct we can support
36
37 namespace dyld3 {
38 namespace closure {
39
40 struct FileInfo {
41 const char* path;
42 const uint8_t* data;
43 const uint64_t length;
44 FileFlags flags;
45 uint64_t mtime;
46 uint64_t inode;
47 };
48
49 class FileSystemMRM : public FileSystem {
50 public:
51 FileSystemMRM() : FileSystem() { }
52
53 bool getRealPath(const char possiblePath[MAXPATHLEN], char realPath[MAXPATHLEN]) const override {
54 Diagnostics diag;
55 std::string resolvedPath = symlinkResolver.realPath(diag, possiblePath);
56 if (diag.hasError()) {
57 diag.verbose("MRM error: %s\n", diag.errorMessage().c_str());
58 diag.clearError();
59 return false;
60 }
61
62 // FIXME: Should we only return real paths of files which point to macho's? For now that is what we are doing
63 auto it = fileMap.find(resolvedPath);
64 if (it == fileMap.end())
65 return false;
66
67 memcpy(realPath, resolvedPath.c_str(), std::min((size_t)MAXPATHLEN, resolvedPath.size() + 1));
68 return true;
69 }
70
71 bool loadFile(const char* path, LoadedFileInfo& info, char realerPath[MAXPATHLEN], void (^error)(const char* format, ...)) const override {
72 Diagnostics diag;
73 std::string resolvedPath = symlinkResolver.realPath(diag, path);
74 if (diag.hasError()) {
75 diag.verbose("MRM error: %s\n", diag.errorMessage().c_str());
76 diag.clearError();
77 return false;
78 }
79
80 auto it = fileMap.find(resolvedPath);
81 if (it == fileMap.end())
82 return false;
83
84 if (resolvedPath == path)
85 realerPath[0] = '\0';
86 else
87 memcpy(realerPath, resolvedPath.c_str(), std::min((size_t)MAXPATHLEN, resolvedPath.size() + 1));
88
89 // The file exists at this exact path. Lets use it!
90 const FileInfo& fileInfo = files[it->second];
91
92 info.fileContent = fileInfo.data;
93 info.fileContentLen = fileInfo.length;
94 info.sliceOffset = 0;
95 info.sliceLen = fileInfo.length;
96 info.inode = fileInfo.inode;
97 info.mtime = fileInfo.mtime;
98 info.unload = nullptr;
99 info.path = path;
100 return true;
101 }
102
103 void unloadFile(const LoadedFileInfo& info) const override {
104 if (info.unload)
105 info.unload(info);
106 }
107
108 void unloadPartialFile(LoadedFileInfo& info, uint64_t keepStartOffset, uint64_t keepLength) const override {
109 // Note we don't actually unload the data here, but we do want to update the offsets for other data structures to track where we are
110 info.fileContent = (const void*)((char*)info.fileContent + keepStartOffset);
111 info.fileContentLen = keepLength;
112 }
113
114 bool fileExists(const char* path, uint64_t* inode=nullptr, uint64_t* mtime=nullptr, bool* issetuid=nullptr) const override {
115 Diagnostics diag;
116 std::string resolvedPath = symlinkResolver.realPath(diag, path);
117 if (diag.hasError()) {
118 diag.verbose("MRM error: %s\n", diag.errorMessage().c_str());
119 diag.clearError();
120 return false;
121 }
122
123 auto it = fileMap.find(resolvedPath);
124 if (it == fileMap.end())
125 return false;
126
127 // The file exists at this exact path. Lets use it!
128 const FileInfo& fileInfo = files[it->second];
129 if (inode)
130 *inode = fileInfo.inode;
131 if (mtime)
132 *mtime = fileInfo.mtime;
133 if (issetuid)
134 *issetuid = false;
135 return true;
136 }
137
138 // MRM file APIs
139 bool addFile(const char* path, uint8_t* data, uint64_t size, Diagnostics& diag, FileFlags fileFlags) {
140 auto iteratorAndInserted = fileMap.insert(std::make_pair(path, files.size()));
141 if (!iteratorAndInserted.second) {
142 diag.error("Already have content for path: '%s'", path);
143 return false;
144 }
145
146 symlinkResolver.addFile(diag, path);
147 if (diag.hasError())
148 return false;
149
150 // on iOS, inode is used to hold hash of path
151 uint64_t hash = 0;
152 for (const char* s = path; *s != '\0'; ++s)
153 hash += hash*4 + *s;
154 uint64_t inode = hash;
155 uint64_t mtime = 0;
156
157 files.push_back((FileInfo){ path, data, size, fileFlags, mtime, inode });
158 return true;
159 }
160
161 bool addSymlink(const char* fromPath, const char* toPath, Diagnostics& diag) {
162 symlinkResolver.addSymlink(diag, fromPath, toPath);
163 return !diag.hasError();
164 }
165
166 void forEachFileInfo(std::function<void(const char* path, FileFlags fileFlags)> lambda) {
167 for (const FileInfo& fileInfo : files)
168 lambda(fileInfo.path, fileInfo.flags);
169 }
170
171 size_t fileCount() const {
172 return files.size();
173 }
174
175 std::vector<DyldSharedCache::FileAlias> getResolvedSymlinks(Diagnostics& diag) {
176 return symlinkResolver.getResolvedSymlinks(diag);
177 }
178
179 private:
180 std::vector<FileInfo> files;
181 std::map<std::string, uint64_t> fileMap;
182 SymlinkResolver symlinkResolver;
183 };
184
185 } // namespace closure
186 } // namespace dyld3
187
188 struct BuildInstance {
189 std::unique_ptr<DyldSharedCache::CreateOptions> options;
190 std::unique_ptr<CacheBuilder> builder;
191 std::vector<CacheBuilder::InputFile> inputFiles;
192 std::vector<const char*> errors;
193 std::vector<const char*> warnings;
194 std::vector<std::string> errorStrings; // Owns the data for the errors
195 std::vector<std::string> warningStrings; // Owns the data for the warnings
196 uint8_t* cacheData = nullptr;
197 uint64_t cacheSize = 0;
198 uint8_t* cacheMapData = nullptr;
199 uint64_t cacheMapSize = 0;
200 std::string cdHash; // Owns the data for the cdHash
201 };
202
203 struct BuildFileResult {
204 std::string path;
205 const uint8_t* data;
206 uint64_t size;
207 };
208
209 struct SharedCacheBuilder {
210 SharedCacheBuilder(const BuildOptions_v1* options);
211 const BuildOptions_v1* options;
212 dyld3::closure::FileSystemMRM fileSystem;
213
214 std::string dylibOrderFileData;
215 std::string dirtyDataOrderFileData;
216
217 // An array of builders and their options as we may have more than one builder for a given device variant.
218 std::vector<BuildInstance> builders;
219
220 // The results from all of the builders
221 // We keep this in a vector to own the data.
222 std::vector<BuildFileResult> fileResults;
223
224 std::vector<std::string> errors;
225 pthread_mutex_t lock;
226
227 enum State {
228 AcceptingFiles,
229 Building,
230 FinishedBuilding
231 };
232
233 State state = AcceptingFiles;
234
235 void runSync(void (^block)()) {
236 pthread_mutex_lock(&lock);
237 block();
238 pthread_mutex_unlock(&lock);
239 }
240
241 __attribute__((format(printf, 2, 3)))
242 void error(const char* format, ...) {
243 va_list list;
244 va_start(list, format);
245 Diagnostics diag;
246 diag.error(format, list);
247 va_end(list);
248
249 errors.push_back(diag.errorMessage());
250 }
251 };
252
253 SharedCacheBuilder::SharedCacheBuilder(const BuildOptions_v1* options) : options(options), lock(PTHREAD_MUTEX_INITIALIZER) {
254
255 }
256
257 void validiateBuildOptions(const BuildOptions_v1* options, SharedCacheBuilder& builder) {
258 if (options->version < kMinBuildVersion) {
259 builder.error("Builder version %llu is less than minimum supported version of %llu", options->version, kMinBuildVersion);
260 }
261 if (options->version > kMaxBuildVersion) {
262 builder.error("Builder version %llu is greater than maximum supported version of %llu", options->version, kMaxBuildVersion);
263 }
264 if (!options->updateName) {
265 builder.error("updateName must not be null");
266 }
267 if (!options->deviceName) {
268 builder.error("deviceName must not be null");
269 }
270 switch (options->disposition) {
271 case Disposition::Unknown:
272 case Disposition::InternalDevelopment:
273 case Disposition::Customer:
274 break;
275 default:
276 builder.error("unknown disposition value");
277 break;
278 }
279 switch (options->platform) {
280 case Platform::unknown:
281 builder.error("platform must not be unknown");
282 break;
283 case Platform::macOS:
284 case Platform::iOS:
285 case Platform::tvOS:
286 case Platform::watchOS:
287 case Platform::bridgeOS:
288 case Platform::iOSMac:
289 case Platform::iOS_simulator:
290 case Platform::tvOS_simulator:
291 case Platform::watchOS_simulator:
292 break;
293 default:
294 builder.error("unknown platform value");
295 break;
296 }
297 if (!options->archs) {
298 builder.error("archs must not be null");
299 }
300 if (!options->numArchs) {
301 builder.error("numArchs must not be 0");
302 }
303 }
304
305 struct SharedCacheBuilder* createSharedCacheBuilder(const BuildOptions_v1* options) {
306 SharedCacheBuilder* builder = new SharedCacheBuilder(options);
307
308 // Check the option struct values are valid
309 validiateBuildOptions(options, *builder);
310
311 return builder;
312 }
313
314 bool addFile(struct SharedCacheBuilder* builder, const char* path, uint8_t* data, uint64_t size, FileFlags fileFlags) {
315 __block bool success = false;
316 builder->runSync(^() {
317 if (builder->state != SharedCacheBuilder::AcceptingFiles) {
318 builder->error("Cannot add file: '%s' as we have already started building", path);
319 return;
320 }
321 size_t pathLength = strlen(path);
322 if (pathLength == 0) {
323 builder->error("Empty path");
324 return;
325 }
326 if (pathLength >= MAXPATHLEN) {
327 builder->error("Path is too long: '%s'", path);
328 return;
329 }
330 if (data == nullptr) {
331 builder->error("Data cannot be null for file: '%s'", path);
332 return;
333 }
334 switch (fileFlags) {
335 case NoFlags:
336 case MustBeInCache:
337 case ShouldBeExcludedFromCacheIfUnusedLeaf:
338 case RequiredClosure:
339 break;
340 case DylibOrderFile:
341 builder->dylibOrderFileData = std::string((char*)data, size);
342 success = true;
343 return;
344 case DirtyDataOrderFile:
345 builder->dirtyDataOrderFileData = std::string((char*)data, size);
346 success = true;
347 return;
348 default:
349 builder->error("unknown file flags value");
350 break;
351 }
352 Diagnostics diag;
353 if (!builder->fileSystem.addFile(path, data, size, diag, fileFlags)) {
354 builder->errors.push_back(diag.errorMessage());
355 return;
356 }
357 success = true;
358 });
359 return success;
360 }
361
362 bool addSymlink(struct SharedCacheBuilder* builder, const char* fromPath, const char* toPath) {
363 __block bool success = false;
364 builder->runSync(^() {
365 if (builder->state != SharedCacheBuilder::AcceptingFiles) {
366 builder->error("Cannot add file: '%s' as we have already started building", fromPath);
367 return;
368 }
369 size_t pathLength = strlen(fromPath);
370 if (pathLength == 0) {
371 builder->error("Empty path");
372 return;
373 }
374 if (pathLength >= MAXPATHLEN) {
375 builder->error("Path is too long: '%s'", fromPath);
376 return;
377 }
378 Diagnostics diag;
379 if (!builder->fileSystem.addSymlink(fromPath, toPath, diag)) {
380 builder->errors.push_back(diag.errorMessage());
381 return;
382 }
383 success = true;
384 });
385 return success;
386 }
387
388 static bool platformExcludeLocalSymbols(Platform platform) {
389 switch (platform) {
390 case Platform::unknown:
391 case Platform::macOS:
392 return false;
393 case Platform::iOS:
394 case Platform::tvOS:
395 case Platform::watchOS:
396 case Platform::bridgeOS:
397 return true;
398 case Platform::iOSMac:
399 case Platform::iOS_simulator:
400 case Platform::tvOS_simulator:
401 case Platform::watchOS_simulator:
402 return false;
403 }
404 }
405
406 static DyldSharedCache::CodeSigningDigestMode platformCodeSigningDigestMode(Platform platform) {
407 switch (platform) {
408 case Platform::unknown:
409 case Platform::macOS:
410 case Platform::iOS:
411 case Platform::tvOS:
412 return DyldSharedCache::SHA256only;
413 case Platform::watchOS:
414 return DyldSharedCache::Agile;
415 case Platform::bridgeOS:
416 case Platform::iOSMac:
417 case Platform::iOS_simulator:
418 case Platform::tvOS_simulator:
419 case Platform::watchOS_simulator:
420 return DyldSharedCache::SHA256only;
421 }
422 }
423
424 static bool platformIsForSimulator(Platform platform) {
425 switch (platform) {
426 case Platform::unknown:
427 case Platform::macOS:
428 case Platform::iOS:
429 case Platform::tvOS:
430 case Platform::watchOS:
431 case Platform::bridgeOS:
432 case Platform::iOSMac:
433 return false;
434 case Platform::iOS_simulator:
435 case Platform::tvOS_simulator:
436 case Platform::watchOS_simulator:
437 return true;
438 }
439 }
440
441 static const char* dispositionName(Disposition disposition) {
442 switch (disposition) {
443 case Disposition::Unknown:
444 return "";
445 case Disposition::InternalDevelopment:
446 return "Internal";
447 case Disposition::Customer:
448 return "Customer";
449 case Disposition::InternalMinDevelopment:
450 return "InternalMinDevelopment";
451 }
452 }
453
454 bool runSharedCacheBuilder(struct SharedCacheBuilder* builder) {
455 __block bool success = false;
456 builder->runSync(^() {
457 if (builder->state != SharedCacheBuilder::AcceptingFiles) {
458 builder->error("Builder has already been run");
459 return;
460 }
461 builder->state = SharedCacheBuilder::Building;
462 if (builder->fileSystem.fileCount() == 0) {
463 builder->error("Cannot run builder with no files");
464 }
465
466 Diagnostics diag;
467 std::vector<DyldSharedCache::FileAlias> aliases = builder->fileSystem.getResolvedSymlinks(diag);
468 if (diag.hasError()) {
469 diag.verbose("Symlink resolver error: %s\n", diag.errorMessage().c_str());
470 }
471
472 if (!builder->errors.empty()) {
473 builder->error("Skipping running shared cache builder due to previous errors");
474 return;
475 }
476
477 __block std::vector<CacheBuilder::InputFile> inputFiles;
478 builder->fileSystem.forEachFileInfo(^(const char* path, FileFlags fileFlags) {
479 CacheBuilder::InputFile::State state = CacheBuilder::InputFile::Unset;
480 switch (fileFlags) {
481 case FileFlags::NoFlags:
482 state = CacheBuilder::InputFile::Unset;
483 break;
484 case FileFlags::MustBeInCache:
485 state = CacheBuilder::InputFile::MustBeIncluded;
486 break;
487 case FileFlags::ShouldBeExcludedFromCacheIfUnusedLeaf:
488 state = CacheBuilder::InputFile::MustBeExcludedIfUnused;
489 break;
490 case FileFlags::RequiredClosure:
491 state = CacheBuilder::InputFile::MustBeIncluded;
492 break;
493 case FileFlags::DylibOrderFile:
494 case FileFlags::DirtyDataOrderFile:
495 builder->error("Order files should not be in the file system");
496 return;
497 }
498 inputFiles.emplace_back((CacheBuilder::InputFile){ path, state });
499 });
500
501 auto addCacheConfiguration = ^(bool isOptimized) {
502 for (uint64_t i = 0; i != builder->options->numArchs; ++i) {
503 auto options = std::make_unique<DyldSharedCache::CreateOptions>((DyldSharedCache::CreateOptions){});
504 const char *cacheSuffix = (isOptimized ? "" : ".development");
505 std::string runtimePath = (builder->options->platform == Platform::macOS) ? "/private/var/db/dyld/" : "/System/Library/Caches/com.apple.dyld/";
506 options->outputFilePath = runtimePath + "dyld_shared_cache_" + builder->options->archs[i] + cacheSuffix;
507 options->outputMapFilePath = options->outputFilePath + ".map";
508 options->archName = builder->options->archs[i];
509 options->platform = (dyld3::Platform)builder->options->platform;
510 options->excludeLocalSymbols = platformExcludeLocalSymbols(builder->options->platform);
511 options->optimizeStubs = isOptimized;
512 options->optimizeObjC = true;
513 options->codeSigningDigestMode = platformCodeSigningDigestMode(builder->options->platform);
514 options->dylibsRemovedDuringMastering = true;
515 options->inodesAreSameAsRuntime = false;
516 options->cacheSupportsASLR = true;
517 options->forSimulator = platformIsForSimulator(builder->options->platform);
518 options->isLocallyBuiltCache = builder->options->isLocallyBuiltCache;
519 options->verbose = builder->options->verboseDiagnostics;
520 options->evictLeafDylibsOnOverflow = true;
521 options->loggingPrefix = std::string(builder->options->deviceName) + dispositionName(builder->options->disposition) + "." + builder->options->archs[i] + cacheSuffix;
522 options->pathPrefixes = { "" };
523 options->dylibOrdering = parseOrderFile(builder->dylibOrderFileData);
524 options->dirtyDataSegmentOrdering = parseOrderFile(builder->dirtyDataOrderFileData);
525
526 auto cacheBuilder = std::make_unique<CacheBuilder>(*options.get(), builder->fileSystem);
527 builder->builders.emplace_back((BuildInstance) { std::move(options), std::move(cacheBuilder), inputFiles });
528 }
529 };
530
531 // Enqueue a cache for each configuration
532 switch (builder->options->disposition) {
533 case Disposition::Unknown:
534 case Disposition::InternalDevelopment:
535 addCacheConfiguration(false);
536 addCacheConfiguration(true);
537 break;
538 case Disposition::Customer:
539 addCacheConfiguration(true);
540 case Disposition::InternalMinDevelopment:
541 addCacheConfiguration(false);
542 }
543
544 // FIXME: This step can run in parallel.
545 for (auto& buildInstance : builder->builders) {
546 CacheBuilder* builder = buildInstance.builder.get();
547 builder->build(buildInstance.inputFiles, aliases);
548
549 // First put the warnings in to a vector to own them.
550 buildInstance.warningStrings.reserve(builder->warnings().size());
551 for (const std::string& warning : builder->warnings())
552 buildInstance.warningStrings.push_back(warning);
553
554 // Then copy to a vector to reference the owner
555 buildInstance.warnings.reserve(buildInstance.warningStrings.size());
556 for (const std::string& warning : buildInstance.warningStrings)
557 buildInstance.warnings.push_back(warning.c_str());
558
559 if (!builder->errorMessage().empty()) {
560 // First put the errors in to a vector to own them.
561 buildInstance.errorStrings.push_back(builder->errorMessage());
562
563 // Then copy to a vector to reference the owner
564 buildInstance.errors.reserve(buildInstance.errorStrings.size());
565 for (const std::string& error : buildInstance.errorStrings)
566 buildInstance.errors.push_back(error.c_str());
567 }
568
569 if (builder->errorMessage().empty()) {
570 builder->writeBuffer(buildInstance.cacheData, buildInstance.cacheSize);
571 builder->writeMapFileBuffer(buildInstance.cacheMapData, buildInstance.cacheMapSize);
572 buildInstance.cdHash = builder->cdHashFirst();
573 }
574 }
575
576 // Now that we have run all of the builds, collect the results
577 // First push file results for each of the shared caches we built
578 __block std::map<std::string, uint32_t> dylibsInCaches;
579 for (auto& buildInstance : builder->builders) {
580 CacheBuilder* cacheBuilder = buildInstance.builder.get();
581 if (!cacheBuilder->errorMessage().empty())
582 continue;
583 builder->fileResults.push_back((BuildFileResult) { buildInstance.options->outputFilePath, buildInstance.cacheData, buildInstance.cacheSize });
584 builder->fileResults.push_back((BuildFileResult) { buildInstance.options->outputMapFilePath, buildInstance.cacheMapData, buildInstance.cacheMapSize });
585
586 cacheBuilder->forEachCacheDylib(^(const std::string &path) {
587 ++dylibsInCaches[path];
588 });
589 }
590
591 // Add entries to tell us to remove all of the dylibs from disk which are in every cache.
592 const size_t numCaches = builder->builders.size();
593 for (const auto& dylibAndCount : dylibsInCaches) {
594 if (dylibAndCount.second == numCaches)
595 builder->fileResults.push_back((BuildFileResult) { dylibAndCount.first, nullptr, 0 });
596 }
597
598 builder->state = SharedCacheBuilder::FinishedBuilding;
599 success = true;
600 });
601 return success;
602 }
603
604 uint64_t getErrorCount(const struct SharedCacheBuilder* builder) {
605 return builder->errors.size();
606 }
607
608 const char* getError(const struct SharedCacheBuilder* builder, uint64_t errorIndex) {
609 if (errorIndex >= builder->errors.size())
610 return nullptr;
611 return builder->errors[errorIndex].c_str();
612 }
613
614 uint64_t getCacheResultCount(const struct SharedCacheBuilder* builder) {
615 return builder->builders.size();
616 }
617
618 void getCacheResult(struct SharedCacheBuilder* builder, uint64_t cacheIndex, BuildResult* result) {
619 if (cacheIndex >= builder->builders.size())
620 return;
621
622 BuildInstance& buildInstance = builder->builders[cacheIndex];
623
624 result->version = 1;
625 result->loggingPrefix = buildInstance.options->loggingPrefix.c_str();
626 result->warnings = buildInstance.warnings.empty() ? nullptr : buildInstance.warnings.data();
627 result->numWarnings = buildInstance.warnings.size();
628 result->errors = buildInstance.errors.empty() ? nullptr : buildInstance.errors.data();
629 result->numErrors = buildInstance.errors.size();
630 result->sharedCachePath = buildInstance.options->outputFilePath.c_str();
631 result->cdHash = buildInstance.cdHash.c_str();
632 }
633
634 uint64_t getFileResultCount(const struct SharedCacheBuilder* builder) {
635 return builder->fileResults.size();
636 }
637
638 void getFileResult(struct SharedCacheBuilder* builder, uint64_t fileIndex, FileResult* result) {
639 if (fileIndex >= builder->fileResults.size())
640 return;
641 const BuildFileResult& buildFileResult = builder->fileResults[fileIndex];
642 *result = (FileResult) { buildFileResult.path.c_str(), buildFileResult.data, buildFileResult.size };
643 }
644
645 void destroySharedCacheBuilder(struct SharedCacheBuilder* builder) {
646 delete builder;
647 }