1 /* -*- mode: C++; c-basic-offset: 4; indent-tabs-mode: nil -*-
3 * Copyright (c) 2016 Apple Inc. All rights reserved.
5 * @APPLE_LICENSE_HEADER_START@
7 * This file contains Original Code and/or Modifications of Original Code
8 * as defined in and that are subject to the Apple Public Source License
9 * Version 2.0 (the 'License'). You may not use this file except in
10 * compliance with the License. Please obtain a copy of the License at
11 * http://www.opensource.apple.com/apsl/ and read it before using this
14 * The Original Code and all software distributed under the License are
15 * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
16 * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
17 * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
18 * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
19 * Please see the License for the specific language governing rights and
20 * limitations under the License.
22 * @APPLE_LICENSE_HEADER_END@
25 #include <sys/types.h>
28 #include <sys/resource.h>
29 #include <mach/mach.h>
30 #include <mach/mach_time.h>
42 #include <sys/param.h>
43 #include <sys/sysctl.h>
44 #include <sys/resource.h>
54 #include <unordered_set>
62 #include <Foundation/NSData.h>
63 #include <Foundation/NSDictionary.h>
64 #include <Foundation/NSPropertyList.h>
65 #include <Foundation/NSString.h>
67 #include "Diagnostics.h"
68 #include "DyldSharedCache.h"
69 #include "FileUtils.h"
70 #include "JSONReader.h"
71 #include "JSONWriter.h"
72 #include "StringUtils.h"
73 #include "mrm_shared_cache_builder.h"
75 #if !__has_feature(objc_arc)
76 #error The use of libdispatch in this files requires it to be compiled with ARC in order to avoid leaks
79 extern char** environ;
81 static dispatch_queue_t build_queue;
83 int runCommandAndWait(Diagnostics& diags, const char* args[])
87 int res = posix_spawn(&pid, args[0], nullptr, nullptr, (char**)args, environ);
89 diags.error("Failed to spawn %s: %s (%d)", args[0], strerror(res), res);
92 res = waitpid(pid, &status, 0);
93 } while (res == -1 && errno == EINTR);
95 if (WIFEXITED(status)) {
96 res = WEXITSTATUS(status);
105 void processRoots(Diagnostics& diags, std::set<std::string>& roots, const char *tempRootsDir)
107 std::set<std::string> processedRoots;
112 for (const auto& root : roots) {
113 res = stat(root.c_str(), &sb);
115 if (res == 0 && S_ISDIR(sb.st_mode)) {
116 processedRoots.insert(root);
120 char tempRootDir[MAXPATHLEN];
121 strlcpy(tempRootDir, tempRootsDir, MAXPATHLEN);
122 strlcat(tempRootDir, "/XXXXXXXX", MAXPATHLEN);
123 mkdtemp(tempRootDir);
125 if (endsWith(root, ".cpio") || endsWith(root, ".cpio.gz") || endsWith(root, ".cpgz") || endsWith(root, ".cpio.bz2") || endsWith(root, ".cpbz2") || endsWith(root, ".pax") || endsWith(root, ".pax.gz") || endsWith(root, ".pgz") || endsWith(root, ".pax.bz2") || endsWith(root, ".pbz2")) {
126 args[0] = (char*)"/usr/bin/ditto";
127 args[1] = (char*)"-x";
128 args[2] = (char*)root.c_str();
129 args[3] = tempRootDir;
131 } else if (endsWith(root, ".tar")) {
132 args[0] = (char*)"/usr/bin/tar";
133 args[1] = (char*)"xf";
134 args[2] = (char*)root.c_str();
135 args[3] = (char*)"-C";
136 args[4] = tempRootDir;
138 } else if (endsWith(root, ".tar.gz") || endsWith(root, ".tgz")) {
139 args[0] = (char*)"/usr/bin/tar";
140 args[1] = (char*)"xzf";
141 args[2] = (char*)root.c_str();
142 args[3] = (char*)"-C";
143 args[4] = tempRootDir;
145 } else if (endsWith(root, ".tar.bz2")
146 || endsWith(root, ".tbz2")
147 || endsWith(root, ".tbz")) {
148 args[0] = (char*)"/usr/bin/tar";
149 args[1] = (char*)"xjf";
150 args[2] = (char*)root.c_str();
151 args[3] = (char*)"-C";
152 args[4] = tempRootDir;
154 } else if (endsWith(root, ".zip")) {
155 args[0] = (char*)"/usr/bin/ditto";
156 args[1] = (char*)"-xk";
157 args[2] = (char*)root.c_str();
158 args[3] = tempRootDir;
161 diags.error("unknown archive type: %s", root.c_str());
165 if (res != runCommandAndWait(diags, args)) {
166 fprintf(stderr, "Could not expand archive %s: %s (%d)", root.c_str(), strerror(res), res);
169 for (auto& existingRoot : processedRoots) {
170 if (existingRoot == tempRootDir)
174 processedRoots.insert(tempRootDir);
177 roots = processedRoots;
180 void writeRootList(const std::string& dstRoot, const std::set<std::string>& roots)
182 if (roots.size() == 0)
185 std::string rootFile = dstRoot + "/roots.txt";
186 FILE* froots = ::fopen(rootFile.c_str(), "w");
190 for (auto& root : roots) {
191 fprintf(froots, "%s\n", root.c_str());
197 BOMCopierCopyOperation filteredCopyIncludingPaths(BOMCopier copier, const char* path, BOMFSObjType type, off_t size)
199 std::string absolutePath = &path[1];
200 void *userData = BOMCopierUserData(copier);
201 std::set<std::string> *cachePaths = (std::set<std::string>*)userData;
202 for (const std::string& cachePath : *cachePaths) {
203 if (startsWith(cachePath, absolutePath))
204 return BOMCopierContinue;
206 if (cachePaths->count(absolutePath)) {
207 return BOMCopierContinue;
209 return BOMCopierSkipFile;
212 static Disposition stringToDisposition(Diagnostics& diags, const std::string& str) {
213 if (diags.hasError())
215 if (str == "Unknown")
217 if (str == "InternalDevelopment")
218 return InternalDevelopment;
219 if (str == "Customer")
221 if (str == "InternalMinDevelopment")
222 return InternalMinDevelopment;
226 static Platform stringToPlatform(Diagnostics& diags, const std::string& str) {
227 if (diags.hasError())
229 if (str == "unknown")
237 if (str == "watchOS")
239 if (str == "bridgeOS")
243 if (str == "UIKitForMac")
245 if (str == "iOS_simulator")
246 return iOS_simulator;
247 if (str == "tvOS_simulator")
248 return tvOS_simulator;
249 if (str == "watchOS_simulator")
250 return watchOS_simulator;
254 static FileFlags stringToFileFlags(Diagnostics& diags, const std::string& str) {
255 if (diags.hasError())
257 if (str == "NoFlags")
259 if (str == "MustBeInCache")
260 return MustBeInCache;
261 if (str == "ShouldBeExcludedFromCacheIfUnusedLeaf")
262 return ShouldBeExcludedFromCacheIfUnusedLeaf;
263 if (str == "RequiredClosure")
264 return RequiredClosure;
265 if (str == "DylibOrderFile")
266 return DylibOrderFile;
267 if (str == "DirtyDataOrderFile")
268 return DirtyDataOrderFile;
272 struct SharedCacheBuilderOptions {
274 std::set<std::string> roots;
275 std::string dylibCacheDir;
276 std::string artifactDir;
278 bool emitDevCaches = true;
279 bool emitElidedDylibs = true;
280 bool listConfigs = false;
281 bool copyRoots = false;
285 std::string emitJSONPath;
286 std::string buildAllPath;
287 std::string resultPath;
288 std::string baselineDifferenceResultPath;
289 std::string baselineCacheMapPath;
290 bool baselineCopyRoots = false;
293 static void loadMRMFiles(Diagnostics& diags,
294 MRMSharedCacheBuilder* sharedCacheBuilder,
295 const std::vector<std::tuple<std::string, std::string, FileFlags>>& inputFiles,
296 std::vector<std::pair<const void*, size_t>>& mappedFiles,
297 const std::set<std::string>& baselineCacheFiles) {
299 for (const std::tuple<std::string, std::string, FileFlags>& inputFile : inputFiles) {
300 const std::string& buildPath = std::get<0>(inputFile);
301 const std::string& runtimePath = std::get<1>(inputFile);
302 FileFlags fileFlags = std::get<2>(inputFile);
304 struct stat stat_buf;
305 int fd = ::open(buildPath.c_str(), O_RDONLY, 0);
307 if (baselineCacheFiles.count(runtimePath)) {
308 diags.error("can't open file '%s', errno=%d\n", buildPath.c_str(), errno);
311 diags.verbose("can't open file '%s', errno=%d\n", buildPath.c_str(), errno);
316 if (fstat(fd, &stat_buf) == -1) {
317 if (baselineCacheFiles.count(runtimePath)) {
318 diags.error("can't stat open file '%s', errno=%d\n", buildPath.c_str(), errno);
322 diags.verbose("can't stat open file '%s', errno=%d\n", buildPath.c_str(), errno);
328 const void* buffer = mmap(NULL, (size_t)stat_buf.st_size, PROT_READ, MAP_PRIVATE, fd, 0);
329 if (buffer == MAP_FAILED) {
330 diags.error("mmap() for file at %s failed, errno=%d\n", buildPath.c_str(), errno);
335 mappedFiles.emplace_back(buffer, (size_t)stat_buf.st_size);
337 addFile(sharedCacheBuilder, runtimePath.c_str(), (uint8_t*)buffer, (size_t)stat_buf.st_size, fileFlags);
341 static void unloadMRMFiles(std::vector<std::pair<const void*, size_t>>& mappedFiles) {
342 for (auto mappedFile : mappedFiles)
343 ::munmap((void*)mappedFile.first, mappedFile.second);
346 static void writeMRMResults(bool cacheBuildSuccess, MRMSharedCacheBuilder* sharedCacheBuilder, const SharedCacheBuilderOptions& options) {
347 if (!cacheBuildSuccess) {
348 uint64_t errorCount = 0;
349 if (const char* const* errors = getErrors(sharedCacheBuilder, &errorCount)) {
350 for (uint64_t i = 0, e = errorCount; i != e; ++i) {
351 const char* errorMessage = errors[i];
352 fprintf(stderr, "ERROR: %s\n", errorMessage);
357 // Now emit each cache we generated, or the errors for them.
358 uint64_t cacheResultCount = 0;
359 if (const CacheResult* const* cacheResults = getCacheResults(sharedCacheBuilder, &cacheResultCount)) {
360 for (uint64_t i = 0, e = cacheResultCount; i != e; ++i) {
361 const CacheResult& result = *(cacheResults[i]);
362 // Always print the warnings if we have roots, even if there are errors
363 if ( (result.numErrors == 0) || !options.roots.empty() ) {
364 for (uint64_t warningIndex = 0; warningIndex != result.numWarnings; ++warningIndex) {
365 fprintf(stderr, "[%s] WARNING: %s\n", result.loggingPrefix, result.warnings[warningIndex]);
368 if (result.numErrors) {
369 for (uint64_t errorIndex = 0; errorIndex != result.numErrors; ++errorIndex) {
370 fprintf(stderr, "[%s] ERROR: %s\n", result.loggingPrefix, result.errors[errorIndex]);
372 cacheBuildSuccess = false;
377 if (!cacheBuildSuccess) {
381 // If we built caches, then write everything out.
382 // TODO: Decide if we should we write any good caches anyway?
383 if (cacheBuildSuccess && !options.dstRoot.empty()) {
384 uint64_t fileResultCount = 0;
385 if (const FileResult* const* fileResults = getFileResults(sharedCacheBuilder, &fileResultCount)) {
386 for (uint64_t i = 0, e = fileResultCount; i != e; ++i) {
387 const FileResult& result = *(fileResults[i]);
389 switch (result.behavior) {
399 const std::string path = options.dstRoot + result.path;
400 std::string pathTemplate = path + "-XXXXXX";
401 size_t templateLen = strlen(pathTemplate.c_str())+2;
402 char pathTemplateSpace[templateLen];
403 strlcpy(pathTemplateSpace, pathTemplate.c_str(), templateLen);
404 int fd = mkstemp(pathTemplateSpace);
406 ::ftruncate(fd, result.size);
407 uint64_t writtenSize = pwrite(fd, result.data, result.size, 0);
408 if ( writtenSize == result.size ) {
409 ::fchmod(fd, S_IRUSR|S_IWUSR|S_IRGRP|S_IROTH); // mkstemp() makes file "rw-------", switch it to "rw-r--r--"
410 if ( ::rename(pathTemplateSpace, path.c_str()) == 0) {
416 fprintf(stderr, "ERROR: could not write file %s\n", pathTemplateSpace);
417 cacheBuildSuccess = false;
420 ::unlink(pathTemplateSpace);
423 fprintf(stderr, "ERROR: could not open file %s\n", pathTemplateSpace);
424 cacheBuildSuccess = false;
431 static void buildCacheFromJSONManifest(Diagnostics& diags, const SharedCacheBuilderOptions& options,
432 const std::string& jsonManifestPath) {
433 dyld3::json::Node manifestNode = dyld3::json::readJSON(diags, jsonManifestPath.c_str());
434 if (diags.hasError())
437 // Top level node should be a map of the options, files, and symlinks.
438 if (manifestNode.map.empty()) {
439 diags.error("Expected map for JSON manifest node\n");
443 // Parse the nodes in the top level manifest node
444 const dyld3::json::Node& versionNode = dyld3::json::getRequiredValue(diags, manifestNode, "version");
445 uint64_t manifestVersion = dyld3::json::parseRequiredInt(diags, versionNode);
446 if (diags.hasError())
449 const uint64_t supportedManifestVersion = 1;
450 if (manifestVersion != supportedManifestVersion) {
451 diags.error("JSON manfiest version of %lld is unsupported. Supported version is %lld\n",
452 manifestVersion, supportedManifestVersion);
455 const dyld3::json::Node& buildOptionsNode = dyld3::json::getRequiredValue(diags, manifestNode, "buildOptions");
456 const dyld3::json::Node& filesNode = dyld3::json::getRequiredValue(diags, manifestNode, "files");
457 const dyld3::json::Node* symlinksNode = dyld3::json::getOptionalValue(diags, manifestNode, "symlinks");
460 const dyld3::json::Node& archsNode = dyld3::json::getRequiredValue(diags, buildOptionsNode, "archs");
461 if (diags.hasError())
463 if (archsNode.array.empty()) {
464 diags.error("Build options archs node is not an array\n");
467 const char* archs[archsNode.array.size()];
468 uint64_t archIndex = 0;
469 for (const dyld3::json::Node& archNode : archsNode.array) {
470 archs[archIndex++] = dyld3::json::parseRequiredString(diags, archNode).c_str();
473 // Parse the rest of the options node.
474 BuildOptions_v1 buildOptions;
475 buildOptions.version = dyld3::json::parseRequiredInt(diags, dyld3::json::getRequiredValue(diags, buildOptionsNode, "version"));
476 buildOptions.updateName = dyld3::json::parseRequiredString(diags, dyld3::json::getRequiredValue(diags, buildOptionsNode, "updateName")).c_str();
477 buildOptions.deviceName = dyld3::json::parseRequiredString(diags, dyld3::json::getRequiredValue(diags, buildOptionsNode, "deviceName")).c_str();
478 buildOptions.disposition = stringToDisposition(diags, dyld3::json::parseRequiredString(diags, dyld3::json::getRequiredValue(diags, buildOptionsNode, "disposition")));
479 buildOptions.platform = stringToPlatform(diags, dyld3::json::parseRequiredString(diags, dyld3::json::getRequiredValue(diags, buildOptionsNode, "platform")));
480 buildOptions.archs = archs;
481 buildOptions.numArchs = archsNode.array.size();
482 buildOptions.verboseDiagnostics = options.debug;
483 buildOptions.isLocallyBuiltCache = true;
485 if (diags.hasError())
488 // Override the disposition if we don't want certaion caches.
489 if (!options.emitDevCaches) {
490 switch (buildOptions.disposition) {
492 // Nothing we can do here as we can't assume what caches are built here.
494 case InternalDevelopment:
495 // This builds both caches, but we don't want dev
496 buildOptions.disposition = Customer;
499 // This is already only the customer cache
501 case InternalMinDevelopment:
502 diags.error("Cannot request no dev cache for InternalMinDevelopment as that is already only a dev cache\n");
507 if (diags.hasError())
510 struct MRMSharedCacheBuilder* sharedCacheBuilder = createSharedCacheBuilder(&buildOptions);
513 if (filesNode.array.empty()) {
514 diags.error("Build options files node is not an array\n");
518 std::vector<std::tuple<std::string, std::string, FileFlags>> inputFiles;
519 for (const dyld3::json::Node& fileNode : filesNode.array) {
520 const std::string& path = dyld3::json::parseRequiredString(diags, dyld3::json::getRequiredValue(diags, fileNode, "path")).c_str();
521 FileFlags fileFlags = stringToFileFlags(diags, dyld3::json::parseRequiredString(diags, dyld3::json::getRequiredValue(diags, fileNode, "flags")));
523 // We can optionally have a sourcePath entry which is the path to get the source content from instead of the install path
524 std::string sourcePath;
525 const dyld3::json::Node* sourcePathNode = dyld3::json::getOptionalValue(diags, fileNode, "sourcePath");
526 if ( sourcePathNode != nullptr ) {
527 if (!sourcePathNode->array.empty()) {
528 diags.error("sourcePath node cannot be an array\n");
531 if (!sourcePathNode->map.empty()) {
532 diags.error("sourcePath node cannot be a map\n");
535 sourcePath = sourcePathNode->value;
540 // Check if one of the -root's has this path
541 bool foundInOverlay = false;
542 for (const std::string& overlay : options.roots) {
544 std::string filePath = overlay + path;
545 if (!stat(filePath.c_str(), &sb)) {
546 foundInOverlay = true;
547 diags.verbose("Taking '%s' from overlay instead of dylib cache\n", path.c_str());
548 inputFiles.push_back({ filePath, path, fileFlags });
556 // Build paths are relative to the build artifact root directory.
557 std::string buildPath;
561 case ShouldBeExcludedFromCacheIfUnusedLeaf:
562 case RequiredClosure:
563 buildPath = "." + sourcePath;
566 case DirtyDataOrderFile:
567 buildPath = "." + sourcePath;
570 inputFiles.push_back({ buildPath, path, fileFlags });
573 if (diags.hasError())
576 // Parse the baseline from the map if we have it
577 std::set<std::string> baselineDylibs;
578 if ( !options.baselineCacheMapPath.empty() ) {
579 dyld3::json::Node mapNode = dyld3::json::readJSON(diags, options.baselineCacheMapPath.c_str());
580 if (diags.hasError())
583 // Top level node should be a map of the version and files
584 if (mapNode.map.empty()) {
585 diags.error("Expected map for JSON cache map node\n");
589 // Parse the nodes in the top level manifest node
590 const dyld3::json::Node& versionNode = dyld3::json::getRequiredValue(diags, mapNode, "version");
591 uint64_t mapVersion = dyld3::json::parseRequiredInt(diags, versionNode);
592 if (diags.hasError())
595 const uint64_t supportedMapVersion = 1;
596 if (mapVersion != supportedMapVersion) {
597 diags.error("JSON map version of %lld is unsupported. Supported version is %lld\n",
598 mapVersion, supportedMapVersion);
603 const dyld3::json::Node& imagesNode = dyld3::json::getRequiredValue(diags, mapNode, "images");
604 if (diags.hasError())
606 if (imagesNode.array.empty()) {
607 diags.error("Images node is not an array\n");
611 for (const dyld3::json::Node& imageNode : imagesNode.array) {
612 const dyld3::json::Node& pathNode = dyld3::json::getRequiredValue(diags, imageNode, "path");
613 if (pathNode.value.empty()) {
614 diags.error("Image path node is not a string\n");
617 baselineDylibs.insert(pathNode.value);
621 std::vector<std::pair<const void*, size_t>> mappedFiles;
622 loadMRMFiles(diags, sharedCacheBuilder, inputFiles, mappedFiles, baselineDylibs);
624 if (diags.hasError())
627 // Parse the symlinks if we have them
629 if (symlinksNode->array.empty()) {
630 diags.error("Build options symlinks node is not an array\n");
633 for (const dyld3::json::Node& symlinkNode : symlinksNode->array) {
634 const std::string& fromPath = dyld3::json::parseRequiredString(diags, dyld3::json::getRequiredValue(diags, symlinkNode, "path")).c_str();
635 const std::string& toPath = dyld3::json::parseRequiredString(diags, dyld3::json::getRequiredValue(diags, symlinkNode, "target")).c_str();
636 addSymlink(sharedCacheBuilder, fromPath.c_str(), toPath.c_str());
640 if (diags.hasError())
643 // Don't create a directory if we are skipping writes, which means we have no dstRoot set
644 if (!options.dstRoot.empty())
645 (void)mkpath_np((options.dstRoot + "/System/Library/Caches/com.apple.dyld/").c_str(), 0755);
647 // Actually build the cache.
648 bool cacheBuildSuccess = runSharedCacheBuilder(sharedCacheBuilder);
650 // Compare this cache to the baseline cache and see if we have any roots to copy over
651 if (!options.baselineDifferenceResultPath.empty() || options.baselineCopyRoots) {
652 std::set<std::string> newDylibs;
653 if (cacheBuildSuccess) {
654 uint64_t fileResultCount = 0;
655 if (const char* const* fileResults = getFilesToRemove(sharedCacheBuilder, &fileResultCount)) {
656 for (uint64_t i = 0; i != fileResultCount; ++i)
657 newDylibs.insert(fileResults[i]);
661 if (options.baselineCopyRoots) {
662 // Work out the set of dylibs in the old cache but not the new one
663 std::set<std::string> dylibsMissingFromNewCache;
664 for (const std::string& baselineDylib : baselineDylibs) {
665 if (!newDylibs.count(baselineDylib))
666 dylibsMissingFromNewCache.insert(baselineDylib);
669 if (!dylibsMissingFromNewCache.empty()) {
670 BOMCopier copier = BOMCopierNewWithSys(BomSys_default());
671 BOMCopierSetUserData(copier, (void*)&dylibsMissingFromNewCache);
672 BOMCopierSetCopyFileStartedHandler(copier, filteredCopyIncludingPaths);
673 std::string dylibCacheRootDir = realFilePath(options.dylibCacheDir);
674 if (dylibCacheRootDir == "") {
675 fprintf(stderr, "Could not find dylib Root directory to copy baseline roots from\n");
678 BOMCopierCopy(copier, dylibCacheRootDir.c_str(), options.dstRoot.c_str());
679 BOMCopierFree(copier);
681 for (const std::string& dylibMissingFromNewCache : dylibsMissingFromNewCache) {
682 diags.verbose("Dylib missing from new cache: '%s'\n", dylibMissingFromNewCache.c_str());
687 if (!options.baselineDifferenceResultPath.empty()) {
688 auto cppToObjStr = [](const std::string& str) {
689 return [NSString stringWithUTF8String:str.c_str()];
692 // Work out the set of dylibs in the cache and taken from the -root
693 NSMutableArray<NSString*>* dylibsFromRoots = [NSMutableArray array];
694 for (auto& root : options.roots) {
695 for (const std::string& dylibInstallName : newDylibs) {
697 std::string filePath = root + "/" + dylibInstallName;
698 if (!stat(filePath.c_str(), &sb)) {
699 [dylibsFromRoots addObject:cppToObjStr(dylibInstallName)];
704 // Work out the set of dylibs in the new cache but not in the baseline cache.
705 NSMutableArray<NSString*>* dylibsMissingFromBaselineCache = [NSMutableArray array];
706 for (const std::string& newDylib : newDylibs) {
707 if (!baselineDylibs.count(newDylib))
708 [dylibsMissingFromBaselineCache addObject:cppToObjStr(newDylib)];
711 NSMutableDictionary* cacheDict = [[NSMutableDictionary alloc] init];
712 cacheDict[@"root-paths-in-cache"] = dylibsFromRoots;
713 cacheDict[@"device-paths-to-delete"] = dylibsMissingFromBaselineCache;
715 NSError* error = nil;
716 NSData* outData = [NSPropertyListSerialization dataWithPropertyList:cacheDict
717 format:NSPropertyListBinaryFormat_v1_0
720 (void)[outData writeToFile:cppToObjStr(options.baselineDifferenceResultPath) atomically:YES];
724 writeMRMResults(cacheBuildSuccess, sharedCacheBuilder, options);
726 destroySharedCacheBuilder(sharedCacheBuilder);
728 unloadMRMFiles(mappedFiles);
731 int main(int argc, const char* argv[])
734 __block Diagnostics diags;
735 SharedCacheBuilderOptions options;
736 std::string jsonManifestPath;
737 char* tempRootsDir = strdup("/tmp/dyld_shared_cache_builder.XXXXXX");
739 mkdtemp(tempRootsDir);
741 for (int i = 1; i < argc; ++i) {
742 const char* arg = argv[i];
744 if (strcmp(arg, "-debug") == 0) {
745 diags = Diagnostics(true);
746 options.debug = true;
747 } else if (strcmp(arg, "-list_configs") == 0) {
748 options.listConfigs = true;
749 } else if (strcmp(arg, "-root") == 0) {
750 std::string realpath = realPath(argv[++i]);
751 if ( realpath.empty() || !fileExists(realpath) ) {
752 fprintf(stderr, "-root path doesn't exist: %s\n", argv[i]);
755 options.roots.insert(realpath);
756 } else if (strcmp(arg, "-copy_roots") == 0) {
757 options.copyRoots = true;
758 } else if (strcmp(arg, "-dylib_cache") == 0) {
759 options.dylibCacheDir = realPath(argv[++i]);
760 } else if (strcmp(arg, "-artifact") == 0) {
761 options.artifactDir = realPath(argv[++i]);
762 } else if (strcmp(arg, "-no_development_cache") == 0) {
763 options.emitDevCaches = false;
764 } else if (strcmp(arg, "-no_overflow_dylibs") == 0) {
765 options.emitElidedDylibs = false;
766 } else if (strcmp(arg, "-development_cache") == 0) {
767 options.emitDevCaches = true;
768 } else if (strcmp(arg, "-overflow_dylibs") == 0) {
769 options.emitElidedDylibs = true;
770 } else if (strcmp(arg, "-mrm") == 0) {
771 options.useMRM = true;
772 } else if (strcmp(arg, "-emit_json") == 0) {
773 options.emitJSONPath = realPath(argv[++i]);
774 } else if (strcmp(arg, "-json_manifest") == 0) {
775 jsonManifestPath = realPath(argv[++i]);
776 } else if (strcmp(arg, "-build_all") == 0) {
777 options.buildAllPath = realPath(argv[++i]);
778 } else if (strcmp(arg, "-dst_root") == 0) {
779 options.dstRoot = realPath(argv[++i]);
780 } else if (strcmp(arg, "-release") == 0) {
781 options.release = argv[++i];
782 } else if (strcmp(arg, "-results") == 0) {
783 options.resultPath = realPath(argv[++i]);
784 } else if (strcmp(arg, "-baseline_diff_results") == 0) {
785 options.baselineDifferenceResultPath = realPath(argv[++i]);
786 } else if (strcmp(arg, "-baseline_copy_roots") == 0) {
787 options.baselineCopyRoots = true;
788 } else if (strcmp(arg, "-baseline_cache_map") == 0) {
789 options.baselineCacheMapPath = realPath(argv[++i]);
792 fprintf(stderr, "unknown option: %s\n", arg);
796 fprintf(stderr, "unknown option: %s\n", arg);
800 (void)options.emitElidedDylibs; // not implemented yet
802 time_t mytime = time(0);
803 fprintf(stderr, "Started: %s", asctime(localtime(&mytime)));
804 processRoots(diags, options.roots, tempRootsDir);
806 struct rlimit rl = { OPEN_MAX, OPEN_MAX };
807 (void)setrlimit(RLIMIT_NOFILE, &rl);
809 if (options.dylibCacheDir.empty() && options.artifactDir.empty() && options.release.empty()) {
810 fprintf(stderr, "you must specify either -dylib_cache, -artifact or -release\n");
812 } else if (!options.dylibCacheDir.empty() && !options.release.empty()) {
813 fprintf(stderr, "you may not use -dylib_cache and -release at the same time\n");
815 } else if (!options.dylibCacheDir.empty() && !options.artifactDir.empty()) {
816 fprintf(stderr, "you may not use -dylib_cache and -artifact at the same time\n");
820 if (jsonManifestPath.empty() && options.buildAllPath.empty()) {
821 fprintf(stderr, "Must specify a -json_manifest path OR a -build_all path\n");
825 if (!options.buildAllPath.empty()) {
826 if (!options.dstRoot.empty()) {
827 fprintf(stderr, "Cannot combine -dst_root and -build_all\n");
830 if (!jsonManifestPath.empty()) {
831 fprintf(stderr, "Cannot combine -json_manifest and -build_all\n");
834 if (!options.baselineDifferenceResultPath.empty()) {
835 fprintf(stderr, "Cannot combine -baseline_diff_results and -build_all\n");
838 if (options.baselineCopyRoots) {
839 fprintf(stderr, "Cannot combine -baseline_copy_roots and -build_all\n");
842 if (!options.baselineCacheMapPath.empty()) {
843 fprintf(stderr, "Cannot combine -baseline_cache_map and -build_all\n");
846 } else if (!options.listConfigs) {
847 if (options.dstRoot.empty()) {
848 fprintf(stderr, "Must specify a valid -dst_root OR -list_configs\n");
852 if (jsonManifestPath.empty()) {
853 fprintf(stderr, "Must specify a -json_manifest path OR -list_configs\n");
858 if (!options.baselineDifferenceResultPath.empty() && (options.roots.size() > 1)) {
859 fprintf(stderr, "Cannot use -baseline_diff_results with more that one -root\n");
863 // Some options don't work with a JSON manifest
864 if (!jsonManifestPath.empty()) {
865 if (!options.resultPath.empty()) {
866 fprintf(stderr, "Cannot use -results with -json_manifest\n");
869 if (!options.baselineDifferenceResultPath.empty() && options.baselineCacheMapPath.empty()) {
870 fprintf(stderr, "Must use -baseline_cache_map with -baseline_diff_results when using -json_manifest\n");
873 if (options.baselineCopyRoots && options.baselineCacheMapPath.empty()) {
874 fprintf(stderr, "Must use -baseline_cache_map with -baseline_copy_roots when using -json_manifest\n");
878 if (!options.baselineCacheMapPath.empty()) {
879 fprintf(stderr, "Cannot use -baseline_cache_map without -json_manifest\n");
884 if (!options.baselineCacheMapPath.empty()) {
885 if (options.baselineDifferenceResultPath.empty() && options.baselineCopyRoots) {
886 fprintf(stderr, "Must use -baseline_cache_map with -baseline_diff_results or -baseline_copy_roots\n");
891 // Find all the JSON files if we use -build_all
892 __block std::vector<std::string> jsonPaths;
893 if (!options.buildAllPath.empty()) {
894 struct stat stat_buf;
895 if (stat(options.buildAllPath.c_str(), &stat_buf) != 0) {
896 fprintf(stderr, "Could not find -build_all path '%s'\n", options.buildAllPath.c_str());
900 if ( (stat_buf.st_mode & S_IFMT) != S_IFDIR ) {
901 fprintf(stderr, "-build_all path is not a directory '%s'\n", options.buildAllPath.c_str());
905 auto processFile = ^(const std::string& path, const struct stat& statBuf) {
906 if ( !endsWith(path, ".json") )
909 jsonPaths.push_back(path);
912 iterateDirectoryTree("", options.buildAllPath,
913 ^(const std::string& dirPath) { return false; },
914 processFile, true /* process files */, false /* recurse */);
916 if (jsonPaths.empty()) {
917 fprintf(stderr, "Didn't find any .json files inside -build_all path: %s\n", options.buildAllPath.c_str());
921 if (options.listConfigs) {
922 for (const std::string& path : jsonPaths) {
923 fprintf(stderr, "Found config: %s\n", path.c_str());
929 if (!options.artifactDir.empty()) {
930 // Find the dylib cache dir from inside the artifact dir
931 struct stat stat_buf;
932 if (stat(options.artifactDir.c_str(), &stat_buf) != 0) {
933 fprintf(stderr, "Could not find artifact path '%s'\n", options.artifactDir.c_str());
936 std::string dir = options.artifactDir + "/AppleInternal/Developer/DylibCaches";
937 if (stat(dir.c_str(), &stat_buf) != 0) {
938 fprintf(stderr, "Could not find artifact path '%s'\n", dir.c_str());
942 if (!options.release.empty()) {
943 // Use the given release
944 options.dylibCacheDir = dir + "/" + options.release + ".dlc";
946 // Find a release directory
947 __block std::vector<std::string> subDirectories;
948 iterateDirectoryTree("", dir, ^(const std::string& dirPath) {
949 subDirectories.push_back(dirPath);
951 }, nullptr, false, false);
953 if (subDirectories.empty()) {
954 fprintf(stderr, "Could not find dlc subdirectories inside '%s'\n", dir.c_str());
958 if (subDirectories.size() > 1) {
959 fprintf(stderr, "Found too many subdirectories inside artifact path '%s'. Use -release to select one\n", dir.c_str());
963 options.dylibCacheDir = subDirectories.front();
967 if (options.dylibCacheDir.empty()) {
968 options.dylibCacheDir = std::string("/AppleInternal/Developer/DylibCaches/") + options.release + ".dlc";
971 //Move into the dir so we can use relative path manifests
972 chdir(options.dylibCacheDir.c_str());
974 dispatch_async(dispatch_get_main_queue(), ^{
975 if (!options.buildAllPath.empty()) {
976 bool requiresConcurrencyLimit = false;
977 dispatch_semaphore_t concurrencyLimit = NULL;
978 // Try build 1 cache per 8GB of RAM
979 uint64_t memSize = 0;
980 size_t sz = sizeof(memSize);
981 if ( sysctlbyname("hw.memsize", &memSize, &sz, NULL, 0) == 0 ) {
982 uint64_t maxThreads = std::max(memSize / 0x200000000ULL, 1ULL);
983 fprintf(stderr, "Detected %lldGb or less of memory, limiting concurrency to %lld threads\n",
984 memSize / (1 << 30), maxThreads);
985 requiresConcurrencyLimit = true;
986 concurrencyLimit = dispatch_semaphore_create(maxThreads);
989 dispatch_apply(jsonPaths.size(), DISPATCH_APPLY_AUTO, ^(size_t index) {
990 // Horrible hack to limit concurrency in low spec build machines.
991 if (requiresConcurrencyLimit) { dispatch_semaphore_wait(concurrencyLimit, DISPATCH_TIME_FOREVER); }
993 const std::string& jsonPath = jsonPaths[index];
994 buildCacheFromJSONManifest(diags, options, jsonPath);
996 if (requiresConcurrencyLimit) { dispatch_semaphore_signal(concurrencyLimit); }
999 buildCacheFromJSONManifest(diags, options, jsonManifestPath);
1002 const char* args[8];
1003 args[0] = (char*)"/bin/rm";
1004 args[1] = (char*)"-rf";
1005 args[2] = (char*)tempRootsDir;
1007 (void)runCommandAndWait(diags, args);
1009 if (diags.hasError()) {
1010 fprintf(stderr, "dyld_shared_cache_builder: error: %s", diags.errorMessage().c_str());
1014 for (const std::string& warn : diags.warnings()) {
1015 fprintf(stderr, "dyld_shared_cache_builder: warning: %s\n", warn.c_str());
1018 // Finally, write the roots.txt to tell us which roots we pulled in
1019 if (!options.dstRoot.empty())
1020 writeRootList(options.dstRoot + "/System/Library/Caches/com.apple.dyld", options.roots);