]> git.saurik.com Git - apple/dyld.git/blob - dyld3/shared-cache/update_dyld_sim_shared_cache.cpp
dyld-551.4.tar.gz
[apple/dyld.git] / dyld3 / shared-cache / update_dyld_sim_shared_cache.cpp
1 /* -*- mode: C++; c-basic-offset: 4; indent-tabs-mode: nil -*-
2 *
3 * Copyright (c) 2016 Apple Inc. All rights reserved.
4 *
5 * @APPLE_LICENSE_HEADER_START@
6 *
7 * This file contains Original Code and/or Modifications of Original Code
8 * as defined in and that are subject to the Apple Public Source License
9 * Version 2.0 (the 'License'). You may not use this file except in
10 * compliance with the License. Please obtain a copy of the License at
11 * http://www.opensource.apple.com/apsl/ and read it before using this
12 * file.
13 *
14 * The Original Code and all software distributed under the License are
15 * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
16 * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
17 * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
18 * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
19 * Please see the License for the specific language governing rights and
20 * limitations under the License.
21 *
22 * @APPLE_LICENSE_HEADER_END@
23 */
24
25 #include <sys/types.h>
26 #include <sys/stat.h>
27 #include <sys/mman.h>
28 #include <mach/mach.h>
29 #include <mach/mach_time.h>
30 #include <limits.h>
31 #include <stdarg.h>
32 #include <stdio.h>
33 #include <stdlib.h>
34 #include <math.h>
35 #include <fcntl.h>
36 #include <dlfcn.h>
37 #include <signal.h>
38 #include <errno.h>
39 #include <assert.h>
40 #include <sys/uio.h>
41 #include <unistd.h>
42 #include <sys/param.h>
43 #include <sys/sysctl.h>
44 #include <sys/resource.h>
45 #include <dirent.h>
46 #include <rootless.h>
47 #include <dscsym.h>
48 #include <dispatch/dispatch.h>
49 #include <pthread/pthread.h>
50
51 #include <algorithm>
52 #include <vector>
53 #include <unordered_set>
54 #include <unordered_set>
55 #include <iostream>
56 #include <fstream>
57
58 #include "MachOParser.h"
59 #include "FileUtils.h"
60 #include "StringUtils.h"
61 #include "DyldSharedCache.h"
62
63
64
65 struct MappedMachOsByCategory
66 {
67 std::string archName;
68 std::vector<DyldSharedCache::MappedMachO> dylibsForCache;
69 std::vector<DyldSharedCache::MappedMachO> otherDylibsAndBundles;
70 std::vector<DyldSharedCache::MappedMachO> mainExecutables;
71 };
72
73 static const char* sSearchDirs[] = {
74 "/bin",
75 "/sbin",
76 "/usr",
77 "/System",
78 };
79
80 static const char* sSkipDirs[] = {
81 "/usr/share",
82 "/usr/local/include",
83 };
84
85
86 static const char* sMacOsAdditions[] = {
87 "/usr/lib/system/libsystem_kernel.dylib",
88 "/usr/lib/system/libsystem_platform.dylib",
89 "/usr/lib/system/libsystem_pthread.dylib",
90 };
91
92
93 static bool verbose = false;
94
95 static bool addIfMachO(const std::string& simRuntimeRootPath, const std::string& runtimePath, const struct stat& statBuf, dyld3::Platform platform, std::vector<MappedMachOsByCategory>& files)
96 {
97 // read start of file to determine if it is mach-o or a fat file
98 std::string fullPath = simRuntimeRootPath + runtimePath;
99 int fd = ::open(fullPath.c_str(), O_RDONLY);
100 if ( fd < 0 )
101 return false;
102 bool result = false;
103 const void* wholeFile = ::mmap(NULL, statBuf.st_size, PROT_READ, MAP_PRIVATE, fd, 0);
104 if ( wholeFile != MAP_FAILED ) {
105 Diagnostics diag;
106 bool usedWholeFile = false;
107 for (MappedMachOsByCategory& file : files) {
108 size_t sliceOffset;
109 size_t sliceLength;
110 bool fatButMissingSlice;
111 const void* slice = MAP_FAILED;
112 if ( dyld3::FatUtil::isFatFileWithSlice(diag, wholeFile, statBuf.st_size, file.archName, sliceOffset, sliceLength, fatButMissingSlice) ) {
113 slice = ::mmap(NULL, sliceLength, PROT_READ, MAP_PRIVATE, fd, sliceOffset);
114 if ( slice != MAP_FAILED ) {
115 //fprintf(stderr, "mapped slice at %p size=0x%0lX, offset=0x%0lX for %s\n", p, len, offset, fullPath.c_str());
116 if ( !dyld3::MachOParser::isValidMachO(diag, file.archName, platform, slice, sliceLength, fullPath.c_str(), false) ) {
117 ::munmap((void*)slice, sliceLength);
118 slice = MAP_FAILED;
119 }
120 }
121 }
122 else if ( !fatButMissingSlice && dyld3::MachOParser::isValidMachO(diag, file.archName, platform, wholeFile, statBuf.st_size, fullPath.c_str(), false) ) {
123 slice = wholeFile;
124 sliceLength = statBuf.st_size;
125 sliceOffset = 0;
126 usedWholeFile = true;
127 //fprintf(stderr, "mapped whole file at %p size=0x%0lX for %s\n", p, len, inputPath.c_str());
128 }
129 if ( slice != MAP_FAILED ) {
130 const mach_header* mh = (mach_header*)slice;
131 dyld3::MachOParser parser(mh);
132 if ( parser.platform() != platform ) {
133 fprintf(stderr, "skipped wrong platform binary: %s\n", fullPath.c_str());
134 result = false;
135 }
136 else {
137 bool sip = true; // assume anything found in the simulator runtime is a platform binary
138 if ( parser.isDynamicExecutable() ) {
139 bool issetuid = (statBuf.st_mode & (S_ISUID|S_ISGID));
140 file.mainExecutables.emplace_back(runtimePath, mh, sliceLength, issetuid, sip, sliceOffset, statBuf.st_mtime, statBuf.st_ino);
141 }
142 else {
143 if ( parser.canBePlacedInDyldCache(runtimePath) ) {
144 file.dylibsForCache.emplace_back(runtimePath, mh, sliceLength, false, sip, sliceOffset, statBuf.st_mtime, statBuf.st_ino);
145 }
146 else {
147 file.otherDylibsAndBundles.emplace_back(runtimePath, mh, sliceLength, false, sip, sliceOffset, statBuf.st_mtime, statBuf.st_ino);
148 }
149 }
150 result = true;
151 }
152 }
153 }
154 if ( !usedWholeFile )
155 ::munmap((void*)wholeFile, statBuf.st_size);
156 }
157 ::close(fd);
158 return result;
159 }
160
161 static void findAllFiles(const std::string& simRuntimeRootPath, dyld3::Platform platform, std::vector<MappedMachOsByCategory>& files)
162 {
163 std::unordered_set<std::string> skipDirs;
164 for (const char* s : sSkipDirs)
165 skipDirs.insert(s);
166
167 for (const char* searchDir : sSearchDirs ) {
168 iterateDirectoryTree(simRuntimeRootPath, searchDir, ^(const std::string& dirPath) { return (skipDirs.count(dirPath) != 0); }, ^(const std::string& path, const struct stat& statBuf) {
169 // ignore files that don't have 'x' bit set (all runnable mach-o files do)
170 const bool hasXBit = ((statBuf.st_mode & S_IXOTH) == S_IXOTH);
171 if ( !hasXBit && !endsWith(path, ".dylib") )
172 return;
173
174 // ignore files too small
175 if ( statBuf.st_size < 0x3000 )
176 return;
177
178 // if the file is mach-o add to list
179 addIfMachO(simRuntimeRootPath, path, statBuf, platform, files);
180 });
181 }
182 }
183
184 static void addMacOSAdditions(std::vector<MappedMachOsByCategory>& allFileSets)
185 {
186 for (const char* addPath : sMacOsAdditions) {
187 struct stat statBuf;
188 if ( stat(addPath, &statBuf) == 0 )
189 addIfMachO("", addPath, statBuf, dyld3::Platform::macOS, allFileSets);
190 }
191 }
192
193
194 static bool dontCache(const std::string& simRuntimeRootPath, const std::string& archName,
195 const std::unordered_set<std::string>& pathsWithDuplicateInstallName,
196 const DyldSharedCache::MappedMachO& aFile, bool warn)
197 {
198 if ( startsWith(aFile.runtimePath, "/usr/lib/system/introspection/") )
199 return true;
200 if ( startsWith(aFile.runtimePath, "/usr/local/") )
201 return true;
202
203 // anything inside a .app bundle is specific to app, so should be in shared cache
204 if ( aFile.runtimePath.find(".app/") != std::string::npos )
205 return true;
206
207 if ( aFile.runtimePath.find("//") != std::string::npos ) {
208 if (warn) fprintf(stderr, "update_dyld_sim_shared_cache: warning: %s double-slash in install name %s\n", archName.c_str(), aFile.runtimePath.c_str());
209 }
210
211 dyld3::MachOParser parser(aFile.mh);
212 const char* installName = parser.installName();
213 if ( (pathsWithDuplicateInstallName.count(aFile.runtimePath) != 0) && (aFile.runtimePath != installName) ) {
214 if (warn) fprintf(stderr, "update_dyld_sim_shared_cache: warning: %s skipping because of duplicate install name %s\n", archName.c_str(), aFile.runtimePath.c_str());
215 return true;
216 }
217
218 if ( aFile.runtimePath != installName ) {
219 // see if install name is a symlink to actual path
220 std::string fullInstall = simRuntimeRootPath + installName;
221 char resolvedPath[PATH_MAX];
222 if ( realpath(fullInstall.c_str(), resolvedPath) != NULL ) {
223 std::string resolvedSymlink = resolvedPath;
224 if ( !simRuntimeRootPath.empty() ) {
225 resolvedSymlink = resolvedSymlink.substr(simRuntimeRootPath.size());
226 }
227 if ( aFile.runtimePath == resolvedSymlink ) {
228 return false;
229 }
230 }
231 if (warn) fprintf(stderr, "update_dyld_sim_shared_cache: warning: %s skipping because of bad install name %s\n", archName.c_str(), aFile.runtimePath.c_str());
232 return true;
233 }
234
235 return false;
236 }
237
238 static void pruneCachedDylibs(const std::string& simRuntimeRootPath, MappedMachOsByCategory& fileSet)
239 {
240 std::unordered_set<std::string> pathsWithDuplicateInstallName;
241
242 std::unordered_map<std::string, std::string> installNameToFirstPath;
243 for (DyldSharedCache::MappedMachO& aFile : fileSet.dylibsForCache) {
244 //fprintf(stderr, "dylib: %s\n", aFile.runtimePath.c_str());
245 dyld3::MachOParser parser(aFile.mh);
246 const char* installName = parser.installName();
247 auto pos = installNameToFirstPath.find(installName);
248 if ( pos == installNameToFirstPath.end() ) {
249 installNameToFirstPath[installName] = aFile.runtimePath;
250 }
251 else {
252 pathsWithDuplicateInstallName.insert(aFile.runtimePath);
253 pathsWithDuplicateInstallName.insert(installNameToFirstPath[installName]);
254 }
255 }
256
257 for (DyldSharedCache::MappedMachO& aFile : fileSet.dylibsForCache) {
258 if ( dontCache(simRuntimeRootPath, fileSet.archName, pathsWithDuplicateInstallName, aFile, true) )
259 fileSet.otherDylibsAndBundles.push_back(aFile);
260 }
261 fileSet.dylibsForCache.erase(std::remove_if(fileSet.dylibsForCache.begin(), fileSet.dylibsForCache.end(),
262 [&](const DyldSharedCache::MappedMachO& aFile) { return dontCache(simRuntimeRootPath, fileSet.archName, pathsWithDuplicateInstallName, aFile, false); }),
263 fileSet.dylibsForCache.end());
264 }
265
266 static bool existingCacheUpToDate(const std::string& existingCache, const std::vector<DyldSharedCache::MappedMachO>& currentDylibs)
267 {
268 // if no existing cache, it is not up-to-date
269 int fd = ::open(existingCache.c_str(), O_RDONLY);
270 if ( fd < 0 )
271 return false;
272
273 // build map of found dylibs
274 std::unordered_map<std::string, const DyldSharedCache::MappedMachO*> currentDylibMap;
275 for (const DyldSharedCache::MappedMachO& aFile : currentDylibs) {
276 //fprintf(stderr, "0x%0llX 0x%0llX %s\n", aFile.inode, aFile.modTime, aFile.runtimePath.c_str());
277 currentDylibMap[aFile.runtimePath] = &aFile;
278 }
279
280 // make sure all dylibs in existing cache have same mtime and inode as found dylib
281 __block bool foundMismatch = false;
282 const uint64_t cacheMapLen = 0x40000000;
283 void *p = ::mmap(NULL, cacheMapLen, PROT_READ, MAP_PRIVATE, fd, 0);
284 if ( p != MAP_FAILED ) {
285 const DyldSharedCache* cache = (DyldSharedCache*)p;
286 cache->forEachImageEntry(^(const char* installName, uint64_t mTime, uint64_t inode) {
287 bool foundMatch = false;
288 auto pos = currentDylibMap.find(installName);
289 if ( pos != currentDylibMap.end() ) {
290 const DyldSharedCache::MappedMachO* foundDylib = pos->second;
291 if ( (foundDylib->inode == inode) && (foundDylib->modTime == mTime) ) {
292 foundMatch = true;
293 }
294 }
295 if ( !foundMatch ) {
296 // use slow path and look for any dylib with a matching inode and mtime
297 bool foundSlow = false;
298 for (const DyldSharedCache::MappedMachO& aFile : currentDylibs) {
299 if ( (aFile.inode == inode) && (aFile.modTime == mTime) ) {
300 foundSlow = true;
301 break;
302 }
303 }
304 if ( !foundSlow ) {
305 foundMismatch = true;
306 if ( verbose )
307 fprintf(stderr, "rebuilding dyld cache because dylib changed: %s\n", installName);
308 }
309 }
310 });
311 ::munmap(p, cacheMapLen);
312 }
313
314 ::close(fd);
315
316 return !foundMismatch;
317 }
318
319
320 inline uint32_t absolutetime_to_milliseconds(uint64_t abstime)
321 {
322 return (uint32_t)(abstime/1000/1000);
323 }
324
325
326 #define TERMINATE_IF_LAST_ARG( s ) \
327 do { \
328 if ( i == argc - 1 ) { \
329 fprintf(stderr, s ); \
330 return 1; \
331 } \
332 } while ( 0 )
333
334 int main(int argc, const char* argv[])
335 {
336 std::string rootPath;
337 std::string dylibListFile;
338 bool force = false;
339 std::string cacheDir;
340 std::unordered_set<std::string> archStrs;
341
342 dyld3::Platform platform = dyld3::Platform::iOS;
343
344 // parse command line options
345 for (int i = 1; i < argc; ++i) {
346 const char* arg = argv[i];
347 if (strcmp(arg, "-debug") == 0) {
348 verbose = true;
349 }
350 else if (strcmp(arg, "-verbose") == 0) {
351 verbose = true;
352 }
353 else if (strcmp(arg, "-tvOS") == 0) {
354 platform = dyld3::Platform::tvOS;
355 }
356 else if (strcmp(arg, "-iOS") == 0) {
357 platform = dyld3::Platform::iOS;
358 }
359 else if (strcmp(arg, "-watchOS") == 0) {
360 platform = dyld3::Platform::watchOS;
361 }
362 else if ( strcmp(arg, "-runtime_dir") == 0 ) {
363 TERMINATE_IF_LAST_ARG("-runtime_dir missing path argument\n");
364 rootPath = argv[++i];
365 }
366 else if (strcmp(arg, "-cache_dir") == 0) {
367 TERMINATE_IF_LAST_ARG("-cache_dir missing path argument\n");
368 cacheDir = argv[++i];
369 }
370 else if (strcmp(arg, "-arch") == 0) {
371 TERMINATE_IF_LAST_ARG("-arch missing argument\n");
372 archStrs.insert(argv[++i]);
373 }
374 else if (strcmp(arg, "-force") == 0) {
375 force = true;
376 }
377 else {
378 //usage();
379 fprintf(stderr, "update_dyld_sim_shared_cache: unknown option: %s\n", arg);
380 return 1;
381 }
382 }
383
384 if ( cacheDir.empty() ) {
385 fprintf(stderr, "missing -cache_dir <path> option to specify directory in which to write cache file(s)\n");
386 return 1;
387 }
388
389 if ( rootPath.empty() ) {
390 fprintf(stderr, "missing -runtime_dir <path> option to specify directory which is root of simulator runtime)\n");
391 return 1;
392 }
393 else {
394 // canonicalize rootPath
395 char resolvedPath[PATH_MAX];
396 if ( realpath(rootPath.c_str(), resolvedPath) != NULL ) {
397 rootPath = resolvedPath;
398 }
399 }
400
401 int err = mkpath_np(cacheDir.c_str(), S_IRWXU | S_IRGRP|S_IXGRP | S_IROTH|S_IXOTH);
402 if ( (err != 0) && (err != EEXIST) ) {
403 fprintf(stderr, "mkpath_np fail: %d", err);
404 return 1;
405 }
406
407 if ( archStrs.empty() ) {
408 switch ( platform ) {
409 case dyld3::Platform::iOS:
410 archStrs.insert("x86_64");
411 break;
412 case dyld3::Platform::tvOS:
413 archStrs.insert("x86_64");
414 break;
415 case dyld3::Platform::watchOS:
416 archStrs.insert("i386");
417 break;
418 case dyld3::Platform::unknown:
419 case dyld3::Platform::macOS:
420 assert(0 && "macOS does not have a simulator");
421 break;
422 case dyld3::Platform::bridgeOS:
423 assert(0 && "bridgeOS does not have a simulator");
424 break;
425 }
426 }
427
428 uint64_t t1 = mach_absolute_time();
429
430 // find all mach-o files for requested architectures
431 __block std::vector<MappedMachOsByCategory> allFileSets;
432 if ( archStrs.count("x86_64") )
433 allFileSets.push_back({"x86_64"});
434 if ( archStrs.count("i386") )
435 allFileSets.push_back({"i386"});
436 findAllFiles(rootPath, platform, allFileSets);
437 addMacOSAdditions(allFileSets);
438 for (MappedMachOsByCategory& fileSet : allFileSets) {
439 pruneCachedDylibs(rootPath, fileSet);
440 }
441
442 uint64_t t2 = mach_absolute_time();
443
444 fprintf(stderr, "time to scan file system and construct lists of mach-o files: %ums\n", absolutetime_to_milliseconds(t2-t1));
445
446 // build all caches in parallel
447 __block bool cacheBuildFailure = false;
448 dispatch_apply(allFileSets.size(), dispatch_get_global_queue(DISPATCH_QUEUE_PRIORITY_DEFAULT, 0), ^(size_t index) {
449 MappedMachOsByCategory& fileSet = allFileSets[index];
450 const std::string outFile = cacheDir + "/dyld_shared_cache_" + fileSet.archName;
451 __block std::unordered_set<std::string> knownMissingDylib;
452
453 DyldSharedCache::MappedMachO (^loader)(const std::string&) = ^DyldSharedCache::MappedMachO(const std::string& runtimePath) {
454 std::string fullPath = rootPath + runtimePath;
455 struct stat statBuf;
456 if ( stat(fullPath.c_str(), &statBuf) == 0 ) {
457 std::vector<MappedMachOsByCategory> mappedFiles;
458 mappedFiles.push_back({fileSet.archName});
459 if ( addIfMachO(rootPath, runtimePath, statBuf, platform, mappedFiles) ) {
460 if ( !mappedFiles.back().dylibsForCache.empty() )
461 return mappedFiles.back().dylibsForCache.back();
462 }
463 }
464 if ( knownMissingDylib.count(runtimePath) == 0 ) {
465 fprintf(stderr, "update_dyld_sim_shared_cache: warning: %s could not use in dylid cache: %s\n", fileSet.archName.c_str(), runtimePath.c_str());
466 knownMissingDylib.insert(runtimePath);
467 }
468 return DyldSharedCache::MappedMachO();
469 };
470 size_t startCount = fileSet.dylibsForCache.size();
471 std::vector<std::pair<DyldSharedCache::MappedMachO, std::set<std::string>>> excludes;
472 DyldSharedCache::verifySelfContained(fileSet.dylibsForCache, loader, excludes);
473 for (size_t i=startCount; i < fileSet.dylibsForCache.size(); ++i) {
474 fprintf(stderr, "update_dyld_sim_shared_cache: warning: %s not found in initial scan, but adding required dylib %s\n", fileSet.archName.c_str(), fileSet.dylibsForCache[i].runtimePath.c_str());
475 }
476 for (auto& exclude : excludes) {
477 std::string reasons = "(\"";
478 for (auto i = exclude.second.begin(); i != exclude.second.end(); ++i) {
479 reasons += *i;
480 if (i != --exclude.second.end()) {
481 reasons += "\", \"";
482 }
483 }
484 reasons += "\")";
485 fprintf(stderr, "update_dyld_shared_cache: warning: %s rejected from cached dylibs: %s (%s)\n", fileSet.archName.c_str(), exclude.first.runtimePath.c_str(), reasons.c_str());
486 fileSet.otherDylibsAndBundles.push_back(exclude.first);
487 }
488
489 // check if cache is already up to date
490 if ( !force ) {
491 if ( existingCacheUpToDate(outFile, fileSet.dylibsForCache) )
492 return;
493 }
494 fprintf(stderr, "make %s cache with %lu dylibs, %lu other dylibs, %lu programs\n", fileSet.archName.c_str(), fileSet.dylibsForCache.size(), fileSet.otherDylibsAndBundles.size(), fileSet.mainExecutables.size());
495
496 // build cache new cache file
497 DyldSharedCache::CreateOptions options;
498 options.archName = fileSet.archName;
499 options.platform = platform;
500 options.excludeLocalSymbols = false;
501 options.optimizeStubs = false;
502 options.optimizeObjC = true;
503 options.codeSigningDigestMode = DyldSharedCache::SHA256only;
504 options.dylibsRemovedDuringMastering = false;
505 options.inodesAreSameAsRuntime = true;
506 options.cacheSupportsASLR = false;
507 options.forSimulator = true;
508 options.verbose = verbose;
509 options.evictLeafDylibsOnOverflow = true;
510 options.pathPrefixes = { rootPath };
511 DyldSharedCache::CreateResults results = DyldSharedCache::create(options, fileSet.dylibsForCache, fileSet.otherDylibsAndBundles, fileSet.mainExecutables);
512
513 // print any warnings
514 for (const std::string& warn : results.warnings) {
515 fprintf(stderr, "update_dyld_sim_shared_cache: warning: %s %s\n", fileSet.archName.c_str(), warn.c_str());
516 }
517 if ( !results.errorMessage.empty() ) {
518 // print error (if one)
519 fprintf(stderr, "update_dyld_sim_shared_cache: %s\n", results.errorMessage.c_str());
520 cacheBuildFailure = true;
521 }
522 else {
523 // save new cache file to disk and write new .map file
524 assert(results.cacheContent != nullptr);
525 if ( !safeSave(results.cacheContent, results.cacheLength, outFile) )
526 cacheBuildFailure = true;
527 if ( !cacheBuildFailure ) {
528 std::string mapStr = results.cacheContent->mapFile();
529 std::string outFileMap = cacheDir + "/dyld_shared_cache_" + fileSet.archName + ".map";
530 safeSave(mapStr.c_str(), mapStr.size(), outFileMap);
531 }
532 // free created cache buffer
533 vm_deallocate(mach_task_self(), (vm_address_t)results.cacheContent, results.cacheLength);
534 }
535 });
536
537 // we could unmap all input files, but tool is about to quit
538
539 return (cacheBuildFailure ? 1 : 0);
540 }
541