dyld-551.3.tar.gz
[apple/dyld.git] / dyld3 / shared-cache / update_dyld_shared_cache.cpp
1 /* -*- mode: C++; c-basic-offset: 4; indent-tabs-mode: nil -*-
2 *
3 * Copyright (c) 2014 Apple Inc. All rights reserved.
4 *
5 * @APPLE_LICENSE_HEADER_START@
6 *
7 * This file contains Original Code and/or Modifications of Original Code
8 * as defined in and that are subject to the Apple Public Source License
9 * Version 2.0 (the 'License'). You may not use this file except in
10 * compliance with the License. Please obtain a copy of the License at
11 * http://www.opensource.apple.com/apsl/ and read it before using this
12 * file.
13 *
14 * The Original Code and all software distributed under the License are
15 * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
16 * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
17 * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
18 * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
19 * Please see the License for the specific language governing rights and
20 * limitations under the License.
21 *
22 * @APPLE_LICENSE_HEADER_END@
23 */
24
25 #include <sys/types.h>
26 #include <sys/stat.h>
27 #include <sys/mman.h>
28 #include <sys/xattr.h>
29 #include <mach/mach.h>
30 #include <mach/mach_time.h>
31 #include <limits.h>
32 #include <stdarg.h>
33 #include <stdio.h>
34 #include <stdlib.h>
35 #include <math.h>
36 #include <fcntl.h>
37 #include <dlfcn.h>
38 #include <signal.h>
39 #include <errno.h>
40 #include <assert.h>
41 #include <sys/uio.h>
42 #include <unistd.h>
43 #include <sys/param.h>
44 #include <sys/sysctl.h>
45 #include <sys/resource.h>
46 #include <dirent.h>
47 #include <rootless.h>
48 #include <dscsym.h>
49 #include <dispatch/dispatch.h>
50 #include <pthread/pthread.h>
51 #include <Bom/Bom.h>
52 #include <CoreFoundation/CoreFoundation.h>
53
54 #include <algorithm>
55 #include <vector>
56 #include <unordered_set>
57 #include <unordered_set>
58 #include <iostream>
59 #include <fstream>
60
61 #include "MachOParser.h"
62 #include "FileUtils.h"
63 #include "StringUtils.h"
64 #include "DyldSharedCache.h"
65
66 struct MappedMachOsByCategory
67 {
68 std::string archName;
69 std::vector<DyldSharedCache::MappedMachO> dylibsForCache;
70 std::vector<DyldSharedCache::MappedMachO> otherDylibsAndBundles;
71 std::vector<DyldSharedCache::MappedMachO> mainExecutables;
72 };
73
74 static const char* sAllowedPrefixes[] = {
75 "/bin/",
76 "/sbin/",
77 "/usr/",
78 "/System",
79 "/Applications/App Store.app/",
80 "/Applications/Automator.app/",
81 "/Applications/Calculator.app/",
82 "/Applications/Calendar.app/",
83 "/Applications/Chess.app/",
84 "/Applications/Contacts.app/",
85 // "/Applications/DVD Player.app/",
86 "/Applications/Dashboard.app/",
87 "/Applications/Dictionary.app/",
88 "/Applications/FaceTime.app/",
89 "/Applications/Font Book.app/",
90 "/Applications/Image Capture.app/",
91 "/Applications/Launchpad.app/",
92 "/Applications/Mail.app/",
93 "/Applications/Maps.app/",
94 "/Applications/Messages.app/",
95 "/Applications/Mission Control.app/",
96 "/Applications/Notes.app/",
97 "/Applications/Photo Booth.app/",
98 // "/Applications/Photos.app/",
99 "/Applications/Preview.app/",
100 "/Applications/QuickTime Player.app/",
101 "/Applications/Reminders.app/",
102 "/Applications/Safari.app/",
103 "/Applications/Siri.app/",
104 "/Applications/Stickies.app/",
105 "/Applications/System Preferences.app/",
106 "/Applications/TextEdit.app/",
107 "/Applications/Time Machine.app/",
108 "/Applications/iBooks.app/",
109 "/Applications/iTunes.app/",
110 "/Applications/Utilities/Activity Monitor.app",
111 "/Applications/Utilities/AirPort Utility.app",
112 "/Applications/Utilities/Audio MIDI Setup.app",
113 "/Applications/Utilities/Bluetooth File Exchange.app",
114 "/Applications/Utilities/Boot Camp Assistant.app",
115 "/Applications/Utilities/ColorSync Utility.app",
116 "/Applications/Utilities/Console.app",
117 "/Applications/Utilities/Digital Color Meter.app",
118 "/Applications/Utilities/Disk Utility.app",
119 "/Applications/Utilities/Grab.app",
120 "/Applications/Utilities/Grapher.app",
121 "/Applications/Utilities/Keychain Access.app",
122 "/Applications/Utilities/Migration Assistant.app",
123 "/Applications/Utilities/Script Editor.app",
124 "/Applications/Utilities/System Information.app",
125 "/Applications/Utilities/Terminal.app",
126 "/Applications/Utilities/VoiceOver Utility.app",
127 "/Library/CoreMediaIO/Plug-Ins/DAL/" // temp until plugins moved or closured working
128 };
129
130 static const char* sDontUsePrefixes[] = {
131 "/usr/share",
132 "/usr/local/",
133 "/System/Library/Assets",
134 "/System/Library/StagedFrameworks",
135 "/System/Library/Kernels/",
136 "/bin/zsh", // until <rdar://31026756> is fixed
137 "/System/Library/Frameworks/CoreServices.framework/Versions/A/Frameworks/Metadata.framework/Versions/A/Support/mdworker", // these load third party plugins
138 "/usr/bin/mdimport", // these load third party plugins
139 };
140
141
142 static bool verbose = false;
143
144
145
146 static bool addIfMachO(const std::string& pathPrefix, const std::string& runtimePath, const struct stat& statBuf, bool requireSIP, std::vector<MappedMachOsByCategory>& files)
147 {
148 // don't precompute closure info for any debug or profile dylibs
149 if ( endsWith(runtimePath, "_profile.dylib") || endsWith(runtimePath, "_debug.dylib") || endsWith(runtimePath, "_profile") || endsWith(runtimePath, "_debug") )
150 return false;
151
152 // read start of file to determine if it is mach-o or a fat file
153 std::string fullPath = pathPrefix + runtimePath;
154 int fd = ::open(fullPath.c_str(), O_RDONLY);
155 if ( fd < 0 )
156 return false;
157 bool result = false;
158 const void* wholeFile = ::mmap(NULL, statBuf.st_size, PROT_READ, MAP_PRIVATE, fd, 0);
159 if ( wholeFile != MAP_FAILED ) {
160 Diagnostics diag;
161 bool usedWholeFile = false;
162 for (MappedMachOsByCategory& file : files) {
163 size_t sliceOffset;
164 size_t sliceLength;
165 bool fatButMissingSlice;
166 const void* slice = MAP_FAILED;
167 if ( dyld3::FatUtil::isFatFileWithSlice(diag, wholeFile, statBuf.st_size, file.archName, sliceOffset, sliceLength, fatButMissingSlice) ) {
168 slice = ::mmap(NULL, sliceLength, PROT_READ, MAP_PRIVATE | MAP_RESILIENT_CODESIGN, fd, sliceOffset);
169 if ( slice != MAP_FAILED ) {
170 //fprintf(stderr, "mapped slice at %p size=0x%0lX, offset=0x%0lX for %s\n", p, len, offset, fullPath.c_str());
171 if ( !dyld3::MachOParser::isValidMachO(diag, file.archName, dyld3::Platform::macOS, slice, sliceLength, fullPath.c_str(), false) ) {
172 ::munmap((void*)slice, sliceLength);
173 slice = MAP_FAILED;
174 }
175 }
176 }
177 else if ( !fatButMissingSlice && dyld3::MachOParser::isValidMachO(diag, file.archName, dyld3::Platform::macOS, wholeFile, statBuf.st_size, fullPath.c_str(), false) ) {
178 slice = wholeFile;
179 sliceLength = statBuf.st_size;
180 sliceOffset = 0;
181 usedWholeFile = true;
182 //fprintf(stderr, "mapped whole file at %p size=0x%0lX for %s\n", p, len, inputPath.c_str());
183 }
184 std::vector<std::string> nonArchWarnings;
185 for (const std::string& warning : diag.warnings()) {
186 if ( !contains(warning, "required architecture") && !contains(warning, "not a dylib") )
187 nonArchWarnings.push_back(warning);
188 }
189 diag.clearWarnings();
190 if ( !nonArchWarnings.empty() ) {
191 fprintf(stderr, "update_dyld_shared_cache: warning: %s for %s: ", file.archName.c_str(), runtimePath.c_str());
192 for (const std::string& warning : nonArchWarnings) {
193 fprintf(stderr, "%s ", warning.c_str());
194 }
195 fprintf(stderr, "\n");
196 }
197 if ( slice != MAP_FAILED ) {
198 const mach_header* mh = (mach_header*)slice;
199 dyld3::MachOParser parser((mach_header*)slice);
200 bool sipProtected = isProtectedBySIP(fd);
201 bool issetuid = false;
202 if ( parser.isDynamicExecutable() ) {
203 // When SIP enabled, only build closures for SIP protected programs
204 if ( !requireSIP || sipProtected ) {
205 //fprintf(stderr, "requireSIP=%d, sipProtected=%d, path=%s\n", requireSIP, sipProtected, fullPath.c_str());
206 issetuid = (statBuf.st_mode & (S_ISUID|S_ISGID));
207 file.mainExecutables.emplace_back(runtimePath, mh, sliceLength, issetuid, sipProtected, sliceOffset, statBuf.st_mtime, statBuf.st_ino);
208 }
209 }
210 else if ( parser.canBePlacedInDyldCache(runtimePath) ) {
211 // when SIP is enabled, only dylib protected by SIP can go in cache
212 if ( !requireSIP || sipProtected )
213 file.dylibsForCache.emplace_back(runtimePath, mh, sliceLength, issetuid, sipProtected, sliceOffset, statBuf.st_mtime, statBuf.st_ino);
214 else
215 file.otherDylibsAndBundles.emplace_back(runtimePath, mh, sliceLength, issetuid, sipProtected, sliceOffset, statBuf.st_mtime, statBuf.st_ino);
216 }
217 else {
218 if ( parser.fileType() == MH_DYLIB ) {
219 std::string installName = parser.installName();
220 if ( startsWith(installName, "@") && !contains(runtimePath, ".app/") ) {
221 if ( startsWith(runtimePath, "/usr/lib/") || startsWith(runtimePath, "/System/Library/") )
222 fprintf(stderr, "update_dyld_shared_cache: warning @rpath install name for system framework: %s\n", runtimePath.c_str());
223 }
224 }
225 file.otherDylibsAndBundles.emplace_back(runtimePath, mh, sliceLength, issetuid, sipProtected, sliceOffset, statBuf.st_mtime, statBuf.st_ino);
226 }
227 result = true;
228 }
229 }
230 if ( !usedWholeFile )
231 ::munmap((void*)wholeFile, statBuf.st_size);
232 }
233 ::close(fd);
234 return result;
235 }
236
237 static void findAllFiles(const std::vector<std::string>& pathPrefixes, bool requireSIP, std::vector<MappedMachOsByCategory>& files)
238 {
239 std::unordered_set<std::string> skipDirs;
240 for (const char* s : sDontUsePrefixes)
241 skipDirs.insert(s);
242
243 __block std::unordered_set<std::string> alreadyUsed;
244 bool multiplePrefixes = (pathPrefixes.size() > 1);
245 for (const std::string& prefix : pathPrefixes) {
246 // get all files from overlay for this search dir
247 for (const char* searchDir : sAllowedPrefixes ) {
248 iterateDirectoryTree(prefix, searchDir, ^(const std::string& dirPath) { return (skipDirs.count(dirPath) != 0); }, ^(const std::string& path, const struct stat& statBuf) {
249 // ignore files that don't have 'x' bit set (all runnable mach-o files do)
250 const bool hasXBit = ((statBuf.st_mode & S_IXOTH) == S_IXOTH);
251 if ( !hasXBit && !endsWith(path, ".dylib") )
252 return;
253
254 // ignore files too small
255 if ( statBuf.st_size < 0x3000 )
256 return;
257
258 // don't add paths already found using previous prefix
259 if ( multiplePrefixes && (alreadyUsed.count(path) != 0) )
260 return;
261
262 // if the file is mach-o, add to list
263 if ( addIfMachO(prefix, path, statBuf, requireSIP, files) ) {
264 if ( multiplePrefixes )
265 alreadyUsed.insert(path);
266 }
267 });
268 }
269 }
270 }
271
272
273 static void findOSFilesViaBOMS(const std::vector<std::string>& pathPrefixes, bool requireSIP, std::vector<MappedMachOsByCategory>& files)
274 {
275 __block std::unordered_set<std::string> runtimePathsFound;
276 for (const std::string& prefix : pathPrefixes) {
277 iterateDirectoryTree(prefix, "/System/Library/Receipts", ^(const std::string&) { return false; }, ^(const std::string& path, const struct stat& statBuf) {
278 if ( !contains(path, "com.apple.pkg.") )
279 return;
280 if ( !endsWith(path, ".bom") )
281 return;
282 std::string fullPath = prefix + path;
283 BOMBom bom = BOMBomOpenWithSys(fullPath.c_str(), false, NULL);
284 if ( bom == nullptr )
285 return;
286 BOMFSObject rootFso = BOMBomGetRootFSObject(bom);
287 if ( rootFso == nullptr ) {
288 BOMBomFree(bom);
289 return;
290 }
291 BOMBomEnumerator e = BOMBomEnumeratorNew(bom, rootFso);
292 if ( e == nullptr ) {
293 fprintf(stderr, "Can't get enumerator for BOM root FSObject\n");
294 return;
295 }
296 BOMFSObjectFree(rootFso);
297 //fprintf(stderr, "using BOM %s\n", path.c_str());
298 while (BOMFSObject fso = BOMBomEnumeratorNext(e)) {
299 if ( BOMFSObjectIsBinaryObject(fso) ) {
300 const char* runPath = BOMFSObjectPathName(fso);
301 if ( (runPath[0] == '.') && (runPath[1] == '/') )
302 ++runPath;
303 if ( runtimePathsFound.count(runPath) == 0 ) {
304 // only add files from sAllowedPrefixes and not in sDontUsePrefixes
305 bool inSearchDir = false;
306 for (const char* searchDir : sAllowedPrefixes ) {
307 if ( strncmp(searchDir, runPath, strlen(searchDir)) == 0 ) {
308 inSearchDir = true;
309 break;
310 }
311 }
312 if ( inSearchDir ) {
313 bool inSkipDir = false;
314 for (const char* skipDir : sDontUsePrefixes) {
315 if ( strncmp(skipDir, runPath, strlen(skipDir)) == 0 ) {
316 inSkipDir = true;
317 break;
318 }
319 }
320 if ( !inSkipDir ) {
321 for (const std::string& prefix2 : pathPrefixes) {
322 struct stat statBuf2;
323 std::string fullPath2 = prefix2 + runPath;
324 if ( stat(fullPath2.c_str(), &statBuf2) == 0 ) {
325 addIfMachO(prefix2, runPath, statBuf2, requireSIP, files);
326 runtimePathsFound.insert(runPath);
327 break;
328 }
329 }
330 }
331 }
332 }
333 }
334 BOMFSObjectFree(fso);
335 }
336
337 BOMBomEnumeratorFree(e);
338 BOMBomFree(bom);
339 });
340 }
341 }
342
343
344 static bool dontCache(const std::string& volumePrefix, const std::string& archName,
345 const std::unordered_set<std::string>& pathsWithDuplicateInstallName,
346 const DyldSharedCache::MappedMachO& aFile, bool warn,
347 const std::unordered_set<std::string>& skipDylibs)
348 {
349 if ( skipDylibs.count(aFile.runtimePath) )
350 return true;
351 if ( startsWith(aFile.runtimePath, "/usr/lib/system/introspection/") )
352 return true;
353 if ( startsWith(aFile.runtimePath, "/System/Library/QuickTime/") )
354 return true;
355 if ( startsWith(aFile.runtimePath, "/System/Library/Tcl/") )
356 return true;
357 if ( startsWith(aFile.runtimePath, "/System/Library/Perl/") )
358 return true;
359 if ( startsWith(aFile.runtimePath, "/System/Library/MonitorPanels/") )
360 return true;
361 if ( startsWith(aFile.runtimePath, "/System/Library/Accessibility/") )
362 return true;
363 if ( startsWith(aFile.runtimePath, "/usr/local/") )
364 return true;
365
366 // anything inside a .app bundle is specific to app, so should not be in shared cache
367 if ( aFile.runtimePath.find(".app/") != std::string::npos )
368 return true;
369
370 if ( archName == "i386" ) {
371 if ( startsWith(aFile.runtimePath, "/System/Library/CoreServices/") )
372 return true;
373 if ( startsWith(aFile.runtimePath, "/System/Library/Extensions/") )
374 return true;
375 }
376
377 if ( aFile.runtimePath.find("//") != std::string::npos ) {
378 if (warn) fprintf(stderr, "update_dyld_shared_cache: warning: %s skipping because of bad install name %s\n", archName.c_str(), aFile.runtimePath.c_str());
379 return true;
380 }
381
382 dyld3::MachOParser parser(aFile.mh);
383 const char* installName = parser.installName();
384 if ( (pathsWithDuplicateInstallName.count(aFile.runtimePath) != 0) && (aFile.runtimePath != installName) ) {
385 if (warn) fprintf(stderr, "update_dyld_shared_cache: warning: %s skipping because of duplicate install name %s\n", archName.c_str(), aFile.runtimePath.c_str());
386 return true;
387 }
388
389 if ( aFile.runtimePath != installName ) {
390 // see if install name is a symlink to actual path
391 std::string fullInstall = volumePrefix + installName;
392 char resolvedPath[PATH_MAX];
393 if ( realpath(fullInstall.c_str(), resolvedPath) != NULL ) {
394 std::string resolvedSymlink = resolvedPath;
395 if ( !volumePrefix.empty() ) {
396 resolvedSymlink = resolvedSymlink.substr(volumePrefix.size());
397 }
398 if ( aFile.runtimePath == resolvedSymlink ) {
399 return false;
400 }
401 }
402 if (warn) fprintf(stderr, "update_dyld_shared_cache: warning: %s skipping because of bad install name %s\n", archName.c_str(), aFile.runtimePath.c_str());
403 return true;
404 }
405 return false;
406 }
407
408 static void pruneCachedDylibs(const std::string& volumePrefix, const std::unordered_set<std::string>& skipDylibs, MappedMachOsByCategory& fileSet)
409 {
410 std::unordered_set<std::string> pathsWithDuplicateInstallName;
411
412 std::unordered_map<std::string, std::string> installNameToFirstPath;
413 for (DyldSharedCache::MappedMachO& aFile : fileSet.dylibsForCache) {
414 dyld3::MachOParser parser(aFile.mh);
415 const char* installName = parser.installName();
416 auto pos = installNameToFirstPath.find(installName);
417 if ( pos == installNameToFirstPath.end() ) {
418 installNameToFirstPath[installName] = aFile.runtimePath;
419 }
420 else {
421 pathsWithDuplicateInstallName.insert(aFile.runtimePath);
422 pathsWithDuplicateInstallName.insert(installNameToFirstPath[installName]);
423 }
424 }
425
426 for (DyldSharedCache::MappedMachO& aFile : fileSet.dylibsForCache) {
427 if ( dontCache(volumePrefix, fileSet.archName, pathsWithDuplicateInstallName, aFile, true, skipDylibs) )
428 fileSet.otherDylibsAndBundles.push_back(aFile);
429 }
430 fileSet.dylibsForCache.erase(std::remove_if(fileSet.dylibsForCache.begin(), fileSet.dylibsForCache.end(),
431 [&](const DyldSharedCache::MappedMachO& aFile) { return dontCache(volumePrefix, fileSet.archName, pathsWithDuplicateInstallName, aFile, false, skipDylibs); }),
432 fileSet.dylibsForCache.end());
433 }
434
435 static void pruneOtherDylibs(const std::string& volumePrefix, MappedMachOsByCategory& fileSet)
436 {
437 // other OS dylibs should not contain dylibs that are embedded in some .app bundle
438 fileSet.otherDylibsAndBundles.erase(std::remove_if(fileSet.otherDylibsAndBundles.begin(), fileSet.otherDylibsAndBundles.end(),
439 [&](const DyldSharedCache::MappedMachO& aFile) { return (aFile.runtimePath.find(".app/") != std::string::npos); }),
440 fileSet.otherDylibsAndBundles.end());
441 }
442
443
444 static void pruneExecutables(const std::string& volumePrefix, MappedMachOsByCategory& fileSet)
445 {
446 // don't build closures for xcode shims in /usr/bin (e.g. /usr/bin/clang) which re-exec themselves to a tool inside Xcode.app
447 fileSet.mainExecutables.erase(std::remove_if(fileSet.mainExecutables.begin(), fileSet.mainExecutables.end(),
448 [&](const DyldSharedCache::MappedMachO& aFile) {
449 if ( !startsWith(aFile.runtimePath, "/usr/bin/") )
450 return false;
451 dyld3::MachOParser parser(aFile.mh);
452 __block bool isXcodeShim = false;
453 parser.forEachDependentDylib(^(const char* loadPath, bool, bool, bool, uint32_t, uint32_t, bool &stop) {
454 if ( strcmp(loadPath, "/usr/lib/libxcselect.dylib") == 0 )
455 isXcodeShim = true;
456 });
457 return isXcodeShim;
458 }), fileSet.mainExecutables.end());
459 }
460
461 static bool existingCacheUpToDate(const std::string& existingCache, const std::vector<DyldSharedCache::MappedMachO>& currentDylibs)
462 {
463 // if no existing cache, it is not up-to-date
464 int fd = ::open(existingCache.c_str(), O_RDONLY);
465 if ( fd < 0 )
466 return false;
467
468 // build map of found dylibs
469 std::unordered_map<std::string, const DyldSharedCache::MappedMachO*> currentDylibMap;
470 for (const DyldSharedCache::MappedMachO& aFile : currentDylibs) {
471 //fprintf(stderr, "0x%0llX 0x%0llX %s\n", aFile.inode, aFile.modTime, aFile.runtimePath.c_str());
472 currentDylibMap[aFile.runtimePath] = &aFile;
473 }
474
475 // make sure all dylibs in existing cache have same mtime and inode as found dylib
476 __block bool foundMismatch = false;
477 const uint64_t cacheMapLen = 0x40000000;
478 void *p = ::mmap(NULL, cacheMapLen, PROT_READ, MAP_PRIVATE, fd, 0);
479 if ( p != MAP_FAILED ) {
480 const DyldSharedCache* cache = (DyldSharedCache*)p;
481 cache->forEachImageEntry(^(const char* installName, uint64_t mTime, uint64_t inode) {
482 bool foundMatch = false;
483 auto pos = currentDylibMap.find(installName);
484 if ( pos != currentDylibMap.end() ) {
485 const DyldSharedCache::MappedMachO* foundDylib = pos->second;
486 if ( (foundDylib->inode == inode) && (foundDylib->modTime == mTime) ) {
487 foundMatch = true;
488 }
489 }
490 if ( !foundMatch ) {
491 // use slow path and look for any dylib with a matching inode and mtime
492 bool foundSlow = false;
493 for (const DyldSharedCache::MappedMachO& aFile : currentDylibs) {
494 if ( (aFile.inode == inode) && (aFile.modTime == mTime) ) {
495 foundSlow = true;
496 break;
497 }
498 }
499 if ( !foundSlow ) {
500 foundMismatch = true;
501 if ( verbose )
502 fprintf(stderr, "rebuilding dyld cache because dylib changed: %s\n", installName);
503 }
504 }
505 });
506 ::munmap(p, cacheMapLen);
507 }
508
509 ::close(fd);
510
511 return !foundMismatch;
512 }
513
514
515 inline uint32_t absolutetime_to_milliseconds(uint64_t abstime)
516 {
517 return (uint32_t)(abstime/1000/1000);
518 }
519
520 static bool runningOnHaswell()
521 {
522 // check system is capable of running x86_64h code
523 struct host_basic_info info;
524 mach_msg_type_number_t count = HOST_BASIC_INFO_COUNT;
525 mach_port_t hostPort = mach_host_self();
526 kern_return_t result = host_info(hostPort, HOST_BASIC_INFO, (host_info_t)&info, &count);
527 mach_port_deallocate(mach_task_self(), hostPort);
528
529 return ( (result == KERN_SUCCESS) && (info.cpu_subtype == CPU_SUBTYPE_X86_64_H) );
530 }
531
532
533
534 #define TERMINATE_IF_LAST_ARG( s ) \
535 do { \
536 if ( i == argc - 1 ) { \
537 fprintf(stderr, s ); \
538 return 1; \
539 } \
540 } while ( 0 )
541
542 int main(int argc, const char* argv[])
543 {
544 std::string rootPath;
545 std::string overlayPath;
546 std::string dylibListFile;
547 bool universal = false;
548 bool force = false;
549 bool searchDisk = false;
550 bool dylibsRemoved = false;
551 std::string cacheDir;
552 std::unordered_set<std::string> archStrs;
553 std::unordered_set<std::string> skipDylibs;
554
555 // parse command line options
556 for (int i = 1; i < argc; ++i) {
557 const char* arg = argv[i];
558 if (strcmp(arg, "-debug") == 0) {
559 verbose = true;
560 }
561 else if (strcmp(arg, "-verbose") == 0) {
562 verbose = true;
563 }
564 else if (strcmp(arg, "-dont_map_local_symbols") == 0) {
565 //We are going to ignore this
566 }
567 else if (strcmp(arg, "-dylib_list") == 0) {
568 TERMINATE_IF_LAST_ARG("-dylib_list missing argument");
569 dylibListFile = argv[++i];
570 }
571 else if ((strcmp(arg, "-root") == 0) || (strcmp(arg, "--root") == 0)) {
572 TERMINATE_IF_LAST_ARG("-root missing path argument\n");
573 rootPath = argv[++i];
574 }
575 else if (strcmp(arg, "-overlay") == 0) {
576 TERMINATE_IF_LAST_ARG("-overlay missing path argument\n");
577 overlayPath = argv[++i];
578 }
579 else if (strcmp(arg, "-cache_dir") == 0) {
580 TERMINATE_IF_LAST_ARG("-cache_dir missing path argument\n");
581 cacheDir = argv[++i];
582 }
583 else if (strcmp(arg, "-arch") == 0) {
584 TERMINATE_IF_LAST_ARG("-arch missing argument\n");
585 archStrs.insert(argv[++i]);
586 }
587 else if (strcmp(arg, "-search_disk") == 0) {
588 searchDisk = true;
589 }
590 else if (strcmp(arg, "-dylibs_removed_in_mastering") == 0) {
591 dylibsRemoved = true;
592 }
593 else if (strcmp(arg, "-force") == 0) {
594 force = true;
595 }
596 else if (strcmp(arg, "-sort_by_name") == 0) {
597 //No-op, we always do this now
598 }
599 else if (strcmp(arg, "-universal_boot") == 0) {
600 universal = true;
601 }
602 else if (strcmp(arg, "-skip") == 0) {
603 TERMINATE_IF_LAST_ARG("-skip missing argument\n");
604 skipDylibs.insert(argv[++i]);
605 }
606 else {
607 //usage();
608 fprintf(stderr, "update_dyld_shared_cache: unknown option: %s\n", arg);
609 return 1;
610 }
611 }
612
613 if ( !rootPath.empty() & !overlayPath.empty() ) {
614 fprintf(stderr, "-root and -overlay cannot be used together\n");
615 return 1;
616 }
617 // canonicalize rootPath
618 if ( !rootPath.empty() ) {
619 char resolvedPath[PATH_MAX];
620 if ( realpath(rootPath.c_str(), resolvedPath) != NULL ) {
621 rootPath = resolvedPath;
622 }
623 // <rdar://problem/33223984> when building closures for boot volume, pathPrefixes should be empty
624 if ( rootPath == "/" ) {
625 rootPath = "";
626 }
627 }
628 // canonicalize overlayPath
629 if ( !overlayPath.empty() ) {
630 char resolvedPath[PATH_MAX];
631 if ( realpath(overlayPath.c_str(), resolvedPath) != NULL ) {
632 overlayPath = resolvedPath;
633 }
634 }
635 //
636 // pathPrefixes for three modes:
637 // 1) no options: { "" } // search only boot volume
638 // 2) -overlay: { overlay, "" } // search overlay, then boot volume
639 // 3) -root: { root } // search only -root volume
640 //
641 std::vector<std::string> pathPrefixes;
642 if ( !overlayPath.empty() )
643 pathPrefixes.push_back(overlayPath);
644 pathPrefixes.push_back(rootPath);
645
646
647 if ( cacheDir.empty() ) {
648 // write cache file into -root or -overlay directory, if used
649 if ( rootPath != "/" )
650 cacheDir = rootPath + MACOSX_DYLD_SHARED_CACHE_DIR;
651 else if ( !overlayPath.empty() )
652 cacheDir = overlayPath + MACOSX_DYLD_SHARED_CACHE_DIR;
653 else
654 cacheDir = MACOSX_DYLD_SHARED_CACHE_DIR;
655 }
656
657 int err = mkpath_np(cacheDir.c_str(), S_IRWXU | S_IRGRP|S_IXGRP | S_IROTH|S_IXOTH);
658 if ( (err != 0) && (err != EEXIST) ) {
659 fprintf(stderr, "mkpath_np fail: %d", err);
660 return 1;
661 }
662
663 if ( archStrs.empty() ) {
664 if ( universal ) {
665 // <rdar://problem/26182089> -universal_boot should make all possible dyld caches
666 archStrs.insert("i386");
667 archStrs.insert("x86_64");
668 archStrs.insert("x86_64h");
669 }
670 else {
671 // just make caches for this machine
672 archStrs.insert("i386");
673 archStrs.insert(runningOnHaswell() ? "x86_64h" : "x86_64");
674 }
675 }
676
677 uint64_t t1 = mach_absolute_time();
678
679 // find all mach-o files for requested architectures
680 bool requireDylibsBeRootlessProtected = isProtectedBySIP(cacheDir);
681 __block std::vector<MappedMachOsByCategory> allFileSets;
682 if ( archStrs.count("x86_64") )
683 allFileSets.push_back({"x86_64"});
684 if ( archStrs.count("x86_64h") )
685 allFileSets.push_back({"x86_64h"});
686 if ( archStrs.count("i386") )
687 allFileSets.push_back({"i386"});
688 if ( searchDisk )
689 findAllFiles(pathPrefixes, requireDylibsBeRootlessProtected, allFileSets);
690 else {
691 std::unordered_set<std::string> runtimePathsFound;
692 findOSFilesViaBOMS(pathPrefixes, requireDylibsBeRootlessProtected, allFileSets);
693 }
694
695 // nothing in OS uses i386 dylibs, so only dylibs used by third party apps need to be in cache
696 for (MappedMachOsByCategory& fileSet : allFileSets) {
697 pruneCachedDylibs(rootPath, skipDylibs, fileSet);
698 pruneOtherDylibs(rootPath, fileSet);
699 pruneExecutables(rootPath, fileSet);
700 }
701
702 uint64_t t2 = mach_absolute_time();
703 if ( verbose ) {
704 if ( searchDisk )
705 fprintf(stderr, "time to scan file system and construct lists of mach-o files: %ums\n", absolutetime_to_milliseconds(t2-t1));
706 else
707 fprintf(stderr, "time to read BOM and construct lists of mach-o files: %ums\n", absolutetime_to_milliseconds(t2-t1));
708 }
709
710 // build caches in parallel on machines with at leat 4GB of RAM
711 uint64_t memSize = 0;
712 size_t sz = sizeof(memSize);;
713 bool buildInParallel = false;
714 if ( sysctlbyname("hw.memsize", &memSize, &sz, NULL, 0) == 0 ) {
715 if ( memSize >= 0x100000000ULL )
716 buildInParallel = true;
717 }
718 dispatch_queue_t dqueue = buildInParallel ? dispatch_get_global_queue(DISPATCH_QUEUE_PRIORITY_DEFAULT, 0)
719 : dispatch_queue_create("serial-queue", DISPATCH_QUEUE_SERIAL);
720
721 // build all caches
722 __block bool cacheBuildFailure = false;
723 __block bool wroteSomeCacheFile = false;
724 dispatch_apply(allFileSets.size(), dqueue, ^(size_t index) {
725 MappedMachOsByCategory& fileSet = allFileSets[index];
726 const std::string outFile = cacheDir + "/dyld_shared_cache_" + fileSet.archName;
727
728 DyldSharedCache::MappedMachO (^loader)(const std::string&) = ^DyldSharedCache::MappedMachO(const std::string& runtimePath) {
729 if ( skipDylibs.count(runtimePath) )
730 return DyldSharedCache::MappedMachO();
731 for (const std::string& prefix : pathPrefixes) {
732 std::string fullPath = prefix + runtimePath;
733 struct stat statBuf;
734 if ( stat(fullPath.c_str(), &statBuf) == 0 ) {
735 std::vector<MappedMachOsByCategory> mappedFiles;
736 mappedFiles.push_back({fileSet.archName});
737 if ( addIfMachO(prefix, runtimePath, statBuf, requireDylibsBeRootlessProtected, mappedFiles) ) {
738 if ( !mappedFiles.back().dylibsForCache.empty() )
739 return mappedFiles.back().dylibsForCache.back();
740 }
741 }
742 }
743 return DyldSharedCache::MappedMachO();
744 };
745 size_t startCount = fileSet.dylibsForCache.size();
746 std::vector<std::pair<DyldSharedCache::MappedMachO, std::set<std::string>>> excludes;
747 DyldSharedCache::verifySelfContained(fileSet.dylibsForCache, loader, excludes);
748 for (size_t i=startCount; i < fileSet.dylibsForCache.size(); ++i) {
749 fprintf(stderr, "update_dyld_shared_cache: warning: %s not in .bom, but adding required dylib %s\n", fileSet.archName.c_str(), fileSet.dylibsForCache[i].runtimePath.c_str());
750 }
751 for (auto& exclude : excludes) {
752 std::string reasons = "(\"";
753 for (auto i = exclude.second.begin(); i != exclude.second.end(); ++i) {
754 reasons += *i;
755 if (i != --exclude.second.end()) {
756 reasons += "\", \"";
757 }
758 }
759 reasons += "\")";
760 fprintf(stderr, "update_dyld_shared_cache: warning: %s rejected from cached dylibs: %s (%s)\n", fileSet.archName.c_str(), exclude.first.runtimePath.c_str(), reasons.c_str());
761 fileSet.otherDylibsAndBundles.push_back(exclude.first);
762 }
763
764 // check if cache is already up to date
765 if ( !force ) {
766 if ( existingCacheUpToDate(outFile, fileSet.dylibsForCache) )
767 return;
768 }
769
770 // add any extra dylibs needed which were not in .bom
771 fprintf(stderr, "update_dyld_shared_cache: %s incorporating %lu OS dylibs, tracking %lu others, building closures for %lu executables\n", fileSet.archName.c_str(), fileSet.dylibsForCache.size(), fileSet.otherDylibsAndBundles.size(), fileSet.mainExecutables.size());
772 //for (const DyldSharedCache::MappedMachO& aFile : fileSet.dylibsForCache) {
773 // fprintf(stderr, " %s\n", aFile.runtimePath.c_str());
774 //}
775
776 // Clear the UUID xattr for the existing cache.
777 // This prevents the existing cache from being used by dyld3 as roots are probably involved
778 if (removexattr(outFile.c_str(), "cacheUUID", 0) != 0) {
779 fprintf(stderr, "update_dyld_shared_cache: warning: failure to remove UUID xattr on shared cache file %s with error %s\n", outFile.c_str(), strerror(errno));
780 }
781
782 // build cache new cache file
783 DyldSharedCache::CreateOptions options;
784 options.archName = fileSet.archName;
785 options.platform = dyld3::Platform::macOS;
786 options.excludeLocalSymbols = false;
787 options.optimizeStubs = false;
788 options.optimizeObjC = true;
789 options.codeSigningDigestMode = DyldSharedCache::SHA256only;
790 options.dylibsRemovedDuringMastering = dylibsRemoved;
791 options.inodesAreSameAsRuntime = true;
792 options.cacheSupportsASLR = (fileSet.archName != "i386");
793 options.forSimulator = false;
794 options.verbose = verbose;
795 options.evictLeafDylibsOnOverflow = true;
796 options.pathPrefixes = pathPrefixes;
797 DyldSharedCache::CreateResults results = DyldSharedCache::create(options, fileSet.dylibsForCache, fileSet.otherDylibsAndBundles, fileSet.mainExecutables);
798
799 // print any warnings
800 for (const std::string& warn : results.warnings) {
801 fprintf(stderr, "update_dyld_shared_cache: warning: %s %s\n", fileSet.archName.c_str(), warn.c_str());
802 }
803 if ( !results.errorMessage.empty() ) {
804 // print error (if one)
805 fprintf(stderr, "update_dyld_shared_cache: %s\n", results.errorMessage.c_str());
806 cacheBuildFailure = true;
807 }
808 else {
809 // save new cache file to disk and write new .map file
810 assert(results.cacheContent != nullptr);
811 if ( !safeSave(results.cacheContent, results.cacheLength, outFile) ) {
812 fprintf(stderr, "update_dyld_shared_cache: could not write dyld cache file %s\n", outFile.c_str());
813 cacheBuildFailure = true;
814 }
815 if ( !cacheBuildFailure ) {
816 uuid_t cacheUUID;
817 results.cacheContent->getUUID(cacheUUID);
818 if (setxattr(outFile.c_str(), "cacheUUID", (const void*)&cacheUUID, sizeof(cacheUUID), 0, XATTR_CREATE) != 0) {
819 fprintf(stderr, "update_dyld_shared_cache: warning: failure to set UUID xattr on shared cache file %s with error %s\n", outFile.c_str(), strerror(errno));
820 }
821 std::string mapStr = results.cacheContent->mapFile();
822 std::string outFileMap = cacheDir + "/dyld_shared_cache_" + fileSet.archName + ".map";
823 safeSave(mapStr.c_str(), mapStr.size(), outFileMap);
824 wroteSomeCacheFile = true;
825 }
826 // free created cache buffer
827 vm_deallocate(mach_task_self(), (vm_address_t)results.cacheContent, results.cacheLength);
828 }
829 });
830
831
832 // Save off spintrace data
833 if ( wroteSomeCacheFile ) {
834 void* h = dlopen("/usr/lib/libdscsym.dylib", 0);
835 if ( h != nullptr ) {
836 typedef int (*dscym_func)(const char*);
837 dscym_func func = (dscym_func)dlsym(h, "dscsym_save_dscsyms_for_current_caches");
838 std::string nuggetRoot = rootPath;
839 if ( nuggetRoot.empty() )
840 nuggetRoot = overlayPath;
841 if ( nuggetRoot.empty() )
842 nuggetRoot = "/";
843 (*func)(nuggetRoot.c_str());
844 }
845 }
846
847
848 // we could unmap all input files, but tool is about to quit
849
850 return (cacheBuildFailure ? 1 : 0);
851 }
852