]> git.saurik.com Git - apple/dyld.git/blob - dyld3/shared-cache/update_dyld_shared_cache.cpp
dyld-733.8.tar.gz
[apple/dyld.git] / dyld3 / shared-cache / update_dyld_shared_cache.cpp
1 /* -*- mode: C++; c-basic-offset: 4; indent-tabs-mode: nil -*-
2 *
3 * Copyright (c) 2014 Apple Inc. All rights reserved.
4 *
5 * @APPLE_LICENSE_HEADER_START@
6 *
7 * This file contains Original Code and/or Modifications of Original Code
8 * as defined in and that are subject to the Apple Public Source License
9 * Version 2.0 (the 'License'). You may not use this file except in
10 * compliance with the License. Please obtain a copy of the License at
11 * http://www.opensource.apple.com/apsl/ and read it before using this
12 * file.
13 *
14 * The Original Code and all software distributed under the License are
15 * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
16 * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
17 * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
18 * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
19 * Please see the License for the specific language governing rights and
20 * limitations under the License.
21 *
22 * @APPLE_LICENSE_HEADER_END@
23 */
24
25 #include <sys/types.h>
26 #include <sys/stat.h>
27 #include <sys/mman.h>
28 #include <mach/mach.h>
29 #include <mach/mach_time.h>
30 #include <mach-o/dyld.h>
31 #include <limits.h>
32 #include <stdarg.h>
33 #include <stdio.h>
34 #include <stdlib.h>
35 #include <math.h>
36 #include <fcntl.h>
37 #include <dlfcn.h>
38 #include <signal.h>
39 #include <errno.h>
40 #include <assert.h>
41 #include <sys/uio.h>
42 #include <unistd.h>
43 #include <sys/param.h>
44 #include <sys/sysctl.h>
45 #include <sys/resource.h>
46 #include <dirent.h>
47 #include <rootless.h>
48 #include <dscsym.h>
49 #include <dispatch/dispatch.h>
50 #include <pthread/pthread.h>
51 #include <Bom/Bom.h>
52 #include <CoreFoundation/CoreFoundation.h>
53
54 #include <algorithm>
55 #include <vector>
56 #include <unordered_set>
57 #include <unordered_set>
58 #include <iostream>
59 #include <fstream>
60
61 #include "FileUtils.h"
62 #include "StringUtils.h"
63 #include "DyldSharedCache.h"
64 #include "MachOFile.h"
65 #include "MachOAnalyzer.h"
66 #include "ClosureFileSystemPhysical.h"
67
68 struct MappedMachOsByCategory
69 {
70 const dyld3::GradedArchs& archs;
71 std::vector<DyldSharedCache::MappedMachO> dylibsForCache;
72 std::vector<DyldSharedCache::MappedMachO> otherDylibsAndBundles;
73 std::vector<DyldSharedCache::MappedMachO> mainExecutables;
74 std::unordered_set<std::string> badZippered;
75 };
76
77 static const char* sAllowedPrefixes[] = {
78 "/bin/",
79 "/sbin/",
80 "/usr/",
81 "/System/",
82 "/Library/Apple/System/",
83 "/Library/Apple/usr/",
84 "/System/Applications/App Store.app/",
85 "/System/Applications/Automator.app/",
86 "/System/Applications/Calculator.app/",
87 "/System/Applications/Calendar.app/",
88 "/System/Applications/Chess.app/",
89 "/System/Applications/Contacts.app/",
90 "/System/Applications/Dashboard.app/",
91 "/System/Applications/Dictionary.app/",
92 "/System/Applications/FaceTime.app/",
93 "/System/Applications/Font Book.app/",
94 "/System/Applications/Image Capture.app/",
95 "/System/Applications/Launchpad.app/",
96 "/System/Applications/Mail.app/",
97 "/System/Applications/Maps.app/",
98 "/System/Applications/Messages.app/",
99 "/System/Applications/Mission Control.app/",
100 "/System/Applications/Notes.app/",
101 "/System/Applications/Photo Booth.app/",
102 "/System/Applications/Preview.app/",
103 "/System/Applications/QuickTime Player.app/",
104 "/System/Applications/Reminders.app/",
105 "/Applications/Safari.app/",
106 "/System/Applications/Siri.app/",
107 "/System/Applications/Stickies.app/",
108 "/System/Applications/System Preferences.app/",
109 "/System/Applications/TextEdit.app/",
110 "/System/Applications/Time Machine.app/",
111 "/System/Applications/iBooks.app/",
112 "/System/Applications/iTunes.app/",
113 "/System/Applications/Utilities/Activity Monitor.app",
114 "/System/Applications/Utilities/AirPort Utility.app",
115 "/System/Applications/Utilities/Audio MIDI Setup.app",
116 "/System/Applications/Utilities/Bluetooth File Exchange.app",
117 "/System/Applications/Utilities/Boot Camp Assistant.app",
118 "/System/Applications/Utilities/ColorSync Utility.app",
119 "/System/Applications/Utilities/Console.app",
120 "/System/Applications/Utilities/Digital Color Meter.app",
121 "/System/Applications/Utilities/Disk Utility.app",
122 "/System/Applications/Utilities/Grab.app",
123 "/System/Applications/Utilities/Grapher.app",
124 "/System/Applications/Utilities/Keychain Access.app",
125 "/System/Applications/Utilities/Migration Assistant.app",
126 "/System/Applications/Utilities/Script Editor.app",
127 "/System/Applications/Utilities/System Information.app",
128 "/System/Applications/Utilities/Terminal.app",
129 "/System/Applications/Utilities/VoiceOver Utility.app",
130 "/Library/CoreMediaIO/Plug-Ins/DAL/" // temp until plugins moved or closured working
131 };
132
133 static const char* sDontUsePrefixes[] = {
134 "/usr/share",
135 "/usr/local/",
136 "/System/Library/Assets",
137 "/System/Library/StagedFrameworks",
138 "/Library/Apple/System/Library/StagedFrameworks",
139 "/System/Library/Kernels/",
140 "/bin/zsh", // until <rdar://31026756> is fixed
141 "/System/Library/Frameworks/CoreServices.framework/Versions/A/Frameworks/Metadata.framework/Versions/A/Support/mdworker", // these load third party plugins
142 "/usr/bin/mdimport", // these load third party plugins
143 };
144
145
146 static bool verbose = false;
147
148
149
150
151 static bool addIfMachO(const dyld3::closure::FileSystem& fileSystem, const std::string& runtimePath, const struct stat& statBuf,
152 bool requireSIP, dev_t rootFS, std::vector<MappedMachOsByCategory>& files)
153 {
154 // don't precompute closure info for any debug or profile dylibs
155 if ( endsWith(runtimePath, "_profile.dylib") || endsWith(runtimePath, "_debug.dylib") || endsWith(runtimePath, "_profile") || endsWith(runtimePath, "_debug") )
156 return false;
157 if ( startsWith(runtimePath, "/usr/lib/system/introspection/") )
158 return false;
159
160 #if !BUILDING_UPDATE_OTHER_DYLD_CACHE_BUILDER
161 // Only use files on the same volume as the boot volume
162 if (statBuf.st_dev != rootFS) {
163 if ( verbose )
164 fprintf(stderr, "update_dyld_shared_cache: warning: skipping overlay file '%s' which is not on the root volume\n", runtimePath.c_str());
165 return false;
166 }
167 #endif
168
169 auto warningHandler = ^(const char* msg) {
170 if ( verbose )
171 fprintf(stderr, "update_dyld_shared_cache: warning: cannot build dlopen closure for '%s' because %s\n", runtimePath.c_str(), msg);
172 };
173
174 bool result = false;
175 for (MappedMachOsByCategory& file : files) {
176 Diagnostics diag;
177 char realerPath[MAXPATHLEN];
178 dyld3::closure::LoadedFileInfo loadedFileInfo = dyld3::MachOAnalyzer::load(diag, fileSystem, runtimePath.c_str(), file.archs, dyld3::Platform::macOS, realerPath);
179 if (diag.hasError() ) {
180 // Try again with iOSMac
181 diag.clearError();
182 loadedFileInfo = dyld3::MachOAnalyzer::load(diag, fileSystem, runtimePath.c_str(), file.archs, dyld3::Platform::iOSMac, realerPath);
183 }
184 const dyld3::MachOAnalyzer* ma = (const dyld3::MachOAnalyzer*)loadedFileInfo.fileContent;
185 if ( ma != nullptr ) {
186 bool issetuid = false;
187 const uint64_t sliceLen = loadedFileInfo.sliceLen;
188 const bool isSipProtected = loadedFileInfo.isSipProtected;
189 if ( ma->isDynamicExecutable() ) {
190 // When SIP enabled, only build closures for SIP protected programs
191 if ( !requireSIP || isSipProtected ) {
192 //fprintf(stderr, "requireSIP=%d, sipProtected=%d, path=%s\n", requireSIP, sipProtected, fullPath.c_str());
193 issetuid = (statBuf.st_mode & (S_ISUID|S_ISGID));
194 file.mainExecutables.emplace_back(runtimePath, ma, sliceLen, issetuid, isSipProtected, loadedFileInfo.sliceOffset, statBuf.st_mtime, statBuf.st_ino);
195 }
196 }
197 else if ( ma->canBePlacedInDyldCache(runtimePath.c_str(), ^(const char* msg) {
198 if (verbose)
199 fprintf(stderr, "update_dyld_shared_cache: warning dylib located at '%s' cannot be placed in cache because: %s\n", runtimePath.c_str(), msg);
200 }) ) {
201 // when SIP is enabled, only dylib protected by SIP can go in cache
202 if ( !requireSIP || isSipProtected )
203 file.dylibsForCache.emplace_back(runtimePath, ma, sliceLen, issetuid, isSipProtected, loadedFileInfo.sliceOffset, statBuf.st_mtime, statBuf.st_ino);
204 else if ( ma->canHavePrecomputedDlopenClosure(runtimePath.c_str(), warningHandler) )
205 file.otherDylibsAndBundles.emplace_back(runtimePath, ma, sliceLen, issetuid, isSipProtected, loadedFileInfo.sliceOffset, statBuf.st_mtime, statBuf.st_ino);
206 }
207 else {
208 if ( ma->isDylib() ) {
209 std::string installName = ma->installName();
210 if ( startsWith(installName, "@") && !contains(runtimePath, ".app/") && !contains(runtimePath, ".xpc/") ) {
211 if ( dyld3::MachOFile::isSharedCacheEligiblePath(runtimePath.c_str()) )
212 fprintf(stderr, "update_dyld_shared_cache: warning @rpath install name for system framework: %s\n", runtimePath.c_str());
213 }
214 }
215 if ( ma->canHavePrecomputedDlopenClosure(runtimePath.c_str(), warningHandler) ) {
216 // Only add a dlopen closure for objc trampolines. The rest should have been shared cache eligible.
217 bool addClosure = false;
218 if ( ma->isDylib() ) {
219 std::string installName = ma->installName();
220 addClosure = installName == "/usr/lib/libobjc-trampolines.dylib";
221 } else {
222 addClosure = true;
223 }
224 if (addClosure)
225 file.otherDylibsAndBundles.emplace_back(runtimePath, ma, sliceLen, issetuid, isSipProtected, loadedFileInfo.sliceOffset, statBuf.st_mtime, statBuf.st_ino);
226 }
227 }
228 result = true;
229 }
230 }
231
232 return result;
233 }
234
235 static void findAllFiles(const dyld3::closure::FileSystem& fileSystem, const std::vector<std::string>& pathPrefixes,
236 bool requireSIP, dev_t rootFS, std::vector<MappedMachOsByCategory>& files)
237 {
238 std::unordered_set<std::string> skipDirs;
239 for (const char* s : sDontUsePrefixes)
240 skipDirs.insert(s);
241
242 __block std::unordered_set<std::string> alreadyUsed;
243 bool multiplePrefixes = (pathPrefixes.size() > 1);
244 for (const std::string& prefix : pathPrefixes) {
245 // get all files from overlay for this search dir
246 for (const char* searchDir : sAllowedPrefixes ) {
247 iterateDirectoryTree(prefix, searchDir, ^(const std::string& dirPath) { return (skipDirs.count(dirPath) != 0); }, ^(const std::string& path, const struct stat& statBuf) {
248 // ignore files that don't have 'x' bit set (all runnable mach-o files do)
249 const bool hasXBit = ((statBuf.st_mode & S_IXOTH) == S_IXOTH);
250 if ( !hasXBit && !endsWith(path, ".dylib") )
251 return;
252
253 // ignore files too small
254 if ( statBuf.st_size < 0x3000 )
255 return;
256
257 // don't add paths already found using previous prefix
258 if ( multiplePrefixes && (alreadyUsed.count(path) != 0) )
259 return;
260
261 // if the file is mach-o, add to list
262 if ( addIfMachO(fileSystem, path, statBuf, requireSIP, rootFS, files) ) {
263 if ( multiplePrefixes )
264 alreadyUsed.insert(path);
265 }
266 });
267 }
268 }
269 }
270
271 static const char* sReceiptLocations[] = {
272 "/System/Library/Receipts",
273 "/Library/Apple/System/Library/Receipts"
274 };
275
276 static void findOSFilesViaBOMS(const dyld3::closure::FileSystem& fileSystem, const std::vector<std::string>& pathPrefixes,
277 bool requireSIP, dev_t rootFS, std::vector<MappedMachOsByCategory>& files)
278 {
279 __block std::unordered_set<std::string> runtimePathsFound;
280 __block bool foundUsableBom = false;
281 for (const std::string& prefix : pathPrefixes) {
282 for (const char* dirToIterate : sReceiptLocations ) {
283 iterateDirectoryTree(prefix, dirToIterate, ^(const std::string&) { return false; }, ^(const std::string& path, const struct stat& statBuf) {
284 if ( !contains(path, "com.apple.pkg.") )
285 return;
286 if ( !endsWith(path, ".bom") )
287 return;
288 std::string fullPath = prefix + path;
289 BOMBom bom = BOMBomOpenWithSys(fullPath.c_str(), false, NULL);
290 if ( bom == nullptr )
291 return;
292 BOMFSObject rootFso = BOMBomGetRootFSObject(bom);
293 if ( rootFso == nullptr ) {
294 BOMBomFree(bom);
295 return;
296 }
297 BOMBomEnumerator e = BOMBomEnumeratorNew(bom, rootFso);
298 if ( e == nullptr ) {
299 fprintf(stderr, "Can't get enumerator for BOM root FSObject\n");
300 return;
301 }
302 BOMFSObjectFree(rootFso);
303 //fprintf(stderr, "using BOM %s\n", path.c_str());
304 foundUsableBom = true;
305 while (BOMFSObject fso = BOMBomEnumeratorNext(e)) {
306 if ( BOMFSObjectIsBinaryObject(fso) ) {
307 const char* runPath = BOMFSObjectPathName(fso);
308 if ( (runPath[0] == '.') && (runPath[1] == '/') )
309 ++runPath;
310 // <rdar://problem/48748330> update_dyld_shared_cache needs to fold away /S/L/Templates/Data
311 if (strncmp(runPath, "/System/Library/Templates/Data/", 31) == 0 )
312 runPath = &runPath[30];
313 if ( runtimePathsFound.count(runPath) == 0 ) {
314 // only add files from sAllowedPrefixes and not in sDontUsePrefixes
315 bool inSearchDir = false;
316 for (const char* searchDir : sAllowedPrefixes ) {
317 if ( strncmp(searchDir, runPath, strlen(searchDir)) == 0 ) {
318 inSearchDir = true;
319 break;
320 }
321 }
322 if ( inSearchDir ) {
323 bool inSkipDir = false;
324 for (const char* skipDir : sDontUsePrefixes) {
325 if ( strncmp(skipDir, runPath, strlen(skipDir)) == 0 ) {
326 inSkipDir = true;
327 break;
328 }
329 }
330 if ( !inSkipDir ) {
331 for (const std::string& prefix2 : pathPrefixes) {
332 struct stat statBuf2;
333 std::string fullPath2 = prefix2 + runPath;
334 if ( stat(fullPath2.c_str(), &statBuf2) == 0 ) {
335 if ( addIfMachO(fileSystem, runPath, statBuf2, requireSIP, rootFS, files) ) {
336 runtimePathsFound.insert(runPath);
337 break;
338 }
339 }
340 }
341 }
342 }
343 }
344 }
345 BOMFSObjectFree(fso);
346 }
347
348 BOMBomEnumeratorFree(e);
349 BOMBomFree(bom);
350 });
351 }
352 }
353
354 if (!foundUsableBom)
355 fprintf(stderr, "update_dyld_shared_cache: warning: No usable BOM files were found in '/System/Library/Receipts'\n");
356 }
357
358
359 static bool dontCache(const std::string& volumePrefix, const std::string& archName,
360 const std::unordered_set<std::string>& pathsWithDuplicateInstallName,
361 const std::unordered_set<std::string>& badZippered,
362 const DyldSharedCache::MappedMachO& aFile, bool warn,
363 const std::unordered_set<std::string>& skipDylibs)
364 {
365 if ( skipDylibs.count(aFile.runtimePath) )
366 return true;
367 if ( startsWith(aFile.runtimePath, "/usr/lib/system/introspection/") )
368 return true;
369 if ( startsWith(aFile.runtimePath, "/System/Library/QuickTime/") )
370 return true;
371 if ( startsWith(aFile.runtimePath, "/System/Library/Tcl/") )
372 return true;
373 if ( startsWith(aFile.runtimePath, "/System/Library/Perl/") )
374 return true;
375 if ( startsWith(aFile.runtimePath, "/System/Library/MonitorPanels/") )
376 return true;
377 if ( startsWith(aFile.runtimePath, "/System/Library/Accessibility/") )
378 return true;
379 if ( startsWith(aFile.runtimePath, "/usr/local/") )
380 return true;
381
382 // anything inside a .app bundle is specific to app, so should not be in shared cache
383 if ( aFile.runtimePath.find(".app/") != std::string::npos )
384 return true;
385
386 if ( archName == "i386" ) {
387 if ( startsWith(aFile.runtimePath, "/System/Library/CoreServices/") )
388 return true;
389 if ( startsWith(aFile.runtimePath, "/System/Library/Extensions/") )
390 return true;
391 }
392
393 if ( aFile.runtimePath.find("//") != std::string::npos ) {
394 if (warn) fprintf(stderr, "update_dyld_shared_cache: warning: %s skipping because of bad install name %s\n", archName.c_str(), aFile.runtimePath.c_str());
395 return true;
396 }
397
398 const char* installName = aFile.mh->installName();
399 if ( (pathsWithDuplicateInstallName.count(aFile.runtimePath) != 0) && (aFile.runtimePath != installName) ) {
400 // <rdar://problem/46431467> if a dylib moves and a symlink is installed into its place, bom iterator will see both and issue a warning
401 struct stat statBuf;
402 bool isSymLink = ( (lstat(aFile.runtimePath.c_str(), &statBuf) == 0) && S_ISLNK(statBuf.st_mode) );
403 if (!isSymLink && warn) fprintf(stderr, "update_dyld_shared_cache: warning: %s skipping because of duplicate install name %s\n", archName.c_str(), aFile.runtimePath.c_str());
404 return true;
405 }
406
407 if (badZippered.count(aFile.runtimePath)) {
408 return true;
409 }
410
411 if ( aFile.runtimePath != installName ) {
412 // see if install name is a symlink to actual path
413 std::string fullInstall = volumePrefix + installName;
414 char resolvedPath[PATH_MAX];
415 if ( realpath(fullInstall.c_str(), resolvedPath) != NULL ) {
416 std::string resolvedSymlink = resolvedPath;
417 if ( !volumePrefix.empty() ) {
418 resolvedSymlink = resolvedSymlink.substr(volumePrefix.size());
419 }
420 if ( aFile.runtimePath == resolvedSymlink ) {
421 return false;
422 }
423 }
424 // <rdar://problem/38000411> also if runtime path is a symlink to install name
425 std::string fullRuntime = volumePrefix + aFile.runtimePath;
426 if ( realpath(fullRuntime.c_str(), resolvedPath) != NULL ) {
427 std::string resolvedSymlink = resolvedPath;
428 if ( !volumePrefix.empty() ) {
429 resolvedSymlink = resolvedSymlink.substr(volumePrefix.size());
430 }
431 if ( resolvedSymlink == installName ) {
432 return false;
433 }
434 }
435 if (warn) fprintf(stderr, "update_dyld_shared_cache: warning: %s skipping because of bad install name %s\n", archName.c_str(), aFile.runtimePath.c_str());
436 return true;
437 }
438 return false;
439 }
440
441 static void pruneCachedDylibs(const std::string& volumePrefix, const std::unordered_set<std::string>& skipDylibs,
442 MappedMachOsByCategory& fileSet, bool warn)
443 {
444 std::unordered_set<std::string> pathsWithDuplicateInstallName;
445
446 std::unordered_map<std::string, std::string> installNameToFirstPath;
447 for (DyldSharedCache::MappedMachO& aFile : fileSet.dylibsForCache) {
448 const char* installName = aFile.mh->installName();
449 auto pos = installNameToFirstPath.find(installName);
450 if ( pos == installNameToFirstPath.end() ) {
451 installNameToFirstPath[installName] = aFile.runtimePath;
452 }
453 else {
454 pathsWithDuplicateInstallName.insert(aFile.runtimePath);
455 pathsWithDuplicateInstallName.insert(installNameToFirstPath[installName]);
456 }
457 }
458
459 std::unordered_map<std::string, std::string> macOSPathToTwinPath;
460 for (const auto& entry : installNameToFirstPath) {
461 if ( startsWith(entry.first, "/System/iOSSupport/") ) {
462 std::string tail = entry.first.substr(18);
463 if ( installNameToFirstPath.count(tail) != 0 ) {
464 macOSPathToTwinPath.insert({ tail, entry.first });
465 }
466 }
467 }
468
469 for (DyldSharedCache::MappedMachO& aFile : fileSet.dylibsForCache) {
470 if ( aFile.mh->isZippered() ) {
471 aFile.mh->forEachDependentDylib(^(const char* loadPath, bool isWeak, bool isReExport, bool isUpward, uint32_t compatVersion, uint32_t curVersion, bool& stop) {
472 auto macOSAndTwinPath = macOSPathToTwinPath.find(loadPath);
473 if ( macOSAndTwinPath != macOSPathToTwinPath.end() ) {
474 if ( warn ) {
475 fprintf(stderr, "update_dyld_shared_cache: warning: evicting UIKitForMac binary: %s as it is linked by zippered binary %s\n",
476 macOSAndTwinPath->second.c_str(), aFile.runtimePath.c_str());
477 }
478 fileSet.badZippered.insert(macOSAndTwinPath->second);
479 }
480 });
481 }
482 }
483
484 for (DyldSharedCache::MappedMachO& aFile : fileSet.dylibsForCache) {
485 if ( dontCache(volumePrefix, fileSet.archs.name(), pathsWithDuplicateInstallName, fileSet.badZippered, aFile, true, skipDylibs) ){
486 // <rdar://problem/46423929> don't build dlopen closures for symlinks to something in the dyld cache
487 if ( pathsWithDuplicateInstallName.count(aFile.runtimePath) == 0 )
488 fileSet.otherDylibsAndBundles.push_back(aFile);
489 }
490 }
491 fileSet.dylibsForCache.erase(std::remove_if(fileSet.dylibsForCache.begin(), fileSet.dylibsForCache.end(),
492 [&](const DyldSharedCache::MappedMachO& aFile) { return dontCache(volumePrefix, fileSet.archs.name(), pathsWithDuplicateInstallName, fileSet.badZippered, aFile, false, skipDylibs); }),
493 fileSet.dylibsForCache.end());
494 }
495
496 static void pruneOtherDylibs(const std::string& volumePrefix, MappedMachOsByCategory& fileSet)
497 {
498 // other OS dylibs should not contain dylibs that are embedded in some .app bundle
499 fileSet.otherDylibsAndBundles.erase(std::remove_if(fileSet.otherDylibsAndBundles.begin(), fileSet.otherDylibsAndBundles.end(),
500 [&](const DyldSharedCache::MappedMachO& aFile) { return (aFile.runtimePath.find(".app/") != std::string::npos); }),
501 fileSet.otherDylibsAndBundles.end());
502 }
503 static void pruneExecutables(const std::string& volumePrefix, MappedMachOsByCategory& fileSet)
504 {
505 // don't build closures for xcode shims in /usr/bin (e.g. /usr/bin/clang) which re-exec themselves to a tool inside Xcode.app
506 fileSet.mainExecutables.erase(std::remove_if(fileSet.mainExecutables.begin(), fileSet.mainExecutables.end(),
507 [&](const DyldSharedCache::MappedMachO& aFile) {
508 if ( !startsWith(aFile.runtimePath, "/usr/bin/") )
509 return false;
510 __block bool isXcodeShim = false;
511 aFile.mh->forEachDependentDylib(^(const char* loadPath, bool, bool, bool, uint32_t, uint32_t, bool &stop) {
512 if ( strcmp(loadPath, "/usr/lib/libxcselect.dylib") == 0 )
513 isXcodeShim = true;
514 });
515 return isXcodeShim;
516 }), fileSet.mainExecutables.end());
517 }
518
519 static bool existingCacheUpToDate(const std::string& existingCache, const std::vector<DyldSharedCache::MappedMachO>& currentDylibs)
520 {
521 // if no existing cache, it is not up-to-date
522 int fd = ::open(existingCache.c_str(), O_RDONLY);
523 if ( fd < 0 )
524 return false;
525 struct stat statbuf;
526 if ( ::fstat(fd, &statbuf) == -1 ) {
527 ::close(fd);
528 return false;
529 }
530
531 // build map of found dylibs
532 std::unordered_map<std::string, const DyldSharedCache::MappedMachO*> currentDylibMap;
533 for (const DyldSharedCache::MappedMachO& aFile : currentDylibs) {
534 //fprintf(stderr, "0x%0llX 0x%0llX %s\n", aFile.inode, aFile.modTime, aFile.runtimePath.c_str());
535 currentDylibMap[aFile.runtimePath] = &aFile;
536 }
537
538 // make sure all dylibs in existing cache have same mtime and inode as found dylib
539 __block bool foundMismatch = false;
540 const uint64_t cacheMapLen = statbuf.st_size;
541 void *p = ::mmap(NULL, cacheMapLen, PROT_READ, MAP_PRIVATE, fd, 0);
542 if ( p != MAP_FAILED ) {
543 const DyldSharedCache* cache = (DyldSharedCache*)p;
544 cache->forEachImageEntry(^(const char* installName, uint64_t mTime, uint64_t inode) {
545 bool foundMatch = false;
546 auto pos = currentDylibMap.find(installName);
547 if ( pos != currentDylibMap.end() ) {
548 const DyldSharedCache::MappedMachO* foundDylib = pos->second;
549 if ( (foundDylib->inode == inode) && (foundDylib->modTime == mTime) ) {
550 foundMatch = true;
551 }
552 }
553 if ( !foundMatch ) {
554 // use slow path and look for any dylib with a matching inode and mtime
555 bool foundSlow = false;
556 for (const DyldSharedCache::MappedMachO& aFile : currentDylibs) {
557 if ( (aFile.inode == inode) && (aFile.modTime == mTime) ) {
558 foundSlow = true;
559 break;
560 }
561 }
562 if ( !foundSlow ) {
563 foundMismatch = true;
564 if ( verbose )
565 fprintf(stderr, "rebuilding dyld cache because dylib changed: %s\n", installName);
566 }
567 }
568 });
569 ::munmap(p, cacheMapLen);
570 }
571
572 ::close(fd);
573
574 return !foundMismatch;
575 }
576
577
578 inline uint32_t absolutetime_to_milliseconds(uint64_t abstime)
579 {
580 return (uint32_t)(abstime/1000/1000);
581 }
582
583 static bool runningOnHaswell()
584 {
585 // check system is capable of running x86_64h code
586 struct host_basic_info info;
587 mach_msg_type_number_t count = HOST_BASIC_INFO_COUNT;
588 mach_port_t hostPort = mach_host_self();
589 kern_return_t result = host_info(hostPort, HOST_BASIC_INFO, (host_info_t)&info, &count);
590 mach_port_deallocate(mach_task_self(), hostPort);
591
592 return ( (result == KERN_SUCCESS) && (info.cpu_subtype == CPU_SUBTYPE_X86_64_H) );
593 }
594
595 #if !BUILDING_UPDATE_OTHER_DYLD_CACHE_BUILDER
596 static std::string currentToolRealPath()
597 {
598 char curToolPath[PATH_MAX];
599 uint32_t curToolPathsize = PATH_MAX;
600 int result = _NSGetExecutablePath(curToolPath, &curToolPathsize);
601 if ( result == 0 ) {
602 char resolvedCurToolPath[PATH_MAX];
603 if ( realpath(curToolPath, resolvedCurToolPath) != NULL )
604 return resolvedCurToolPath;
605 else
606 return curToolPath;
607 }
608 return "/usr/bin/update_dyld_shared_cache";
609 }
610 #endif
611
612 #define TERMINATE_IF_LAST_ARG( s ) \
613 do { \
614 if ( i == argc - 1 ) { \
615 fprintf(stderr, s ); \
616 return 1; \
617 } \
618 } while ( 0 )
619
620 int main(int argc, const char* argv[], const char* envp[])
621 {
622 std::string rootPath;
623 std::string overlayPath;
624 std::string dylibListFile;
625 bool universal = false;
626 bool force = false;
627 bool searchDisk = false;
628 bool dylibsRemoved = false;
629 std::string cacheDir;
630 std::unordered_set<std::string> archStrs;
631 std::unordered_set<std::string> skipDylibs;
632
633 // parse command line options
634 for (int i = 1; i < argc; ++i) {
635 const char* arg = argv[i];
636 if (strcmp(arg, "-debug") == 0) {
637 verbose = true;
638 }
639 else if (strcmp(arg, "-verbose") == 0) {
640 verbose = true;
641 }
642 else if (strcmp(arg, "-dont_map_local_symbols") == 0) {
643 //We are going to ignore this
644 }
645 else if (strcmp(arg, "-dylib_list") == 0) {
646 TERMINATE_IF_LAST_ARG("-dylib_list missing argument");
647 dylibListFile = argv[++i];
648 }
649 else if ((strcmp(arg, "-root") == 0) || (strcmp(arg, "--root") == 0)) {
650 TERMINATE_IF_LAST_ARG("-root missing path argument\n");
651 rootPath = argv[++i];
652 }
653 else if (strcmp(arg, "-overlay") == 0) {
654 TERMINATE_IF_LAST_ARG("-overlay missing path argument\n");
655 overlayPath = argv[++i];
656 }
657 else if (strcmp(arg, "-cache_dir") == 0) {
658 TERMINATE_IF_LAST_ARG("-cache_dir missing path argument\n");
659 cacheDir = argv[++i];
660 }
661 else if (strcmp(arg, "-arch") == 0) {
662 TERMINATE_IF_LAST_ARG("-arch missing argument\n");
663 archStrs.insert(argv[++i]);
664 }
665 else if (strcmp(arg, "-search_disk") == 0) {
666 searchDisk = true;
667 }
668 else if (strcmp(arg, "-dylibs_removed_in_mastering") == 0) {
669 dylibsRemoved = true;
670 }
671 else if (strcmp(arg, "-force") == 0) {
672 force = true;
673 }
674 else if (strcmp(arg, "-sort_by_name") == 0) {
675 //No-op, we always do this now
676 }
677 else if (strcmp(arg, "-universal_boot") == 0) {
678 universal = true;
679 }
680 else if (strcmp(arg, "-skip") == 0) {
681 TERMINATE_IF_LAST_ARG("-skip missing argument\n");
682 skipDylibs.insert(argv[++i]);
683 }
684 else {
685 //usage();
686 fprintf(stderr, "update_dyld_shared_cache: unknown option: %s\n", arg);
687 return 1;
688 }
689 }
690
691 if ( !rootPath.empty() & !overlayPath.empty() ) {
692 fprintf(stderr, "-root and -overlay cannot be used together\n");
693 return 1;
694 }
695 // canonicalize rootPath
696 if ( !rootPath.empty() ) {
697 char resolvedPath[PATH_MAX];
698 if ( realpath(rootPath.c_str(), resolvedPath) != NULL ) {
699 rootPath = resolvedPath;
700 }
701 // <rdar://problem/33223984> when building closures for boot volume, pathPrefixes should be empty
702 if ( rootPath == "/" ) {
703 rootPath = "";
704 }
705 }
706 // canonicalize overlayPath
707 if ( !overlayPath.empty() ) {
708 char resolvedPath[PATH_MAX];
709 if ( realpath(overlayPath.c_str(), resolvedPath) != NULL ) {
710 overlayPath = resolvedPath;
711 }
712 }
713
714 #if !BUILDING_UPDATE_OTHER_DYLD_CACHE_BUILDER
715 // <rdar://problem/36362221> update_dyld_shared_cache -root should re-exec() itself to a newer version
716 std::string newTool;
717 if ( !rootPath.empty() )
718 newTool = rootPath + "/usr/bin/update_dyld_shared_cache_root_mode";
719 else if ( !overlayPath.empty() )
720 newTool = overlayPath + "/usr/bin/update_dyld_shared_cache";
721 if ( !newTool.empty() ) {
722 struct stat newToolStatBuf;
723 if ( stat(newTool.c_str(), &newToolStatBuf) == 0 ) {
724 // don't re-exec if we are already running that tool
725 if ( newTool != currentToolRealPath() ) {
726 argv[0] = newTool.c_str();
727 execve(newTool.c_str(), (char**)argv, (char**)envp);
728 fprintf(stderr, "update_dyld_shared_cache: error: could not find '%s/usr/bin/update_dyld_shared_cache_root_mode' in target volume\n", rootPath.c_str());
729 return 1;
730 }
731 }
732 if ( !rootPath.empty() ) {
733 // could be old macOS dmg, try old tool name
734 newTool = rootPath + "/usr/bin/update_dyld_shared_cache";
735 if ( stat(newTool.c_str(), &newToolStatBuf) == 0 ) {
736 // don't re-exec if we are already running that tool
737 if ( newTool != currentToolRealPath() ) {
738 argv[0] = newTool.c_str();
739 execve(newTool.c_str(), (char**)argv, (char**)envp);
740 }
741 }
742 fprintf(stderr, "update_dyld_shared_cache: error: could not find '%s/usr/bin/update_dyld_shared_cache_root_mode' in target volume\n", rootPath.c_str());
743 return 1;
744 }
745 }
746 #else
747 if ( rootPath.empty() ) {
748 fprintf(stderr, "update_dyld_shared_cache_root_mode: error: -root option missing\n");
749 return 1;
750 }
751 #endif
752
753 // Find the boot volume so that we can ensure all overlays are on the same volume
754 struct stat rootStatBuf;
755 if ( stat(rootPath == "" ? "/" : rootPath.c_str(), &rootStatBuf) != 0 ) {
756 fprintf(stderr, "update_dyld_shared_cache: error: could not stat root file system because '%s'\n", strerror(errno));
757 return 1;
758 }
759 dev_t rootFS = rootStatBuf.st_dev;
760
761
762 //
763 // pathPrefixes for three modes:
764 // 1) no options: { "" } // search only boot volume
765 // 2) -overlay: { overlay, "" } // search overlay, then boot volume
766 // 3) -root: { root } // search only -root volume
767 //
768 std::vector<std::string> pathPrefixes;
769 if ( !overlayPath.empty() ) {
770 // Only add the overlay path if it exists, and is the same volume as the root
771 struct stat overlayStatBuf;
772 if ( stat(overlayPath.c_str(), &overlayStatBuf) != 0 ) {
773 fprintf(stderr, "update_dyld_shared_cache: warning: ignoring overlay dir '%s' because '%s'\n", overlayPath.c_str(), strerror(errno));
774 overlayPath.clear();
775 }
776 else {
777 char resolvedOverlayPath[PATH_MAX];
778 if ( realpath(overlayPath.c_str(), resolvedOverlayPath) != NULL ) {
779 overlayPath = resolvedOverlayPath;
780 }
781 else {
782 fprintf(stderr, "update_dyld_shared_cache: warning: ignoring overlay dir '%s' because realpath() failed\n", overlayPath.c_str());
783 overlayPath.clear();
784 }
785 }
786 if ( !overlayPath.empty() )
787 pathPrefixes.push_back(overlayPath);
788 }
789 pathPrefixes.push_back(rootPath);
790
791 // build FileSystem object
792 const char* fsRoot = rootPath.empty() ? nullptr : rootPath.c_str();
793 const char* fsOverlay = overlayPath.empty() ? nullptr : overlayPath.c_str();
794 dyld3::closure::FileSystemPhysical fileSystem(fsRoot, fsOverlay);
795
796 // normalize output directory
797 if ( cacheDir.empty() ) {
798 // if -cache_dir is not specified, then write() will eventually fail if we are not running as root
799 if ( geteuid() != 0 ) {
800 fprintf(stderr, "update_dyld_shared_cache: must be run as root (sudo)\n");
801 return 1;
802 }
803
804 // write cache file into -root or -overlay directory, if used
805 if ( rootPath != "/" )
806 cacheDir = rootPath + MACOSX_DYLD_SHARED_CACHE_DIR;
807 else if ( !overlayPath.empty() )
808 cacheDir = overlayPath + MACOSX_DYLD_SHARED_CACHE_DIR;
809 else
810 cacheDir = MACOSX_DYLD_SHARED_CACHE_DIR;
811 }
812 int err = mkpath_np(cacheDir.c_str(), S_IRWXU | S_IRGRP|S_IXGRP | S_IROTH|S_IXOTH);
813 if ( (err != 0) && (err != EEXIST) ) {
814 fprintf(stderr, "update_dyld_shared_cache: could not access cache dir: mkpath_np(%s) failed errno=%d\n", cacheDir.c_str(), err);
815 return 1;
816 }
817 // make sure cacheDir is always a real path, so it can be checked later to see if it changed
818 char resolvedCachePath[PATH_MAX];
819 ::realpath(cacheDir.c_str(), resolvedCachePath);
820 cacheDir = resolvedCachePath;
821
822 #if BUILDING_UPDATE_OTHER_DYLD_CACHE_BUILDER
823 bool requireDylibsBeRootlessProtected = false;
824 #else
825 bool requireDylibsBeRootlessProtected = isProtectedBySIPExceptDyld(cacheDir);
826 if ( requireDylibsBeRootlessProtected && !overlayPath.empty() && !isProtectedBySIP(overlayPath.c_str()) ) {
827 fprintf(stderr, "update_dyld_shared_cache: warning: ignoring overlay dir '%s' because it is not SIP protected\n", overlayPath.c_str());
828 overlayPath.clear();
829 pathPrefixes.clear();
830 pathPrefixes.push_back(rootPath);
831 }
832 #endif
833
834 if ( archStrs.empty() ) {
835 // <rdar://44190126> check if OS has enough i386 to make a shared cache
836 char realerPath[MAXPATHLEN];
837 Diagnostics testDiag;
838 const char* foundationPath = "/System/Library/Frameworks/Foundation.framework/Versions/C/Foundation";
839 dyld3::closure::LoadedFileInfo foundationInfo = dyld3::MachOAnalyzer::load(testDiag, fileSystem, foundationPath, dyld3::GradedArchs::i386, dyld3::Platform::macOS, realerPath);
840 bool foundationHas32bit = (foundationInfo.fileContent != NULL);
841 if ( foundationHas32bit )
842 fileSystem.unloadFile(foundationInfo);
843
844 if ( universal ) {
845 // <rdar://problem/26182089> -universal_boot should make all possible dyld caches
846 if ( foundationHas32bit )
847 archStrs.insert("i386");
848 archStrs.insert("x86_64");
849 archStrs.insert("x86_64h");
850 }
851 else {
852 // just make caches for this machine
853 if ( foundationHas32bit )
854 archStrs.insert("i386");
855 archStrs.insert(runningOnHaswell() ? "x86_64h" : "x86_64");
856 }
857 }
858
859 uint64_t t1 = mach_absolute_time();
860
861 // find all mach-o files for requested architectures
862 __block std::vector<MappedMachOsByCategory> allFileSets;
863 if ( archStrs.count("x86_64") )
864 allFileSets.push_back({dyld3::GradedArchs::x86_64});
865 if ( archStrs.count("x86_64h") )
866 allFileSets.push_back({dyld3::GradedArchs::x86_64h});
867 if ( archStrs.count("i386") )
868 allFileSets.push_back({dyld3::GradedArchs::i386});
869 if ( searchDisk )
870 findAllFiles(fileSystem, pathPrefixes, requireDylibsBeRootlessProtected, rootFS, allFileSets);
871 else {
872 std::unordered_set<std::string> runtimePathsFound;
873 findOSFilesViaBOMS(fileSystem, pathPrefixes, requireDylibsBeRootlessProtected, rootFS, allFileSets);
874 }
875
876 // nothing in OS uses i386 dylibs, so only dylibs used by third party apps need to be in cache
877 for (MappedMachOsByCategory& fileSet : allFileSets) {
878 pruneCachedDylibs(rootPath, skipDylibs, fileSet, verbose);
879 pruneOtherDylibs(rootPath, fileSet);
880 pruneExecutables(rootPath, fileSet);
881 }
882
883 uint64_t t2 = mach_absolute_time();
884 if ( verbose ) {
885 if ( searchDisk )
886 fprintf(stderr, "time to scan file system and construct lists of mach-o files: %ums\n", absolutetime_to_milliseconds(t2-t1));
887 else
888 fprintf(stderr, "time to read BOM and construct lists of mach-o files: %ums\n", absolutetime_to_milliseconds(t2-t1));
889 }
890
891 // build caches in parallel on machines with at leat 4GB of RAM
892 uint64_t memSize = 0;
893 size_t sz = sizeof(memSize);;
894 bool buildInParallel = false;
895 if ( sysctlbyname("hw.memsize", &memSize, &sz, NULL, 0) == 0 ) {
896 if ( memSize >= 0x100000000ULL )
897 buildInParallel = true;
898 }
899 dispatch_queue_t dqueue = buildInParallel ? dispatch_get_global_queue(DISPATCH_QUEUE_PRIORITY_DEFAULT, 0)
900 : dispatch_queue_create("serial-queue", DISPATCH_QUEUE_SERIAL);
901
902 // build all caches
903 __block bool cacheBuildFailure = false;
904 __block bool wroteSomeCacheFile = false;
905 dispatch_apply(allFileSets.size(), dqueue, ^(size_t index) {
906 MappedMachOsByCategory& fileSet = allFileSets[index];
907 const std::string outFile = cacheDir + "/dyld_shared_cache_" + fileSet.archs.name();
908
909 DyldSharedCache::MappedMachO (^loader)(const std::string&) = ^DyldSharedCache::MappedMachO(const std::string& runtimePath) {
910 if ( skipDylibs.count(runtimePath) )
911 return DyldSharedCache::MappedMachO();
912 if (fileSet.badZippered.count(runtimePath)) {
913 return DyldSharedCache::MappedMachO();
914 }
915 for (const std::string& prefix : pathPrefixes) {
916 std::string fullPath = prefix + runtimePath;
917 struct stat statBuf;
918 if ( stat(fullPath.c_str(), &statBuf) == 0 ) {
919 char resolvedPath[PATH_MAX];
920 if ( realpath(fullPath.c_str(), resolvedPath) != NULL ) {
921 std::string resolvedSymlink = resolvedPath;
922 if ( !rootPath.empty() ) {
923 resolvedSymlink = resolvedSymlink.substr(rootPath.size());
924 }
925 if ( (runtimePath != resolvedSymlink) && !contains(runtimePath, "InputContext") ) { //HACK remove InputContext when fixed
926 // path requested is a symlink path, check if real path already loaded
927 for (const DyldSharedCache::MappedMachO& aDylibMapping : fileSet.dylibsForCache) {
928 if ( aDylibMapping.runtimePath == resolvedSymlink ) {
929 if ( verbose )
930 fprintf(stderr, "verifySelfContained, redirect %s to %s\n", runtimePath.c_str(), aDylibMapping.runtimePath.c_str());
931 return aDylibMapping;
932 }
933 }
934 }
935 }
936
937 std::vector<MappedMachOsByCategory> mappedFiles;
938 mappedFiles.push_back({fileSet.archs});
939 if ( addIfMachO(fileSystem, runtimePath, statBuf, requireDylibsBeRootlessProtected, rootFS, mappedFiles) ) {
940 if ( !mappedFiles.back().dylibsForCache.empty() ) {
941 if ( verbose )
942 fprintf(stderr, "verifySelfContained, add %s\n", mappedFiles.back().dylibsForCache.back().runtimePath.c_str());
943 return mappedFiles.back().dylibsForCache.back();
944 }
945 }
946 }
947 }
948 return DyldSharedCache::MappedMachO();
949 };
950 size_t startCount = fileSet.dylibsForCache.size();
951 std::vector<std::pair<DyldSharedCache::MappedMachO, std::set<std::string>>> excludes;
952 DyldSharedCache::verifySelfContained(fileSet.dylibsForCache, fileSet.badZippered, loader, excludes);
953 for (size_t i=startCount; i < fileSet.dylibsForCache.size(); ++i) {
954 fprintf(stderr, "update_dyld_shared_cache: warning: %s not in .bom, but adding required dylib %s\n", fileSet.archs.name(), fileSet.dylibsForCache[i].runtimePath.c_str());
955 }
956 for (auto& exclude : excludes) {
957 std::string reasons = "(\"";
958 for (auto i = exclude.second.begin(); i != exclude.second.end(); ++i) {
959 reasons += *i;
960 if (i != --exclude.second.end()) {
961 reasons += "\", \"";
962 }
963 }
964 reasons += "\")";
965 fprintf(stderr, "update_dyld_shared_cache: warning: %s rejected from cached dylibs: %s (%s)\n", fileSet.archs.name(), exclude.first.runtimePath.c_str(), reasons.c_str());
966 fileSet.otherDylibsAndBundles.push_back(exclude.first);
967 }
968
969 // check if cache is already up to date
970 if ( !force ) {
971 if ( existingCacheUpToDate(outFile, fileSet.dylibsForCache) )
972 return;
973 }
974
975 // add any extra dylibs needed which were not in .bom
976 fprintf(stderr, "update_dyld_shared_cache: %s incorporating %lu OS dylibs, tracking %lu others, building closures for %lu executables\n",
977 fileSet.archs.name(), fileSet.dylibsForCache.size(), fileSet.otherDylibsAndBundles.size(), fileSet.mainExecutables.size());
978 //for (const DyldSharedCache::MappedMachO& aFile : fileSet.otherDylibsAndBundles) {
979 // fprintf(stderr, " %s\n", aFile.runtimePath.c_str());
980 //}
981
982
983 // build cache new cache file
984 DyldSharedCache::CreateOptions options;
985 options.outputFilePath = outFile;
986 options.outputMapFilePath = cacheDir + "/dyld_shared_cache_" + fileSet.archs.name() + ".map";
987 options.archs = &fileSet.archs;
988 options.platform = dyld3::Platform::macOS;
989 options.excludeLocalSymbols = false;
990 options.optimizeStubs = false;
991 options.optimizeObjC = true;
992 options.codeSigningDigestMode = DyldSharedCache::SHA256only;
993 options.dylibsRemovedDuringMastering = dylibsRemoved;
994 options.inodesAreSameAsRuntime = true;
995 options.cacheSupportsASLR = (&fileSet.archs != &dyld3::GradedArchs::i386);
996 options.forSimulator = false;
997 options.isLocallyBuiltCache = true;
998 options.verbose = verbose;
999 options.evictLeafDylibsOnOverflow = true;
1000 DyldSharedCache::CreateResults results = DyldSharedCache::create(options, fileSystem, fileSet.dylibsForCache, fileSet.otherDylibsAndBundles, fileSet.mainExecutables);
1001
1002 // print any warnings
1003 for (const std::string& warn : results.warnings) {
1004 fprintf(stderr, "update_dyld_shared_cache: warning: %s %s\n", fileSet.archs.name(), warn.c_str());
1005 }
1006 if ( results.errorMessage.empty() ) {
1007 wroteSomeCacheFile = true;
1008 }
1009 else {
1010 fprintf(stderr, "update_dyld_shared_cache: %s\n", results.errorMessage.c_str());
1011 cacheBuildFailure = true;
1012 }
1013 });
1014
1015
1016 // Save off spintrace data
1017 if ( wroteSomeCacheFile ) {
1018 void* h = dlopen("/usr/lib/libdscsym.dylib", 0);
1019 if ( h != nullptr ) {
1020 typedef int (*dscym_func)(const char*);
1021 dscym_func func = (dscym_func)dlsym(h, "dscsym_save_dscsyms_for_current_caches");
1022 std::string nuggetRoot = rootPath;
1023 if ( nuggetRoot.empty() )
1024 nuggetRoot = overlayPath;
1025 if ( nuggetRoot.empty() )
1026 nuggetRoot = "/";
1027 (*func)(nuggetRoot.c_str());
1028 }
1029 }
1030
1031
1032 // we could unmap all input files, but tool is about to quit
1033
1034 return (cacheBuildFailure ? 1 : 0);
1035 }
1036