1 /* -*- mode: C++; c-basic-offset: 4; indent-tabs-mode: nil -*-
3 * Copyright (c) 2016 Apple Inc. All rights reserved.
5 * @APPLE_LICENSE_HEADER_START@
7 * This file contains Original Code and/or Modifications of Original Code
8 * as defined in and that are subject to the Apple Public Source License
9 * Version 2.0 (the 'License'). You may not use this file except in
10 * compliance with the License. Please obtain a copy of the License at
11 * http://www.opensource.apple.com/apsl/ and read it before using this
14 * The Original Code and all software distributed under the License are
15 * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
16 * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
17 * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
18 * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
19 * Please see the License for the specific language governing rights and
20 * limitations under the License.
22 * @APPLE_LICENSE_HEADER_END@
25 #include <sys/types.h>
28 #include <mach/mach.h>
29 #include <mach/mach_time.h>
42 #include <sys/param.h>
43 #include <sys/sysctl.h>
44 #include <sys/resource.h>
48 #include <dispatch/dispatch.h>
49 #include <pthread/pthread.h>
53 #include <unordered_set>
54 #include <unordered_set>
58 #include "MachOParser.h"
59 #include "FileUtils.h"
60 #include "StringUtils.h"
61 #include "DyldSharedCache.h"
65 struct MappedMachOsByCategory
68 std::vector
<DyldSharedCache::MappedMachO
> dylibsForCache
;
69 std::vector
<DyldSharedCache::MappedMachO
> otherDylibsAndBundles
;
70 std::vector
<DyldSharedCache::MappedMachO
> mainExecutables
;
73 static const char* sSearchDirs
[] = {
80 static const char* sSkipDirs
[] = {
86 static const char* sMacOsAdditions
[] = {
87 "/usr/lib/system/libsystem_kernel.dylib",
88 "/usr/lib/system/libsystem_platform.dylib",
89 "/usr/lib/system/libsystem_pthread.dylib",
93 static bool verbose
= false;
95 static bool addIfMachO(const std::string
& simRuntimeRootPath
, const std::string
& runtimePath
, const struct stat
& statBuf
, dyld3::Platform platform
, std::vector
<MappedMachOsByCategory
>& files
)
97 // read start of file to determine if it is mach-o or a fat file
98 std::string fullPath
= simRuntimeRootPath
+ runtimePath
;
99 int fd
= ::open(fullPath
.c_str(), O_RDONLY
);
103 const void* wholeFile
= ::mmap(NULL
, statBuf
.st_size
, PROT_READ
, MAP_PRIVATE
, fd
, 0);
104 if ( wholeFile
!= MAP_FAILED
) {
106 bool usedWholeFile
= false;
107 for (MappedMachOsByCategory
& file
: files
) {
110 bool fatButMissingSlice
;
111 const void* slice
= MAP_FAILED
;
112 if ( dyld3::FatUtil::isFatFileWithSlice(diag
, wholeFile
, statBuf
.st_size
, file
.archName
, sliceOffset
, sliceLength
, fatButMissingSlice
) ) {
113 slice
= ::mmap(NULL
, sliceLength
, PROT_READ
, MAP_PRIVATE
, fd
, sliceOffset
);
114 if ( slice
!= MAP_FAILED
) {
115 //fprintf(stderr, "mapped slice at %p size=0x%0lX, offset=0x%0lX for %s\n", p, len, offset, fullPath.c_str());
116 if ( !dyld3::MachOParser::isValidMachO(diag
, file
.archName
, platform
, slice
, sliceLength
, fullPath
.c_str(), false) ) {
117 ::munmap((void*)slice
, sliceLength
);
122 else if ( !fatButMissingSlice
&& dyld3::MachOParser::isValidMachO(diag
, file
.archName
, platform
, wholeFile
, statBuf
.st_size
, fullPath
.c_str(), false) ) {
124 sliceLength
= statBuf
.st_size
;
126 usedWholeFile
= true;
127 //fprintf(stderr, "mapped whole file at %p size=0x%0lX for %s\n", p, len, inputPath.c_str());
129 if ( slice
!= MAP_FAILED
) {
130 const mach_header
* mh
= (mach_header
*)slice
;
131 dyld3::MachOParser
parser(mh
);
132 if ( parser
.platform() != platform
) {
133 fprintf(stderr
, "skipped wrong platform binary: %s\n", fullPath
.c_str());
137 bool sip
= true; // assume anything found in the simulator runtime is a platform binary
138 if ( parser
.isDynamicExecutable() ) {
139 bool issetuid
= (statBuf
.st_mode
& (S_ISUID
|S_ISGID
));
140 file
.mainExecutables
.emplace_back(runtimePath
, mh
, sliceLength
, issetuid
, sip
, sliceOffset
, statBuf
.st_mtime
, statBuf
.st_ino
);
143 if ( parser
.canBePlacedInDyldCache(runtimePath
) ) {
144 file
.dylibsForCache
.emplace_back(runtimePath
, mh
, sliceLength
, false, sip
, sliceOffset
, statBuf
.st_mtime
, statBuf
.st_ino
);
147 file
.otherDylibsAndBundles
.emplace_back(runtimePath
, mh
, sliceLength
, false, sip
, sliceOffset
, statBuf
.st_mtime
, statBuf
.st_ino
);
154 if ( !usedWholeFile
)
155 ::munmap((void*)wholeFile
, statBuf
.st_size
);
161 static void findAllFiles(const std::string
& simRuntimeRootPath
, dyld3::Platform platform
, std::vector
<MappedMachOsByCategory
>& files
)
163 std::unordered_set
<std::string
> skipDirs
;
164 for (const char* s
: sSkipDirs
)
167 for (const char* searchDir
: sSearchDirs
) {
168 iterateDirectoryTree(simRuntimeRootPath
, searchDir
, ^(const std::string
& dirPath
) { return (skipDirs
.count(dirPath
) != 0); }, ^(const std::string
& path
, const struct stat
& statBuf
) {
169 // ignore files that don't have 'x' bit set (all runnable mach-o files do)
170 const bool hasXBit
= ((statBuf
.st_mode
& S_IXOTH
) == S_IXOTH
);
171 if ( !hasXBit
&& !endsWith(path
, ".dylib") )
174 // ignore files too small
175 if ( statBuf
.st_size
< 0x3000 )
178 // if the file is mach-o add to list
179 addIfMachO(simRuntimeRootPath
, path
, statBuf
, platform
, files
);
184 static void addMacOSAdditions(std::vector
<MappedMachOsByCategory
>& allFileSets
)
186 for (const char* addPath
: sMacOsAdditions
) {
188 if ( stat(addPath
, &statBuf
) == 0 )
189 addIfMachO("", addPath
, statBuf
, dyld3::Platform::macOS
, allFileSets
);
194 static bool dontCache(const std::string
& simRuntimeRootPath
, const std::string
& archName
,
195 const std::unordered_set
<std::string
>& pathsWithDuplicateInstallName
,
196 const DyldSharedCache::MappedMachO
& aFile
, bool warn
)
198 if ( startsWith(aFile
.runtimePath
, "/usr/lib/system/introspection/") )
200 if ( startsWith(aFile
.runtimePath
, "/usr/local/") )
203 // anything inside a .app bundle is specific to app, so should be in shared cache
204 if ( aFile
.runtimePath
.find(".app/") != std::string::npos
)
207 if ( aFile
.runtimePath
.find("//") != std::string::npos
) {
208 if (warn
) fprintf(stderr
, "update_dyld_sim_shared_cache: warning: %s double-slash in install name %s\n", archName
.c_str(), aFile
.runtimePath
.c_str());
211 dyld3::MachOParser
parser(aFile
.mh
);
212 const char* installName
= parser
.installName();
213 if ( (pathsWithDuplicateInstallName
.count(aFile
.runtimePath
) != 0) && (aFile
.runtimePath
!= installName
) ) {
214 if (warn
) fprintf(stderr
, "update_dyld_sim_shared_cache: warning: %s skipping because of duplicate install name %s\n", archName
.c_str(), aFile
.runtimePath
.c_str());
218 if ( aFile
.runtimePath
!= installName
) {
219 // see if install name is a symlink to actual path
220 std::string fullInstall
= simRuntimeRootPath
+ installName
;
221 char resolvedPath
[PATH_MAX
];
222 if ( realpath(fullInstall
.c_str(), resolvedPath
) != NULL
) {
223 std::string resolvedSymlink
= resolvedPath
;
224 if ( !simRuntimeRootPath
.empty() ) {
225 resolvedSymlink
= resolvedSymlink
.substr(simRuntimeRootPath
.size());
227 if ( aFile
.runtimePath
== resolvedSymlink
) {
231 if (warn
) fprintf(stderr
, "update_dyld_sim_shared_cache: warning: %s skipping because of bad install name %s\n", archName
.c_str(), aFile
.runtimePath
.c_str());
238 static void pruneCachedDylibs(const std::string
& simRuntimeRootPath
, MappedMachOsByCategory
& fileSet
)
240 std::unordered_set
<std::string
> pathsWithDuplicateInstallName
;
242 std::unordered_map
<std::string
, std::string
> installNameToFirstPath
;
243 for (DyldSharedCache::MappedMachO
& aFile
: fileSet
.dylibsForCache
) {
244 //fprintf(stderr, "dylib: %s\n", aFile.runtimePath.c_str());
245 dyld3::MachOParser
parser(aFile
.mh
);
246 const char* installName
= parser
.installName();
247 auto pos
= installNameToFirstPath
.find(installName
);
248 if ( pos
== installNameToFirstPath
.end() ) {
249 installNameToFirstPath
[installName
] = aFile
.runtimePath
;
252 pathsWithDuplicateInstallName
.insert(aFile
.runtimePath
);
253 pathsWithDuplicateInstallName
.insert(installNameToFirstPath
[installName
]);
257 for (DyldSharedCache::MappedMachO
& aFile
: fileSet
.dylibsForCache
) {
258 if ( dontCache(simRuntimeRootPath
, fileSet
.archName
, pathsWithDuplicateInstallName
, aFile
, true) )
259 fileSet
.otherDylibsAndBundles
.push_back(aFile
);
261 fileSet
.dylibsForCache
.erase(std::remove_if(fileSet
.dylibsForCache
.begin(), fileSet
.dylibsForCache
.end(),
262 [&](const DyldSharedCache::MappedMachO
& aFile
) { return dontCache(simRuntimeRootPath
, fileSet
.archName
, pathsWithDuplicateInstallName
, aFile
, false); }),
263 fileSet
.dylibsForCache
.end());
266 static bool existingCacheUpToDate(const std::string
& existingCache
, const std::vector
<DyldSharedCache::MappedMachO
>& currentDylibs
)
268 // if no existing cache, it is not up-to-date
269 int fd
= ::open(existingCache
.c_str(), O_RDONLY
);
273 // build map of found dylibs
274 std::unordered_map
<std::string
, const DyldSharedCache::MappedMachO
*> currentDylibMap
;
275 for (const DyldSharedCache::MappedMachO
& aFile
: currentDylibs
) {
276 //fprintf(stderr, "0x%0llX 0x%0llX %s\n", aFile.inode, aFile.modTime, aFile.runtimePath.c_str());
277 currentDylibMap
[aFile
.runtimePath
] = &aFile
;
280 // make sure all dylibs in existing cache have same mtime and inode as found dylib
281 __block
bool foundMismatch
= false;
282 const uint64_t cacheMapLen
= 0x40000000;
283 void *p
= ::mmap(NULL
, cacheMapLen
, PROT_READ
, MAP_PRIVATE
, fd
, 0);
284 if ( p
!= MAP_FAILED
) {
285 const DyldSharedCache
* cache
= (DyldSharedCache
*)p
;
286 cache
->forEachImageEntry(^(const char* installName
, uint64_t mTime
, uint64_t inode
) {
287 bool foundMatch
= false;
288 auto pos
= currentDylibMap
.find(installName
);
289 if ( pos
!= currentDylibMap
.end() ) {
290 const DyldSharedCache::MappedMachO
* foundDylib
= pos
->second
;
291 if ( (foundDylib
->inode
== inode
) && (foundDylib
->modTime
== mTime
) ) {
296 // use slow path and look for any dylib with a matching inode and mtime
297 bool foundSlow
= false;
298 for (const DyldSharedCache::MappedMachO
& aFile
: currentDylibs
) {
299 if ( (aFile
.inode
== inode
) && (aFile
.modTime
== mTime
) ) {
305 foundMismatch
= true;
307 fprintf(stderr
, "rebuilding dyld cache because dylib changed: %s\n", installName
);
311 ::munmap(p
, cacheMapLen
);
316 return !foundMismatch
;
320 inline uint32_t absolutetime_to_milliseconds(uint64_t abstime
)
322 return (uint32_t)(abstime
/1000/1000);
326 #define TERMINATE_IF_LAST_ARG( s ) \
328 if ( i == argc - 1 ) { \
329 fprintf(stderr, s ); \
334 int main(int argc
, const char* argv
[])
336 std::string rootPath
;
337 std::string dylibListFile
;
339 std::string cacheDir
;
340 std::unordered_set
<std::string
> archStrs
;
342 dyld3::Platform platform
= dyld3::Platform::iOS
;
344 // parse command line options
345 for (int i
= 1; i
< argc
; ++i
) {
346 const char* arg
= argv
[i
];
347 if (strcmp(arg
, "-debug") == 0) {
350 else if (strcmp(arg
, "-verbose") == 0) {
353 else if (strcmp(arg
, "-tvOS") == 0) {
354 platform
= dyld3::Platform::tvOS
;
356 else if (strcmp(arg
, "-iOS") == 0) {
357 platform
= dyld3::Platform::iOS
;
359 else if (strcmp(arg
, "-watchOS") == 0) {
360 platform
= dyld3::Platform::watchOS
;
362 else if ( strcmp(arg
, "-runtime_dir") == 0 ) {
363 TERMINATE_IF_LAST_ARG("-runtime_dir missing path argument\n");
364 rootPath
= argv
[++i
];
366 else if (strcmp(arg
, "-cache_dir") == 0) {
367 TERMINATE_IF_LAST_ARG("-cache_dir missing path argument\n");
368 cacheDir
= argv
[++i
];
370 else if (strcmp(arg
, "-arch") == 0) {
371 TERMINATE_IF_LAST_ARG("-arch missing argument\n");
372 archStrs
.insert(argv
[++i
]);
374 else if (strcmp(arg
, "-force") == 0) {
379 fprintf(stderr
, "update_dyld_sim_shared_cache: unknown option: %s\n", arg
);
384 if ( cacheDir
.empty() ) {
385 fprintf(stderr
, "missing -cache_dir <path> option to specify directory in which to write cache file(s)\n");
389 if ( rootPath
.empty() ) {
390 fprintf(stderr
, "missing -runtime_dir <path> option to specify directory which is root of simulator runtime)\n");
394 // canonicalize rootPath
395 char resolvedPath
[PATH_MAX
];
396 if ( realpath(rootPath
.c_str(), resolvedPath
) != NULL
) {
397 rootPath
= resolvedPath
;
401 int err
= mkpath_np(cacheDir
.c_str(), S_IRWXU
| S_IRGRP
|S_IXGRP
| S_IROTH
|S_IXOTH
);
402 if ( (err
!= 0) && (err
!= EEXIST
) ) {
403 fprintf(stderr
, "mkpath_np fail: %d", err
);
407 if ( archStrs
.empty() ) {
408 switch ( platform
) {
409 case dyld3::Platform::iOS
:
410 archStrs
.insert("x86_64");
412 case dyld3::Platform::tvOS
:
413 archStrs
.insert("x86_64");
415 case dyld3::Platform::watchOS
:
416 archStrs
.insert("i386");
418 case dyld3::Platform::unknown
:
419 case dyld3::Platform::macOS
:
420 assert(0 && "macOS does not have a simulator");
422 case dyld3::Platform::bridgeOS
:
423 assert(0 && "bridgeOS does not have a simulator");
428 uint64_t t1
= mach_absolute_time();
430 // find all mach-o files for requested architectures
431 __block
std::vector
<MappedMachOsByCategory
> allFileSets
;
432 if ( archStrs
.count("x86_64") )
433 allFileSets
.push_back({"x86_64"});
434 if ( archStrs
.count("i386") )
435 allFileSets
.push_back({"i386"});
436 findAllFiles(rootPath
, platform
, allFileSets
);
437 addMacOSAdditions(allFileSets
);
438 for (MappedMachOsByCategory
& fileSet
: allFileSets
) {
439 pruneCachedDylibs(rootPath
, fileSet
);
442 uint64_t t2
= mach_absolute_time();
444 fprintf(stderr
, "time to scan file system and construct lists of mach-o files: %ums\n", absolutetime_to_milliseconds(t2
-t1
));
446 // build all caches in parallel
447 __block
bool cacheBuildFailure
= false;
448 dispatch_apply(allFileSets
.size(), dispatch_get_global_queue(DISPATCH_QUEUE_PRIORITY_DEFAULT
, 0), ^(size_t index
) {
449 MappedMachOsByCategory
& fileSet
= allFileSets
[index
];
450 const std::string outFile
= cacheDir
+ "/dyld_shared_cache_" + fileSet
.archName
;
451 __block
std::unordered_set
<std::string
> knownMissingDylib
;
453 DyldSharedCache::MappedMachO (^loader
)(const std::string
&) = ^DyldSharedCache::MappedMachO(const std::string
& runtimePath
) {
454 std::string fullPath
= rootPath
+ runtimePath
;
456 if ( stat(fullPath
.c_str(), &statBuf
) == 0 ) {
457 std::vector
<MappedMachOsByCategory
> mappedFiles
;
458 mappedFiles
.push_back({fileSet
.archName
});
459 if ( addIfMachO(rootPath
, runtimePath
, statBuf
, platform
, mappedFiles
) ) {
460 if ( !mappedFiles
.back().dylibsForCache
.empty() )
461 return mappedFiles
.back().dylibsForCache
.back();
464 if ( knownMissingDylib
.count(runtimePath
) == 0 ) {
465 fprintf(stderr
, "update_dyld_sim_shared_cache: warning: %s could not use in dylid cache: %s\n", fileSet
.archName
.c_str(), runtimePath
.c_str());
466 knownMissingDylib
.insert(runtimePath
);
468 return DyldSharedCache::MappedMachO();
470 size_t startCount
= fileSet
.dylibsForCache
.size();
471 std::vector
<std::pair
<DyldSharedCache::MappedMachO
, std::set
<std::string
>>> excludes
;
472 DyldSharedCache::verifySelfContained(fileSet
.dylibsForCache
, loader
, excludes
);
473 for (size_t i
=startCount
; i
< fileSet
.dylibsForCache
.size(); ++i
) {
474 fprintf(stderr
, "update_dyld_sim_shared_cache: warning: %s not found in initial scan, but adding required dylib %s\n", fileSet
.archName
.c_str(), fileSet
.dylibsForCache
[i
].runtimePath
.c_str());
476 for (auto& exclude
: excludes
) {
477 std::string reasons
= "(\"";
478 for (auto i
= exclude
.second
.begin(); i
!= exclude
.second
.end(); ++i
) {
480 if (i
!= --exclude
.second
.end()) {
485 fprintf(stderr
, "update_dyld_shared_cache: warning: %s rejected from cached dylibs: %s (%s)\n", fileSet
.archName
.c_str(), exclude
.first
.runtimePath
.c_str(), reasons
.c_str());
486 fileSet
.otherDylibsAndBundles
.push_back(exclude
.first
);
489 // check if cache is already up to date
491 if ( existingCacheUpToDate(outFile
, fileSet
.dylibsForCache
) )
494 fprintf(stderr
, "make %s cache with %lu dylibs, %lu other dylibs, %lu programs\n", fileSet
.archName
.c_str(), fileSet
.dylibsForCache
.size(), fileSet
.otherDylibsAndBundles
.size(), fileSet
.mainExecutables
.size());
496 // build cache new cache file
497 DyldSharedCache::CreateOptions options
;
498 options
.archName
= fileSet
.archName
;
499 options
.platform
= platform
;
500 options
.excludeLocalSymbols
= false;
501 options
.optimizeStubs
= false;
502 options
.optimizeObjC
= true;
503 options
.codeSigningDigestMode
= DyldSharedCache::SHA256only
;
504 options
.dylibsRemovedDuringMastering
= false;
505 options
.inodesAreSameAsRuntime
= true;
506 options
.cacheSupportsASLR
= false;
507 options
.forSimulator
= true;
508 options
.verbose
= verbose
;
509 options
.evictLeafDylibsOnOverflow
= true;
510 options
.pathPrefixes
= { rootPath
};
511 DyldSharedCache::CreateResults results
= DyldSharedCache::create(options
, fileSet
.dylibsForCache
, fileSet
.otherDylibsAndBundles
, fileSet
.mainExecutables
);
513 // print any warnings
514 for (const std::string
& warn
: results
.warnings
) {
515 fprintf(stderr
, "update_dyld_sim_shared_cache: warning: %s %s\n", fileSet
.archName
.c_str(), warn
.c_str());
517 if ( !results
.errorMessage
.empty() ) {
518 // print error (if one)
519 fprintf(stderr
, "update_dyld_sim_shared_cache: %s\n", results
.errorMessage
.c_str());
520 cacheBuildFailure
= true;
523 // save new cache file to disk and write new .map file
524 assert(results
.cacheContent
!= nullptr);
525 if ( !safeSave(results
.cacheContent
, results
.cacheLength
, outFile
) )
526 cacheBuildFailure
= true;
527 if ( !cacheBuildFailure
) {
528 std::string mapStr
= results
.cacheContent
->mapFile();
529 std::string outFileMap
= cacheDir
+ "/dyld_shared_cache_" + fileSet
.archName
+ ".map";
530 safeSave(mapStr
.c_str(), mapStr
.size(), outFileMap
);
532 // free created cache buffer
533 vm_deallocate(mach_task_self(), (vm_address_t
)results
.cacheContent
, results
.cacheLength
);
537 // we could unmap all input files, but tool is about to quit
539 return (cacheBuildFailure
? 1 : 0);