From: Apple Date: Wed, 29 Mar 2017 20:04:43 +0000 (+0000) Subject: dyld-433.5.tar.gz X-Git-Tag: macos-10124^0 X-Git-Url: https://git.saurik.com/apple/dyld.git/commitdiff_plain/d113e8b5ffc80ec29316b2eb968dfaa7502c1da8 dyld-433.5.tar.gz --- diff --git a/bin/expand.pl b/bin/expand.pl new file mode 100755 index 0000000..b21ef3f --- /dev/null +++ b/bin/expand.pl @@ -0,0 +1,66 @@ +#!/usr/bin/perl + +use strict; + + +my $sdk = $ENV{"SDKROOT"}; +my $availCmd = $sdk . "/usr/local/libexec/availability.pl"; + +sub expandVersions +{ + my $macroPrefix = shift; + my $availArg = shift; + + my $cmd = $availCmd . " " . $availArg; + my $versionList = `$cmd`; + my $tmp = $versionList; + while ($tmp =~ m/^\s*([\S]+)(.*)$/) { + my $vers = $1; + $tmp = $2; + + my $major = 0; + my $minor = 0; + my $revision = 0; + my $uvers; + + if ($vers =~ m/^(\d+)$/) { + $major = $1; + $uvers = sprintf("%d_0", $major); + } elsif ($vers =~ m/^(\d+).(\d+)$/) { + $major = $1; + $minor = $2; + $uvers = sprintf("%d_%d", $major, $minor); + } elsif ($vers =~ m/^(\d+).(\d+).(\d+)$/) { + $major = $1; + $minor = $2; + $revision = $3; + if ($revision == 0) { + $uvers = sprintf("%d_%d", $major, $minor); + } + else { + $uvers = sprintf("%d_%d_%d", $major, $minor, $revision); + } + } + printf "#define %s%-18s 0x00%02X%02X%02X\n", $macroPrefix, $uvers, $major, $minor, $revision; + } +} + + + + +while() +{ + if(m/^\/\/\@MAC_VERSION_DEFS\@$/) { + expandVersions("DYLD_MACOSX_VERSION_", "--macosx"); + } + elsif(m/^\/\/\@IOS_VERSION_DEFS\@$/) { + expandVersions("DYLD_IOS_VERSION_", "--ios"); + } + elsif(m/^\/\/\@WATCHOS_VERSION_DEFS\@$/) { + expandVersions("DYLD_WATCHOS_VERSION_", "--watchos"); + } + else { + print $_; + } +} + diff --git a/doc/man/man1/dyld.1 b/doc/man/man1/dyld.1 index 4c9e61d..ad1cec3 100644 --- a/doc/man/man1/dyld.1 +++ b/doc/man/man1/dyld.1 @@ -16,8 +16,6 @@ DYLD_VERSIONED_LIBRARY_PATH .br DYLD_PRINT_TO_FILE .br -DYLD_ROOT_PATH -.br DYLD_SHARED_REGION .br DYLD_INSERT_LIBRARIES @@ -152,10 +150,6 @@ logging output (triggered by DYLD_PRINT_* settings) to file descriptor 2 (which is usually stderr). But this setting causes the dynamic linker to write logging output to the specified file. .TP -.B DYLD_ROOT_PATH -This is a colon separated list of directories. The dynamic linker will prepend each of -this directory paths to every image access until a file is found. -.TP .B DYLD_SHARED_REGION This can be "use" (the default), "avoid", or "private". Setting it to "avoid" tells dyld to not use the shared cache. All OS dylibs are loaded diff --git a/dyld.xcodeproj/project.pbxproj b/dyld.xcodeproj/project.pbxproj index 89a7c1f..49d05ff 100644 --- a/dyld.xcodeproj/project.pbxproj +++ b/dyld.xcodeproj/project.pbxproj @@ -33,6 +33,7 @@ F908137111D3FB5000626CC1 /* usr|local|include|mach-o */, F908137211D3FB5000626CC1 /* usr|share|man|man1 */, F908137311D3FB5000626CC1 /* usr|share|man|man3 */, + F96D19701D7F62C3007AF3CE /* Install dyld_priv.h */, ); dependencies = ( F9B4D78012AD9736000605A6 /* PBXTargetDependency */, @@ -119,7 +120,6 @@ F908135D11D3FACD00626CC1 /* dyld-interposing.h in usr|local|include|mach-o */ = {isa = PBXBuildFile; fileRef = F918691408B16D2500E0F9DB /* dyld-interposing.h */; }; F908135E11D3FACD00626CC1 /* dyld_cache_format.h in usr|local|include|mach-o */ = {isa = PBXBuildFile; fileRef = F93937400A94FC4700070A07 /* dyld_cache_format.h */; }; F908135F11D3FACD00626CC1 /* dyld_gdb.h in usr|local|include|mach-o */ = {isa = PBXBuildFile; fileRef = F9ED4CE80630A80600DF4E74 /* dyld_gdb.h */; }; - F908136011D3FACD00626CC1 /* dyld_priv.h in usr|local|include|mach-o */ = {isa = PBXBuildFile; fileRef = F9ED4CE90630A80600DF4E74 /* dyld_priv.h */; }; F908136411D3FB0300626CC1 /* dyld.1 in usr|share|man|man1 */ = {isa = PBXBuildFile; fileRef = EF799FE9070D27BB00F78484 /* dyld.1 */; }; F908136811D3FB3A00626CC1 /* dladdr.3 in usr|share|man|man3 */ = {isa = PBXBuildFile; fileRef = EF799FEB070D27BB00F78484 /* dladdr.3 */; }; F908136911D3FB3A00626CC1 /* dlclose.3 in usr|share|man|man3 */ = {isa = PBXBuildFile; fileRef = EF799FEC070D27BB00F78484 /* dlclose.3 */; }; @@ -359,7 +359,6 @@ F908135E11D3FACD00626CC1 /* dyld_cache_format.h in usr|local|include|mach-o */, F9FF8C161C69B080009F8A53 /* dyld_process_info.h in usr|local|include|mach-o */, F908135F11D3FACD00626CC1 /* dyld_gdb.h in usr|local|include|mach-o */, - F908136011D3FACD00626CC1 /* dyld_priv.h in usr|local|include|mach-o */, ); name = "usr|local|include|mach-o"; runOnlyForDeploymentPostprocessing = 1; @@ -485,6 +484,7 @@ F958D4751C7FCD4A00A0B199 /* dyld_process_info_internal.h */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.h; name = dyld_process_info_internal.h; path = src/dyld_process_info_internal.h; sourceTree = ""; }; F958D4761C7FCD4A00A0B199 /* dyld_process_info_notify.cpp */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.cpp.cpp; name = dyld_process_info_notify.cpp; path = src/dyld_process_info_notify.cpp; sourceTree = ""; }; F95C95160E994796007B7CB8 /* MachOTrie.hpp */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.cpp.h; path = MachOTrie.hpp; sourceTree = ""; }; + F96D19711D7F63EE007AF3CE /* expand.pl */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = text.script.perl; name = expand.pl; path = bin/expand.pl; sourceTree = ""; }; F971DD131A4A0E0700BBDD52 /* base.xcconfig */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = text.xcconfig; path = base.xcconfig; sourceTree = ""; }; F971DD141A4A0E0700BBDD52 /* dyld.xcconfig */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = text.xcconfig; path = dyld.xcconfig; sourceTree = ""; }; F971DD151A4A0E0700BBDD52 /* libdyld.xcconfig */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = text.xcconfig; path = libdyld.xcconfig; sourceTree = ""; }; @@ -810,6 +810,7 @@ F9ED4CBE0630A7B100DF4E74 /* include */ = { isa = PBXGroup; children = ( + F96D19711D7F63EE007AF3CE /* expand.pl */, F95090D01C5AB89A0031F81D /* dyld_process_info.h */, F918691408B16D2500E0F9DB /* dyld-interposing.h */, F98D274C0AA79D7400416316 /* dyld_images.h */, @@ -942,6 +943,7 @@ buildConfigurationList = F9D8C7DD087B087300E93EFB /* Build configuration list for PBXNativeTarget "dyld" */; buildPhases = ( F9D050C811DD701A00FB0A29 /* configure archives */, + F96D19A31D91D733007AF3CE /* make dyld_priv.h */, F9ED4C950630A76000DF4E74 /* Sources */, F907E2490FA6469000BFEDBD /* install iPhone file */, F9213B3F18BFC9CB001CB6E8 /* simulator entitlement */, @@ -1140,6 +1142,38 @@ shellScript = "if [[ \"${PLATFORM_NAME}\" == *simulator* ]]\nthen\n\tcd ${DSTROOT}/${INSTALL_PATH}\n\tln -s libdyld.dylib libdyld_sim.dylib\nfi\n"; showEnvVarsInLog = 0; }; + F96D19701D7F62C3007AF3CE /* Install dyld_priv.h */ = { + isa = PBXShellScriptBuildPhase; + buildActionMask = 8; + files = ( + ); + inputPaths = ( + ); + name = "Install dyld_priv.h"; + outputPaths = ( + ); + runOnlyForDeploymentPostprocessing = 1; + shellPath = /bin/sh; + shellScript = "${SRCROOT}/bin/expand.pl < ${SRCROOT}/include/mach-o/dyld_priv.h > ${DSTROOT}/${INSTALL_PATH_PREFIX}/usr/local/include/mach-o/dyld_priv.h\n"; + showEnvVarsInLog = 0; + }; + F96D19A31D91D733007AF3CE /* make dyld_priv.h */ = { + isa = PBXShellScriptBuildPhase; + buildActionMask = 2147483647; + files = ( + ); + inputPaths = ( + "$(SRCROOT)/include/mach-o/dyld_priv.h", + ); + name = "make dyld_priv.h"; + outputPaths = ( + "$(DERIVED_FILE_DIR)/mach-o/dyld_priv.h", + ); + runOnlyForDeploymentPostprocessing = 0; + shellPath = /bin/sh; + shellScript = "mkdir -p ${DERIVED_FILE_DIR}/mach-o\n${SRCROOT}/bin/expand.pl < ${SRCROOT}/include/mach-o/dyld_priv.h > ${DERIVED_FILE_DIR}/mach-o/dyld_priv.h\n"; + showEnvVarsInLog = 0; + }; F991E3030FF1A4EC0082CCC9 /* do not install duplicates */ = { isa = PBXShellScriptBuildPhase; buildActionMask = 8; @@ -1909,6 +1943,7 @@ GCC_WARN_TYPECHECK_CALLS_TO_PRINTF = YES; GCC_WARN_UNINITIALIZED_AUTOS = NO; HEADER_SEARCH_PATHS = ( + "$(DERIVED_FILE_DIR)/**", ./include, "./launch-cache", ); @@ -1959,6 +1994,7 @@ GCC_WARN_SHADOW = YES; GCC_WARN_TYPECHECK_CALLS_TO_PRINTF = YES; HEADER_SEARCH_PATHS = ( + "$(DERIVED_FILE_DIR)/**", ./include, "./launch-cache", ); diff --git a/include/mach-o/dyld_priv.h b/include/mach-o/dyld_priv.h index db108fd..2cd4006 100644 --- a/include/mach-o/dyld_priv.h +++ b/include/mach-o/dyld_priv.h @@ -184,50 +184,12 @@ extern const struct mach_header* dyld_image_header_containing_address(const void // Convienence constants for return values from dyld_get_sdk_version() and friends. -#define DYLD_MACOSX_VERSION_10_4 0x000A0400 -#define DYLD_MACOSX_VERSION_10_5 0x000A0500 -#define DYLD_MACOSX_VERSION_10_6 0x000A0600 -#define DYLD_MACOSX_VERSION_10_7 0x000A0700 -#define DYLD_MACOSX_VERSION_10_8 0x000A0800 -#define DYLD_MACOSX_VERSION_10_9 0x000A0900 -#define DYLD_MACOSX_VERSION_10_10 0x000A0A00 -#define DYLD_MACOSX_VERSION_10_11 0x000A0B00 -#define DYLD_MACOSX_VERSION_10_12 0x000A0C00 - -#define DYLD_IOS_VERSION_2_0 0x00020000 -#define DYLD_IOS_VERSION_2_1 0x00020100 -#define DYLD_IOS_VERSION_2_2 0x00020200 -#define DYLD_IOS_VERSION_3_0 0x00030000 -#define DYLD_IOS_VERSION_3_1 0x00030100 -#define DYLD_IOS_VERSION_3_2 0x00030200 -#define DYLD_IOS_VERSION_4_0 0x00040000 -#define DYLD_IOS_VERSION_4_1 0x00040100 -#define DYLD_IOS_VERSION_4_2 0x00040200 -#define DYLD_IOS_VERSION_4_3 0x00040300 -#define DYLD_IOS_VERSION_5_0 0x00050000 -#define DYLD_IOS_VERSION_5_1 0x00050100 -#define DYLD_IOS_VERSION_6_0 0x00060000 -#define DYLD_IOS_VERSION_6_1 0x00060100 -#define DYLD_IOS_VERSION_7_0 0x00070000 -#define DYLD_IOS_VERSION_7_1 0x00070100 -#define DYLD_IOS_VERSION_8_0 0x00080000 -#define DYLD_IOS_VERSION_8_1 0x00080100 -#define DYLD_IOS_VERSION_8_2 0x00080200 -#define DYLD_IOS_VERSION_8_3 0x00080300 -#define DYLD_IOS_VERSION_8_4 0x00080400 -#define DYLD_IOS_VERSION_9_0 0x00090000 -#define DYLD_IOS_VERSION_9_1 0x00090100 -#define DYLD_IOS_VERSION_9_2 0x00090200 -#define DYLD_IOS_VERSION_9_3 0x00090300 -#define DYLD_IOS_VERSION_10_0 0x000A0000 - - -#define DYLD_WATCHOS_VERSION_1_0 0x00010000 -#define DYLD_WATCHOS_VERSION_2_0 0x00020000 -#define DYLD_WATCHOS_VERSION_2_1 0x00020100 -#define DYLD_WATCHOS_VERSION_2_2 0x00020200 -#define DYLD_WATCHOS_VERSION_3_0 0x00030000 +//@MAC_VERSION_DEFS@ + +//@IOS_VERSION_DEFS@ + +//@WATCHOS_VERSION_DEFS@ // @@ -388,6 +350,15 @@ extern bool _dyld_get_image_uuid(const struct mach_header* mh, uuid_t uuid); extern bool _dyld_get_shared_cache_uuid(uuid_t uuid); +// +// Returns the start address of the dyld cache in the process and sets length to the size of the cache. +// Returns NULL if the process is not using a dyld shared cache +// +// Exists in Mac OS X 10.13 and later +// Exists in iOS 11.0 and later +extern const void* _dyld_get_shared_cache_range(size_t* length); + + // // When dyld must terminate a process because of a required dependent dylib diff --git a/include/mach-o/dyld_process_info.h b/include/mach-o/dyld_process_info.h index 67bf25d..8e7629f 100644 --- a/include/mach-o/dyld_process_info.h +++ b/include/mach-o/dyld_process_info.h @@ -122,6 +122,10 @@ extern dyld_process_info_notify _dyld_process_info_notify(task_t task, dispatch_ void (^notify)(bool unload, uint64_t timestamp, uint64_t machHeader, const uuid_t uuid, const char* path), void (^notifyExit)(), kern_return_t* kernelError); +// add block to call right before main() is entered. +// does nothing if process is already in main(). +extern void _dyld_process_info_notify_main(dyld_process_info_notify objc, void (^notifyMain)()); + // stop notifications and invalid dyld_process_info_notify object extern void _dyld_process_info_notify_release(dyld_process_info_notify object); diff --git a/include/objc-shared-cache.h b/include/objc-shared-cache.h index ef5c109..6f29230 100644 --- a/include/objc-shared-cache.h +++ b/include/objc-shared-cache.h @@ -114,13 +114,13 @@ static uint64_t lookup8( uint8_t *k, size_t length, uint64_t level); // Perfect hash code is at the end of this file. -struct perfect_hash { +struct __attribute__((packed)) perfect_hash { uint32_t capacity; uint32_t occupied; uint32_t shift; uint32_t mask; uint64_t salt; - + uint32_t scramble[256]; uint8_t *tab; // count == mask+1; free with delete[] @@ -159,7 +159,7 @@ static perfect_hash make_perfect(const string_map& strings); // Precomputed perfect hash table of strings. // Base class for precomputed selector table and class table. // Edit objc-sel-table.s if you change this structure. -struct objc_stringhash_t { +struct __attribute__((packed)) objc_stringhash_t { uint32_t capacity; uint32_t occupied; uint32_t shift; @@ -167,7 +167,7 @@ struct objc_stringhash_t { uint32_t unused1; // was zero uint32_t unused2; // alignment pad uint64_t salt; - + uint32_t scramble[256]; uint8_t tab[0]; /* tab[mask+1] (always power-of-2) */ // uint8_t checkbytes[capacity]; /* check byte for each string */ diff --git a/interlinked-dylibs/BindAllImages.cpp b/interlinked-dylibs/BindAllImages.cpp index 8bd9e6e..0fbc09c 100644 --- a/interlinked-dylibs/BindAllImages.cpp +++ b/interlinked-dylibs/BindAllImages.cpp @@ -43,6 +43,8 @@ #include "dyld_cache_config.h" +#include "MachOProxy.h" + #if !NEW_CACHE_FILE_FORMAT #include "CacheFileAbstraction.hpp" #endif @@ -57,31 +59,18 @@ namespace { template class BindInfo { public: - BindInfo(void* cacheBuffer, macho_header

* mh); + BindInfo(void* cacheBuffer, const std::map>& segmentMap, macho_header

* mh, MachOProxy* proxy); void setReExports(const std::unordered_map*>& dylibPathToBindInfo); void setDependentDylibs(const std::unordered_map*>& dylibPathToBindInfo); void bind(const std::unordered_map*>& dylibPathToBindInfo, std::vector& pointersForASLR); - static void bindAllImagesInCache(void* cacheBuffer, const std::unordered_map& dylibPathToMachHeader, std::vector& pointersForASLR); - - void addExportsToGlobalMap(std::unordered_map*>& reverseMap); + static void bindAllImagesInCache(void* cacheBuffer, const std::vector dylibs, const std::map>& segmentMap, std::vector& pointersForASLR); private: typedef typename P::uint_t pint_t; typedef typename P::E E; - struct SymbolInfo { - SymbolInfo() { } - pint_t address = 0; - bool isResolver = false; - bool isAbsolute = false; - bool isSymbolReExport = false; - bool isThreadLocal = false; - int reExportDylibIndex = 0; - std::string reExportName; - }; - void bindImmediates(const std::unordered_map*>& dylibPathToBindInfo, std::vector& pointersForASLR); void bindLazyPointers(const std::unordered_map*>& dylibPathToBindInfo, std::vector& pointersForASLR); @@ -101,6 +90,7 @@ private: void* _cacheBuffer; macho_header

* _mh; + MachOProxy* _proxy; const uint8_t* _linkeditBias; const char* _installName; const macho_symtab_command

* _symTabCmd; @@ -110,7 +100,7 @@ private: std::vector _segSizes; std::vector _segCacheOffsets; std::vector*>_segCmds; - std::unordered_map _exports; + const std::map>& _segmentMap; std::vector _reExportedDylibNames; std::vector*> _reExportedDylibs; std::vector*> _dependentDylibs; @@ -119,10 +109,17 @@ private: ResolverToBlessedLazyPointerMap _resolverBlessedMap; }; - template -BindInfo

::BindInfo(void* cacheBuffer, macho_header

* mh) - : _cacheBuffer(cacheBuffer), _mh(mh), _linkeditBias((uint8_t*)cacheBuffer), _symTabCmd(nullptr), _dynSymTabCmd(nullptr), _dyldInfo(nullptr), _baseAddress(0) +BindInfo

::BindInfo(void* cacheBuffer, const std::map>& segmentMap, macho_header

* mh, MachOProxy* proxy) + : _cacheBuffer(cacheBuffer) + , _mh(mh) + , _segmentMap(segmentMap) + , _proxy(proxy) + , _linkeditBias((uint8_t*)cacheBuffer) + , _symTabCmd(nullptr) + , _dynSymTabCmd(nullptr) + , _dyldInfo(nullptr) + , _baseAddress(0) { macho_segment_command

* segCmd; macho_dylib_command

* dylibCmd; @@ -180,35 +177,6 @@ BindInfo

::BindInfo(void* cacheBuffer, macho_header

* mh) if ( !ExportInfoTrie::parseTrie(exportsStart, exportsEnd, exports) ) { terminate("malformed exports trie in %s", _installName); } - - for(const ExportInfoTrie::Entry& entry : exports) { - _exports[entry.name].address = (pint_t)entry.info.address + _baseAddress; - switch ( entry.info.flags & EXPORT_SYMBOL_FLAGS_KIND_MASK ) { - case EXPORT_SYMBOL_FLAGS_KIND_REGULAR: - if ( (entry.info.flags & EXPORT_SYMBOL_FLAGS_STUB_AND_RESOLVER) ) { - _exports[entry.name].isResolver = true; - } - if ( entry.info.flags & EXPORT_SYMBOL_FLAGS_REEXPORT ) { - SymbolInfo& info = _exports[entry.name]; - info.isSymbolReExport = true; - info.reExportDylibIndex = (int)entry.info.other; - if ( !entry.info.importName.empty()) - info.reExportName = entry.info.importName; - } - break; - case EXPORT_SYMBOL_FLAGS_KIND_THREAD_LOCAL: - _exports[entry.name].isThreadLocal = true; - break; - case EXPORT_SYMBOL_FLAGS_KIND_ABSOLUTE: - _exports[entry.name].isAbsolute = true; - _exports[entry.name].address = (pint_t)entry.info.address; - break; - default: - terminate("non-regular symbol binding not supported for %s in %s", entry.name.c_str(), _installName); - break; - } - } - } template @@ -460,7 +428,6 @@ void BindInfo

::bindLocation(uint8_t segmentIndex, uint64_t segmentOffset, uin pint_t lpVMAddr = targetBinder->findBlessedLazyPointerFor(symbolName); // switch stub to use shared lazy pointer to reduce dirty pages this->switchStubToUseSharedLazyPointer(symbolName, lpVMAddr); - return; } @@ -493,13 +460,13 @@ template bool BindInfo

::findExportedSymbolAddress(const char* symbolName, const std::unordered_map*>& dylibPathToBindInfo, pint_t* address, BindInfo

** foundIn, bool* isResolverSymbol, bool* isAbsolute) { - auto pos = _exports.find(symbolName); - if ( pos != _exports.end() ) { - if ( pos->second.isSymbolReExport ) { + auto info = _proxy->symbolInfo(symbolName); + if (info != nullptr) { + if (info->isSymbolReExport) { const char* importName = symbolName; - if ( !pos->second.reExportName.empty() ) - importName = pos->second.reExportName.c_str(); - std::string& depPath = _dependentPaths[pos->second.reExportDylibIndex-1]; + if (!info->reExportName.empty()) + importName = info->reExportName.c_str(); + std::string& depPath = _dependentPaths[info->reExportDylibIndex - 1]; auto pos2 = dylibPathToBindInfo.find(depPath); if ( pos2 != dylibPathToBindInfo.end() ) { BindInfo

* reExportFrom = pos2->second; @@ -509,10 +476,11 @@ bool BindInfo

::findExportedSymbolAddress(const char* symbolName, const std::u verboseLog("findExportedSymbolAddress(%s) => ???\n", symbolName); } } - *address = pos->second.address; + + *address = (pint_t)_proxy->addressOf(symbolName, _segmentMap); *foundIn = this; - *isResolverSymbol = pos->second.isResolver; - *isAbsolute = pos->second.isAbsolute; + *isResolverSymbol = info->isResolver; + *isAbsolute = info->isAbsolute; //verboseLog("findExportedSymbolAddress(%s) => 0x0%llX\n", symbolName, (uint64_t)*address); return true; } @@ -524,26 +492,6 @@ bool BindInfo

::findExportedSymbolAddress(const char* symbolName, const std::u return false; } -template -void BindInfo

::addExportsToGlobalMap(std::unordered_map*>& reverseMap) -{ - for (const auto& expEntry : _exports) { - const std::string& symName = expEntry.first; - auto pos = reverseMap.find(symName); - if ( pos == reverseMap.end() ) { - reverseMap[symName] = this; - } - else { - BindInfo

* other = pos->second; - if ( expEntry.second.isSymbolReExport ) - continue; - if ( other->_exports[symName].isSymbolReExport ) - continue; - //warning("symbol '%s' exported from %s and %s\n", symName.c_str(), this->_installName, other->_installName); - } - } -} - template typename P::uint_t BindInfo

::findBlessedLazyPointerFor(const std::string& resolverSymbolName) { @@ -557,18 +505,17 @@ typename P::uint_t BindInfo

::findBlessedLazyPointerFor(const std::string& res // if this symbol is re-exported from another dylib, look there bool thisDylibImplementsResolver = false; - auto pos = _exports.find(resolverSymbolName); - if ( pos != _exports.end() ) { - const SymbolInfo& info = pos->second; - if ( info.isSymbolReExport ) { + const auto info = _proxy->symbolInfo(resolverSymbolName); + if (info) { + if (info->isSymbolReExport) { std::string reImportName = resolverSymbolName; - if ( !info.reExportName.empty() ) - reImportName = info.reExportName; - if ( info.reExportDylibIndex > _dependentDylibs.size() ) { - warning("dylib index for re-exported symbol %s too large (%d) in %s", resolverSymbolName.c_str(), info.reExportDylibIndex, _installName); + if (!info->reExportName.empty()) + reImportName = info->reExportName; + if (info->reExportDylibIndex > _dependentDylibs.size()) { + warning("dylib index for re-exported symbol %s too large (%d) in %s", resolverSymbolName.c_str(), info->reExportDylibIndex, _installName); } else { - BindInfo

* reExportedFrom = _dependentDylibs[info.reExportDylibIndex-1]; + BindInfo

* reExportedFrom = _dependentDylibs[info->reExportDylibIndex - 1]; if ( log ) verboseLog( "following re-export of %s in %s, to %s in %s", resolverSymbolName.c_str(), _installName, reImportName.c_str(), reExportedFrom->_installName); pint_t lp = reExportedFrom->findBlessedLazyPointerFor(reImportName); if ( lp != 0 ) { @@ -577,7 +524,7 @@ typename P::uint_t BindInfo

::findBlessedLazyPointerFor(const std::string& res } } } - if ( info.isResolver ) + if (info->isResolver) thisDylibImplementsResolver = true; } @@ -700,19 +647,23 @@ void BindInfo

::switchArmStubsLazyPointer(uint8_t* stubMappedAddress, pint_t s E::set32(instructions[3], betterOffset); } - template -void BindInfo

::bindAllImagesInCache(void* cacheBuffer, const std::unordered_map& dylibPathToMachHeader, std::vector& pointersForASLR) +void BindInfo

::bindAllImagesInCache(void* cacheBuffer, const std::vector dylibs, const std::map>& segmentMap, std::vector& pointersForASLR) { - // build BindInfo object for each dylib - std::unordered_map*, BindInfo

*> headersToBindInfo; std::unordered_map*> dylibPathToBindInfo; - for (const auto& entry: dylibPathToMachHeader) { - macho_header

* mh = (macho_header

*)entry.second; - if ( headersToBindInfo.count(mh) == 0 ) - headersToBindInfo[mh] = new BindInfo

(cacheBuffer, mh); - dylibPathToBindInfo[entry.first] = headersToBindInfo[mh]; - } + std::unordered_map*, BindInfo

*> headersToBindInfo; + for (auto& dylib : dylibs) { + auto dylibI = segmentMap.find(dylib); + assert(dylibI != segmentMap.end()); + macho_header

* mh = (macho_header

*)((uint8_t*)cacheBuffer + dylibI->second[0].cacheFileOffset); + if (headersToBindInfo.count(mh) == 0) { + headersToBindInfo[mh] = new BindInfo

(cacheBuffer, segmentMap, mh, dylib); + } + dylibPathToBindInfo[dylib->installName] = headersToBindInfo[mh]; + for (const auto& alias : dylib->installNameAliases) { + dylibPathToBindInfo[alias] = headersToBindInfo[mh]; + } + } // chain re-exported dylibs for (const auto& entry: headersToBindInfo) { @@ -725,12 +676,6 @@ void BindInfo

::bindAllImagesInCache(void* cacheBuffer, const std::unordered_m entry.second->bind(dylibPathToBindInfo, pointersForASLR); } - // look for exported symbol collisions - std::unordered_map*> reverseMap; - for (const auto& entry: headersToBindInfo) { - entry.second->addExportsToGlobalMap(reverseMap); - } - // clean up for (const auto& entry: headersToBindInfo) { delete entry.second; @@ -740,17 +685,16 @@ void BindInfo

::bindAllImagesInCache(void* cacheBuffer, const std::unordered_m } // anonymous namespace - -void SharedCache::bindAllImagesInCache(const std::unordered_map& dylibPathToMachHeader, std::vector& pointersForASLR) +void SharedCache::bindAllImagesInCache(const std::vector dylibs, const std::map>& segmentMap, std::vector& pointersForASLR) { switch ( _arch.arch ) { case CPU_TYPE_ARM: case CPU_TYPE_I386: - BindInfo>::bindAllImagesInCache(_buffer.get(), dylibPathToMachHeader, pointersForASLR); + BindInfo>::bindAllImagesInCache(_buffer.get(), dylibs, segmentMap, pointersForASLR); break; case CPU_TYPE_X86_64: case CPU_TYPE_ARM64: - BindInfo>::bindAllImagesInCache(_buffer.get(), dylibPathToMachHeader, pointersForASLR); + BindInfo>::bindAllImagesInCache(_buffer.get(), dylibs, segmentMap, pointersForASLR); break; default: terminate("unsupported arch 0x%08X", _arch.arch); diff --git a/interlinked-dylibs/FileCache.cpp b/interlinked-dylibs/FileCache.cpp index 3ccb9a1..4178426 100644 --- a/interlinked-dylibs/FileCache.cpp +++ b/interlinked-dylibs/FileCache.cpp @@ -157,7 +157,7 @@ std::string normalize_absolute_file_path(const std::string &path) { FileCache fileCache; FileCache::FileCache(void) { - cache_queue = dispatch_queue_create("com.apple.dyld.cache.cache", DISPATCH_QUEUE_SERIAL); + cache_queue = dispatch_queue_create("com.apple.dyld.cache.cache", dispatch_queue_attr_make_with_qos_class(DISPATCH_QUEUE_SERIAL, QOS_CLASS_USER_INITIATED, 0)); } @@ -172,59 +172,87 @@ void FileCache::preflightCache(const std::string& path) cacheBuilderDispatchAsync(cache_queue, [=] { std::string normalizedPath = normalize_absolute_file_path(path); if (entries.count(normalizedPath) == 0) { - fill(normalizedPath); + entries[normalizedPath] = fill(normalizedPath); } }); } std::tuple FileCache::cacheLoad(const std::string path) { + bool found = false; + std::tuple retval; std::string normalizedPath = normalize_absolute_file_path(path); - cacheBuilderDispatchSync(cache_queue, [=] { - if ( entries.count(normalizedPath) == 0 ) - fill(normalizedPath); + cacheBuilderDispatchSync(cache_queue, [=, &found, &retval] { + auto entry = entries.find(normalizedPath); + if (entry != entries.end()) { + retval = entry->second; + found = true; + } }); - return entries[normalizedPath]; -} + if (!found) { + auto info = fill(normalizedPath); + cacheBuilderDispatchSync(cache_queue, [=, &found, &retval] { + auto entry = entries.find(normalizedPath); + if (entry != entries.end()) { + retval = entry->second; + }else { + retval = entries[normalizedPath] = info; + retval = info; + } + }); + } + return retval; +} //FIXME error handling -void FileCache::fill(const std::string& path) { +std::tuple FileCache::fill(const std::string& path) { struct stat stat_buf; int fd = ::open(path.c_str(), O_RDONLY, 0); if ( fd == -1 ) { verboseLog("can't open file '%s', errno=%d", path.c_str(), errno); - entries[path] = std::make_tuple((uint8_t*)(-1), stat_buf, false); - return; + return std::make_tuple((uint8_t*)(-1), stat_buf, false); } if ( fstat(fd, &stat_buf) == -1) { verboseLog("can't stat open file '%s', errno=%d", path.c_str(), errno); - entries[path] = std::make_tuple((uint8_t*)(-1), stat_buf, false); ::close(fd); - return; + return std::make_tuple((uint8_t*)(-1), stat_buf, false); } if ( stat_buf.st_size < 4096 ) { verboseLog("file too small '%s'", path.c_str()); - entries[path] = std::make_tuple((uint8_t*)(-1), stat_buf, false); ::close(fd); - return; + return std::make_tuple((uint8_t*)(-1), stat_buf, false); } bool rootlessProtected = isProtectedBySIP(path, fd); void* buffer_ptr = mmap(NULL, stat_buf.st_size, PROT_READ, MAP_PRIVATE, fd, 0); if (buffer_ptr == MAP_FAILED) { - verboseLog("mmap() for shared cache at %s failed, errno=%d", path.c_str(), errno); - entries[path] = std::make_tuple((uint8_t*)(-1), stat_buf, false); + verboseLog("mmap() for file at %s failed, errno=%d", path.c_str(), errno); ::close(fd); - return; + return std::make_tuple((uint8_t*)(-1), stat_buf, false); } - entries[path] = std::make_tuple((uint8_t*)buffer_ptr, stat_buf, rootlessProtected); ::close(fd); + + //PERF-HACK: touch bits of the MachO before we need them to speed things up on a spinning disk + madvise((void*)buffer_ptr, 4096, MADV_WILLNEED); + + //if it is fat we need to touch each architecture + const fat_header* fh = (fat_header*)buffer_ptr; + if ( OSSwapBigToHostInt32( fh->magic ) == FAT_MAGIC ) { + const fat_arch* slices = (const fat_arch*)( (char*)fh + sizeof( fat_header ) ); + const uint32_t sliceCount = OSSwapBigToHostInt32( fh->nfat_arch ); + for ( uint32_t i = 0; i < sliceCount; ++i ) { + uint32_t fileOffset = OSSwapBigToHostInt32( slices[i].offset ); + madvise((void*)((uint8_t *)buffer_ptr+fileOffset), 4096, MADV_WILLNEED); + } + } + + return std::make_tuple((uint8_t*)buffer_ptr, stat_buf, rootlessProtected); } diff --git a/interlinked-dylibs/Logging.cpp b/interlinked-dylibs/Logging.cpp index 0f744ef..296b551 100644 --- a/interlinked-dylibs/Logging.cpp +++ b/interlinked-dylibs/Logging.cpp @@ -32,7 +32,6 @@ #include "Logging.h" -#define MAX_LOG_STR_LEN (512) //const char* kDispatchQueueLogNameKey = "kDispatchQueueLogNameKey"; const char* kDispatchWarningArrayKey = "kDispatchWarningArrayKey"; @@ -142,27 +141,24 @@ void setReturnNonZeroOnTerminate() void queued_print(FILE* __restrict fd, const char* str) { - const char* qstr = strdup(str); - dispatch_async(getLogQueue(), ^{ - (void)fprintf(fd, "%s", qstr); - free((void*)qstr); + (void)fprintf(fd, "%s", str); + free((void*)str); }); } #define VLOG(header) \ va_list list; \ va_start(list, format); \ - char temp[MAX_LOG_STR_LEN]; \ - vsprintf(temp, format, list); \ + char *temp, *temp2; \ + vasprintf(&temp, format, list); \ auto ctx = getLoggingContext(); \ - char temp2[MAX_LOG_STR_LEN]; \ if (ctx && !ctx->name().empty()) { \ - snprintf(temp2, MAX_LOG_STR_LEN, "[%s] %s%s\n", ctx->name().c_str(), header, \ - temp); \ + asprintf(&temp2, "[%s] %s%s\n", ctx->name().c_str(), header, temp); \ } else { \ - snprintf(temp2, MAX_LOG_STR_LEN, "%s%s\n", header, temp); \ + asprintf(&temp2, "%s%s\n", header, temp); \ } \ + free(temp); \ queued_print(stderr, temp2); \ va_end(list); @@ -188,14 +184,13 @@ void warning(const char* format, ...) va_list list; va_start(list, format); - char temp[MAX_LOG_STR_LEN]; - vsprintf(temp, format, list); - char* blockTemp = strdup(temp); + char* blockTemp; + vasprintf(&blockTemp, format, list); auto ctx = getLoggingContext(); if (ctx) { for (auto& target : ctx->targets().second) { - ctx->targets().first->configurations[target.first].architectures[target.second].results.warnings.push_back(blockTemp); + ctx->targets().first->configuration(target.first).architecture(target.second).results.warnings.push_back(blockTemp); } } @@ -218,7 +213,7 @@ void terminate(const char* format, ...) if (ctx) { // We are a work in a logging context, throw - throw std::string(temp); + throw std::string(temp2); } else { // We are in general handing, let the loggging queue darain and exit dispatch_sync(getLogQueue(), ^{ diff --git a/interlinked-dylibs/Logging.h b/interlinked-dylibs/Logging.h index db4c992..807261a 100644 --- a/interlinked-dylibs/Logging.h +++ b/interlinked-dylibs/Logging.h @@ -93,11 +93,11 @@ std::function* heapSafe(BodyFtor&& Body, std::shared_ptrtargets(); - for (auto& target : warningTargets.second) { - warningTargets.first->configurations[target.first].architectures[target.second].results.failure = exception; - } if (context) { + WarningTargets warningTargets = context->targets(); + for (auto& target : warningTargets.second) { + warningTargets.first->configuration(target.first).architecture(target.second).results.failure = exception; + } context->taint(); } } catch (...) { diff --git a/interlinked-dylibs/MachOProxy.cpp b/interlinked-dylibs/MachOProxy.cpp index e1923f8..a7418b9 100644 --- a/interlinked-dylibs/MachOProxy.cpp +++ b/interlinked-dylibs/MachOProxy.cpp @@ -12,18 +12,24 @@ #include "mega-dylib-utils.h" #include "Logging.h" +#include "Trie.hpp" #include "MachOProxy.h" +#ifndef EXPORT_SYMBOL_FLAGS_KIND_ABSOLUTE +#define EXPORT_SYMBOL_FLAGS_KIND_ABSOLUTE 0x02 +#endif + namespace { -std::map mapMachOFile( const std::string& path ) { - std::map retval; +std::vector mapMachOFile(const std::string& buildPath, const std::string& path) +{ + std::vector retval; const uint8_t* p = (uint8_t*)( -1 ); struct stat stat_buf; bool rootless; - std::tie( p, stat_buf, rootless ) = fileCache.cacheLoad( path ); + std::tie(p, stat_buf, rootless) = fileCache.cacheLoad(buildPath); - if ( p == (uint8_t*)( -1 ) ) { + if (p == (uint8_t*)(-1)) { return retval; } @@ -41,19 +47,105 @@ std::map mapMachOFile( const std::string& path ) { const mach_header* th = (mach_header*)(p+fileOffset); if ( ( OSSwapLittleToHostInt32( th->magic ) == MH_MAGIC ) || ( OSSwapLittleToHostInt32( th->magic ) == MH_MAGIC_64 ) ) { uint32_t fileSize = static_cast( stat_buf.st_size ); - retval[stringForArch( arch )] = new MachOProxy( path, stat_buf.st_ino, stat_buf.st_mtime, fileOffset, fileSize, rootless ); + retval.push_back(new MachOProxy(buildPath, path, stringForArch(arch), stat_buf.st_ino, stat_buf.st_mtime, fileOffset, fileSize, rootless)); + //retval[stringForArch( arch )] = new MachOProxy( path, stat_buf.st_ino, stat_buf.st_mtime, fileOffset, fileSize, rootless ); } } } else if ( ( OSSwapLittleToHostInt32( mh->magic ) == MH_MAGIC ) || ( OSSwapLittleToHostInt32( mh->magic ) == MH_MAGIC_64 ) ) { ArchPair arch( OSSwapLittleToHostInt32( mh->cputype ), OSSwapLittleToHostInt32( mh->cpusubtype ) ); uint32_t fileOffset = OSSwapBigToHostInt32( 0 ); uint32_t fileSize = static_cast( stat_buf.st_size ); - retval[stringForArch( arch )] = new MachOProxy( path, stat_buf.st_ino, stat_buf.st_mtime, fileOffset, fileSize, rootless ); + retval.push_back(new MachOProxy(buildPath, path, stringForArch(arch), stat_buf.st_ino, stat_buf.st_mtime, fileOffset, fileSize, rootless)); + //retval[stringForArch( arch )] = new MachOProxy( path, stat_buf.st_ino, stat_buf.st_mtime, fileOffset, fileSize, rootless ); } else { // warning( "file '%s' is not contain requested a MachO", path.c_str() ); } return retval; } + +} /* Anonymous namespace */ + +template +std::vector MachOProxy::dependencies() +{ + const uint8_t* buffer = getBuffer(); + const macho_header

* mh = (const macho_header

*)buffer; + const macho_load_command

* const cmds = (macho_load_command

*)((uint8_t*)mh + sizeof(macho_header

)); + const uint32_t cmd_count = mh->ncmds(); + const macho_load_command

* cmd = cmds; + std::vector retval; + + for (uint32_t i = 0; i < cmd_count; ++i) { + switch (cmd->cmd()) { + case LC_LOAD_DYLIB: + case LC_LOAD_WEAK_DYLIB: + case LC_REEXPORT_DYLIB: + case LC_LOAD_UPWARD_DYLIB: { + macho_dylib_command

* dylib = (macho_dylib_command

*)cmd; + std::string depName = dylib->name(); + + retval.push_back(depName); + } break; + } + cmd = (const macho_load_command

*)(((uint8_t*)cmd) + cmd->cmdsize()); + } + + return retval; +} + +template +std::vector MachOProxy::reexports() +{ + const uint8_t* buffer = getBuffer(); + const macho_header

* mh = (const macho_header

*)buffer; + const macho_load_command

* const cmds = (macho_load_command

*)((uint8_t*)mh + sizeof(macho_header

)); + const uint32_t cmd_count = mh->ncmds(); + const macho_load_command

* cmd = cmds; + std::vector retval; + + for (uint32_t i = 0; i < cmd_count; ++i) { + switch (cmd->cmd()) { + case LC_REEXPORT_DYLIB: { + macho_dylib_command

* dylib = (macho_dylib_command

*)cmd; + std::string depName = dylib->name(); + + retval.push_back(depName); + } break; + } + cmd = (const macho_load_command

*)(((uint8_t*)cmd) + cmd->cmdsize()); + } + + return retval; +} + +std::vector MachOProxy::dependencies() +{ + switch (archForString(arch).arch) { + case CPU_TYPE_ARM: + case CPU_TYPE_I386: + return dependencies>(); + case CPU_TYPE_X86_64: + case CPU_TYPE_ARM64: + return dependencies>(); + break; + default: + return std::vector(); + } +} + +std::vector MachOProxy::reexports() +{ + switch (archForString(arch).arch) { + case CPU_TYPE_ARM: + case CPU_TYPE_I386: + return reexports>(); + case CPU_TYPE_X86_64: + case CPU_TYPE_ARM64: + return reexports>(); + break; + default: + return std::vector(); + } } template @@ -66,13 +158,17 @@ std::string MachOProxy::machoParser(bool ignoreUncacheableDylibsInExecutables) const macho_symtab_command

* symTab = nullptr; const macho_dysymtab_command

* dynSymTab = nullptr; const macho_load_command

* const cmds = (macho_load_command

*)((uint8_t*)mh + sizeof(macho_header

)); + const macho_dyld_info_command

* dyldInfo = nullptr; const uint32_t cmd_count = mh->ncmds(); const macho_load_command

* cmd = cmds; - _dylib = (mh->filetype() == MH_DYLIB); - _executable = (mh->filetype() == MH_EXECUTE); - if (mh->filetype() == MH_DYLIB_STUB) { + uint64_t baseAddr = 0; + _filetype = mh->filetype(); + if (_filetype == MH_DYLIB_STUB) { return "stub dylib"; } + if (_filetype == MH_DSYM) { + return "DSYM"; + } for (uint32_t i = 0; i < cmd_count; ++i) { switch (cmd->cmd()) { case LC_ID_DYLIB: { @@ -85,10 +181,11 @@ std::string MachOProxy::machoParser(bool ignoreUncacheableDylibsInExecutables) } installName = dylib->name(); installNameOffsetInTEXT = (uint32_t)((uint8_t*)cmd - buffer) + dylib->name_offset(); + addAlias(path); } break; case LC_UUID: { const macho_uuid_command

* uuidCmd = (macho_uuid_command

*)cmd; - ::memcpy(uuid, uuidCmd->uuid(), sizeof(uuid_t)); + uuid = UUID(uuidCmd->uuid()); } break; case LC_LOAD_DYLIB: case LC_LOAD_WEAK_DYLIB: @@ -96,20 +193,20 @@ std::string MachOProxy::machoParser(bool ignoreUncacheableDylibsInExecutables) case LC_LOAD_UPWARD_DYLIB: { macho_dylib_command

* dylib = (macho_dylib_command

*)cmd; std::string depName = dylib->name(); - if ( _executable && ignoreUncacheableDylibsInExecutables && !has_prefix(depName, "/usr/lib/") && !has_prefix(depName, "/System/Library/") ) { + if ( isExecutable() && ignoreUncacheableDylibsInExecutables && !has_prefix(depName, "/usr/lib/") && !has_prefix(depName, "/System/Library/") ) { // in update_dyld_shared_cache don't warn if root executable links with something not eligible for shared cache break; } else if ( depName[0] != '/' ) { return "linked against a dylib whose -install_name was non-absolute (e.g. @rpath)"; } - dependencies.insert(depName); } break; case macho_segment_command

::CMD: { const macho_segment_command

* segCmd = (macho_segment_command

*)cmd; - MachOProxy::Segment seg; + MachOProxySegment seg; seg.name = segCmd->segname(); seg.size = align(segCmd->vmsize(), 12); + seg.vmaddr = segCmd->vmaddr(); seg.diskSize = (uint32_t)segCmd->filesize(); seg.fileOffset = (uint32_t)segCmd->fileoff(); seg.protection = segCmd->initprot(); @@ -127,6 +224,9 @@ std::string MachOProxy::machoParser(bool ignoreUncacheableDylibsInExecutables) seg.p2align = 12; } segments.push_back(seg); + if (seg.name == "__TEXT") { + baseAddr = seg.vmaddr; + } } break; case LC_SEGMENT_SPLIT_INFO: hasSplitSegInfo = true; @@ -139,20 +239,82 @@ std::string MachOProxy::machoParser(bool ignoreUncacheableDylibsInExecutables) break; case LC_DYLD_INFO: case LC_DYLD_INFO_ONLY: + dyldInfo = (macho_dyld_info_command

*)cmd; hasDylidInfo = true; break; } cmd = (const macho_load_command

*)(((uint8_t*)cmd) + cmd->cmdsize()); } - if (!_dylib) { - return ""; - } + identifier = uuid; if (!hasDylidInfo) { return "built for old OS"; } + if (dyldInfo && dyldInfo->bind_size() != 0) { + _bind_offset = dyldInfo->bind_off(); + _bind_size = dyldInfo->bind_size(); + } + + if (dyldInfo && dyldInfo->lazy_bind_size() != 0) { + _lazy_bind_offset = dyldInfo->lazy_bind_off(); + _lazy_bind_size = dyldInfo->lazy_bind_size(); + } + + // if no export info, no _exports map to build + if (dyldInfo && dyldInfo->export_size() != 0) { + std::vector exports; + const uint8_t* exportsStart = &buffer[dyldInfo->export_off()]; + const uint8_t* exportsEnd = &exportsStart[dyldInfo->export_size()]; + if (!ExportInfoTrie::parseTrie(exportsStart, exportsEnd, exports)) { + terminate("malformed exports trie in %s", path.c_str()); + } + + for (const ExportInfoTrie::Entry& entry : exports) { + if (!_exports[entry.name].isAbsolute) { + for (const auto& seg : segments) { + if (seg.size > 0 && (seg.vmaddr - baseAddr) <= entry.info.address && entry.info.address < (seg.vmaddr - baseAddr) + seg.size) { + _exports[entry.name].segmentOffset = entry.info.address - (seg.vmaddr - baseAddr); + _exports[entry.name].segmentName = seg.name; + break; + } + } + } else { + _exports[entry.name].segmentOffset = (uint64_t)entry.info.address; + _exports[entry.name].segmentName = ""; + } + + switch (entry.info.flags & EXPORT_SYMBOL_FLAGS_KIND_MASK) { + case EXPORT_SYMBOL_FLAGS_KIND_REGULAR: + if ((entry.info.flags & EXPORT_SYMBOL_FLAGS_STUB_AND_RESOLVER)) { + _exports[entry.name].isResolver = true; + } + if (entry.info.flags & EXPORT_SYMBOL_FLAGS_REEXPORT) { + SymbolInfo& info = _exports[entry.name]; + info.isSymbolReExport = true; + info.reExportDylibIndex = (int)entry.info.other; + if (!entry.info.importName.empty()) + info.reExportName = entry.info.importName; + } + break; + case EXPORT_SYMBOL_FLAGS_KIND_THREAD_LOCAL: + _exports[entry.name].isThreadLocal = true; + break; + case EXPORT_SYMBOL_FLAGS_KIND_ABSOLUTE: + _exports[entry.name].isAbsolute = true; + break; + default: + terminate("non-regular symbol binding not supported for %s in %s", entry.name.c_str(), path.c_str()); + break; + } + } + } + + if (!isDylib()) { + return ""; + } + if ((mh->flags() & MH_TWOLEVEL) == 0) { return "built with -flat_namespace"; } @@ -192,54 +354,142 @@ std::string MachOProxy::machoParser(bool ignoreUncacheableDylibsInExecutables) const bool MachOProxy::isDylib() { - return _dylib; + return (_filetype == MH_DYLIB); } const bool MachOProxy::isExecutable() { - return _executable; + return (_filetype == MH_EXECUTE); } -std::map MachOProxy::findDylibInfo(const std::string& path, bool warnOnProblems, bool ignoreUncacheableDylibsInExecutables) { - std::map slices = mapMachOFile( path ); - std::vector errorSlices; +static std::map identifierMap; +std::map, MachOProxy*> archMap; +static dispatch_queue_t identifierQueue; + +MachOProxy* MachOProxy::forIdentifier(const ImageIdentifier& identifier, const std::string preferredArch) +{ + auto i = identifierMap.find(identifier); + // We need an identifier + if (i == identifierMap.end()) + return nullptr; + + // Is the identifier the arch we want? + if (i->second->arch == preferredArch) + return i->second; + + // Fallback to a slow path to try to find a best fit + return forInstallnameAndArch(i->second->installName, preferredArch); +} + +MachOProxy* MachOProxy::forInstallnameAndArch(const std::string& installname, const std::string& arch) +{ + auto i = archMap.find(std::make_pair(installname, arch)); + if (i == archMap.end()) + i = archMap.find(std::make_pair(installname, fallbackArchStringForArchString(arch))); + if (i != archMap.end()) + return i->second; + return nullptr; +} + +void MachOProxy::mapDependencies() +{ + // Build a complete map of all installname/alias,archs to their proxies + runOnAllProxies(false, [&](MachOProxy* proxy) { + archMap[std::make_pair(proxy->path, proxy->arch)] = proxy; + for (auto& alias : proxy->installNameAliases) { + archMap[std::make_pair(alias, proxy->arch)] = proxy; + } + }); + + //Wire up the dependencies + runOnAllProxies(false, [&](MachOProxy* proxy) { + auto dependencyInstallnames = proxy->dependencies(); + for (auto dependencyInstallname : dependencyInstallnames) { + auto dependencyProxy = forInstallnameAndArch(dependencyInstallname, proxy->arch); + if (dependencyProxy == nullptr) { + proxy->error = "Missing dependency: " + dependencyInstallname; + } else { + proxy->requiredIdentifiers.push_back(dependencyProxy->identifier); + dependencyProxy->dependentIdentifiers.push_back(proxy->identifier); + } + } + + auto reexportInstallnames = proxy->reexports(); + for (auto reexportInstallname : reexportInstallnames) { + auto reexportProxy = forInstallnameAndArch(reexportInstallname, proxy->arch); + if (reexportProxy == nullptr) { + proxy->error = "Missing reexport dylib: " + reexportInstallname; + } else { + proxy->_reexportProxies.push_back(reexportProxy); + } + } + + }); +} + +void MachOProxy::runOnAllProxies(bool concurrently, std::function lambda) +{ + dispatch_group_t runGroup = dispatch_group_create(); + dispatch_queue_t runQueue = dispatch_get_global_queue(QOS_CLASS_USER_INITIATED, NULL); + + for (auto& identifier : identifierMap) { + if (concurrently) { + cacheBuilderDispatchGroupAsync(runGroup, runQueue, [&] { + lambda(identifier.second); + }); + } else { + lambda(identifier.second); + } + } + + dispatch_group_wait(runGroup, DISPATCH_TIME_FOREVER); +} + +std::map MachOProxy::loadProxies(const std::string& buildPath, const std::string& path, bool warnOnProblems, bool ignoreUncacheableDylibsInExecutables) +{ + std::vector slices = mapMachOFile(buildPath, path); + std::map retval; for ( auto& slice : slices ) { std::string errorMessage; verboseLog( "analyzing file '%s'", path.c_str() ); - switch ( archForString( slice.first ).arch ) { + switch (archForString(slice->arch).arch) { case CPU_TYPE_ARM: case CPU_TYPE_I386: - errorMessage = slice.second->machoParser>(ignoreUncacheableDylibsInExecutables); + errorMessage = slice->machoParser>(ignoreUncacheableDylibsInExecutables); break; case CPU_TYPE_X86_64: case CPU_TYPE_ARM64: - errorMessage = slice.second->machoParser>(ignoreUncacheableDylibsInExecutables); + errorMessage = slice->machoParser>(ignoreUncacheableDylibsInExecutables); break; default: - errorMessage = "unsupported arch '" + slice.first + "'"; + errorMessage = "unsupported arch '" + slice->arch + "'"; break; } - if ( !errorMessage.empty() ) { - if ( warnOnProblems ) - warning( "%s (%s)", errorMessage.c_str(), path.c_str() ); - errorSlices.push_back( slice.first ); + if (errorMessage.empty()) { + static dispatch_once_t onceToken; + dispatch_once(&onceToken, ^{ + identifierQueue = dispatch_queue_create("com.apple.dyld.cache.metabom.ids", DISPATCH_QUEUE_SERIAL); + }); + retval[slice->arch] = slice; + dispatch_sync(identifierQueue, ^{ + identifierMap[slice->identifier] = slice; + }); + } else { + if (warnOnProblems) + warning("%s (%s)", errorMessage.c_str(), path.c_str()); } } - for ( const auto& slice : errorSlices ) { - slices.erase( slice ); - } - - return slices; + return retval; } const uint8_t* MachOProxy::getBuffer() { const uint8_t* p = (uint8_t*)( -1 ); struct stat stat_buf; bool rootless; - std::tie(p, stat_buf,rootless) = fileCache.cacheLoad(path); + std::tie(p, stat_buf, rootless) = fileCache.cacheLoad(buildPath); return p + fatFileOffset; } diff --git a/interlinked-dylibs/MachOProxy.h b/interlinked-dylibs/MachOProxy.h index 7a7f620..4781504 100644 --- a/interlinked-dylibs/MachOProxy.h +++ b/interlinked-dylibs/MachOProxy.h @@ -17,49 +17,143 @@ #include +#include "mega-dylib-utils.h" + +struct MachOProxy; + +struct MachOProxySegment { + std::string name; + uint64_t size; + uint64_t sizeOfSections; + uint64_t vmaddr; + uint32_t diskSize; + uint32_t fileOffset; + uint8_t p2align; + uint8_t protection; +}; + struct MachOProxy { - MachOProxy(const std::string& p, ino_t i, time_t t, uint32_t o, uint32_t s, bool r) : - path(p), fatFileOffset(o), fileSize(s), lastModTime(t), inode(i), installNameOffsetInTEXT(0), rootlessProtected(r) { - bzero( &uuid, sizeof( uuid_t ) ); + MachOProxy(const std::string& bp, const std::string& p, const std::string& a, ino_t i, time_t t, uint32_t o, uint32_t s, bool r) + : buildPath(bp) + , path(p) + , arch(a) + , fatFileOffset(o) + , fileSize(s) + , lastModTime(t) + , inode(i) + , installNameOffsetInTEXT(0) + , rootlessProtected(r) + , _bind_offset(0) + , _bind_size(0) + , queue(dispatch_queue_create("com.apple.dyld.proxy", NULL)) + { } - struct Segment { - std::string name; - uint64_t size; - uint64_t sizeOfSections; - uint32_t diskSize; - uint32_t fileOffset; - uint8_t p2align; - uint8_t protection; + struct SymbolInfo { + SymbolInfo() {} + std::string segmentName; + uint64_t segmentOffset = 0; + bool isResolver = false; + bool isAbsolute = false; + bool isSymbolReExport = false; + bool isThreadLocal = false; + int reExportDylibIndex = 0; + std::string reExportName; }; - const std::string path; - const uint32_t fatFileOffset; - const uint32_t fileSize; - const time_t lastModTime; - const ino_t inode; - const bool rootlessProtected; - std::string installName; - std::set installNameAliases; - uint32_t installNameOffsetInTEXT; - std::set dependencies; - std::set dependents; - uuid_t uuid; - std::vector segments; + const std::string buildPath; + const std::string path; + const std::string arch; + const uint32_t fatFileOffset; + const uint32_t fileSize; + const time_t lastModTime; + const ino_t inode; + const bool rootlessProtected; + dispatch_queue_t queue; + std::string installName; + std::set installNameAliases; + uint32_t installNameOffsetInTEXT; + std::string error; + std::vector requiredIdentifiers; + std::vector dependentIdentifiers; + UUID uuid; + ImageIdentifier identifier; + std::vector segments; const uint8_t* getBuffer(); + const uint8_t* getBindStart() { return &(getBuffer())[_bind_offset]; } + const uint8_t* getBindEnd() { return &(getBuffer())[_bind_offset + _bind_size]; } + const uint8_t* getLazyBindStart() { return &(getBuffer())[_lazy_bind_offset]; } + const uint8_t* getLazyBindEnd() { return &(getBuffer())[_lazy_bind_offset + _lazy_bind_size]; } + const bool isDylib(); const bool isExecutable(); bool addAlias(const std::string& alias); + static void mapDependencies(); + + const uint64_t addressOf(const std::string& symbol, const std::map>& segmentMap) + { + auto info = symbolInfo(symbol); + assert(info != nullptr); + if (info->isAbsolute) + return info->segmentOffset; + auto proxyI = segmentMap.find(this); + assert(proxyI != segmentMap.end()); + + for (const auto& seg : proxyI->second) { + if (seg.base->name == info->segmentName) { + assert(!info->segmentName.empty()); + return seg.address + info->segmentOffset; + } + } + + return 0; + } - static std::map findDylibInfo(const std::string& path, bool warnOnProblems=false, bool ignoreUncacheableDylibsInExecutables=false); + SymbolInfo* symbolInfo(const std::string& symbol) + { + auto i = _exports.find(symbol); + if (i != _exports.end()) + return &i->second; + return nullptr; + } + + bool providesSymbol(const std::string& symbol) + { + if (_exports.find(symbol) != _exports.end()) + return true; + + for (const auto& proxy : _reexportProxies) { + if (proxy->providesSymbol(symbol)) + return true; + } + return false; + } + static std::map loadProxies(const std::string& prefix, const std::string& path, bool warnOnProblems = false, bool ignoreUncacheableDylibsInExecutables = false); + static void runOnAllProxies(bool concurrently, std::function lambda); + static MachOProxy* forIdentifier(const ImageIdentifier& identifier, const std::string preferredArch); + static MachOProxy* forInstallnameAndArch(const std::string& installname, const std::string& arch); + + std::vector dependencies(); + std::vector reexports(); private: - bool _dylib; - bool _executable; + uint32_t _filetype; + std::map _exports; + uint32_t _bind_offset; + uint32_t _bind_size; + uint32_t _lazy_bind_offset; + uint32_t _lazy_bind_size; + std::vector _reexportProxies; template std::string machoParser(bool ignoreUncacheableDylibsInExecutables); + + template + std::vector dependencies(); + + template + std::vector reexports(); }; diff --git a/interlinked-dylibs/Manifest.h b/interlinked-dylibs/Manifest.h index bface3c..6611ef0 100644 --- a/interlinked-dylibs/Manifest.h +++ b/interlinked-dylibs/Manifest.h @@ -17,9 +17,14 @@ #include #include +#include -struct SharedCache; struct MachOProxy; + +extern void terminate(const char* format, ...) __printflike(1, 2) __attribute__((noreturn)); +extern std::string toolDir(); + +struct SharedCache; struct Manifest; struct Manifest { @@ -33,22 +38,16 @@ struct Manifest { File( MachOProxy* P ) : proxy( P ) {} }; - struct FileSet { - std::map dylibs; - std::map executables; - }; - struct Anchor { - std::string installname; + ImageIdentifier identifier; bool required; - - Anchor( const std::string& IN ) : installname( IN ) {} + Anchor( const ImageIdentifier& I ) : identifier( I ) {} }; struct SegmentInfo { - std::string name; - uint64_t startAddr; - uint64_t endAddr; + std::string name; + uint64_t startAddr; + uint64_t endAddr; }; struct SegmentInfoHasher { @@ -65,143 +64,189 @@ struct Manifest { struct DylibInfo { bool included; std::string exclusionInfo; - uuid_t uuid; + UUID uuid; + std::string installname; std::vector segments; DylibInfo(void) : included(true) {} - void exclude(const std::string& reason) { - included = false; - exclusionInfo = reason; - } }; struct Results { - std::string failure; - std::map dylibs; - std::vector warnings; - CacheInfo developmentCache; - CacheInfo productionCache; - }; + std::string failure; + std::map dylibs; + std::vector warnings; + CacheInfo developmentCache; + CacheInfo productionCache; + DylibInfo& dylibForInstallname(const std::string& installname) + { + auto i = find_if(dylibs.begin(), dylibs.end(), [&installname](std::pair d) { return d.second.installname == installname; }); + assert(i != dylibs.end()); + return i->second; + } + void exclude(MachOProxy* proxy, const std::string& reason); + }; struct Architecture { std::vector anchors; mutable Results results; - //FIXME: Gross - std::unordered_map> dependents; - - uint64_t hash(void) const { - if (!_hash) { - for (auto& dylib : results.dylibs) { - if (dylib.second.included) { - _hash ^= std::hash()(dylib.first); - //HACK to get some of the UUID into the hash - _hash ^= std::hash()(*(uint64_t *)(&dylib.second.uuid[0])); - } - }; - } - - return _hash; - } - bool equivalent(const Architecture& O) const { - if (hash() != O.hash()) { - return false; - } - for (auto& dylib : results.dylibs) { - if (dylib.second.included) { - auto Odylib = O.results.dylibs.find(dylib.first); - if (Odylib == O.results.dylibs.end() - || Odylib->second.included == false - || memcmp(&Odylib->second.uuid[0], &dylib.second.uuid[0], sizeof(uuid_t)) != 0) - return false; - } - } - //Iterate over O.results to make sure we included all the same things - for (auto Odylib : O.results.dylibs) { - if (Odylib.second.included) { - auto dylib = results.dylibs.find(Odylib.first); - if (dylib == results.dylibs.end() - || dylib->second.included == false) - return false; - } - } - return true; - } - private: - mutable uint64_t _hash = 0; - }; + bool operator==(const Architecture& O) const + { + for (auto& dylib : results.dylibs) { + if (dylib.second.included) { + auto Odylib = O.results.dylibs.find(dylib.first); + if (Odylib == O.results.dylibs.end() + || Odylib->second.included == false + || Odylib->second.uuid != dylib.second.uuid) + return false; + } + } + + for (const auto& Odylib : O.results.dylibs) { + if (Odylib.second.included) { + auto dylib = results.dylibs.find(Odylib.first); + if (dylib == results.dylibs.end() + || dylib->second.included == false + || dylib->second.uuid != Odylib.second.uuid) + return false; + } + } + + return true; + } + + bool operator!=(const Architecture& other) const { return !(*this == other); } + }; struct Configuration { - std::string platformName; - std::string metabomTag; - std::set metabomExcludeTags; - std::set metabomRestrictTags; - std::set restrictedInstallnames; - std::map architectures; - - uint64_t hash( void ) const { - if (!_hash) { - _hash ^= std::hash()(architectures.size()); - // We want the preliminary info here to make dedupe decisions - for (auto& arch : architectures) { - _hash ^= arch.second.hash(); - }; - } - return _hash; - } - - //Used for dedupe - bool equivalent(const Configuration& O) const { - if (hash() != O.hash()) - return false; - for (const auto& arch : architectures) { - if (O.architectures.count(arch.first) == 0) - return false; - if (!arch.second.equivalent(O.architectures.find(arch.first)->second)) - return false; - } - - return true; - } - private: - mutable uint64_t _hash = 0; + std::string platformName; + std::string metabomTag; + std::set metabomExcludeTags; + std::set metabomRestrictTags; + std::set restrictedInstallnames; + std::map architectures; + + bool operator==(const Configuration& O) const + { + return architectures == O.architectures; + } + + bool operator!=(const Configuration& other) const { return !(*this == other); } + + const Architecture& architecture(const std::string& architecture) const + { + assert(architectures.find(architecture) != architectures.end()); + return architectures.find(architecture)->second; + } + + void forEachArchitecture(std::function lambda) + { + for (const auto& architecutre : architectures) { + lambda(architecutre.first); + } + } }; - std::map architectureFiles; - std::map projects; - std::map configurations; - std::string dylibOrderFile; - std::string dirtyDataOrderFile; - std::string metabomFile; - std::string build; - // FIXME every needs to adopt platform string for v5 - std::string platform; - uint32_t manifest_version; - bool normalized; - - Manifest( void ) {} + const std::map& projects() + { + return _projects; + } + + const Configuration& configuration(const std::string& configuration) const + { + assert(_configurations.find(configuration) != _configurations.end()); + return _configurations.find(configuration)->second; + } + + void forEachConfiguration(std::function lambda) + { + for (const auto& configuration : _configurations) { + lambda(configuration.first); + } + } + + void addProjectSource(const std::string& project, const std::string& source, bool first = false) + { + auto& sources = _projects[project].sources; + if (std::find(sources.begin(), sources.end(), source) == sources.end()) { + if (first) { + sources.insert(sources.begin(), source); + } else { + sources.push_back(source); + } + } + } + + const std::string projectPath(const std::string& projectName) + { + auto project = _projects.find(projectName); + if (project == _projects.end()) + return ""; + if (project->second.sources.size() == 0) + return ""; + return project->second.sources[0]; + } + + + const bool empty(void) { + for (const auto& configuration : _configurations) { + if (configuration.second.architectures.size() != 0) + return false; + } + return true; + } + + const std::string dylibOrderFile() const { return _dylibOrderFile; }; + void setDylibOrderFile(const std::string& dylibOrderFile) { _dylibOrderFile = dylibOrderFile; }; + + const std::string dirtyDataOrderFile() const { return _dirtyDataOrderFile; }; + void setDirtyDataOrderFile(const std::string& dirtyDataOrderFile) { _dirtyDataOrderFile = dirtyDataOrderFile; }; + + const std::string metabomFile() const { return _metabomFile; }; + void setMetabomFile(const std::string& metabomFile) { _metabomFile = metabomFile; }; + + const std::string platform() const { return _platform; }; + void setPlatform(const std::string& platform) { _platform = platform; }; + + const std::string& build() const { return _build; }; + void setBuild(const std::string& build) { _build = build; }; + const uint32_t version() const { return _manifestVersion; }; + void setVersion(const uint32_t manifestVersion) { _manifestVersion = manifestVersion; }; + bool normalized; + + Manifest(void) {} + Manifest(const std::set& archs, const std::string& overlayPath, const std::string& rootPath, const std::set& paths); #if BOM_SUPPORT - Manifest( const std::string& path ); - Manifest( const std::string& path, const std::set& overlays ); + Manifest(const std::string& path); + Manifest(const std::string& path, const std::set& overlays); #endif - void write( const std::string& path ); - void canonicalize( void ); - void calculateClosure( bool enforeceRootless ); - void pruneClosure(); - bool sameContentsAsCacheAtPath( const std::string& configuration, const std::string& architecture, - const std::string& path ) const; - MachOProxy* dylibProxy( const std::string& installname, const std::string& arch ); - MachOProxy* removeLargestLeafDylib( const std::string& configuration, const std::string& architecture ); + void write(const std::string& path); + void canonicalize(void); + void calculateClosure(bool enforeceRootless); + bool sameContentsAsCacheAtPath(const std::string& configuration, const std::string& architecture, + const std::string& path) const; + void remove(const std::string& config, const std::string& arch); + MachOProxy* removeLargestLeafDylib(const std::string& configuration, const std::string& architecture); + bool checkLinks(); + void runConcurrently(dispatch_queue_t queue, dispatch_semaphore_t concurrencyLimitingSemaphore, std::function lambda); + bool filterForConfig(const std::string& configName); private: - void removeDylib( MachOProxy* proxy, const std::string& reason, const std::string& configuration, const std::string& architecture, - std::unordered_set& processedInstallnames ); - File* dylibForInstallName( const std::string& installname, const std::string& arch ); - void calculateClosure( const std::string& configuration, const std::string& architecture); - void pruneClosure(const std::string& configuration, const std::string& architecture); - void canonicalizeDylib( const std::string& installname ); - template - void canonicalizeDylib( const std::string& installname, const uint8_t* p ); - void addImplicitAliases( void ); + uint32_t _manifestVersion; + std::string _build; + std::string _dylibOrderFile; + std::string _dirtyDataOrderFile; + std::string _metabomFile; + std::string _platform; + std::map _projects; + std::map _configurations; + void removeDylib(MachOProxy* proxy, const std::string& reason, const std::string& configuration, const std::string& architecture, + std::unordered_set& processedIdentifiers); + void calculateClosure(const std::string& configuration, const std::string& architecture); + void canonicalizeDylib(const std::string& installname); + template + void canonicalizeDylib(const std::string& installname, const uint8_t* p); + void addImplicitAliases(void); + MachOProxy* dylibProxy(const std::string& installname, const std::string& arch); }; diff --git a/interlinked-dylibs/Manifest.mm b/interlinked-dylibs/Manifest.mm index 34e81ec..dad8f43 100644 --- a/interlinked-dylibs/Manifest.mm +++ b/interlinked-dylibs/Manifest.mm @@ -32,351 +32,462 @@ extern "C" { #include #include -#include "Manifest.h" #include "dsc_iterator.h" - +#include "MachOProxy.h" #include "mega-dylib-utils.h" +#include "Manifest.h" + namespace { - //FIXME this should be in a class - static bool rootless = true; +//FIXME this should be in a class +static bool rootless = true; +static inline NSString* cppToObjStr(const std::string& str) { return [NSString stringWithUTF8String:str.c_str()]; } - static inline NSString* cppToObjStr(const std::string& str) { return [NSString stringWithUTF8String:str.c_str()]; } +std::string fileExists(const std::string& path) +{ + const uint8_t* p = (uint8_t*)(-1); + struct stat stat_buf; -#if BOM_SUPPORT - std::string effectivePath(const std::string source, const std::string target) - { - if (target[0] == '/') - return normalize_absolute_file_path(target); + std::tie(p, stat_buf, rootless) = fileCache.cacheLoad(path); + if (p != (uint8_t*)(-1)) { + return normalize_absolute_file_path(path); + } - std::size_t found = source.find_last_of('/'); - return normalize_absolute_file_path(source.substr(0, found) + "/" + target); - } + return ""; +} + +} /* Anonymous namespace */ + +void Manifest::Results::exclude(MachOProxy* proxy, const std::string& reason) +{ + dylibs[proxy->identifier].uuid = proxy->uuid; + dylibs[proxy->identifier].installname = proxy->installName; + dylibs[proxy->identifier].included = false; + dylibs[proxy->identifier].exclusionInfo = reason; +} - std::string checkSymlink(const std::string path, const std::pair& symlink, const std::set& directories) - { - if (directories.count(symlink.second) == 0) { - if (path == symlink.second) - return symlink.first; +Manifest::Manifest(const std::set& archs, const std::string& overlayPath, const std::string& rootPath, const std::set& paths) +{ + std::set processedPaths; + std::set unprocessedPaths = paths; + std::set pathsToProcess; + std::set_difference(unprocessedPaths.begin(), unprocessedPaths.end(), processedPaths.begin(), processedPaths.end(), + std::inserter(pathsToProcess, pathsToProcess.begin())); + while (!pathsToProcess.empty()) { + for (const std::string path : pathsToProcess) { + processedPaths.insert(path); + std::string fullPath; + if (rootPath != "/") { + // with -root, only look in the root path volume + fullPath = fileExists(rootPath + path); } else { - auto res = std::mismatch(symlink.second.begin(), symlink.second.end(), path.begin()); - if (res.first == symlink.second.end()) { - std::string alias = normalize_absolute_file_path(symlink.first + std::string(res.second, path.end())); - return alias; + // with -overlay, look first in overlay dir + if (!overlayPath.empty()) + fullPath = fileExists(overlayPath + path); + // if not in overlay, look in boot volume + if (fullPath.empty()) + fullPath = fileExists(path); + } + if (fullPath.empty()) + continue; + auto proxies = MachOProxy::loadProxies(fullPath, path); + + for (const auto& arch : archs) { + auto proxyI = proxies.find(arch); + if (proxyI == proxies.end()) + proxyI = proxies.find(fallbackArchStringForArchString(arch)); + if (proxyI == proxies.end()) + continue; + + auto dependecies = proxyI->second->dependencies(); + for (const auto& dependency : dependecies) { + unprocessedPaths.insert(dependency); } + _configurations["localhost"].architectures[arch].anchors.push_back(proxyI->second->identifier); } - return ""; - } -#endif + + //Stuff } + pathsToProcess.clear(); + std::set_difference(unprocessedPaths.begin(), unprocessedPaths.end(), processedPaths.begin(), processedPaths.end(), + std::inserter(pathsToProcess, pathsToProcess.begin())); + } + MachOProxy::mapDependencies(); +} + #if BOM_SUPPORT - Manifest::Manifest(const std::string& path) - : Manifest(path, std::set()) - { - } +Manifest::Manifest(const std::string& path) + : Manifest(path, std::set()) +{ +} - Manifest::Manifest(const std::string& path, const std::set& overlays) - { - NSMutableDictionary* manifestDict = [NSMutableDictionary dictionaryWithContentsOfFile:cppToObjStr(path)]; - std::map metabomTagMap; - std::map> metabomExcludeTagMap; - std::map> metabomRestrictedTagMap; - metabomFile = [manifestDict[@"metabomFile"] UTF8String]; - - for (NSString* project in manifestDict[@"projects"]) { - for (NSString* source in manifestDict[@"projects"][project]) { - projects[[project UTF8String]].sources.push_back([source UTF8String]); - } - } +Manifest::Manifest(const std::string& path, const std::set& overlays) +{ + NSMutableDictionary* manifestDict = [NSMutableDictionary dictionaryWithContentsOfFile:cppToObjStr(path)]; + std::map metabomTagMap; + std::map> metabomExcludeTagMap; + std::map> metabomRestrictedTagMap; + std::vector> configProxies; - for (NSString* configuration in manifestDict[@"configurations"]) { - std::string configStr = [configuration UTF8String]; - std::string configTag = [manifestDict[@"configurations"][configuration][@"metabomTag"] UTF8String]; - metabomTagMap[configTag] = configStr; + setMetabomFile([manifestDict[@"metabomFile"] UTF8String]); - if (manifestDict[@"configurations"][configuration][@"metabomExcludeTags"]) { - for (NSString* excludeTag in manifestDict[@"configurations"][configuration][@"metabomExcludeTags"]) { - metabomExcludeTagMap[configStr].insert([excludeTag UTF8String]); - configurations[configStr].metabomExcludeTags.insert([excludeTag UTF8String]); - } - } + for (NSString* project in manifestDict[@"projects"]) { + for (NSString* source in manifestDict[@"projects"][project]) { + addProjectSource([project UTF8String], [source UTF8String]); + } + } - if (manifestDict[@"configurations"][configuration][@"metabomRestrictTags"]) { - for (NSString* restrictTag in manifestDict[@"configurations"][configuration][@"metabomRestrictTags"]) { - metabomRestrictedTagMap[configStr].insert([restrictTag UTF8String]); - configurations[configStr].metabomRestrictTags.insert([restrictTag UTF8String]); - } - } + for (NSString* configuration in manifestDict[@"configurations"]) { + std::string configStr = [configuration UTF8String]; + std::string configTag = [manifestDict[@"configurations"][configuration][@"metabomTag"] UTF8String]; + metabomTagMap[configTag] = configStr; - configurations[configStr].metabomTag = configTag; - configurations[configStr].platformName = - [manifestDict[@"configurations"][configuration][@"platformName"] UTF8String]; + if (manifestDict[@"configurations"][configuration][@"metabomExcludeTags"]) { + for (NSString* excludeTag in manifestDict[@"configurations"][configuration][@"metabomExcludeTags"]) { + metabomExcludeTagMap[configStr].insert([excludeTag UTF8String]); + _configurations[configStr].metabomExcludeTags.insert([excludeTag UTF8String]); } + } - manifest_version = [manifestDict[@"manifest-version"] unsignedIntValue]; - build = [manifestDict[@"build"] UTF8String]; - if (manifestDict[@"dylibOrderFile"]) { - dylibOrderFile = [manifestDict[@"dylibOrderFile"] UTF8String]; - } - if (manifestDict[@"dirtyDataOrderFile"]) { - dirtyDataOrderFile = [manifestDict[@"dirtyDataOrderFile"] UTF8String]; + if (manifestDict[@"configurations"][configuration][@"metabomRestrictTags"]) { + for (NSString* restrictTag in manifestDict[@"configurations"][configuration][@"metabomRestrictTags"]) { + metabomRestrictedTagMap[configStr].insert([restrictTag UTF8String]); + _configurations[configStr].metabomRestrictTags.insert([restrictTag UTF8String]); } + } - auto metabom = MBMetabomOpen(metabomFile.c_str(), false); - auto metabomEnumerator = MBIteratorNewWithPath(metabom, ".", ""); - MBEntry entry; - - std::map symlinks; - std::set directories; + _configurations[configStr].metabomTag = configTag; + _configurations[configStr].platformName = + [manifestDict[@"configurations"][configuration][@"platformName"] UTF8String]; + } - // FIXME error handling (NULL metabom) + setVersion([manifestDict[@"manifest-version"] unsignedIntValue]); + setBuild([manifestDict[@"build"] UTF8String]); + if (manifestDict[@"dylibOrderFile"]) { + setDylibOrderFile([manifestDict[@"dylibOrderFile"] UTF8String]); + } + if (manifestDict[@"dirtyDataOrderFile"]) { + setDirtyDataOrderFile([manifestDict[@"dirtyDataOrderFile"] UTF8String]); + } - while ((entry = MBIteratorNext(metabomEnumerator))) { - auto fsObject = MBEntryGetFSObject(entry); - std::string entryPath = BOMFSObjectPathName(fsObject); + auto metabom = MBMetabomOpen(metabomFile().c_str(), false); + auto metabomEnumerator = MBIteratorNewWithPath(metabom, ".", ""); + MBEntry entry; + + auto bomSemaphore = dispatch_semaphore_create(32); + auto bomGroup = dispatch_group_create(); + auto bomQueue = dispatch_queue_create("com.apple.dyld.cache.metabom.bom", dispatch_queue_attr_make_with_qos_class(DISPATCH_QUEUE_SERIAL, QOS_CLASS_USER_INITIATED, 0)); + auto archQueue = dispatch_queue_create("com.apple.dyld.cache.metabom.arch", dispatch_queue_attr_make_with_qos_class(DISPATCH_QUEUE_SERIAL, QOS_CLASS_USER_INITIATED, 0)); + auto manifestQueue = dispatch_queue_create("com.apple.dyld.cache.metabom.arch", dispatch_queue_attr_make_with_qos_class(DISPATCH_QUEUE_CONCURRENT, QOS_CLASS_USER_INITIATED, 0)); + + // FIXME error handling (NULL metabom) + + //First we iterate through the bom and build our objects + + while ((entry = MBIteratorNext(metabomEnumerator))) { + dispatch_semaphore_wait(bomSemaphore, DISPATCH_TIME_FOREVER); + cacheBuilderDispatchGroupAsync(bomGroup, manifestQueue, [this, &bomSemaphore, &archQueue, &bomGroup, &bomQueue, &metabom, entry, &overlays, &metabomTagMap, &metabomRestrictedTagMap, &metabomExcludeTagMap, &manifestDict, &configProxies] { + BOMFSObject fsObject = nullptr; + std::string entryPath; + BOMFSObjType entryType; + cacheBuilderDispatchSync(bomQueue, [&entry, &fsObject, &entryPath, &entryType] { + fsObject = MBEntryGetFSObject(entry); + entryPath = BOMFSObjectPathName(fsObject); if (entryPath[0] == '.') { entryPath.erase(0, 1); } - auto entryType = BOMFSObjectType(fsObject); - - switch (entryType) { - case BOMFileType: { - MBTag tag; - auto tagCount = MBEntryGetNumberOfProjectTags(entry); - - if (!BOMFSObjectIsBinaryObject(fsObject)) - break; + entryType = BOMFSObjectType(fsObject); + }); - if (tagCount == 0) { - break; - } else if (tagCount == 1) { - MBEntryGetProjectTags(entry, &tag); - } else { - MBTag* tags = (MBTag*)malloc(sizeof(MBTag) * tagCount); - MBEntryGetProjectTags(entry, tags); - - //Sigh, we can have duplicate entries for the same tag, so build a set to work with - std::set tagStrs; - std::map tagStrMap; - for (auto i = 0; i < tagCount; ++i) { + MBTag tag; + auto tagCount = MBEntryGetNumberOfProjectTags(entry); + if ( entryType == BOMFileType && BOMFSObjectIsBinaryObject(fsObject) && MBEntryGetNumberOfProjectTags(entry) != 0 && tagCount != 0 ) { + if (tagCount == 1) { + MBEntryGetProjectTags(entry, &tag); + } else { + MBTag* tags = (MBTag*)malloc(sizeof(MBTag) * tagCount); + MBEntryGetProjectTags(entry, tags); + + //Sigh, we can have duplicate entries for the same tag, so build a set to work with + std::set tagStrs; + std::map tagStrMap; + for (auto i = 0; i < tagCount; ++i) { + cacheBuilderDispatchSync(bomQueue, [i, &metabom, &tagStrs, &tagStrMap, &tags] { tagStrs.insert(MBMetabomGetProjectForTag(metabom, tags[i])); tagStrMap.insert(std::make_pair(MBMetabomGetProjectForTag(metabom, tags[i]), tags[i])); - } + }); + } - if (tagStrs.size() > 1) { - std::string projects; - for (const auto& tagStr : tagStrs) { - if (!projects.empty()) - projects += ", "; + if (tagStrs.size() > 1) { + std::string projects; + for (const auto& tagStr : tagStrs) { + if (!projects.empty()) + projects += ", "; - projects += "'" + tagStr + "'"; - } - warning("Bom entry '%s' is claimed by multiple projects: %s, taking first entry", entryPath.c_str(), projects.c_str()); + projects += "'" + tagStr + "'"; } - tag = tagStrMap[*tagStrs.begin()]; - free(tags); + warning("Bom entry '%s' is claimed by multiple projects: %s, taking first entry", entryPath.c_str(), projects.c_str()); } + tag = tagStrMap[*tagStrs.begin()]; + free(tags); + } - std::string projectName = MBMetabomGetProjectForTag(metabom, tag); + std::string projectName; + cacheBuilderDispatchSync(bomQueue, [&projectName, &metabom, &tag] { + projectName = MBMetabomGetProjectForTag(metabom, tag); + }); - // FIXME we need to actually walk down the searchpaths - auto project = projects.find(projectName); - if (project == projects.end()) - break; - if (project->second.sources.size() == 0) + std::map proxies; + for (const auto& overlay : overlays) { + proxies = MachOProxy::loadProxies(overlay + "/" + entryPath, entryPath); + if (proxies.size() > 0) break; - std::string projectPath = project->second.sources[0]; - std::map proxies; + } - for (const auto& overlay : overlays) { - proxies = MachOProxy::findDylibInfo(overlay + "/" + entryPath); - if (proxies.size() > 0) - break; - } + if (proxies.size() == 0) { + proxies = MachOProxy::loadProxies(projectPath(projectName) + "/" + entryPath, entryPath); + } + + tagCount = MBEntryGetNumberOfPackageTags(entry); + MBTag* tags = (MBTag*)malloc(sizeof(MBTag) * tagCount); + MBEntryGetPackageTags(entry, tags); + std::set tagStrs; - if (proxies.size() == 0) { - proxies = MachOProxy::findDylibInfo(projectPath + "/" + entryPath); + cacheBuilderDispatchSync(bomQueue, [&] { + for (auto i = 0; i < tagCount; ++i) { + tagStrs.insert(MBMetabomGetPackageForTag(metabom, tags[i])); } + }); - for (auto& proxy : proxies) { - assert(proxy.second != nullptr); - if (proxy.second->isExecutable()) { - architectureFiles[proxy.first].executables.insert(std::make_pair(entryPath, File(proxy.second))); + for (auto& proxy : proxies) { + for (const auto& tagStr : tagStrs) { + // Does the configuration exist + auto configuration = metabomTagMap.find(tagStr); + if (configuration == metabomTagMap.end()) continue; + auto restrictions = metabomRestrictedTagMap.find(configuration->second); + if (restrictions != metabomRestrictedTagMap.end() && !is_disjoint(restrictions->second, tagStrs)) { + _configurations[configuration->second].restrictedInstallnames.insert(proxy.second->installName); } - if (!proxy.second->isDylib()) + // Is the configuration excluded + auto exclusions = metabomExcludeTagMap.find(configuration->second); + if (exclusions != metabomExcludeTagMap.end() && !is_disjoint(exclusions->second, tagStrs)) { continue; - assert(proxy.second->installName != ""); - proxy.second->addAlias(entryPath); - architectureFiles[proxy.first].dylibs.insert( - std::make_pair(proxy.second->installName, File(proxy.second))); - auto tagCount = MBEntryGetNumberOfPackageTags(entry); - MBTag* tags = (MBTag*)malloc(sizeof(MBTag) * tagCount); - MBEntryGetPackageTags(entry, tags); - std::set tagStrs; - - for (auto i = 0; i < tagCount; ++i) { - tagStrs.insert(MBMetabomGetPackageForTag(metabom, tags[i])); } - - for (const auto& tagStr : tagStrs) { - // Does the configuration exist - auto configuration = metabomTagMap.find(tagStr); - if (configuration == metabomTagMap.end()) - continue; - - auto restrictions = metabomRestrictedTagMap.find(configuration->second); - if (restrictions != metabomRestrictedTagMap.end() && !is_disjoint(restrictions->second, tagStrs)) { - configurations[configuration->second].restrictedInstallnames.insert(proxy.second->installName); - } - // Is the configuration excluded - auto exclusions = metabomExcludeTagMap.find(configuration->second); - if (exclusions != metabomExcludeTagMap.end() && !is_disjoint(exclusions->second, tagStrs)) - continue; - + cacheBuilderDispatchGroupAsync(bomGroup, archQueue, [this, &manifestDict, &configProxies, configuration, proxy, tagStr] { if ([manifestDict[@"configurations"][cppToObjStr(configuration->second)][@"architectures"] - containsObject:cppToObjStr(proxy.first)]) { - configurations[configuration->second.c_str()].architectures[proxy.first].anchors.push_back( - proxy.second->installName); + containsObject:cppToObjStr(proxy.second->arch)]) { + _configurations[configuration->second].architectures[proxy.second->arch].anchors.push_back(proxy.second->identifier); } - } - - free(tags); + }); } - } break; - case BOMSymlinkType: { - if (!has_prefix(entryPath, "/usr/lib/") && !has_prefix(entryPath, "/System/Library/")) - break; - const char* target = BOMFSObjectSymlinkTarget(fsObject); - if (target) { - symlinks[entryPath] = effectivePath(entryPath, target); - } - } break; - case BOMDirectoryType: { - if (!has_prefix(entryPath, "/usr/lib/") && !has_prefix(entryPath, "/System/Library/")) - break; - directories.insert(entryPath); - } break; - default: - break; } - } - - MBIteratorFree(metabomEnumerator); - MBMetabomFree(metabom); - - dispatch_queue_t symlinkQueue = dispatch_get_global_queue(QOS_CLASS_DEFAULT, NULL); - dispatch_group_t symlinkGroup = dispatch_group_create(); - - for (auto& fileSet : architectureFiles) { - cacheBuilderDispatchGroupAsync(symlinkGroup, symlinkQueue, [&] { - for (auto& file : fileSet.second.dylibs) { - bool aliasAdded = true; - auto proxy = file.second.proxy; - - while (aliasAdded) { - aliasAdded = false; + } + dispatch_semaphore_signal(bomSemaphore); + }); + } - for (auto& symlink : symlinks) { - std::set newAliases; - auto alias = checkSymlink(proxy->installName, symlink, directories); - if (alias != "") { - newAliases.insert(alias); - } + dispatch_group_wait(bomGroup, DISPATCH_TIME_FOREVER); + MBIteratorFree(metabomEnumerator); + MBMetabomFree(metabom); + MachOProxy::mapDependencies(); +} - for (auto& existingAlias : proxy->installNameAliases) { - alias = checkSymlink(existingAlias, symlink, directories); - if (alias != "") { - newAliases.insert(alias); - } - } +#endif - for (auto& alias : newAliases) { - if (proxy->addAlias(alias)) { - aliasAdded = true; - } - } - } +template +bool checkLink(MachOProxy* proxy, const uint8_t* p, const uint8_t* end) +{ + bool retval = true; + std::vector dylibs = proxy->dependencies(); + + std::string symbolName; + int libraryOrdinal = 0; + bool weakImport = false; + bool done = false; + + while (!done && (p < end)) { + uint8_t immediate = *p & BIND_IMMEDIATE_MASK; + uint8_t opcode = *p & BIND_OPCODE_MASK; + ++p; + switch (opcode) { + case BIND_OPCODE_DONE: + done = true; + break; + case BIND_OPCODE_SET_DYLIB_ORDINAL_IMM: + libraryOrdinal = immediate; + break; + case BIND_OPCODE_SET_DYLIB_ORDINAL_ULEB: + libraryOrdinal = (int)read_uleb128(p, end); + break; + case BIND_OPCODE_SET_DYLIB_SPECIAL_IMM: + // the special ordinals are negative numbers + if (immediate == 0) + libraryOrdinal = 0; + else { + int8_t signExtended = BIND_OPCODE_MASK | immediate; + libraryOrdinal = signExtended; + } + break; + case BIND_OPCODE_SET_SYMBOL_TRAILING_FLAGS_IMM: + weakImport = ((immediate & BIND_SYMBOL_FLAGS_WEAK_IMPORT) != 0); + symbolName = (char*)p; + while (*p != '\0') + ++p; + ++p; + break; + case BIND_OPCODE_SET_TYPE_IMM: + break; + case BIND_OPCODE_SET_ADDEND_SLEB: + (void)read_sleb128(p, end); + break; + case BIND_OPCODE_SET_SEGMENT_AND_OFFSET_ULEB: + case BIND_OPCODE_ADD_ADDR_ULEB: + (void)read_uleb128(p, end); + break; + case BIND_OPCODE_DO_BIND_ULEB_TIMES_SKIPPING_ULEB: + (void)read_uleb128(p, end); + case BIND_OPCODE_DO_BIND_ADD_ADDR_ULEB: + (void)read_uleb128(p, end); + case BIND_OPCODE_DO_BIND_ADD_ADDR_IMM_SCALED: + case BIND_OPCODE_DO_BIND: { + if (libraryOrdinal <= 0) + break; + if ( libraryOrdinal > dylibs.size() ) { + warning("Illegal library ordinal (%d) in dylib %s bind opcode (max ordinal %lu)", libraryOrdinal, proxy->path.c_str(), dylibs.size()); + retval = false; + } + else { + auto dependencyProxy = MachOProxy::forInstallnameAndArch(dylibs[libraryOrdinal - 1], proxy->arch); + if (!weakImport && (!dependencyProxy || !dependencyProxy->providesSymbol(symbolName))) { + warning("Could not find symbol %s in dylib %s for %s", symbolName.c_str(), dylibs[libraryOrdinal - 1].c_str(), proxy->path.c_str()); + retval = false; } } - }); + } break; + default: + warning("bad bind opcode in binary 0x%02X in %s", *p, proxy->path.c_str()); } - dispatch_group_wait(symlinkGroup, DISPATCH_TIME_FOREVER); + } - for (auto& fileSet : architectureFiles) { - for (auto& file : fileSet.second.dylibs) { - auto proxy = file.second.proxy; + return retval; +} - for (const auto& dependency : proxy->dependencies) { - auto dependencyProxy = dylibProxy(dependency, fileSet.first); - if (dependencyProxy == nullptr) - break; +bool checkLink(MachOProxy* proxy) +{ + switch (archForString(proxy->arch).arch) { + case CPU_TYPE_ARM: + case CPU_TYPE_I386: + return (checkLink>(proxy, proxy->getBindStart(), proxy->getBindEnd()) + && checkLink>(proxy, proxy->getLazyBindStart(), proxy->getLazyBindEnd())); + case CPU_TYPE_ARM64: + case CPU_TYPE_X86_64: + return (checkLink>(proxy, proxy->getBindStart(), proxy->getBindEnd()) + && checkLink>(proxy, proxy->getLazyBindStart(), proxy->getLazyBindEnd())); + default: + terminate("unsupported arch 0x%08X", archForString(proxy->arch).arch); + } +} - dependencyProxy->dependents.insert(proxy->installName); - } +bool Manifest::checkLinks() +{ + dispatch_queue_t linkCheckQueue = dispatch_get_global_queue(QOS_CLASS_DEFAULT, NULL); + dispatch_semaphore_t linkCheckSemphore = dispatch_semaphore_create(32); + + dispatch_group_t linkCheckGroup = dispatch_group_create(); + + runConcurrently(linkCheckQueue, linkCheckSemphore, [this](const std::string configuration, const std::string architecture) { + for (const auto& anchor : this->configuration(configuration).architecture(architecture).anchors) { + const auto identifier = anchor.identifier; + const auto proxy = MachOProxy::forIdentifier(identifier, architecture); + if (proxy->isExecutable()) { + checkLink(proxy); } } -} -#endif + }); -void Manifest::calculateClosure( bool enforceRootless ) { - rootless = enforceRootless; + dispatch_group_wait(linkCheckGroup, DISPATCH_TIME_FOREVER); - for ( auto& config : configurations ) { - for ( auto& arch : config.second.architectures ) { - calculateClosure( config.first, arch.first ); + return true; +} + +void Manifest::runConcurrently(dispatch_queue_t queue, dispatch_semaphore_t concurrencyLimitingSemaphore, std::function lambda) +{ + dispatch_group_t runGroup = dispatch_group_create(); + for (auto& config : _configurations) { + for (auto& architecture : config.second.architectures) { + dispatch_semaphore_wait(concurrencyLimitingSemaphore, DISPATCH_TIME_FOREVER); + cacheBuilderDispatchGroupAsync(runGroup, queue, [&] { + WarningTargets targets; + targets.first = this; + targets.second.insert(std::make_pair(config.first, architecture.first)); + auto ctx = std::make_shared(config.first + "/" + architecture.first, targets); + setLoggingContext(ctx); + lambda(config.first, architecture.first); + dispatch_semaphore_signal(concurrencyLimitingSemaphore); + }); } } -} -Manifest::File* Manifest::dylibForInstallName( const std::string& installname, const std::string& arch ) { - auto archIter = architectureFiles.find( arch ); - if ( archIter == architectureFiles.end() ) return nullptr; + dispatch_group_wait(runGroup, DISPATCH_TIME_FOREVER); +} - auto& files = archIter->second.dylibs; - auto dylibIterator = files.find( installname ); +bool Manifest::filterForConfig(const std::string& configName) +{ + for (const auto configuration : _configurations) { + if (configName == configuration.first) { + std::map filteredConfigs; + filteredConfigs[configName] = configuration.second; - if ( dylibIterator != files.end() ) return &dylibIterator->second; + _configurations = filteredConfigs; - for ( auto& candidate : files ) { - if ( candidate.second.proxy->installNameAliases.count( installname ) > 0 ) { - dylibIterator = files.find( candidate.first ); - return &dylibIterator->second; - } + for (auto& arch : configuration.second.architectures) { + arch.second.results = Manifest::Results(); + } + return true; } - // Check if we can fallback to an interworkable architecture - std::string fallbackArchStr = fallbackArchStringForArchString( arch ); - if ( !fallbackArchStr.empty() ) { - return dylibForInstallName( installname, fallbackArchStr ); - } - - return nullptr; + } + return false; } +void Manifest::calculateClosure(bool enforceRootless) +{ + auto closureSemaphore = dispatch_semaphore_create(32); + auto closureGroup = dispatch_group_create(); + auto closureQueue = dispatch_queue_create("com.apple.dyld.cache.closure", dispatch_queue_attr_make_with_qos_class(DISPATCH_QUEUE_CONCURRENT, QOS_CLASS_USER_INITIATED, 0)); + rootless = enforceRootless; -MachOProxy* Manifest::dylibProxy( const std::string& installname, const std::string& arch ) { - auto dylib = dylibForInstallName( installname, arch ); - - if ( dylib != nullptr ) { - assert( dylib->proxy != nullptr ); - return dylib->proxy; + for (auto& config : _configurations) { + for (auto& arch : config.second.architectures) { + dispatch_semaphore_wait(closureSemaphore, DISPATCH_TIME_FOREVER); + cacheBuilderDispatchGroupAsync(closureGroup, closureQueue, [&] { + calculateClosure(config.first, arch.first); + dispatch_semaphore_signal(closureSemaphore); + }); + } } - return nullptr; + dispatch_group_wait(closureGroup, DISPATCH_TIME_FOREVER); +} + +void Manifest::remove(const std::string& config, const std::string& arch) +{ + if (_configurations.count(config)) + _configurations[config].architectures.erase(arch); } bool Manifest::sameContentsAsCacheAtPath(const std::string& configuration, const std::string& architecture, const std::string& path) const { - __block std::set>> cacheDylibs; - std::set>> manifestDylibs; - struct stat statbuf; - if ( ::stat(path.c_str(), &statbuf) == -1 ) { - // don't warn if there is no existing cache file - if ( errno != ENOENT ) - warning("stat() failed for dyld shared cache at %s, errno=%d", path.c_str(), errno); - return false; + std::set> cacheDylibs; + std::set> manifestDylibs; + struct stat statbuf; + if (::stat(path.c_str(), &statbuf) == -1) { + // don't warn if there is no existing cache file + if (errno != ENOENT) + warning("stat() failed for dyld shared cache at %s, errno=%d", path.c_str(), errno); + return false; } int cache_fd = ::open(path.c_str(), O_RDONLY); @@ -393,89 +504,73 @@ Manifest::sameContentsAsCacheAtPath(const std::string& configuration, const std: } ::close(cache_fd); - if (configurations.count(configuration) == 0 - || configurations.find(configuration)->second.architectures.count(architecture) == 0) - return false; - - for (auto& dylib : configurations.find(configuration)->second.architectures.find(architecture)->second.results.dylibs) { - if ( dylib.second.included == true) { - std::pair> dylibPair; - dylibPair.first = dylib.first; - bcopy((const void *)&dylib.second.uuid[0], &dylibPair.second[0], sizeof(uuid_t)); - manifestDylibs.insert(dylibPair); - auto file = architectureFiles.find(architecture)->second.dylibs.find(dylib.first); - if (file != architectureFiles.find(architecture)->second.dylibs.end()) { - for ( auto& alias : file->second.proxy->installNameAliases ) { - std::pair> aliasPair; - aliasPair.first = alias; - bcopy((const void *)&dylib.second.uuid[0], &aliasPair.second[0], sizeof(uuid_t)); - manifestDylibs.insert(aliasPair); - } - } - } - } - - (void)dyld_shared_cache_iterate(mappedCache, (uint32_t)statbuf.st_size, - ^(const dyld_shared_cache_dylib_info* dylibInfo, const dyld_shared_cache_segment_info* segInfo){ - std::pair> dylibPair; - dylibPair.first = dylibInfo->path; - bcopy((const void *)&dylibInfo->uuid[0], &dylibPair.second[0], sizeof(uuid_t)); - cacheDylibs.insert(dylibPair); - }); + if (_configurations.count(configuration) == 0 + || _configurations.find(configuration)->second.architectures.count(architecture) == 0) + return false; + + Architecture existingArch; + (void)dyld_shared_cache_iterate(mappedCache, (uint32_t)statbuf.st_size, + [&existingArch, &architecture](const dyld_shared_cache_dylib_info* dylibInfo, const dyld_shared_cache_segment_info* segInfo) { + UUID uuid = *dylibInfo->uuid; + DylibInfo info; + info.uuid = uuid; + existingArch.results.dylibs[ImageIdentifier(uuid)] = info; + }); - return (manifestDylibs == cacheDylibs); + return (existingArch == _configurations.find(configuration)->second.architectures.find(architecture)->second); } -void Manifest::removeDylib( MachOProxy* proxy, const std::string& reason, const std::string& configuration, - const std::string& architecture, std::unordered_set& processedInstallnames ) { - auto configIter = configurations.find( configuration ); - if ( configIter == configurations.end() ) return; +void Manifest::removeDylib(MachOProxy* proxy, const std::string& reason, const std::string& configuration, + const std::string& architecture, std::unordered_set& processedIdentifiers) +{ + auto configIter = _configurations.find(configuration); + if (configIter == _configurations.end()) + return; auto archIter = configIter->second.architectures.find( architecture ); if ( archIter == configIter->second.architectures.end() ) return; auto& archManifest = archIter->second; - if ( archManifest.results.dylibs.count( proxy->installName ) == 0 ) { - bcopy( &proxy->uuid[0], &archManifest.results.dylibs[proxy->installName].uuid[0], sizeof( uuid_t ) ); - processedInstallnames.insert( proxy->installName ); + if (archManifest.results.dylibs.count(proxy->identifier) == 0) { + archManifest.results.dylibs[proxy->identifier].uuid = proxy->uuid; + archManifest.results.dylibs[proxy->identifier].installname = proxy->installName; + processedIdentifiers.insert(proxy->identifier); } - archManifest.results.dylibs[proxy->installName].exclude( reason ); + archManifest.results.exclude(MachOProxy::forIdentifier(proxy->identifier, architecture), reason); - processedInstallnames.insert( proxy->installName ); - for ( auto& alias : proxy->installNameAliases ) { - processedInstallnames.insert( alias ); - } + processedIdentifiers.insert(proxy->identifier); - for ( const auto& dependent : proxy->dependents ) { - auto dependentProxy = dylibProxy( dependent, architecture ); - auto dependentResultIter = archManifest.results.dylibs.find( dependentProxy->installName ); + for (const auto& dependent : proxy->dependentIdentifiers) { + auto dependentProxy = MachOProxy::forIdentifier(dependent, architecture); + auto dependentResultIter = archManifest.results.dylibs.find(dependentProxy->identifier); if ( dependentProxy && ( dependentResultIter == archManifest.results.dylibs.end() || dependentResultIter->second.included == true ) ) { - removeDylib( dependentProxy, "Missing dependency: " + proxy->installName, configuration, architecture, - processedInstallnames ); + removeDylib(dependentProxy, "Missing dependency: " + proxy->installName, configuration, architecture, + processedIdentifiers); } } } MachOProxy* Manifest::removeLargestLeafDylib( const std::string& configuration, const std::string& architecture ) { - std::set activeInstallnames; + std::set activeIdentifiers; std::set leafDylibs; - auto configIter = configurations.find( configuration ); - if ( configIter == configurations.end() ) terminate( "Internal error" ); + auto configIter = _configurations.find(configuration); + if (configIter == _configurations.end()) + terminate("Internal error"); ; auto archIter = configIter->second.architectures.find( architecture ); if ( archIter == configIter->second.architectures.end() ) terminate( "Internal error" ); ; for ( const auto& dylibInfo : archIter->second.results.dylibs ) { if ( dylibInfo.second.included ) { - activeInstallnames.insert( dylibInfo.first ); + activeIdentifiers.insert(dylibInfo.first); } } - for ( const auto& installname : activeInstallnames ) { - auto dylib = dylibProxy( installname, architecture ); + for (const auto& identifier : activeIdentifiers) { + auto dylib = MachOProxy::forIdentifier(identifier, architecture); bool dependents = false; - for ( const auto& depedent : dylib->dependents ) { - if ( depedent != dylib->installName && activeInstallnames.count( depedent ) ) { + for (const auto& depedent : dylib->dependentIdentifiers) { + if (depedent != identifier && activeIdentifiers.count(depedent)) { dependents = true; break; } @@ -493,134 +588,67 @@ MachOProxy* Manifest::removeLargestLeafDylib( const std::string& configuration, largestLeafDylib = dylib; } } - std::unordered_set empty; + std::unordered_set empty; removeDylib( largestLeafDylib, "VM space overflow", configuration, architecture, empty ); return largestLeafDylib; } -static void recursiveInvalidate(const std::string& invalidName, std::unordered_map>& usesOf, std::unordered_set& unusableInstallNames) -{ - if ( unusableInstallNames.count(invalidName) ) - return; - unusableInstallNames.insert(invalidName); - for (const std::string& name : usesOf[invalidName] ) { - recursiveInvalidate(name, usesOf, unusableInstallNames); - } -} - -void Manifest::pruneClosure() -{ - for (auto& config : configurations) { - for (auto& arch : config.second.architectures) { - pruneClosure(config.first, arch.first); - } - } -} - -void Manifest::pruneClosure(const std::string& configuration, const std::string& architecture) -{ - auto configIter = configurations.find(configuration); - if ( configIter == configurations.end() ) - return; - auto archIter = configIter->second.architectures.find(architecture); - if ( archIter == configIter->second.architectures.end() ) - return; - auto& archManifest = archIter->second; - - // build reverse dependency map and list of excluded dylibs - std::unordered_map> reverseDep; - std::unordered_set unusableStart; - for (const auto& dylib : archManifest.results.dylibs) { - const std::string dylibInstallName = dylib.first; - if ( dylib.second.included ) { - if ( MachOProxy* proxy = dylibProxy(dylibInstallName, architecture) ) { - for (const std::string& dependentPath : proxy->dependencies) { - reverseDep[dependentPath].insert(dylibInstallName); - } - } - } - else { - unusableStart.insert(dylibInstallName); - } - } - - // mark unusable, all dylibs depending on the initially unusable dylibs - std::unordered_set newUnusable; - for (const std::string& unusable : unusableStart) { - recursiveInvalidate(unusable, reverseDep, newUnusable); - } - - // remove unusable dylibs from manifest - std::unordered_set dummy; - for (const std::string& unusable : newUnusable) { - if ( MachOProxy* proxy = dylibProxy(unusable, architecture) ) - removeDylib(proxy, "Missing dependency: " + unusable, configuration, architecture, dummy); - warning("can't use: %s because dependent dylib cannot be used", unusable.c_str()); - } -} - void Manifest::calculateClosure( const std::string& configuration, const std::string& architecture ) { - auto& archManifest = configurations[configuration].architectures[architecture]; - auto archFileIter = architectureFiles.find( architecture ); - assert( archFileIter != architectureFiles.end() ); - auto files = archFileIter->second.dylibs; - - std::unordered_set newInstallnames; + auto& archManifest = _configurations[configuration].architectures[architecture]; + std::unordered_set newIdentifiers; for ( auto& anchor : archManifest.anchors ) { - newInstallnames.insert(anchor.installname); - } + newIdentifiers.insert(anchor.identifier); + } - std::unordered_set processedInstallnames; + std::unordered_set processedIdentifiers; - while (!newInstallnames.empty()) { - std::unordered_set installnamesToProcess = newInstallnames; - newInstallnames.clear(); + while (!newIdentifiers.empty()) { + std::unordered_set identifiersToProcess = newIdentifiers; + newIdentifiers.clear(); - for (const std::string& installname : installnamesToProcess) { - if (processedInstallnames.count(installname) > 0) { - continue; - } + for (const auto& identifier : identifiersToProcess) { + if (processedIdentifiers.count(identifier) > 0) { + continue; + } - auto proxy = dylibProxy( installname, architecture ); + auto proxy = MachOProxy::forIdentifier(identifier, architecture); - if ( proxy == nullptr ) { + if (proxy == nullptr) { // No path - archManifest.results.dylibs[installname].exclude( "Could not find file for install name" ); - warning("Could not find file for install name (%s)", installname.c_str()); continue; } - if (configurations[configuration].restrictedInstallnames.count(installname) != 0) { - removeDylib(proxy, "Dylib '" + installname + "' removed due to explict restriction", configuration, architecture, - processedInstallnames); + // HACK: This is a policy decision we may want to revisit. + if (!proxy->isDylib()) { continue; } - // Validate we have all are depedencies - for ( const auto& dependency : proxy->dependencies ) { - if ( !dylibProxy( dependency, architecture ) ) { - removeDylib( proxy, "Missing dependency: " + dependency, configuration, architecture, - processedInstallnames ); - break; - } + + if (!proxy->error.empty()) { + archManifest.results.exclude(proxy, proxy->error); + processedIdentifiers.insert(proxy->identifier); + continue; } - // assert(info->installName == installname); - if ( archManifest.results.dylibs.count( proxy->installName ) == 0 ) { - bcopy( &proxy->uuid[0], &archManifest.results.dylibs[proxy->installName].uuid[0], sizeof( uuid_t ) ); - processedInstallnames.insert( proxy->installName ); + if (proxy->isDylib()) { + if (_configurations[configuration].restrictedInstallnames.count(proxy->installName) != 0) { + removeDylib(proxy, "Dylib '" + proxy->installName + "' removed due to explict restriction", configuration, architecture, + processedIdentifiers); + continue; + } - auto fileIter = files.find( proxy->installName ); - if ( fileIter != files.end() ) { - for ( auto& aliasName : fileIter->second.proxy->installNameAliases ) { - processedInstallnames.insert( aliasName ); - } + if (archManifest.results.dylibs.count(proxy->identifier) == 0) { + archManifest.results.dylibs[proxy->identifier].uuid = proxy->uuid; + archManifest.results.dylibs[proxy->identifier].installname = proxy->installName; + archManifest.results.dylibs[proxy->identifier].included = true; + + processedIdentifiers.insert(proxy->identifier); } } - for ( const auto& dependency : proxy->dependencies ) { - if ( processedInstallnames.count( dependency ) == 0 ) { - newInstallnames.insert( dependency ); + for (const auto& dependency : proxy->requiredIdentifiers) { + if (processedIdentifiers.count(dependency) == 0) { + newIdentifiers.insert(dependency); } } } @@ -635,17 +663,17 @@ void Manifest::write( const std::string& path ) { NSMutableDictionary* configurationsDict = [[NSMutableDictionary alloc] init]; NSMutableDictionary* resultsDict = [[NSMutableDictionary alloc] init]; - cacheDict[@"manifest-version"] = @( manifest_version ); - cacheDict[@"build"] = cppToObjStr( build ); - cacheDict[@"dylibOrderFile"] = cppToObjStr( dylibOrderFile ); - cacheDict[@"dirtyDataOrderFile"] = cppToObjStr( dirtyDataOrderFile ); - cacheDict[@"metabomFile"] = cppToObjStr( metabomFile ); + cacheDict[@"manifest-version"] = @(version()); + cacheDict[@"build"] = cppToObjStr(build()); + cacheDict[@"dylibOrderFile"] = cppToObjStr(dylibOrderFile()); + cacheDict[@"dirtyDataOrderFile"] = cppToObjStr(dirtyDataOrderFile()); + cacheDict[@"metabomFile"] = cppToObjStr(metabomFile()); cacheDict[@"projects"] = projectDict; cacheDict[@"results"] = resultsDict; cacheDict[@"configurations"] = configurationsDict; - for ( const auto& project : projects ) { + for (const auto& project : projects()) { NSMutableArray* sources = [[NSMutableArray alloc] init]; for ( const auto& source : project.second.sources ) { @@ -655,7 +683,7 @@ void Manifest::write( const std::string& path ) { projectDict[cppToObjStr( project.first )] = sources; } - for ( auto& configuration : configurations ) { + for (auto& configuration : _configurations) { NSMutableArray* archArray = [[NSMutableArray alloc] init]; for ( auto& arch : configuration.second.architectures ) { [archArray addObject:cppToObjStr( arch.first )]; @@ -674,7 +702,7 @@ void Manifest::write( const std::string& path ) { }; } - for ( auto& configuration : configurations ) { + for (auto& configuration : _configurations) { NSMutableDictionary* archResultsDict = [[NSMutableDictionary alloc] init]; for ( auto& arch : configuration.second.architectures ) { NSMutableDictionary* dylibsDict = [[NSMutableDictionary alloc] init]; @@ -684,49 +712,49 @@ void Manifest::write( const std::string& path ) { NSString *prodCDHash = cppToObjStr(arch.second.results.productionCache.cdHash); NSString *devCDHash = cppToObjStr(arch.second.results.developmentCache.cdHash); - for ( auto& dylib : arch.second.results.dylibs ) { - NSMutableDictionary* dylibDict = [[NSMutableDictionary alloc] init]; - if ( dylib.second.included ) { - NSMutableDictionary* segments = [[NSMutableDictionary alloc] init]; - dylibDict[@"included"] = @YES; - for ( auto& segment : dylib.second.segments ) { - segments[cppToObjStr( segment.name )] = - @{ @"startAddr" : @( segment.startAddr ), - @"endAddr" : @( segment.endAddr ) }; - } - dylibDict[@"segments"] = segments; - } else { - dylibDict[@"included"] = @NO; - dylibDict[@"exclusionInfo"] = cppToObjStr(dylib.second.exclusionInfo); - } - dylibsDict[cppToObjStr( dylib.first )] = dylibDict; - } + for ( auto& dylib : arch.second.results.dylibs ) { + NSMutableDictionary* dylibDict = [[NSMutableDictionary alloc] init]; + if ( dylib.second.included ) { + NSMutableDictionary* segments = [[NSMutableDictionary alloc] init]; + dylibDict[@"included"] = @YES; + for ( auto& segment : dylib.second.segments ) { + segments[cppToObjStr( segment.name )] = + @{ @"startAddr" : @( segment.startAddr ), + @"endAddr" : @( segment.endAddr ) }; + } + dylibDict[@"segments"] = segments; + } else { + dylibDict[@"included"] = @NO; + dylibDict[@"exclusionInfo"] = cppToObjStr(dylib.second.exclusionInfo); + } + dylibsDict[cppToObjStr( dylib.second.installname )] = dylibDict; + } - for ( auto& region : arch.second.results.developmentCache.regions ) { - devRegionsDict[cppToObjStr( region.name )] = - @{ @"startAddr" : @( region.startAddr ), - @"endAddr" : @( region.endAddr ) }; - } + for ( auto& region : arch.second.results.developmentCache.regions ) { + devRegionsDict[cppToObjStr( region.name )] = + @{ @"startAddr" : @( region.startAddr ), + @"endAddr" : @( region.endAddr ) }; + } - for ( auto& region : arch.second.results.productionCache.regions ) { - prodRegionsDict[cppToObjStr( region.name )] = - @{ @"startAddr" : @( region.startAddr ), - @"endAddr" : @( region.endAddr ) }; - } + for ( auto& region : arch.second.results.productionCache.regions ) { + prodRegionsDict[cppToObjStr( region.name )] = + @{ @"startAddr" : @( region.startAddr ), + @"endAddr" : @( region.endAddr ) }; + } - for ( auto& warning : arch.second.results.warnings ) { - [warningsArray addObject:cppToObjStr( warning )]; - } + for ( auto& warning : arch.second.results.warnings ) { + [warningsArray addObject:cppToObjStr( warning )]; + } - BOOL built = arch.second.results.failure.empty(); - archResultsDict[cppToObjStr( arch.first )] = @{ - @"dylibs" : dylibsDict, - @"built" : @( built ), - @"failure" : cppToObjStr( arch.second.results.failure ), - @"productionCache" : @{@"cdhash" : prodCDHash, @"regions" : prodRegionsDict}, - @"developmentCache" : @{@"cdhash" : devCDHash, @"regions" : devRegionsDict}, - @"warnings" : warningsArray - }; + BOOL built = arch.second.results.failure.empty(); + archResultsDict[cppToObjStr( arch.first )] = @{ + @"dylibs" : dylibsDict, + @"built" : @( built ), + @"failure" : cppToObjStr( arch.second.results.failure ), + @"productionCache" : @{@"cdhash" : prodCDHash, @"regions" : prodRegionsDict}, + @"developmentCache" : @{@"cdhash" : devCDHash, @"regions" : devRegionsDict}, + @"warnings" : warningsArray + }; } resultsDict[cppToObjStr( configuration.first )] = archResultsDict; } diff --git a/interlinked-dylibs/MultiCacheBuilder.h b/interlinked-dylibs/MultiCacheBuilder.h index 9bba12d..cd29ebd 100644 --- a/interlinked-dylibs/MultiCacheBuilder.h +++ b/interlinked-dylibs/MultiCacheBuilder.h @@ -40,7 +40,6 @@ struct MultiCacheBuilder { void logStats(); private: - void runOnManifestConcurrently(std::function lambda); void buildCache(const std::string cachePath, const std::set configurations, const std::string architecture, bool development); void write_cache(std::string cachePath, const std::set& configurations, const std::string& architecture, std::shared_ptr cache, bool developmentCache); }; diff --git a/interlinked-dylibs/MultiCacheBuilder.mm b/interlinked-dylibs/MultiCacheBuilder.mm index d1c7bab..4658cbc 100644 --- a/interlinked-dylibs/MultiCacheBuilder.mm +++ b/interlinked-dylibs/MultiCacheBuilder.mm @@ -23,6 +23,7 @@ #include "mega-dylib-utils.h" #include "Logging.h" +#include "MachOProxy.h" #include "MultiCacheBuilder.h" @@ -155,44 +156,42 @@ void MultiCacheBuilder::write_cache(std::string cachePath, const std::set configurations, const std::string architecture, bool development) { - auto& configResults = _manifest.configurations[*configurations.begin()].architectures[architecture].results.dylibs; + auto& configResults = _manifest.configuration(*configurations.begin()).architecture(architecture).results.dylibs; if ( _skipBuilds ) { log( "Build Skipped" ); for ( auto& config : configurations ) { for ( auto& dylib : configResults ) { - _manifest.configurations[config].architectures[architecture].results.dylibs[dylib.first].exclude( - "All dylibs excluded" ); + _manifest.configuration(config).architecture(architecture).results.exclude(MachOProxy::forIdentifier(dylib.first, architecture), "All dylibs excluded"); } } return; } - Manifest::Architecture arch; std::vector> dylibs; std::vector emptyList; std::shared_ptr cache = std::make_shared(_manifest, *configurations.begin(), architecture); for (auto& config : configurations) { - auto& results = _manifest.configurations[config].architectures[architecture].results.dylibs; + auto& results = _manifest.configuration(config).architecture(architecture).results; - for (auto& dylib : configResults) { - if (dylib.second.included == false - && results.count(dylib.first) - && results[dylib.first].included == true) { - results[dylib.first].exclude(dylib.second.exclusionInfo); - } + for (auto& dylib : configResults) { + if (dylib.second.included == false + && results.dylibs.count(dylib.first) + && results.dylibs[dylib.first].included == true) { + results.exclude(MachOProxy::forIdentifier(dylib.first, architecture), dylib.second.exclusionInfo); + } } } - if (development) { - cache->buildForDevelopment(cachePath); - } else { - cache->buildForProduction(cachePath); - } + if (development) { + cache->buildForDevelopment(cachePath); + } else { + cache->buildForProduction(cachePath); + } - std::vector regionStartAddresses; + std::vector regionStartAddresses; std::vector regionSizes; std::vector regionFileOffsets; @@ -207,95 +206,74 @@ void MultiCacheBuilder::buildCache(const std::string cachePath, const std::setforEachImage([&](const void* machHeader, const char* installName, time_t mtime, - ino_t inode, const std::vector& segments) { - for (auto& seg : segments) { - uint64_t vmAddr = 0; - for (int i=0; i < regionSizes.size(); ++i) { - if ( (seg.fileOffset >= regionFileOffsets[i]) && (seg.fileOffset < (regionFileOffsets[i]+regionSizes[i])) ) { - vmAddr = regionStartAddresses[i] + seg.fileOffset - regionFileOffsets[i]; - } - } - for (auto& config : configurations) { - _manifest.configurations[config].architectures[architecture].results.dylibs[installName].segments.push_back({seg.name, vmAddr, vmAddr+seg.size}); - if (_manifest.configurations[config].architectures[architecture].results.dylibs[installName].segments.size() == 0) { - warning("Attempting to write info for excluded dylib"); - _manifest.configurations[config].architectures[architecture].results.dylibs[installName].exclude("Internal Error"); - } - } - } - }); - if (development) { - verboseLog("developement cache size = %llu", cache->fileSize()); - } else { - verboseLog("production cache size = %llu", cache->fileSize()); - } + cache->forEachImage([&](const void* machHeader, const char* installName, time_t mtime, + ino_t inode, const std::vector& segments) { + for (auto& seg : segments) { + uint64_t vmAddr = 0; + for (int i = 0; i < regionSizes.size(); ++i) { + if ((seg.fileOffset >= regionFileOffsets[i]) && (seg.fileOffset < (regionFileOffsets[i] + regionSizes[i]))) { + vmAddr = regionStartAddresses[i] + seg.fileOffset - regionFileOffsets[i]; + } + } + + for (auto& config : configurations) { + _manifest.configuration(config).architecture(architecture).results.dylibForInstallname(installName).segments.push_back({ seg.name, vmAddr, vmAddr + seg.size }); + if (_manifest.configuration(config).architecture(architecture).results.dylibForInstallname(installName).segments.size() == 0) { + warning("Attempting to write info for non-existent dylib"); + } + } + } + }); + if (development) { + verboseLog("developement cache size = %llu", cache->fileSize()); + } else { + verboseLog("production cache size = %llu", cache->fileSize()); + } if ( cache->vmSize()+align(cache->vmSize()/200, sharedRegionRegionAlignment(archForString(architecture))) > sharedRegionRegionSize(archForString(architecture))) { warning("shared cache will not fit in shared regions address space. Overflow amount: %llu", cache->vmSize() + align(cache->vmSize() / 200, sharedRegionRegionAlignment(archForString(architecture))) - sharedRegionRegionSize(archForString(architecture))); return; } - write_cache(cachePath, configurations, architecture, cache, development); - for (auto& config : configurations) { + write_cache(cachePath, configurations, architecture, cache, development); + for (auto& config : configurations) { if (development) { - _manifest.configurations[config].architectures[architecture].results.developmentCache.cdHash = cache->cdHashString(); - } - else { - _manifest.configurations[config].architectures[architecture].results.productionCache.cdHash = cache->cdHashString(); - } - } -} - -void MultiCacheBuilder::runOnManifestConcurrently(std::function lambda) -{ - dispatch_group_t runGroup = dispatch_group_create(); - for (auto& config : _manifest.configurations) { - for (auto& architecture : config.second.architectures) { - dispatch_semaphore_wait(_concurrencyLimitingSemaphore, DISPATCH_TIME_FOREVER); - cacheBuilderDispatchGroupAsync(runGroup, _buildQueue, [&] { - WarningTargets targets; - targets.first = &_manifest; - targets.second.insert(std::make_pair(config.first, architecture.first)); - auto ctx = std::make_shared(config.first + "/" + architecture.first, targets); - setLoggingContext(ctx); - lambda(config.first, architecture.first); - dispatch_semaphore_signal(_concurrencyLimitingSemaphore); - }); + _manifest.configuration(config).architecture(architecture).results.developmentCache.cdHash = cache->cdHashString(); + } else { + _manifest.configuration(config).architecture(architecture).results.productionCache.cdHash = cache->cdHashString(); } - } - - dispatch_group_wait(runGroup, DISPATCH_TIME_FOREVER); + } } void MultiCacheBuilder::buildCaches(std::string masterDstRoot) { if (_bniMode) { std::vector> dedupedCacheSets; - for (auto& config : _manifest.configurations) { - bool dupeFound = false; - - for (auto& cacheSet : dedupedCacheSets) { - if (config.second.equivalent(_manifest.configurations[*cacheSet.begin()])) { - cacheSet.insert(config.first); - dupeFound = true; - break; - } - } + _manifest.forEachConfiguration([&dedupedCacheSets, this](const std::string& configName) { + auto config = _manifest.configuration(configName); + bool dupeFound = false; + + for (auto& cacheSet : dedupedCacheSets) { + if (config == _manifest.configuration(*cacheSet.begin())) { + cacheSet.insert(configName); + dupeFound = true; + break; + } + } - if (!dupeFound) { - std::set temp; - temp.insert(config.first); - dedupedCacheSets.push_back(temp); - } - } + if (!dupeFound) { + std::set temp; + temp.insert(configName); + dedupedCacheSets.push_back(temp); + } + }); - for (auto& cacheSet : dedupedCacheSets) { + for (auto& cacheSet : dedupedCacheSets) { //FIXME we may want to consider moving to hashes of UUID sets std::string setName; @@ -326,8 +304,8 @@ void MultiCacheBuilder::buildCaches(std::string masterDstRoot) { } } - for (auto& arch : _manifest.configurations[*cacheSet.begin()].architectures) { - dispatch_semaphore_wait(_concurrencyLimitingSemaphore, DISPATCH_TIME_FOREVER); + for (auto& arch : _manifest.configuration(*cacheSet.begin()).architectures) { + dispatch_semaphore_wait(_concurrencyLimitingSemaphore, DISPATCH_TIME_FOREVER); cacheBuilderDispatchGroupAsync(_writeGroup, _buildQueue, [=] { WarningTargets targets; targets.first = &_manifest; @@ -358,20 +336,22 @@ void MultiCacheBuilder::buildCaches(std::string masterDstRoot) { dispatch_group_wait(_writeGroup, DISPATCH_TIME_FOREVER); #if BOM_SUPPORT - if ( !_skipWrites ) { - for ( auto& configuration : _manifest.configurations ) { - std::vector prodBomPaths; - std::vector devBomPaths; - - for (auto& arch : configuration.second.architectures) { - std::string cachePath = "dyld_shared_cache_" + arch.first; - prodBomPaths.push_back(cachePath); - cachePath += ".development"; - devBomPaths.push_back(cachePath); - dispatch_group_enter(_writeGroup); + if (!_skipWrites) { + _manifest.forEachConfiguration([this, &masterDstRoot](const std::string& configName) { + auto config = _manifest.configuration(configName); + // for ( auto& configuration : _manifest.configurations ) { + std::vector prodBomPaths; + std::vector devBomPaths; + + for (auto& arch : config.architectures) { + std::string cachePath = "dyld_shared_cache_" + arch.first; + prodBomPaths.push_back(cachePath); + cachePath += ".development"; + devBomPaths.push_back(cachePath); + dispatch_group_enter(_writeGroup); cacheBuilderDispatchAsync(_writeQueue, [=] { char buffer[MAXPATHLEN]; - sprintf(buffer, "%s/Boms/%s.prod.bom", masterDstRoot.c_str(), configuration.first.c_str()); + sprintf(buffer, "%s/Boms/%s.prod.bom", masterDstRoot.c_str(), configName.c_str()); BOMBom bom = BOMBomNew(buffer); insertCacheDirInBom(bom); for (auto& path : prodBomPaths) { @@ -379,7 +359,7 @@ void MultiCacheBuilder::buildCaches(std::string masterDstRoot) { } BOMBomFree(bom); - sprintf(buffer, "%s/Boms/%s.dev.bom", masterDstRoot.c_str(), configuration.first.c_str()); + sprintf(buffer, "%s/Boms/%s.dev.bom", masterDstRoot.c_str(), configName.c_str()); bom = BOMBomNew(buffer); insertCacheDirInBom(bom); for (auto& path : devBomPaths) { @@ -387,7 +367,7 @@ void MultiCacheBuilder::buildCaches(std::string masterDstRoot) { } BOMBomFree(bom); - sprintf(buffer, "%s/Boms/%s.full.bom", masterDstRoot.c_str(), configuration.first.c_str()); + sprintf(buffer, "%s/Boms/%s.full.bom", masterDstRoot.c_str(), configName.c_str()); bom = BOMBomNew(buffer); insertCacheDirInBom(bom); for (auto& path : prodBomPaths) { @@ -400,32 +380,32 @@ void MultiCacheBuilder::buildCaches(std::string masterDstRoot) { dispatch_group_leave(_writeGroup); }); } - } - } + }); + } #endif /* BOM_SUPPORT */ - } else { - runOnManifestConcurrently( - [&](const std::string configuration, const std::string architecture) { - cacheBuilderDispatchGroupAsync(_writeGroup, _buildQueue, [=] { - std::set configurations; - configurations.insert( configuration ); - // FIXME hacky, we make implicit assumptions about dev vs non-dev and layout depending on the flags - if ( _buildRoot ) { - int err = mkpath_np( ( masterDstRoot + "/System/Library/Caches/com.apple.dyld/" ).c_str(), 0755 ); - - if ( err != 0 && err != EEXIST ) { - terminate( "mkpath_np fail: %d", err ); - } - buildCache(masterDstRoot + "/System/Library/Caches/com.apple.dyld/dyld_shared_cache_" + architecture, - configurations, architecture, false); - buildCache(masterDstRoot + "/System/Library/Caches/com.apple.dyld/dyld_shared_cache_" + architecture + ".development", - configurations, architecture, true); - } else { - buildCache(masterDstRoot + "/dyld_shared_cache_" + architecture, configurations, architecture, true); + } else { + _manifest.runConcurrently(_buildQueue, _concurrencyLimitingSemaphore, + [&](const std::string configuration, const std::string architecture) { + cacheBuilderDispatchGroupAsync(_writeGroup, _buildQueue, [=] { + std::set configurations; + configurations.insert( configuration ); + // FIXME hacky, we make implicit assumptions about dev vs non-dev and layout depending on the flags + if ( _buildRoot ) { + int err = mkpath_np( ( masterDstRoot + "/System/Library/Caches/com.apple.dyld/" ).c_str(), 0755 ); + + if ( err != 0 && err != EEXIST ) { + terminate( "mkpath_np fail: %d", err ); } - }); + buildCache(masterDstRoot + "/System/Library/Caches/com.apple.dyld/dyld_shared_cache_" + architecture, + configurations, architecture, false); + buildCache(masterDstRoot + "/System/Library/Caches/com.apple.dyld/dyld_shared_cache_" + architecture + ".development", + configurations, architecture, true); + } else { + buildCache(masterDstRoot + "/dyld_shared_cache_" + architecture, configurations, architecture, true); + } }); - dispatch_group_wait(_writeGroup, DISPATCH_TIME_FOREVER); + }); + dispatch_group_wait(_writeGroup, DISPATCH_TIME_FOREVER); } int err = sync_volume_np(masterDstRoot.c_str(), SYNC_VOLUME_FULLSYNC | SYNC_VOLUME_WAIT); diff --git a/interlinked-dylibs/OptimizerBranches.cpp b/interlinked-dylibs/OptimizerBranches.cpp index c982e95..110b123 100644 --- a/interlinked-dylibs/OptimizerBranches.cpp +++ b/interlinked-dylibs/OptimizerBranches.cpp @@ -43,6 +43,7 @@ #include +#include "MachOProxy.h" static const bool verbose = false; @@ -1259,11 +1260,11 @@ void SharedCache::bypassStubs(const std::vector& branchPoolStartAddrs) // construct a StubOptimizer for each image std::vector*> optimizers; - forEachImage([&](const void* mh, const char*, time_t, ino_t, const std::vector&) { + forEachImage([&](const void* mh, const char*, time_t, ino_t, const std::vector&) { optimizers.push_back(new StubOptimizer

(_buffer.get(), (macho_header

*)mh)); }); - // construct a BranchPoolDylib for each pool + // construct a BranchPoolDylib for each pool std::vector*> pools; if ( _arch.arch == CPU_TYPE_ARM64 ) { @@ -1283,8 +1284,8 @@ void SharedCache::bypassStubs(const std::vector& branchPoolStartAddrs) } }); uint64_t lastLinkEditRegionUsedOffset = 0; - forEachImage([&](const void* mh, const char*, time_t, ino_t, const std::vector& segs) { - for (MachOProxy::Segment seg : segs) { + forEachImage([&](const void* mh, const char*, time_t, ino_t, const std::vector& segs) { + for (const auto& seg : segs) { if ( seg.name != "__LINKEDIT" ) continue; if ( seg.fileOffset >= lastLinkEditRegionUsedOffset ) diff --git a/interlinked-dylibs/OptimizerLinkedit.cpp b/interlinked-dylibs/OptimizerLinkedit.cpp index e446c43..a3b8f4a 100644 --- a/interlinked-dylibs/OptimizerLinkedit.cpp +++ b/interlinked-dylibs/OptimizerLinkedit.cpp @@ -1148,7 +1148,7 @@ void SharedCache::optimizeLinkedit(bool dontMapLocalSymbols, bool addAccelerator { // construct a LinkeditOptimizer for each image std::vector*> optimizers; - forEachImage([&](const void* mh, const char*, time_t, ino_t, const std::vector&) { + forEachImage([&](const void* mh, const char*, time_t, ino_t, const std::vector&) { optimizers.push_back(new LinkeditOptimizer

(_buffer.get(), (macho_header

*)mh)); }); // add optimizer for each branch pool diff --git a/interlinked-dylibs/OptimizerObjC.cpp b/interlinked-dylibs/OptimizerObjC.cpp index 23c4474..7b1eef3 100644 --- a/interlinked-dylibs/OptimizerObjC.cpp +++ b/interlinked-dylibs/OptimizerObjC.cpp @@ -478,7 +478,7 @@ void optimizeObjC(SharedCache& cache, std::vector& pointersForASLR, bool const macho_section

*optPointerListSection = nullptr; std::vector*> objcDylibs; cache.forEachImage([&](const void* machHeader, const char* installName, - time_t, ino_t, const std::vector& segments) { + time_t, ino_t, const std::vector& segments) { const macho_header

* mh = (const macho_header

*)machHeader; if ( strstr(installName, "/libobjc.") != nullptr ) { optROSection = mh->getSection("__TEXT", "__objc_opt_ro"); diff --git a/interlinked-dylibs/SharedCache.cpp b/interlinked-dylibs/SharedCache.cpp index acabc4c..55ee8ae 100644 --- a/interlinked-dylibs/SharedCache.cpp +++ b/interlinked-dylibs/SharedCache.cpp @@ -52,6 +52,8 @@ #include #include +#include "MachOProxy.h" + #include "OptimizerBranches.h" #include "CacheFileAbstraction.hpp" @@ -184,18 +186,26 @@ std::string fallbackArchStringForArchString( const std::string& archStr ) { } SharedCache::SharedCache(Manifest& manifest, - const std::string& configuration, const std::string& architecture) : - _manifest(manifest), _arch(archForString(architecture)), - _archManifest(manifest.configurations.find(configuration)->second.architectures.find(architecture)->second), _buffer(nullptr), - _fileSize(0), _vmSize(0), _aliasCount(0), _slideInfoFileOffset(0), _slideInfoBufferSize(0) { + const std::string& configuration, const std::string& architecture) + : _manifest(manifest) + , _arch(archForString(architecture)) + , _archManifest(manifest.configuration(configuration).architecture(architecture)) + , _buffer(nullptr) + , _fileSize(0) + , _vmSize(0) + , _aliasCount(0) + , _slideInfoFileOffset(0) + , _slideInfoBufferSize(0) +{ auto maxCacheVMSize = sharedRegionRegionSize(_arch); - for ( auto& includedDylib : _archManifest.results.dylibs ) { - if (includedDylib.second.included) { + for (auto& includedIdentifier : _archManifest.results.dylibs) { + if (includedIdentifier.second.included) { //assert(manifest.dylibs.count(includedDylib.first) > 0); //assert(manifest.dylibs.find(includedDylib.first)->second.proxies.count(architecture) > 0); - MachOProxy* proxy = _manifest.dylibProxy( includedDylib.first, architecture ); + MachOProxy* proxy = MachOProxy::forIdentifier(includedIdentifier.first, architecture); assert(proxy != nullptr); + assert(proxy->isDylib()); _dylibs.push_back(proxy); } } @@ -211,9 +221,9 @@ SharedCache::SharedCache(Manifest& manifest, _aliasCount += dylib->installNameAliases.size(); } - sortDylibs(_manifest.dylibOrderFile); - if ( !_manifest.dirtyDataOrderFile.empty() ) - loadDirtyDataOrderFile(_manifest.dirtyDataOrderFile); + sortDylibs(_manifest.dylibOrderFile()); + if (!_manifest.dirtyDataOrderFile().empty()) + loadDirtyDataOrderFile(_manifest.dirtyDataOrderFile()); assignSegmentAddresses(); if ( _vmSize > maxCacheVMSize ) @@ -304,7 +314,7 @@ void SharedCache::buildForDevelopment(const std::string& cachePath) { std::vector emptyBranchPoolOffsets; buildUnoptimizedCache(); optimizeObjC(false/*not production*/); - if (_manifest.platform == "osx") { + if (_manifest.platform() == "osx") { optimizeLinkedit(false, false, emptyBranchPoolOffsets); } else { optimizeLinkedit(true, false, emptyBranchPoolOffsets); @@ -324,7 +334,7 @@ void SharedCache::buildForDevelopment(const std::string& cachePath) { }); _vmSize = endAddr - sharedRegionStartExecutableAddress(_arch); - if (_manifest.platform == "osx") { + if (_manifest.platform() == "osx") { appendCodeSignature("release"); } else { appendCodeSignature("development"); @@ -398,12 +408,12 @@ bool SharedCache::writeCacheMapFile(const std::string& mapPath) { std::unordered_set seenHeaders; forEachImage([&](const void* machHeader, const char* installName, time_t mtime, - ino_t inode, const std::vector& segments) { + ino_t inode, const std::vector& segments) { if ( !seenHeaders.count(machHeader) ) { seenHeaders.insert(machHeader); fprintf(fmap, "%s\n", installName); - for (const MachOProxy::Segment& seg : segments) { + for (const auto& seg : segments) { uint64_t vmAddr = 0; for (int i=0; i < regionSizes.size(); ++i) { if ( (seg.fileOffset >= regionFileOffsets[i]) && (seg.fileOffset < (regionFileOffsets[i]+regionSizes[i])) ) { @@ -415,15 +425,14 @@ bool SharedCache::writeCacheMapFile(const std::string& mapPath) { } }); - ::fclose(fmap); return true; } template -std::vector getSegments(const void* cacheBuffer, const void* machHeader) +std::vector getSegments(const void* cacheBuffer, const void* machHeader) { - std::vector result; + std::vector result; macho_header

* mh = (macho_header

*)machHeader; const uint32_t cmd_count = mh->ncmds(); const macho_load_command

* cmd = (macho_load_command

*)((uint8_t*)mh + sizeof(macho_header

)); @@ -431,9 +440,11 @@ std::vector getSegments(const void* cacheBuffer, const void if ( cmd->cmd() != macho_segment_command

::CMD ) continue; macho_segment_command

* segCmd = (macho_segment_command

*)cmd; - MachOProxy::Segment seg; + MachOProxySegment seg; + seg.name = segCmd->segname(); seg.name = segCmd->segname(); seg.size = segCmd->vmsize(); + seg.vmaddr = segCmd->vmaddr(); seg.diskSize = (uint32_t)segCmd->filesize(); seg.fileOffset = (uint32_t)segCmd->fileoff(); seg.protection = segCmd->initprot(); @@ -783,8 +794,14 @@ void SharedCache::assignSegmentAddresses() uint64_t endReadOnlyAddress = align(addr, sharedRegionRegionAlignment(_arch)); _readOnlyRegion.size = endReadOnlyAddress - _readOnlyRegion.address; _fileSize = _readOnlyRegion.fileOffset + _readOnlyRegion.size; + + // FIXME: Confirm these numbers for all platform/arch combos // assume LINKEDIT optimzation reduces LINKEDITs to %40 of original size - _vmSize = _readOnlyRegion.address+(_readOnlyRegion.size * 2/5) - _textRegion.address; + if (_manifest.platform() == "osx") { + _vmSize = _readOnlyRegion.address + (_readOnlyRegion.size * 9 / 10) - _textRegion.address; + } else { + _vmSize = _readOnlyRegion.address + (_readOnlyRegion.size * 2 / 5) - _textRegion.address; + } } uint64_t SharedCache::pathHash(const char* path) @@ -1329,7 +1346,7 @@ void SharedCache::writeCacheHeader(void) for (auto& dylib : _dylibs) { auto textSeg = _segmentMap[dylib][0]; images->set_address(textSeg.address); - if (_manifest.platform == "osx") { + if (_manifest.platform() == "osx") { images->set_modTime(dylib->lastModTime); images->set_inode(dylib->inode); } else { @@ -1345,7 +1362,7 @@ void SharedCache::writeCacheHeader(void) if (!dylib->installNameAliases.empty()) { for (const std::string& alias : dylib->installNameAliases) { images->set_address(_segmentMap[dylib][0].address); - if (_manifest.platform == "osx") { + if (_manifest.platform() == "osx") { images->set_modTime(dylib->lastModTime); images->set_inode(dylib->inode); } else { @@ -1368,9 +1385,9 @@ void SharedCache::writeCacheHeader(void) // write text image array and image names pool at same time for (auto& dylib : _dylibs) { - textImages->set_uuid(dylib->uuid); + textImages->set_uuid(dylib->uuid.get()); textImages->set_loadAddress(_segmentMap[dylib][0].address); - textImages->set_textSegmentSize((uint32_t)dylib->segments[0].size); + textImages->set_textSegmentSize((uint32_t)_segmentMap[dylib].front().cacheSegSize); textImages->set_pathOffset(stringOffset); ::strcpy((char*)&buffer[stringOffset], dylib->installName.c_str()); stringOffset += dylib->installName.size()+1; @@ -1403,34 +1420,19 @@ void SharedCache::rebaseAll(void) void SharedCache::bindAll(void) { - std::unordered_map dylibPathToMachHeader; - for (auto& dylib : _dylibs) { - void* mh = (uint8_t*)_buffer.get() + _segmentMap[dylib][0].cacheFileOffset; - dylibPathToMachHeader[dylib->installName] = mh; - for (const std::string& path : dylib->installNameAliases) { - if (path != dylib->installName) { - dylibPathToMachHeader[path] = mh; - } - } - } - - bindAllImagesInCache(dylibPathToMachHeader, _pointersForASLR); + bindAllImagesInCache(_dylibs, _segmentMap, _pointersForASLR); } void SharedCache::writeCacheSegments(void) { uint8_t* cacheBytes = (uint8_t*)_buffer.get(); for (auto& dylib : _dylibs) { - struct stat stat_buf; - const uint8_t* srcDylib; - bool rootless; + const uint8_t* srcDylib = dylib->getBuffer(); - std::tie(srcDylib, stat_buf, rootless) = fileCache.cacheLoad(dylib->path); for (auto& seg : _segmentMap[dylib]) { - uint32_t segFileOffset = dylib->fatFileOffset + seg.base->fileOffset; uint64_t copySize = std::min(seg.cacheSegSize, (uint64_t)seg.base->diskSize); verboseLog("copy segment %12s (0x%08llX bytes) to %p (logical addr 0x%llX) for %s", seg.base->name.c_str(), copySize, &cacheBytes[seg.cacheFileOffset], seg.address, dylib->installName.c_str()); - ::memcpy(&cacheBytes[seg.cacheFileOffset], &srcDylib[segFileOffset], copySize); + ::memcpy(&cacheBytes[seg.cacheFileOffset], &srcDylib[seg.base->fileOffset], copySize); } } } @@ -1443,7 +1445,7 @@ void SharedCache::appendCodeSignature(const std::string& suffix) uint8_t dscHashType = CS_HASHTYPE_SHA1; uint8_t dscHashSize = CS_HASH_SIZE_SHA1; uint32_t dscDigestFormat = kCCDigestSHA1; - if ( _manifest.platform == "osx" ) { + if (_manifest.platform() == "osx") { dscHashType = CS_HASHTYPE_SHA256; dscHashSize = CS_HASH_SIZE_SHA256; dscDigestFormat = kCCDigestSHA256; diff --git a/interlinked-dylibs/dyld_shared_cache_builder.mm b/interlinked-dylibs/dyld_shared_cache_builder.mm index 075a5ed..0ce8429 100644 --- a/interlinked-dylibs/dyld_shared_cache_builder.mm +++ b/interlinked-dylibs/dyld_shared_cache_builder.mm @@ -61,10 +61,12 @@ #include -#include "Manifest.h" +#include "mega-dylib-utils.h" + #include "MultiCacheBuilder.h" -#include "mega-dylib-utils.h" +#include "MachOProxy.h" +#include "Manifest.h" #include "Logging.h" #if !__has_feature(objc_arc) @@ -185,6 +187,15 @@ bool writeRootList( const std::string &dstRoot, const std::set &roo return true; } +std::string realPath(const std::string& path) { + char resolvedPath[PATH_MAX]; + if (realpath( path.c_str(), &resolvedPath[0]) != nullptr) { + return resolvedPath; + } else { + return ""; + } +} + std::set cachePaths; BOMCopierCopyOperation filteredCopy(BOMCopier copier, const char *path, BOMFSObjType type, off_t size) { @@ -198,7 +209,6 @@ BOMCopierCopyOperation filteredCopy(BOMCopier copier, const char *path, BOMFSObj int main (int argc, const char * argv[]) { @autoreleasepool { std::set roots; - std::vector inputRoots; std::string dylibCacheDir; std::string release; bool emitDevCaches = true; @@ -223,13 +233,12 @@ int main (int argc, const char * argv[]) { } else if (strcmp(arg, "-list_configs") == 0) { listConfigs = true; } else if (strcmp(arg, "-root") == 0) { - std::string root = argv[++i]; - inputRoots.push_back(root); + std::string root = realPath(argv[++i]); processRoot(root, roots); } else if (strcmp(arg, "-copy_roots") == 0) { copyRoots = true; } else if (strcmp(arg, "-dylib_cache") == 0) { - dylibCacheDir = argv[++i]; + dylibCacheDir = realPath(argv[++i]); } else if (strcmp(arg, "-no_development_cache") == 0) { emitDevCaches = false; } else if (strcmp(arg, "-no_overflow_dylibs") == 0) { @@ -239,11 +248,11 @@ int main (int argc, const char * argv[]) { } else if (strcmp(arg, "-overflow_dylibs") == 0) { emitElidedDylibs = true; } else if (strcmp(arg, "-dst_root") == 0) { - dstRoot = argv[++i]; + dstRoot = realPath(argv[++i]); } else if (strcmp(arg, "-release") == 0) { release = argv[++i]; } else if (strcmp(arg, "-results") == 0) { - resultPath = argv[++i]; + resultPath = realPath(argv[++i]); } else { //usage(); terminate("unknown option: %s\n", arg); @@ -279,63 +288,50 @@ int main (int argc, const char * argv[]) { auto manifest = Manifest(dylibCacheDir + "/Manifest.plist", roots); - if (manifest.build.empty()) { - terminate("No manifest found at '%s/Manifest.plist'\n", dylibCacheDir.c_str()); + if (manifest.build().empty()) { + terminate("No manifest found at '%s/Manifest.plist'\n", dylibCacheDir.c_str()); } - log("Building Caches for %s", manifest.build.c_str()); + log("Building Caches for %s", manifest.build().c_str()); - if (listConfigs) { - for (auto& config : manifest.configurations) { - printf("%s\n", config.first.c_str()); - } - exit(0); + if (listConfigs) { + manifest.forEachConfiguration([](const std::string& configName) { + printf("%s\n", configName.c_str()); + }); + exit(0); } - std::map filteredConfigs; - - for (auto& config : manifest.configurations) { - if (config.first == configuration) { - filteredConfigs[config.first] = config.second; - - for (auto& arch : filteredConfigs[config.first].architectures) { - arch.second.results = Manifest::Results(); - } - } - } - - if ( filteredConfigs.empty() ) { - terminate( "No config %s. Please run with -list_configs to see configurations available for this %s.\n", - configuration.c_str(), manifest.build.c_str() ); - } - - manifest.configurations = filteredConfigs; - manifest.calculateClosure(false); - - // FIXME: Plumb through no_development - - std::shared_ptr builder = std::make_shared( manifest, false, false, true ); - builder->buildCaches(dstRoot); - writeRootList(dstRoot, roots); - - if (copyRoots) { - for (auto& config : manifest.configurations) { - for (auto& arch : config.second.architectures) { - for (auto& dylib : arch.second.results.dylibs) { - if (dylib.second.included) { - MachOProxy *proxy = manifest.dylibProxy( dylib.first, arch.first ); - cachePaths.insert( proxy->installName ); - for ( auto &alias : proxy->installNameAliases ) { - cachePaths.insert(alias); - } - } - } - } - } - - BOMCopier copier = BOMCopierNewWithSys(BomSys_default()); - BOMCopierSetCopyFileStartedHandler(copier, filteredCopy); - for (auto& root : roots) { - BOMCopierCopy(copier, root.c_str(), dstRoot.c_str()); + if (!manifest.filterForConfig(configuration)) { + terminate("No config %s. Please run with -list_configs to see configurations available for this %s.\n", + configuration.c_str(), manifest.build().c_str()); + } + manifest.calculateClosure(false); + manifest.checkLinks(); + + // FIXME: Plumb through no_development + + std::shared_ptr builder = std::make_shared(manifest, false, false, true); + builder->buildCaches(dstRoot); + writeRootList(dstRoot, roots); + + if (copyRoots) { + manifest.forEachConfiguration([&manifest](const std::string& configName) { + for (auto& arch : manifest.configuration(configName).architectures) { + for (auto& dylib : arch.second.results.dylibs) { + if (dylib.second.included) { + MachOProxy* proxy = MachOProxy::forIdentifier(dylib.first, arch.first); + cachePaths.insert(proxy->installName); + for (auto& alias : proxy->installNameAliases) { + cachePaths.insert(alias); + } + } + } + } + }); + + BOMCopier copier = BOMCopierNewWithSys(BomSys_default()); + BOMCopierSetCopyFileStartedHandler(copier, filteredCopy); + for (auto& root : roots) { + BOMCopierCopy(copier, root.c_str(), dstRoot.c_str()); } BOMCopierFree(copier); } @@ -381,7 +377,7 @@ int main (int argc, const char * argv[]) { dumpLogAndExit(); } - dispatch_main(); + dispatch_main(); - return 0; + return 0; } diff --git a/interlinked-dylibs/mega-dylib-utils.h b/interlinked-dylibs/mega-dylib-utils.h index 0482ffa..16fc4ab 100644 --- a/interlinked-dylibs/mega-dylib-utils.h +++ b/interlinked-dylibs/mega-dylib-utils.h @@ -39,19 +39,76 @@ #include #include #include +#include #include #include #include #include #include +#ifndef UUID +#include + +struct UUID { + UUID() {} + UUID(const UUID& other) { uuid_copy(&_bytes[0], &other._bytes[0]); } + UUID(const uuid_t other_uuid) { uuid_copy(&_bytes[0], other_uuid); } + bool operator<(const UUID& other) const { return uuid_compare(&_bytes[0], &other._bytes[0]) < 0; } + bool operator==(const UUID& other) const { return uuid_compare(&_bytes[0], &other._bytes[0]) == 0; } + bool operator!=(const UUID& other) const { return !(*this == other); } + + size_t hash() const { + size_t retval = 0; + for (auto i = 0; i < 16/sizeof(size_t); ++i) { + retval ^= ((size_t *)(&_bytes[0]))[i]; + } + return retval; + } + const unsigned char* get() const { return &_bytes[0]; }; +private: + std::array _bytes; +}; + +struct ImageIdentifier { + ImageIdentifier() {} + ImageIdentifier(const UUID &U) : _uuid(U) {} + size_t hash() const { return _uuid.hash(); } + bool operator<(const ImageIdentifier& other) const { return _uuid < other._uuid; } + bool operator==(const ImageIdentifier& other) const { return _uuid == other._uuid; } + bool operator!=(const ImageIdentifier& other) const { return !(*this == other); } + +private: + UUID _uuid; +}; + +namespace std { +template <> +struct hash { + size_t operator()(const UUID& x) const + { + return x.hash(); + } +}; + +template <> +struct hash { + size_t operator()(const ImageIdentifier& x) const + { + return x.hash(); + } +}; +} + +#endif + #include "CacheFileAbstraction.hpp" #include "MachOFileAbstraction.hpp" #include "Manifest.h" -#include "MachOProxy.h" +struct MachOProxy; +struct MachOProxySegment; struct SharedCache; struct FileCache { @@ -60,7 +117,7 @@ struct FileCache { void preflightCache(const std::string& path); void preflightCache(const std::unordered_set &paths); private: - void fill(const std::string& path); + std::tuple fill(const std::string& path); std::unordered_map> entries; dispatch_queue_t cache_queue; @@ -85,10 +142,15 @@ std::string fallbackArchStringForArchString( const std::string& archStr ); struct SharedCache { struct SegmentInfo { - SegmentInfo(const MachOProxy::Segment* seg) - : base(seg), address(0), cacheFileOffset(0), cacheSegSize(0) { } + SegmentInfo(const MachOProxySegment* seg) + : base(seg) + , address(0) + , cacheFileOffset(0) + , cacheSegSize(0) + { + } - const MachOProxy::Segment* base; + const MachOProxySegment* base; uint64_t address; uint64_t cacheFileOffset; uint64_t cacheSegSize; @@ -121,7 +183,8 @@ struct SharedCache { bool writeCacheMapFile(const std::string& mapPath); typedef std::function& segments)> DylibHandler; + const std::vector& segments)> + DylibHandler; // Calls lambda once per image in the cache void forEachImage(DylibHandler handler); @@ -143,7 +206,7 @@ private: // Once all a dylib's segments are copied into a cache, this function will adjust the contents of // the TEXT, DATA, and LINKEDIT segments in the cache to be correct for their new addresses. - void bindAllImagesInCache(const std::unordered_map& dylibPathToMachHeader, std::vector& pointersForASLR); + void bindAllImagesInCache(const std::vector dylibs, const std::map>& segmentMap, std::vector& pointersForASLR); // After adjustImageForNewSegmentLocations() is called to rebase all segments, this function can be called to // bind all symbols to their new addresses @@ -160,7 +223,7 @@ private: void optimizeObjC(bool forProduction); void writeSlideInfoV2(void); - void buildUnoptimizedCache(); + void buildUnoptimizedCache(void); void appendCodeSignature(const std::string& suffix); template void buildForDevelopment(const std::string& cachePath); template void buildForProduction(const std::string& cachePath); @@ -185,7 +248,7 @@ private: ArchPair _arch; std::vector _dylibs; std::shared_ptr _buffer; - std::unordered_map> _segmentMap; + std::map> _segmentMap; std::string archName(); @@ -213,7 +276,6 @@ std::string dirpath(const std::string& path); std::string toolDir(); bool isProtectedBySIP(const std::string& path, int fd=-1); - template inline bool is_disjoint(const Set1& set1, const Set2& set2) { @@ -244,7 +306,6 @@ inline bool has_prefix(const std::string& str, const std::string& prefix) return std::mismatch(prefix.begin(), prefix.end(), str.begin()).first == prefix.end(); } - #define NEW_CACHE_FILE_FORMAT 0 #endif // __MEGA_DYLIB_UTILS_H__ diff --git a/interlinked-dylibs/multi_dyld_shared_cache_builder.mm b/interlinked-dylibs/multi_dyld_shared_cache_builder.mm index 7aec8dc..b21264f 100644 --- a/interlinked-dylibs/multi_dyld_shared_cache_builder.mm +++ b/interlinked-dylibs/multi_dyld_shared_cache_builder.mm @@ -56,10 +56,12 @@ #include #include +#include "mega-dylib-utils.h" + #include "MultiCacheBuilder.h" #include "Manifest.h" +#include "MachOProxy.h" -#include "mega-dylib-utils.h" #include "Logging.h" #if !__has_feature(objc_arc) @@ -106,45 +108,34 @@ bool copyFile(const std::string& from, const std::string& to) void createArtifact(Manifest& manifest, const std::string& dylibCachePath, bool includeExecutables) { mkpath_np(dylibCachePath.c_str(), 0755); - (void)copyFile(manifest.dylibOrderFile, dylibCachePath + "Metadata/dylibOrderFile.txt"); - (void)copyFile(manifest.dirtyDataOrderFile, dylibCachePath + "Metadata/dirtyDataOrderFile.txt"); - (void)copyFile(manifest.metabomFile, dylibCachePath + "Metadata/metabom.bom"); + (void)copyFile(manifest.dylibOrderFile(), dylibCachePath + "Metadata/dylibOrderFile.txt"); + (void)copyFile(manifest.dirtyDataOrderFile(), dylibCachePath + "Metadata/dirtyDataOrderFile.txt"); + (void)copyFile(manifest.metabomFile(), dylibCachePath + "Metadata/metabom.bom"); std::set copied; - - for (auto archFiles : manifest.architectureFiles) { - for (auto& file : archFiles.second.dylibs) { - std::string installname = file.first; - if (copied.count(installname) > 0) { - continue; - } - (void)copyFile(file.second.proxy->path, normalize_absolute_file_path(dylibCachePath + "/Root/" + file.first)); - copied.insert(installname); + MachOProxy::runOnAllProxies(false, [&](MachOProxy* proxy) { + if (copied.count(proxy->path) > 0) { + return; } - if (includeExecutables) { - for (auto& file : archFiles.second.executables) { - std::string installname = file.first; - if (copied.count(installname) > 0) { - continue; - } - (void)copyFile(file.second.proxy->path, normalize_absolute_file_path(dylibCachePath + "/Root/" + file.first)); - copied.insert(installname); - } - } - } - + if (!includeExecutables && !proxy->isDylib()) + return; + (void)copyFile(proxy->buildPath, normalize_absolute_file_path(dylibCachePath + "/Root/" + proxy->path)); + copied.insert(proxy->path); + }); + + // HACK for 10.e + (void)symlink("libstdc++.6.0.9.dylib", (dylibCachePath + "/Root/usr/lib/libstdc++.6.dylib").c_str()); + (void)symlink("libstdc++.6.0.9.dylib", (dylibCachePath + "/Root/usr/lib/libstdc++.dylib").c_str()); log("Artifact dylibs copied"); } void addArtifactPaths(Manifest &manifest) { - manifest.dylibOrderFile = "./Metadata/dylibOrderFile.txt"; - manifest.dirtyDataOrderFile = "./Metadata/dirtyDataOrderFile.txt"; - manifest.metabomFile = "./Metadata/metabom.bom"; + manifest.setDylibOrderFile("./Metadata/dylibOrderFile.txt"); + manifest.setDirtyDataOrderFile("./Metadata/dirtyDataOrderFile.txt"); + manifest.setMetabomFile("./Metadata/metabom.bom"); - for ( auto& projects : manifest.projects ) { - if ( projects.second.sources[0] != "./Root/" ) { - projects.second.sources.insert( projects.second.sources.begin(), "./Root/" ); - } + for (auto& projects : manifest.projects()) { + manifest.addProjectSource(projects.first, "./Root", true); } } @@ -161,7 +152,6 @@ int main (int argc, const char * argv[]) { bool skipBuilds = false; bool preflight = false; std::string manifestPath; - time_t mytime = time(0); log("Started: %s", asctime(localtime(&mytime))); @@ -208,17 +198,17 @@ int main (int argc, const char * argv[]) { skipBuilds = true; } - if (manifest.build.empty()) { - terminate("No version found in manifest"); + if (manifest.build().empty()) { + terminate("No version found in manifest"); } - log("Building Caches for %s", manifest.build.c_str()); + log("Building Caches for %s", manifest.build().c_str()); - if ( masterDstRoot.empty() ) { + if ( masterDstRoot.empty() ) { terminate("-master_dst_root required path argument"); } - if (manifest.manifest_version < 4) { - terminate("must specify valid manifest file"); + if (manifest.version() < 4) { + terminate("must specify valid manifest file"); } struct rlimit rl = {OPEN_MAX, OPEN_MAX}; @@ -228,31 +218,24 @@ int main (int argc, const char * argv[]) { (void)mkpath_np((masterDstRoot + "/Boms/").c_str(), 0755); } - if (manifest.dylibOrderFile.empty()) { - manifest.dylibOrderFile = toolDir() + "/dylib-order.txt"; - } - - if (manifest.dirtyDataOrderFile.empty()) { - manifest.dirtyDataOrderFile = toolDir() + "/dirty-data-segments-order.txt"; - } - auto dylibCacheCtx = std::make_shared("DylibCache"); setLoggingContext(dylibCacheCtx); - if (!skipWrites) { + if (!skipWrites && !skipBuilds) { cacheBuilderDispatchGroupAsync(build_group, build_queue, [&] { createArtifact(manifest, masterDstRoot + "/Artifact.dlc/", true); }); if (!dylibCacheDir.empty()) { cacheBuilderDispatchGroupAsync(build_group, build_queue, [&] { - createArtifact(manifest, dylibCacheDir + "/AppleInternal/Developer/DylibCaches/" + manifest.build + ".dlc/", false); + createArtifact(manifest, dylibCacheDir + "/AppleInternal/Developer/DylibCaches/" + manifest.build() + ".dlc/", false); }); } } setLoggingContext(defaultCtx); manifest.calculateClosure(false); + manifest.checkLinks(); std::shared_ptr builder = std::make_shared(manifest, true, skipWrites, false, skipBuilds); dispatch_group_async(build_group, build_queue, [&] { builder->buildCaches(masterDstRoot); }); dispatch_group_wait(build_group, DISPATCH_TIME_FOREVER); @@ -271,7 +254,7 @@ int main (int argc, const char * argv[]) { manifest.write(masterDstRoot + "/Artifact.dlc/Manifest.plist"); if (!dylibCacheDir.empty()) { - manifest.write(dylibCacheDir + kDylibCachePrefix + manifest.build + ".dlc/Manifest.plist"); + manifest.write(dylibCacheDir + kDylibCachePrefix + manifest.build() + ".dlc/Manifest.plist"); } setLoggingContext(defaultCtx); } @@ -285,5 +268,3 @@ int main (int argc, const char * argv[]) { return 0; } - - diff --git a/interlinked-dylibs/update_dyld_shared_cache.mm b/interlinked-dylibs/update_dyld_shared_cache.mm index 46f9e3a..bd7fbea 100644 --- a/interlinked-dylibs/update_dyld_shared_cache.mm +++ b/interlinked-dylibs/update_dyld_shared_cache.mm @@ -54,13 +54,12 @@ extern "C" { #include #include -#include "MachOProxy.h" -#include "manifest.h" #include "mega-dylib-utils.h" +#include "MultiCacheBuilder.h" +#include "MachOProxy.h" +#include "Manifest.h" #include "Logging.h" -#import "MultiCacheBuilder.h" - #if !__has_feature(objc_arc) #error The use of libdispatch in this files requires it to be compiled with ARC in order to avoid leaks #endif @@ -202,75 +201,6 @@ bool improvePath(const char* volumeRootPath, const std::vector& ove return tryPath("", path, foundPath, aliases); } -std::string fileExists( const std::string& path ) { - const uint8_t* p = (uint8_t*)( -1 ); - struct stat stat_buf; - bool rootless; - - std::tie( p, stat_buf, rootless ) = fileCache.cacheLoad( path ); - if ( p != (uint8_t*)( -1 ) ) { - return normalize_absolute_file_path( path ); - } - - return ""; -} - -void populateManifest(Manifest& manifest, std::set archs, const std::string& overlayPath, - const std::string& rootPath, const std::set& paths) { - for ( const auto& arch : archs ) { - auto fallback = fallbackArchStringForArchString(arch); - std::set allArchs = archs; - std::set processedPaths; - std::set unprocessedPaths = paths; - std::set pathsToProcess; - std::set_difference( unprocessedPaths.begin(), unprocessedPaths.end(), processedPaths.begin(), processedPaths.end(), - std::inserter( pathsToProcess, pathsToProcess.begin() ) ); - while ( !pathsToProcess.empty() ) { - for (const std::string path : pathsToProcess) { - processedPaths.insert(path); - std::string fullPath; - if ( rootPath != "/" ) { - // with -root, only look in the root path volume - fullPath = fileExists(rootPath + path); - } - else { - // with -overlay, look first in overlay dir - if ( !overlayPath.empty() ) - fullPath = fileExists(overlayPath + path); - // if not in overlay, look in boot volume - if ( fullPath.empty() ) - fullPath = fileExists(path); - } - if ( fullPath.empty() ) - continue; - auto proxies = MachOProxy::findDylibInfo(fullPath, true, true); - auto proxy = proxies.find(arch); - if (proxy == proxies.end()) - proxy = proxies.find(fallback); - if (proxy == proxies.end()) - continue; - - for ( const auto& dependency : proxy->second->dependencies ) { - unprocessedPaths.insert( dependency ); - } - - if ( proxy->second->installName.empty() ) { - continue; - } - - proxy->second->addAlias( path ); - manifest.architectureFiles[arch].dylibs.insert(std::make_pair(proxy->second->installName, - Manifest::File(proxy->second))); - manifest.configurations["localhost"].architectures[arch].anchors.push_back( proxy->second->installName ); - } - - pathsToProcess.clear(); - std::set_difference( unprocessedPaths.begin(), unprocessedPaths.end(), processedPaths.begin(), processedPaths.end(), - std::inserter( pathsToProcess, pathsToProcess.begin() ) ); - } - } -} - static bool runningOnHaswell() { // check system is capable of running x86_64h code @@ -397,39 +327,37 @@ int main(int argc, const char* argv[]) terminate("mkpath_np fail: %d", err); } - Manifest manifest; - - std::set paths; + std::set paths; - if ( !dylibListFile.empty() ) { - if ( !parsePathsFile( dylibListFile, paths ) ) { - terminate( "could not build intiial paths\n" ); - } - } else if ( !buildInitialPaths( rootPath, overlayPath, paths ) ) { + if ( !dylibListFile.empty() ) { + if ( !parsePathsFile( dylibListFile, paths ) ) { terminate( "could not build intiial paths\n" ); } + } else if ( !buildInitialPaths( rootPath, overlayPath, paths ) ) { + terminate( "could not build intiial paths\n" ); + } + + Manifest manifest(archStrs, overlayPath, rootPath, paths); - manifest.platform = platform; - populateManifest( manifest, archStrs, overlayPath, rootPath, paths ); + manifest.setPlatform(platform); - // If the path we are writing to is trusted then our sources need to be trusted - // Can't update the update_dyld_shared_cache on a non-boot volume - bool requireDylibsBeRootlessProtected = isProtectedBySIP(cacheDir); - manifest.calculateClosure( requireDylibsBeRootlessProtected ); - manifest.pruneClosure(); + // If the path we are writing to is trusted then our sources need to be trusted + // Can't update the update_dyld_shared_cache on a non-boot volume + bool requireDylibsBeRootlessProtected = isProtectedBySIP(cacheDir); + manifest.calculateClosure( requireDylibsBeRootlessProtected ); for (const std::string& archStr : archStrs) { std::string cachePath = cacheDir + "/dyld_shared_cache_" + archStr; - if ( manifest.sameContentsAsCacheAtPath("localhost", archStr, cachePath) && !force ) { - manifest.configurations["localhost"].architectures.erase(archStr); + if (!force && manifest.sameContentsAsCacheAtPath("localhost", archStr, cachePath)) { + manifest.remove("localhost", archStr); verboseLog("%s is already up to date", cachePath.c_str()); } } - - // If caches already up to date, do nothing - if ( manifest.configurations["localhost"].architectures.empty() ) + + if (manifest.empty()) { dumpLogAndExit(false); - + } + // build caches std::shared_ptr builder = std::make_shared(manifest, false, false, false, false, requireDylibsBeRootlessProtected); builder->buildCaches(cacheDir); @@ -445,4 +373,3 @@ int main(int argc, const char* argv[]) dispatch_main(); } - diff --git a/src/ImageLoaderMegaDylib.cpp b/src/ImageLoaderMegaDylib.cpp index f3612d6..2e45f21 100644 --- a/src/ImageLoaderMegaDylib.cpp +++ b/src/ImageLoaderMegaDylib.cpp @@ -283,6 +283,18 @@ unsigned ImageLoaderMegaDylib::findImageIndex(const LinkContext& context, const } } } + + // handle symlinks embedded in load commands + char resolvedPath[PATH_MAX]; + realpath(path, resolvedPath); + int realpathErrno = errno; + // If realpath() resolves to a path which does not exist on disk, errno is set to ENOENT + if ( (realpathErrno == ENOENT) || (realpathErrno == 0) ) { + if ( strcmp(resolvedPath, path) != 0 ) + return findImageIndex(context, resolvedPath); + } + + dyld::throwf("no cache image with name (%s)", path); } diff --git a/src/dyld.cpp b/src/dyld.cpp index cda4944..4aff26a 100644 --- a/src/dyld.cpp +++ b/src/dyld.cpp @@ -312,7 +312,7 @@ static OSSpinLock sDynamicReferencesLock = 0; static bool sLogToFile = false; #endif static char sLoadingCrashMessage[1024] = "dyld: launch, loading dependent libraries"; - +static bool sSafeMode = false; static _dyld_objc_notify_mapped sNotifyObjCMapped; static _dyld_objc_notify_init sNotifyObjCInit; static _dyld_objc_notify_unmapped sNotifyObjCUnmapped; @@ -779,7 +779,7 @@ static void notifyMonitoringDyld(bool unloading, unsigned portSlot, unsigned ima header->imageCount = imageCount; header->imagesOffset = sizeof(dyld_process_info_notify_header); header->stringsOffset = sizeof(dyld_process_info_notify_header) + entriesSize; - header->timestamp = mach_absolute_time(); + header->timestamp = dyld::gProcessInfo->infoArrayChangeTimestamp; dyld_process_info_image_entry* entries = (dyld_process_info_image_entry*)&buffer[header->imagesOffset]; char* const pathPoolStart = (char*)&buffer[header->stringsOffset]; char* pathPool = pathPoolStart; @@ -834,6 +834,38 @@ static void notifyMonitoringDyld(bool unloading, unsigned portSlot, unsigned ima } } +static void notifyMonitoringDyldMain() +{ + for (int slot=0; slot < DYLD_MAX_PROCESS_INFO_NOTIFY_COUNT; ++slot) { + if ( dyld::gProcessInfo->notifyPorts[slot] != 0 ) { + if ( sNotifyReplyPorts[slot] == 0 ) { + if ( !mach_port_allocate(mach_task_self(), MACH_PORT_RIGHT_RECEIVE, &sNotifyReplyPorts[slot]) ) + mach_port_insert_right(mach_task_self(), sNotifyReplyPorts[slot], sNotifyReplyPorts[slot], MACH_MSG_TYPE_MAKE_SEND); + //dyld::log("allocated reply port %d\n", sNotifyReplyPorts[slot]); + } + //dyld::log("found port to send to\n"); + mach_msg_header_t h; + h.msgh_bits = MACH_MSGH_BITS(MACH_MSG_TYPE_COPY_SEND,MACH_MSG_TYPE_MAKE_SEND); // MACH_MSG_TYPE_MAKE_SEND_ONCE + h.msgh_id = DYLD_PROCESS_INFO_NOTIFY_MAIN_ID; + h.msgh_local_port = sNotifyReplyPorts[slot]; + h.msgh_remote_port = dyld::gProcessInfo->notifyPorts[slot]; + h.msgh_reserved = 0; + h.msgh_size = (mach_msg_size_t)sizeof(mach_msg_header_t); + //dyld::log("sending to port[%d]=%d, size=%d, reply port=%d, id=0x%X\n", slot, dyld::gProcessInfo->notifyPorts[slot], h.msgh_size, sNotifyReplyPorts[slot], h.msgh_id); + kern_return_t sendResult = mach_msg(&h, MACH_SEND_MSG | MACH_RCV_MSG | MACH_SEND_TIMEOUT, h.msgh_size, h.msgh_size, sNotifyReplyPorts[slot], 100, MACH_PORT_NULL); + //dyld::log("send result = 0x%X, msg_id=%d, msg_size=%d\n", sendResult, h.msgh_id, h.msgh_size); + if ( sendResult == MACH_SEND_INVALID_DEST ) { + // sender is not responding, detatch + //dyld::log("process requesting notification gone. deallocation send port %d and receive port %d\n", dyld::gProcessInfo->notifyPorts[slot], sNotifyReplyPorts[slot]); + mach_port_deallocate(mach_task_self(), dyld::gProcessInfo->notifyPorts[slot]); + mach_port_deallocate(mach_task_self(), sNotifyReplyPorts[slot]); + dyld::gProcessInfo->notifyPorts[slot] = 0; + sNotifyReplyPorts[slot] = 0; + } + } + } +} + #define MAX_KERNEL_IMAGES_PER_CALL (100) static void flushKernelNotifications(bool loading, bool force, std::array& kernelInfos, uint32_t &kernelInfoCount) { @@ -1903,7 +1935,7 @@ void processDyldEnvironmentVariable(const char* key, const char* value, const ch else if ( strcmp(key, "DYLD_PRINT_CODE_SIGNATURES") == 0 ) { gLinkContext.verboseCodeSignatures = true; } - else if ( strcmp(key, "DYLD_SHARED_REGION") == 0 ) { + else if ( (strcmp(key, "DYLD_SHARED_REGION") == 0) && !sSafeMode ) { if ( strcmp(value, "private") == 0 ) { gLinkContext.sharedRegionMode = ImageLoader::kUsePrivateSharedRegion; } @@ -1921,10 +1953,10 @@ void processDyldEnvironmentVariable(const char* key, const char* value, const ch } } #if DYLD_SHARED_CACHE_SUPPORT - else if ( strcmp(key, "DYLD_SHARED_CACHE_DIR") == 0 ) { + else if ( (strcmp(key, "DYLD_SHARED_CACHE_DIR") == 0) && !sSafeMode ) { sSharedCacheDir = value; } - else if ( strcmp(key, "DYLD_SHARED_CACHE_DONT_VALIDATE") == 0 ) { + else if ( (strcmp(key, "DYLD_SHARED_CACHE_DONT_VALIDATE") == 0) && !sSafeMode ) { sSharedCacheIgnoreInodeAndTimeStamp = true; } #endif @@ -1960,7 +1992,7 @@ void processDyldEnvironmentVariable(const char* key, const char* value, const ch } #endif #if !TARGET_IPHONE_SIMULATOR - else if ( (strcmp(key, "DYLD_PRINT_TO_FILE") == 0) && (mainExecutableDir == NULL) ) { + else if ( (strcmp(key, "DYLD_PRINT_TO_FILE") == 0) && (mainExecutableDir == NULL) && !sSafeMode ) { int fd = open(value, O_WRONLY | O_CREAT | O_APPEND, 0644); if ( fd != -1 ) { sLogfile = fd; @@ -3422,6 +3454,23 @@ static ImageLoader* loadPhase3(const char* path, const char* orgPath, const Load return loadPhase4(path, orgPath, context, cacheIndex, exceptions); } +static ImageLoader* loadPhase2cache(const char* path, const char *orgPath, const LoadContext& context, unsigned& cacheIndex, std::vector* exceptions) { + ImageLoader* image = NULL; +#if !TARGET_IPHONE_SIMULATOR + if ( exceptions != NULL) { + char resolvedPath[PATH_MAX]; + realpath(path, resolvedPath); + int myerr = errno; + // If realpath() resolves to a path which does not exist on disk, errno is set to ENOENT + if ( (myerr == ENOENT) || (myerr == 0) ) + { + image = loadPhase4(resolvedPath, orgPath, context, cacheIndex, exceptions); + } + } +#endif + return image; +} + // try search paths static ImageLoader* loadPhase2(const char* path, const char* orgPath, const LoadContext& context, @@ -3441,6 +3490,9 @@ static ImageLoader* loadPhase2(const char* path, const char* orgPath, const Load strcat(npath, frameworkPartialPath); //dyld::log("dyld: fallback framework path used: %s() -> loadPhase4(\"%s\", ...)\n", __func__, npath); image = loadPhase4(npath, orgPath, context, cacheIndex, exceptions); + // Look in the cache if appropriate + if ( image == NULL) + image = loadPhase2cache(npath, orgPath, context, cacheIndex, exceptions); if ( image != NULL ) return image; } @@ -3458,6 +3510,9 @@ static ImageLoader* loadPhase2(const char* path, const char* orgPath, const Load strcat(libpath, libraryLeafName); //dyld::log("dyld: fallback library path used: %s() -> loadPhase4(\"%s\", ...)\n", __func__, libpath); image = loadPhase4(libpath, orgPath, context, cacheIndex, exceptions); + // Look in the cache if appropriate + if ( image == NULL) + image = loadPhase2cache(libpath, orgPath, context, cacheIndex, exceptions); if ( image != NULL ) return image; } @@ -3575,32 +3630,10 @@ ImageLoader* load(const char* path, const LoadContext& context, unsigned& cacheI // try all path permutations and try open() until first success std::vector exceptions; image = loadPhase0(path, orgPath, context, cacheIndex, &exceptions); -#if __IPHONE_OS_VERSION_MIN_REQUIRED && DYLD_SHARED_CACHE_SUPPORT && !TARGET_IPHONE_SIMULATOR +#if !TARGET_IPHONE_SIMULATOR // support symlinks on disk to a path in dyld shared cache - if ( (image == NULL) && cacheablePath(path) && !context.dontLoad ) { - char resolvedPath[PATH_MAX]; - realpath(path, resolvedPath); - int myerr = errno; - // If realpath() resolves to a path which does not exist on disk, errno is set to ENOENT - if ( (myerr == ENOENT) || (myerr == 0) ) - { - // see if this image is in shared cache - const macho_header* mhInCache; - const char* pathInCache; - long slideInCache; - if ( findInSharedCacheImage(resolvedPath, false, NULL, &mhInCache, &pathInCache, &slideInCache) ) { - struct stat stat_buf; - bzero(&stat_buf, sizeof(stat_buf)); - try { - image = ImageLoaderMachO::instantiateFromCache(mhInCache, pathInCache, slideInCache, stat_buf, gLinkContext); - image = checkandAddImage(image, context); - } - catch (...) { - image = NULL; - } - } - } - } + if ( image == NULL) + image = loadPhase2cache(path, orgPath, context, cacheIndex, &exceptions); #endif CRSetCrashLogMessage2(NULL); if ( image != NULL ) { @@ -4002,6 +4035,7 @@ static void mapSharedCache() return; } } + dyld::gProcessInfo->sharedCacheBaseAddress = cacheBaseAddress; // check if cache file is slidable const dyld_cache_header* header = sSharedCache; if ( (header->mappingOffset >= 0x48) && (header->slideInfoSize != 0) ) { @@ -4011,10 +4045,9 @@ static void mapSharedCache() const uint8_t* preferedLoadAddress = (uint8_t*)(long)(mappings[0].address); sSharedCacheSlide = loadedAddress - preferedLoadAddress; dyld::gProcessInfo->sharedCacheSlide = sSharedCacheSlide; - dyld::gProcessInfo->sharedCacheBaseAddress = cacheBaseAddress; //dyld::log("sSharedCacheSlide=0x%08lX, loadedAddress=%p, preferedLoadAddress=%p\n", sSharedCacheSlide, loadedAddress, preferedLoadAddress); } - // if cache has a uuid, copy it + // if cache has a uuid, copy it if ( header->mappingOffset >= 0x68 ) { memcpy(dyld::gProcessInfo->sharedCacheUUID, header->uuid, 16); } @@ -4778,14 +4811,27 @@ bool dlopenFromCache(const char* path, int mode, void** handle) { if ( sAllCacheImagesProxy == NULL ) return false; + char fallbackPath[PATH_MAX]; bool result = sAllCacheImagesProxy->dlopenFromCache(gLinkContext, path, mode, handle); if ( !result && (strchr(path, '/') == NULL) ) { // POSIX says you can call dlopen() with a leaf name (e.g. dlopen("libz.dylb")) - char fallbackPath[PATH_MAX]; strcpy(fallbackPath, "/usr/lib/"); strlcat(fallbackPath, path, PATH_MAX); result = sAllCacheImagesProxy->dlopenFromCache(gLinkContext, fallbackPath, mode, handle); + if ( !result ) + path = fallbackPath; } + if ( !result ) { + // leaf name could be a symlink + char resolvedPath[PATH_MAX]; + realpath(path, resolvedPath); + int realpathErrno = errno; + // If realpath() resolves to a path which does not exist on disk, errno is set to ENOENT + if ( (realpathErrno == ENOENT) || (realpathErrno == 0) ) { + result = sAllCacheImagesProxy->dlopenFromCache(gLinkContext, resolvedPath, mode, handle); + } + } + return result; } @@ -5345,9 +5391,10 @@ static void configureProcessRestrictions(const macho_header* mainExecutableMH) if ( issetugid() || hasRestrictedSegment(mainExecutableMH) ) { gLinkContext.processIsRestricted = true; } + bool usingSIP = (csr_check(CSR_ALLOW_TASK_FOR_PID) != 0); if ( csops(0, CS_OPS_STATUS, &flags, sizeof(flags)) != -1 ) { // On OS X CS_RESTRICT means the program was signed with entitlements - if ( ((flags & CS_RESTRICT) == CS_RESTRICT) && (csr_check(CSR_ALLOW_TASK_FOR_PID) != 0) ) { + if ( ((flags & CS_RESTRICT) == CS_RESTRICT) && usingSIP ) { gLinkContext.processIsRestricted = true; } // Library Validation loosens searching but requires everything to be code signed @@ -5355,6 +5402,7 @@ static void configureProcessRestrictions(const macho_header* mainExecutableMH) gLinkContext.processIsRestricted = false; //gLinkContext.requireCodeSignature = true; gLinkContext.processUsingLibraryValidation = true; + sSafeMode = usingSIP; } } #endif @@ -5893,8 +5941,10 @@ reloadAllImages: #endif char dyldPathBuffer[MAXPATHLEN+1]; int len = proc_regionfilename(getpid(), (uint64_t)(long)addressInDyld, dyldPathBuffer, MAXPATHLEN); - if ( (len != 0) && (strcmp(dyldPathBuffer, gProcessInfo->dyldPath) != 0) ) { - gProcessInfo->dyldPath = strdup(dyldPathBuffer); + if ( len > 0 ) { + dyldPathBuffer[len] = '\0'; // proc_regionfilename() does not zero terminate returned string + if ( strcmp(dyldPathBuffer, gProcessInfo->dyldPath) != 0 ) + gProcessInfo->dyldPath = strdup(dyldPathBuffer); } // load any inserted libraries @@ -6017,6 +6067,10 @@ reloadAllImages: // run all initializers initializeMainExecutable(); #endif + + // notify any montoring proccesses that this process is about to enter main() + notifyMonitoringDyldMain(); + // find entry point for main executable result = (uintptr_t)sMainExecutable->getThreadPC(); if ( result != 0 ) { diff --git a/src/dyldAPIs.cpp b/src/dyldAPIs.cpp index 31a5a8a..c87aaff 100644 --- a/src/dyldAPIs.cpp +++ b/src/dyldAPIs.cpp @@ -160,6 +160,7 @@ static struct dyld_func dyld_funcs[] = { {"__dyld_is_memory_immutable", (void*)_dyld_is_memory_immutable }, {"__dyld_objc_notify_register", (void*)_dyld_objc_notify_register }, {"__dyld_get_shared_cache_uuid", (void*)_dyld_get_shared_cache_uuid }, + {"__dyld_get_shared_cache_range", (void*)_dyld_get_shared_cache_range }, // deprecated @@ -2032,3 +2033,17 @@ bool _dyld_get_shared_cache_uuid(uuid_t uuid) { return dyld::sharedCacheUUID(uuid); } + +const void* _dyld_get_shared_cache_range(size_t* length) +{ +#if DYLD_SHARED_CACHE_SUPPORT + uintptr_t cacheEndAddr = (dyld_shared_cache_ranges.ranges[2].start + dyld_shared_cache_ranges.ranges[2].length); + *length = cacheEndAddr - dyld_shared_cache_ranges.ranges[0].start; + return (void*)(dyld_shared_cache_ranges.ranges[0].start); +#else + return NULL; +#endif +} + + + diff --git a/src/dyldAPIsInLibSystem.cpp b/src/dyldAPIsInLibSystem.cpp index 46c5c3a..35abc2a 100644 --- a/src/dyldAPIsInLibSystem.cpp +++ b/src/dyldAPIsInLibSystem.cpp @@ -517,22 +517,22 @@ static uint32_t deriveSDKVersFromDylibs(const mach_header* mh) #if __IPHONE_OS_VERSION_MIN_REQUIRED static const DylibToOSMapping foundationMapping[] = { - { PACKED_VERSION(678,24,0), DYLD_IOS_VERSION_2_0 }, - { PACKED_VERSION(678,26,0), DYLD_IOS_VERSION_2_1 }, - { PACKED_VERSION(678,29,0), DYLD_IOS_VERSION_2_2 }, - { PACKED_VERSION(678,47,0), DYLD_IOS_VERSION_3_0 }, - { PACKED_VERSION(678,51,0), DYLD_IOS_VERSION_3_1 }, - { PACKED_VERSION(678,60,0), DYLD_IOS_VERSION_3_2 }, - { PACKED_VERSION(751,32,0), DYLD_IOS_VERSION_4_0 }, - { PACKED_VERSION(751,37,0), DYLD_IOS_VERSION_4_1 }, - { PACKED_VERSION(751,49,0), DYLD_IOS_VERSION_4_2 }, - { PACKED_VERSION(751,58,0), DYLD_IOS_VERSION_4_3 }, - { PACKED_VERSION(881,0,0), DYLD_IOS_VERSION_5_0 }, - { PACKED_VERSION(890,1,0), DYLD_IOS_VERSION_5_1 }, - { PACKED_VERSION(992,0,0), DYLD_IOS_VERSION_6_0 }, - { PACKED_VERSION(993,0,0), DYLD_IOS_VERSION_6_1 }, - { PACKED_VERSION(1038,14,0),DYLD_IOS_VERSION_7_0 }, - { PACKED_VERSION(0,0,0), DYLD_IOS_VERSION_7_0 } + { PACKED_VERSION(678,24,0), 0x00020000 }, + { PACKED_VERSION(678,26,0), 0x00020100 }, + { PACKED_VERSION(678,29,0), 0x00020200 }, + { PACKED_VERSION(678,47,0), 0x00030000 }, + { PACKED_VERSION(678,51,0), 0x00030100 }, + { PACKED_VERSION(678,60,0), 0x00030200 }, + { PACKED_VERSION(751,32,0), 0x00040000 }, + { PACKED_VERSION(751,37,0), 0x00040100 }, + { PACKED_VERSION(751,49,0), 0x00040200 }, + { PACKED_VERSION(751,58,0), 0x00040300 }, + { PACKED_VERSION(881,0,0), 0x00050000 }, + { PACKED_VERSION(890,1,0), 0x00050100 }, + { PACKED_VERSION(992,0,0), 0x00060000 }, + { PACKED_VERSION(993,0,0), 0x00060100 }, + { PACKED_VERSION(1038,14,0),0x00070000 }, + { PACKED_VERSION(0,0,0), 0x00070000 } // We don't need to expand this table because all recent // binaries have LC_VERSION_MIN_ load command. }; @@ -554,13 +554,13 @@ static uint32_t deriveSDKVersFromDylibs(const mach_header* mh) // a new last entry needs to be added and the previous zero // updated to the GM dylib version. static const DylibToOSMapping libSystemMapping[] = { - { PACKED_VERSION(88,1,3), DYLD_MACOSX_VERSION_10_4 }, - { PACKED_VERSION(111,0,0), DYLD_MACOSX_VERSION_10_5 }, - { PACKED_VERSION(123,0,0), DYLD_MACOSX_VERSION_10_6 }, - { PACKED_VERSION(159,0,0), DYLD_MACOSX_VERSION_10_7 }, - { PACKED_VERSION(169,3,0), DYLD_MACOSX_VERSION_10_8 }, - { PACKED_VERSION(1197,0,0), DYLD_MACOSX_VERSION_10_9 }, - { PACKED_VERSION(0,0,0), DYLD_MACOSX_VERSION_10_9 } + { PACKED_VERSION(88,1,3), 0x000A0400 }, + { PACKED_VERSION(111,0,0), 0x000A0500 }, + { PACKED_VERSION(123,0,0), 0x000A0600 }, + { PACKED_VERSION(159,0,0), 0x000A0700 }, + { PACKED_VERSION(169,3,0), 0x000A0800 }, + { PACKED_VERSION(1197,0,0), 0x000A0900 }, + { PACKED_VERSION(0,0,0), 0x000A0900 } // We don't need to expand this table because all recent // binaries have LC_VERSION_MIN_ load command. }; @@ -1607,6 +1607,16 @@ bool _dyld_get_shared_cache_uuid(uuid_t uuid) return p(uuid); } +const void* _dyld_get_shared_cache_range(size_t* length) +{ + DYLD_NO_LOCK_THIS_BLOCK; + static const void* (*p)(size_t*) = NULL; + + if(p == NULL) + _dyld_func_lookup("__dyld_get_shared_cache_range", (void**)&p); + return p(length); +} + bool dyld_process_is_restricted() { diff --git a/src/dyld_process_info.cpp b/src/dyld_process_info.cpp index 849f17c..51f7ad4 100644 --- a/src/dyld_process_info.cpp +++ b/src/dyld_process_info.cpp @@ -227,14 +227,22 @@ dyld_process_info_base* dyld_process_info_base::makeSuspended(task_t task, kern_ if ( result != KERN_SUCCESS ) break; if ( info.protection == (VM_PROT_READ|VM_PROT_EXECUTE) ) { - if ( mainExecutableAddress == 0 ) { + // read start of vm region to verify it is a mach header + mach_vm_size_t readSize = sizeof(mach_header_64); + mach_header_64 mhBuffer; + if ( mach_vm_read_overwrite(task, address, sizeof(mach_header_64), (vm_address_t)&mhBuffer, &readSize) != KERN_SUCCESS ) + continue; + if ( (mhBuffer.magic != MH_MAGIC) && (mhBuffer.magic != MH_MAGIC_64) ) + continue; + // now know the region is the start of a mach-o file + if ( mhBuffer.filetype == MH_EXECUTE ) { mainExecutableAddress = address; int len = proc_regionfilename(pid, mainExecutableAddress, mainExecutablePathBuffer, PATH_MAX); if ( len != 0 ) mainExecutablePathBuffer[len] = '\0'; ++imageCount; } - else if ( dyldAddress == 0 ) { + else if ( mhBuffer.filetype == MH_DYLINKER ) { dyldAddress = address; int len = proc_regionfilename(pid, dyldAddress, dyldPathBuffer, PATH_MAX); if ( len != 0 ) @@ -500,7 +508,7 @@ void dyld_process_info_base::forEachSegment(uint64_t machHeaderAddress, void (^c // Implementation that works with existing dyld data structures -dyld_process_info _dyld_process_info_create(task_t task, uint64_t timestamp, kern_return_t* kr) +static dyld_process_info _dyld_process_info_create_inner(task_t task, uint64_t timestamp, kern_return_t* kr) { if ( kr != NULL ) *kr = KERN_SUCCESS; @@ -588,10 +596,16 @@ dyld_process_info _dyld_process_info_create(task_t task, uint64_t timestamp, ker imageCount = MIN(imageCount, 8192); // read image array + if ( allImageInfo64.infoArray == 0 ) { + // dyld is in middle of updating image list, try again + return NULL; + } dyld_image_info_64 imageArray64[imageCount]; if ( kern_return_t r = mach_vm_read_overwrite(task, allImageInfo64.infoArray, imageArraySize, (vm_address_t)&imageArray64, &readSize) ) { - if ( kr != NULL ) + // if image array moved, try whole thing again + if ( kr != NULL ) { *kr = r; + } return NULL; } // normalize by expanding 32-bit image_infos into 64-bit ones @@ -607,10 +621,47 @@ dyld_process_info _dyld_process_info_create(task_t task, uint64_t timestamp, ker } // create object based on local copy of all image infos and image array - return dyld_process_info_base::make(task, allImageInfo64, imageArray64, kr); + dyld_process_info result = dyld_process_info_base::make(task, allImageInfo64, imageArray64, kr); + + // verify nothing has changed by re-reading all_image_infos struct and checking timestamp + if ( result != NULL ) { + dyld_all_image_infos_64 allImageInfo64again; + readSize = task_dyld_info.all_image_info_size; + if ( kern_return_t r = mach_vm_read_overwrite(task, task_dyld_info.all_image_info_addr, task_dyld_info.all_image_info_size, (vm_address_t)&allImageInfo64again, &readSize) ) { + if ( kr != NULL ) + *kr = r; + free((void*)result); + return NULL; + } + uint64_t doneTimeStamp = allImageInfo64again.infoArrayChangeTimestamp; + if ( task_dyld_info.all_image_info_format == TASK_DYLD_ALL_IMAGE_INFO_32 ) { + const dyld_all_image_infos_32* allImageInfo32 = (dyld_all_image_infos_32*)&allImageInfo64again; + doneTimeStamp = allImageInfo32->infoArrayChangeTimestamp; + } + if ( allImageInfo64.infoArrayChangeTimestamp != doneTimeStamp ) { + // image list has changed since we started reading it + // throw out what we have and start over + free((void*)result); + result = nullptr; + } + } + + return result; } +dyld_process_info _dyld_process_info_create(task_t task, uint64_t timestamp, kern_return_t* kr) +{ + // Other process may be loading and unloading as we read its memory, which can cause a read failure + // Retry if something fails + for (int i=0; i < 100; ++i) { + if ( dyld_process_info result = _dyld_process_info_create_inner( task, timestamp, kr) ) + return result; + + } + return NULL; +} + void _dyld_process_info_get_state(dyld_process_info info, dyld_process_state_info* stateInfo) { *stateInfo = *info->stateInfo(); diff --git a/src/dyld_process_info_internal.h b/src/dyld_process_info_internal.h index 23f614b..057278a 100644 --- a/src/dyld_process_info_internal.h +++ b/src/dyld_process_info_internal.h @@ -109,6 +109,7 @@ struct dyld_image_info_64 { #define DYLD_PROCESS_INFO_NOTIFY_MAX_BUFFER_SIZE (32*1024) #define DYLD_PROCESS_INFO_NOTIFY_LOAD_ID 0x1000 #define DYLD_PROCESS_INFO_NOTIFY_UNLOAD_ID 0x2000 +#define DYLD_PROCESS_INFO_NOTIFY_MAIN_ID 0x3000 struct dyld_process_info_image_entry { diff --git a/src/dyld_process_info_notify.cpp b/src/dyld_process_info_notify.cpp index 3cfa788..760f06c 100644 --- a/src/dyld_process_info_notify.cpp +++ b/src/dyld_process_info_notify.cpp @@ -37,6 +37,7 @@ typedef void (^Notify)(bool unload, uint64_t timestamp, uint64_t machHeader, const uuid_t uuid, const char* path); typedef void (^NotifyExit)(); +typedef void (^NotifyMain)(); // @@ -48,6 +49,7 @@ struct __attribute__((visibility("hidden"))) dyld_process_info_notify_base ~dyld_process_info_notify_base(); uint32_t& retainCount() const { return _retainCount; } + void setNotifyMain(NotifyMain notifyMain) const { _notifyMain = notifyMain; } private: dyld_process_info_notify_base(dispatch_queue_t queue, Notify notify, NotifyExit notifyExit, task_t task); @@ -61,6 +63,7 @@ private: dispatch_queue_t _queue; Notify _notify; NotifyExit _notifyExit; + mutable NotifyMain _notifyMain; task_t _targetTask; dispatch_source_t _machSource; uint64_t _portAddressInTarget; @@ -70,7 +73,7 @@ private: dyld_process_info_notify_base::dyld_process_info_notify_base(dispatch_queue_t queue, Notify notify, NotifyExit notifyExit, task_t task) - : _retainCount(1), _queue(queue), _notify(notify), _notifyExit(notifyExit), _targetTask(task), _machSource(NULL), _portAddressInTarget(0), _sendPortInTarget(0), _receivePortInMonitor(0) + : _retainCount(1), _queue(queue), _notify(notify), _notifyExit(notifyExit), _notifyMain(NULL), _targetTask(task), _machSource(NULL), _portAddressInTarget(0), _sendPortInTarget(0), _receivePortInMonitor(0) { dispatch_retain(_queue); } @@ -188,6 +191,20 @@ void dyld_process_info_notify_base::setMachSourceOnQueue() replyHeader.msgh_size = sizeof(replyHeader); mach_msg(&replyHeader, MACH_SEND_MSG | MACH_SEND_TIMEOUT, replyHeader.msgh_size, 0, MACH_PORT_NULL, 100, MACH_PORT_NULL); } + else if ( h->msgh_id == DYLD_PROCESS_INFO_NOTIFY_MAIN_ID ) { + if ( _notifyMain != NULL ) { + _notifyMain(); + } + // reply to dyld, so it can continue + mach_msg_header_t replyHeader; + replyHeader.msgh_bits = MACH_MSGH_BITS(MACH_MSG_TYPE_COPY_SEND, MACH_MSG_TYPE_MAKE_SEND); + replyHeader.msgh_id = 0; + replyHeader.msgh_local_port = MACH_PORT_NULL; + replyHeader.msgh_remote_port = h->msgh_remote_port; + replyHeader.msgh_reserved = 0; + replyHeader.msgh_size = sizeof(replyHeader); + mach_msg(&replyHeader, MACH_SEND_MSG | MACH_SEND_TIMEOUT, replyHeader.msgh_size, 0, MACH_PORT_NULL, 100, MACH_PORT_NULL); + } else if ( h->msgh_id == MACH_NOTIFY_PORT_DELETED ) { mach_port_t deadPort = ((mach_port_deleted_notification_t *)h)->not_port; //fprintf(stderr, "received message id=MACH_NOTIFY_PORT_DELETED, size=%d, deadPort=%d\n", h->msgh_size, deadPort); @@ -292,6 +309,11 @@ dyld_process_info_notify _dyld_process_info_notify(task_t task, dispatch_queue_t return dyld_process_info_notify_base::make(task, queue, notify, notifyExit, kr); } +void _dyld_process_info_notify_main(dyld_process_info_notify object, void (^notifyMain)()) +{ + object->setNotifyMain(notifyMain); +} + void _dyld_process_info_notify_retain(dyld_process_info_notify object) { object->retainCount() += 1; diff --git a/testing/build_tests.py b/testing/build_tests.py index fd7874a..4bb4482 100755 --- a/testing/build_tests.py +++ b/testing/build_tests.py @@ -75,15 +75,18 @@ def buildTestCase(testCaseDirectives, testCaseSourceDir, toolsDir, sdkDir, minOs compilerSearchOptions = " -isysroot " + sdkDir + " -I" + sdkDir + "/System/Library/Frameworks/System.framework/PrivateHeaders" if minOsOptionsName == "mmacosx-version-min": taskForPidCommand = "touch " + envEnableCommand = "touch " else: taskForPidCommand = "codesign --force --sign - --entitlements " + testCaseSourceDir + "/../../task_for_pid_entitlement.plist " + envEnableCommand = "codesign --force --sign - --entitlements " + testCaseSourceDir + "/../../get_task_allow_entitlement.plist " buildSubs = { "CC": toolsDir + "/usr/bin/clang " + archOptions + " -" + minOsOptionsName + "=" + str(minOS) + compilerSearchOptions, "CXX": toolsDir + "/usr/bin/clang++ " + archOptions + " -" + minOsOptionsName + "=" + str(minOS) + compilerSearchOptions, "BUILD_DIR": testCaseDestDirBuild, "RUN_DIR": testCaseDestDirRun, "TEMP_DIR": scratchDir, - "TASK_FOR_PID_ENABLE": taskForPidCommand + "TASK_FOR_PID_ENABLE": taskForPidCommand, + "DYLD_ENV_VARS_ENABLE": envEnableCommand } os.makedirs(testCaseDestDirBuild) os.chdir(testCaseSourceDir) @@ -91,9 +94,12 @@ def buildTestCase(testCaseDirectives, testCaseSourceDir, toolsDir, sdkDir, minOs for line in testCaseDirectives["BUILD"]: cmd = string.Template(line).safe_substitute(buildSubs) print >> sys.stderr, cmd - cmdList = [] - cmdList = string.split(cmd) - result = subprocess.call(cmdList) + if "&&" in cmd: + result = subprocess.call(cmd, shell=True) + else: + cmdList = [] + cmdList = string.split(cmd) + result = subprocess.call(cmdList) if result: return result shutil.rmtree(scratchDir, ignore_errors=True) @@ -172,7 +178,7 @@ if __name__ == "__main__": mytest["Command"].append("./run.sh") for runline in testCaseDirectives["RUN"]: if "$SUDO" in runline: - mytest["AsRoot"] = 1 + mytest["AsRoot"] = True if testCaseDirectives["RUN_TIMEOUT"]: mytest["Timeout"] = testCaseDirectives["RUN_TIMEOUT"] allTests.append(mytest) diff --git a/testing/get_task_allow_entitlement.plist b/testing/get_task_allow_entitlement.plist new file mode 100644 index 0000000..bd10085 --- /dev/null +++ b/testing/get_task_allow_entitlement.plist @@ -0,0 +1,8 @@ + + + + + get-task-allow + + + diff --git a/testing/test-cases/crt-vars-libSystem.dtest/main.c b/testing/test-cases/crt-vars-libSystem.dtest/main.c new file mode 100644 index 0000000..6eb9381 --- /dev/null +++ b/testing/test-cases/crt-vars-libSystem.dtest/main.c @@ -0,0 +1,100 @@ + +// BUILD: $CC main.c -o $BUILD_DIR/crt-vars-libSystem.exe + +// RUN: ./crt-vars-libSystem.exe + +#include +#include +#include +#include +#include + +// This struct is passed as fifth parameter to libSystem.dylib's initializer so it record +// the address of crt global variables. +struct ProgramVars +{ + const void* mh; + int* NXArgcPtr; + char*** NXArgvPtr; + char*** environPtr; + char** __prognamePtr; +}; + + +// global variables defeined in crt1.o +extern char** NXArgv; +extern int NXArgc; +extern char** environ; +extern char* __progname; + + +static const struct ProgramVars* sVars; + +void __attribute__((constructor)) +myInit(int argc, const char* argv[], const char* envp[], const char* apple[], const struct ProgramVars* vars) +{ + sVars = vars; +} + + +int main(int argc, const char* argv[]) +{ + printf("[BEGIN] crt-vars-libSystem\n"); + bool success = true; + + if ( _NSGetArgv() != &NXArgv ) { + printf("[FAIL] crt-libSystem: _NSGetArgv() != &NXArgv (%p!=%p) for %s", _NSGetArgv(), &NXArgv, argv[0]); + success = false; + } + + if ( _NSGetArgc() != &NXArgc ) { + printf("[FAIL] crt-libSystem: _NSGetArgc() != &NXArgc (%p!=%p) for %s", _NSGetArgc(), &NXArgc, argv[0]); + success = false; + } + + if ( _NSGetEnviron() != &environ ) { + printf("[FAIL] crt-libSystem: _NSGetEnviron() != &environv (%p!=%p) for %s", _NSGetEnviron(), &environ, argv[0]); + success = false; + } + + if ( _NSGetProgname() != &__progname ) { + printf("[FAIL] crt-libSystem: _NSGetProgname() != &__progname (%p!=%p) for %s", _NSGetProgname(), &__progname, argv[0]); + success = false; + } + + if ( _NSGetMachExecuteHeader() != &_mh_execute_header ) { + printf("[FAIL] crt-libSystem: _NSGetMachExecuteHeader() != &_mh_execute_headerv (%p!=%p) for %s", _NSGetMachExecuteHeader(), &_mh_execute_header, argv[0]); + success = false; + } + + if ( sVars->NXArgvPtr != &NXArgv ) { + printf("[FAIL] crt-libSystem: sVars->NXArgvPtr != &NXArg (%p!=%p) for %s", sVars->NXArgvPtr, &NXArgv, argv[0]); + success = false; + } + + if ( sVars->NXArgcPtr != &NXArgc ) { + printf("[FAIL] crt-libSystem: sVars->NXArgcPtr != &NXArgc (%p!=%p) for %s", sVars->NXArgcPtr, &NXArgc, argv[0]); + success = false; + } + + if ( sVars->environPtr != &environ ) { + printf("[FAIL] crt-libSystem: sVars->environPtr != &environ (%p!=%p) for %s", sVars->environPtr, &environ, argv[0]); + success = false; + } + + if ( sVars->__prognamePtr != &__progname ) { + printf("[FAIL] crt-libSystem: sVars->__prognamePtr != &__progname (%p!=%p) for %s", sVars->__prognamePtr, &__progname, argv[0]); + success = false; + } + + if ( sVars->mh != &_mh_execute_header ) { + printf("[FAIL] crt-libSystem: sVars->mh != &_mh_execute_header (%p!=%p) for %s", sVars->mh, &_mh_execute_header, argv[0]); + success = false; + } + + if ( success ) + printf("[PASS] crt-vars-libSystem\n"); + + return 0; +} + diff --git a/testing/test-cases/dladdr-basic.dtest/main-no-syms.c b/testing/test-cases/dladdr-basic.dtest/main-no-syms.c new file mode 100644 index 0000000..c79c78a --- /dev/null +++ b/testing/test-cases/dladdr-basic.dtest/main-no-syms.c @@ -0,0 +1,38 @@ + +// BUILD: $CC main-no-syms.c -o $BUILD_DIR/dladdr-stripped.exe +// BUILD: strip $BUILD_DIR/dladdr-stripped.exe + +// RUN: ./dladdr-stripped.exe + + +#include +#include +#include +#include +#include + + + +/// +/// verify dladdr() returns NULL for a symbol name in a fully stripped +/// main executable (and not _mh_execute_header+nnn). +/// + +int main() +{ + printf("[BEGIN] dladdr-stripped\n"); + + Dl_info info; + if ( dladdr(&main, &info) == 0 ) { + printf("[FAIL] dladdr(&main, xx) failed"); + return 0; + } + + if ( info.dli_sname != NULL ){ + printf("[FAIL] dladdr() returned: \"%s\" instead of NULL", info.dli_sname); + return 0; + } + + printf("[PASS] dladdr-stripped\n"); + return 0; +} diff --git a/testing/test-cases/dladdr-basic.dtest/main.c b/testing/test-cases/dladdr-basic.dtest/main.c new file mode 100644 index 0000000..d047353 --- /dev/null +++ b/testing/test-cases/dladdr-basic.dtest/main.c @@ -0,0 +1,129 @@ + +// BUILD: $CC main.c -o $BUILD_DIR/dladdr-basic.exe + +// RUN: ./dladdr-basic.exe + +#include +#include +#include +#include +#include + + +int bar() +{ + return 2; +} + +static int foo() +{ + return 3; +} + +__attribute__((visibility("hidden"))) int hide() +{ + return 4; +} + +// checks global symbol +static void verifybar() +{ + Dl_info info; + if ( dladdr(&bar, &info) == 0 ) { + printf("[FAIL] dladdr(&bar, xx) failed"); + exit(0); + } + if ( strcmp(info.dli_sname, "bar") != 0 ) { + printf("[FAIL] dladdr()->dli_sname is \"%s\" instead of \"bar\"", info.dli_sname); + exit(0); + } + if ( info.dli_saddr != &bar) { + printf("[FAIL] dladdr()->dli_saddr is not &bar"); + exit(0); + } + if ( info.dli_fbase != dyld_image_header_containing_address(&bar) ) { + printf("[FAIL] dladdr()->dli_fbase is not image that contains &bar"); + exit(0); + } +} + +// checks local symbol +static void verifyfoo() +{ + Dl_info info; + if ( dladdr(&foo, &info) == 0 ) { + printf("[FAIL] dladdr(&foo, xx) failed"); + exit(0); + } + if ( strcmp(info.dli_sname, "foo") != 0 ) { + printf("[FAIL] dladdr()->dli_sname is \"%s\" instead of \"foo\"", info.dli_sname); + exit(0); + } + if ( info.dli_saddr != &foo) { + printf("[FAIL] dladdr()->dli_saddr is not &foo"); + exit(0); + } + if ( info.dli_fbase != dyld_image_header_containing_address(&foo) ) { + printf("[FAIL] dladdr()->dli_fbase is not image that contains &foo"); + exit(0); + } +} + +// checks hidden symbol +static void verifyhide() +{ + Dl_info info; + if ( dladdr(&hide, &info) == 0 ) { + printf("[FAIL] dladdr(&hide, xx) failed"); + exit(0); + } + if ( strcmp(info.dli_sname, "hide") != 0 ) { + printf("[FAIL] dladdr()->dli_sname is \"%s\" instead of \"hide\"", info.dli_sname); + exit(0); + } + if ( info.dli_saddr != &hide) { + printf("[FAIL] dladdr()->dli_saddr is not &hide"); + exit(0); + } + if ( info.dli_fbase != dyld_image_header_containing_address(&hide) ) { + printf("[FAIL] dladdr()->dli_fbase is not image that contains &hide"); + exit(0); + } +} + +// checks dylib symbol +static void verifymalloc() +{ + Dl_info info; + if ( dladdr(&malloc, &info) == 0 ) { + printf("[FAIL] dladdr(&malloc, xx) failed"); + exit(0); + } + if ( strcmp(info.dli_sname, "malloc") != 0 ) { + printf("[FAIL] dladdr()->dli_sname is \"%s\" instead of \"malloc\"", info.dli_sname); + exit(0); + } + if ( info.dli_saddr != &malloc) { + printf("[FAIL] dladdr()->dli_saddr is not &malloc"); + exit(0); + } + if ( info.dli_fbase != dyld_image_header_containing_address(&malloc) ) { + printf("[FAIL] dladdr()->dli_fbase is not image that contains &malloc"); + exit(0); + } +} + + +int main() +{ + printf("[BEGIN] dladdr-basic\n"); + verifybar(); + verifyhide(); + verifyfoo(); + verifymalloc(); + + + printf("[PASS] dladdr-basic\n"); + return 0; +} + diff --git a/testing/test-cases/dlopen-RTLD_NOLOAD.dtest/foo.c b/testing/test-cases/dlopen-RTLD_NOLOAD.dtest/foo.c new file mode 100644 index 0000000..13457f2 --- /dev/null +++ b/testing/test-cases/dlopen-RTLD_NOLOAD.dtest/foo.c @@ -0,0 +1,5 @@ +int foo() +{ + return 10; +} + diff --git a/testing/test-cases/dlopen-RTLD_NOLOAD.dtest/init-a.c b/testing/test-cases/dlopen-RTLD_NOLOAD.dtest/init-a.c new file mode 100644 index 0000000..237f51b --- /dev/null +++ b/testing/test-cases/dlopen-RTLD_NOLOAD.dtest/init-a.c @@ -0,0 +1,22 @@ + + +#include + +extern bool inInitB; +extern bool doneInitB; + +bool initsInWrongOrder = false; +bool doneInitA = false; + +__attribute__((constructor)) +void initA() +{ + if ( inInitB ) + initsInWrongOrder = true; + doneInitA = true; +} + +bool allInitsDone() +{ + return doneInitA && doneInitB; +} diff --git a/testing/test-cases/dlopen-RTLD_NOLOAD.dtest/init-b.c b/testing/test-cases/dlopen-RTLD_NOLOAD.dtest/init-b.c new file mode 100644 index 0000000..5ade0fb --- /dev/null +++ b/testing/test-cases/dlopen-RTLD_NOLOAD.dtest/init-b.c @@ -0,0 +1,25 @@ + +#include +#include +#include +#include + +bool doneInitB = false; +bool inInitB = false; + + +__attribute__((constructor)) +void initB() +{ + inInitB = true; + + // "upward" link to libInitA.dylib + void* handle = dlopen("libInitA.dylib", RTLD_NOLOAD); + if ( handle == NULL ) { + printf("[FAIL] dlopen-RTLD_NOLOAD-in-initializer: dlopen(libInitA.dylib, RTLD_NOLOAD) failed but it should have worked: %s\n", dlerror()); + return; + } + inInitB = false; + + doneInitB = true; +} diff --git a/testing/test-cases/dlopen-RTLD_NOLOAD.dtest/init-main.c b/testing/test-cases/dlopen-RTLD_NOLOAD.dtest/init-main.c new file mode 100644 index 0000000..6218689 --- /dev/null +++ b/testing/test-cases/dlopen-RTLD_NOLOAD.dtest/init-main.c @@ -0,0 +1,33 @@ + +// BUILD: $CC init-b.c -dynamiclib -install_name $RUN_DIR/libInitB.dylib -o $BUILD_DIR/libInitB.dylib +// BUILD: $CC init-a.c -dynamiclib -install_name $RUN_DIR/libInitA.dylib $BUILD_DIR/libInitB.dylib -o $BUILD_DIR/libInitA.dylib +// BUILD: $CC init-main.c $BUILD_DIR/libInitA.dylib -o $BUILD_DIR/dlopen-RTLD_NOLOAD-in-initializer.exe + +// RUN: ./dlopen-RTLD_NOLOAD-in-initializer.exe + +#include +#include +#include +#include +#include + + +extern bool initsInWrongOrder; +extern bool allInitsDone(); + +int main() +{ + printf("[BEGIN] dlopen-RTLD_NOLOAD-in-initializer\n"); + + /// + /// This tests that using RTLD_NOLOAD in an initializer does not trigger out of order initializers + /// + if ( initsInWrongOrder ) + printf("[FAIL] dlopen-RTLD_NOLOAD-in-initializer: wrong init order\n"); + else if ( !allInitsDone() ) + printf("[FAIL] dlopen-RTLD_NOLOAD-in-initializer: all initializers not run\n"); + else + printf("[PASS] dlopen-RTLD_NOLOAD-in-initializer\n"); + + return 0; +} diff --git a/testing/test-cases/dlopen-RTLD_NOLOAD.dtest/main.c b/testing/test-cases/dlopen-RTLD_NOLOAD.dtest/main.c new file mode 100644 index 0000000..56aa42c --- /dev/null +++ b/testing/test-cases/dlopen-RTLD_NOLOAD.dtest/main.c @@ -0,0 +1,53 @@ + +// BUILD: $CC foo.c -dynamiclib -install_name $RUN_DIR/libfoo.dylib -o $BUILD_DIR/libfoo.dylib +// BUILD: $CC main.c $BUILD_DIR/libfoo.dylib -o $BUILD_DIR/dlopen-RTLD_NOLOAD-basic.exe +// BUILD: cd $BUILD_DIR && ln -s libfoo.dylib libfoo-sym.dylib + +// RUN: ./dlopen-RTLD_NOLOAD-basic.exe + +#include +#include +#include +#include + + +int main() +{ + printf("[BEGIN] dlopen-RTLD_NOLOAD-basic\n"); + + /// + /// This tests that RTLD_NOLOAD finds existing dylib statically linked + /// + void* handle = dlopen("libfoo.dylib", RTLD_NOLOAD); + if ( handle == NULL ) { + printf("[FAIL] dlopen-RTLD_NOLOAD-basic: dlopen(libfoo.dylib, RTLD_NOLOAD) failed but it should have worked: %s\n", dlerror()); + return 0; + } + void* sym = dlsym(handle, "foo"); + if ( sym == NULL ) { + printf("[FAIL] dlopen-RTLD_NOLOAD-basic: dlsym(handle, \"foo\") failed but it should have worked: %s\n", dlerror()); + return 0; + } + + /// + /// This tests that RTLD_NOLOAD verifies that non-existant dylib returns NULL + /// + void* handle2 = dlopen("libfobbulate.dylib", RTLD_NOLOAD); + if ( handle2 != NULL ) { + printf("[FAIL] dlopen-RTLD_NOLOAD-basic: dlopen(libfobbulate.dylib, RTLD_NOLOAD) succeeded but it should have failed\n"); + return 0; + } + + + /// + /// This tests that RTLD_NOLOAD finds symlink to existing dylib + /// + void* handle3 = dlopen("libfoo-sym.dylib", RTLD_NOLOAD); + if ( handle3 == NULL ) { + printf("[FAIL] dlopen-RTLD_NOLOAD-basic: dlopen(libfoo-sym.dylib, RTLD_NOLOAD) failed but it should have worked: %s\n", dlerror()); + return 0; + } + + printf("[PASS] dlopen-RTLD_NOLOAD-basic\n"); + return 0; +} diff --git a/testing/test-cases/dlopen-realpath.dtest/main.c b/testing/test-cases/dlopen-realpath.dtest/main.c new file mode 100644 index 0000000..b34e008 --- /dev/null +++ b/testing/test-cases/dlopen-realpath.dtest/main.c @@ -0,0 +1,41 @@ + +// BUILD: $CC main.c -o $BUILD_DIR/dlopen-realpath.exe +// BUILD: cd $BUILD_DIR && ln -s ./IOKit.framework/IOKit IOKit && ln -s /System/Library/Frameworks/IOKit.framework IOKit.framework + +// RUN: ./dlopen-realpath.exe + +#include +#include + + +static void tryImage(const char* path) +{ + printf("[BEGIN] dlopen-realpath %s\n", path); + void* handle = dlopen(path, RTLD_LAZY); + if ( handle == NULL ) { + printf("dlerror(): %s\n", dlerror()); + printf("[FAIL] dlopen-realpath %s\n", path); + return; + } + + int result = dlclose(handle); + if ( result != 0 ) { + printf("dlclose() returned %c\n", result); + printf("[FAIL] dlopen-realpath %s\n", path); + return; + } + + printf("[PASS] dlopen-realpath %s\n", path); +} + + + +int main() +{ + tryImage("./IOKit.framework/IOKit"); + tryImage("./././IOKit/../IOKit.framework/IOKit"); + tryImage("./IOKit"); + + return 0; +} + diff --git a/testing/test-cases/dyld_process_info_notify.dtest/main.c b/testing/test-cases/dyld_process_info_notify.dtest/main.c index 04d391d..e04e618 100644 --- a/testing/test-cases/dyld_process_info_notify.dtest/main.c +++ b/testing/test-cases/dyld_process_info_notify.dtest/main.c @@ -92,6 +92,10 @@ static bool monitor(task_t task, bool disconnectEarly, bool attachLate) __block bool sawlibSystem = false; __block bool gotTerminationNotice = false; __block bool gotEarlyNotice = false; + __block bool gotMainNotice = false; + __block bool gotMainNoticeBeforeAllInitialDylibs = false; + __block bool gotFooNoticeBeforeMain = false; + __block int libFooLoadCount = 0; __block int libFooUnloadCount = 0; dispatch_semaphore_t taskDone = dispatch_semaphore_create(0); @@ -108,6 +112,8 @@ static bool monitor(task_t task, bool disconnectEarly, bool attachLate) if ( strstr(path, "/libSystem") != NULL ) sawlibSystem = true; if ( strstr(path, "/libfoo.dylib") != NULL ) { + if ( !gotMainNotice ) + gotFooNoticeBeforeMain = true; if ( unload ) ++libFooUnloadCount; else @@ -136,6 +142,14 @@ static bool monitor(task_t task, bool disconnectEarly, bool attachLate) return false; } + // register for notification that it is entrying main() + _dyld_process_info_notify_main(handle, ^{ + //fprintf(stderr, "target entering main()\n"); + gotMainNotice = true; + if ( !sawMainExecutable || !sawlibSystem ) + gotMainNoticeBeforeAllInitialDylibs = true; + }); + // if process suspends itself, wait until it has done so if ( attachLate ) wait_util_task_suspended(task); @@ -158,6 +172,21 @@ static bool monitor(task_t task, bool disconnectEarly, bool attachLate) return false; } + if ( !gotMainNotice ) { + fprintf(stderr, "did not get notification of main()\n"); + return false; + } + + if ( gotMainNoticeBeforeAllInitialDylibs ) { + fprintf(stderr, "notification of main() arrived before all initial dylibs\n"); + return false; + } + + if ( gotFooNoticeBeforeMain ) { + fprintf(stderr, "notification of main() arrived after libfoo load notice\n"); + return false; + } + if ( !attachLate && !sawlibSystem ) { fprintf(stderr, "did not get load notification of libSystem\n"); return false; diff --git a/testing/test-cases/dyld_process_info_unload.dtest/foo.c b/testing/test-cases/dyld_process_info_unload.dtest/foo.c new file mode 100644 index 0000000..85e6cd8 --- /dev/null +++ b/testing/test-cases/dyld_process_info_unload.dtest/foo.c @@ -0,0 +1 @@ +void foo() {} diff --git a/testing/test-cases/dyld_process_info_unload.dtest/main.c b/testing/test-cases/dyld_process_info_unload.dtest/main.c new file mode 100644 index 0000000..3a8bfd5 --- /dev/null +++ b/testing/test-cases/dyld_process_info_unload.dtest/main.c @@ -0,0 +1,129 @@ + +// BUILD: $CC target.c -o $BUILD_DIR/target.exe +// BUILD: $CC foo.c -o $BUILD_DIR/libfoo.dylib -dynamiclib -install_name $RUN_DIR/libfoo.dylib +// BUILD: $CC main.c -o $BUILD_DIR/dyld_process_info_unload.exe +// BUILD: $TASK_FOR_PID_ENABLE $BUILD_DIR/dyld_process_info_unload.exe + +// RUN: $SUDO ./dyld_process_info_unload.exe $RUN_DIR/target.exe + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + + +extern char** environ; + +#if __x86_64__ + cpu_type_t otherArch[] = { CPU_TYPE_I386 }; +#elif __i386__ + cpu_type_t otherArch[] = { CPU_TYPE_X86_64 }; +#elif __arm64__ + cpu_type_t otherArch[] = { CPU_TYPE_ARM }; +#elif __arm__ + cpu_type_t otherArch[] = { CPU_TYPE_ARM64 }; +#endif + +static task_t launchTest(const char* testProgPath, bool launchOtherArch, bool launchSuspended) +{ + posix_spawnattr_t attr; + if ( posix_spawnattr_init(&attr) != 0 ) { + printf("[FAIL] dyld_process_info posix_spawnattr_init()\n"); + exit(0); + } + if ( launchSuspended ) { + if ( posix_spawnattr_setflags(&attr, POSIX_SPAWN_START_SUSPENDED) != 0 ) { + printf("[FAIL] dyld_process_info POSIX_SPAWN_START_SUSPENDED\n"); + exit(0); + } + } + if ( launchOtherArch ) { + size_t copied; + if ( posix_spawnattr_setbinpref_np(&attr, 1, otherArch, &copied) != 0 ) { + printf("[FAIL] dyld_process_info posix_spawnattr_setbinpref_np()\n"); + exit(0); + } + } + + pid_t childPid; + const char* argv[] = { testProgPath, NULL }; + int psResult = posix_spawn(&childPid, testProgPath, NULL, &attr, (char**)argv, environ); + if ( psResult != 0 ) { + printf("[FAIL] dyld_process_info posix_spawn(%s) failed, err=%d\n", testProgPath, psResult); + exit(0); + } + //printf("child pid=%d\n", childPid); + + task_t childTask = 0; + if ( task_for_pid(mach_task_self(), childPid, &childTask) != KERN_SUCCESS ) { + printf("[FAIL] dyld_process_info task_for_pid()\n"); + kill(childPid, SIGKILL); + exit(0); + } + + // wait until process is up and has suspended itself + struct task_basic_info info; + do { + unsigned count = TASK_BASIC_INFO_COUNT; + kern_return_t kr = task_info(childTask, TASK_BASIC_INFO, (task_info_t)&info, &count); + sleep(1); + } while ( info.suspend_count == 0 ); + + return childTask; +} + +static bool alwaysGetImages(task_t task, bool launchedSuspended) +{ + int failCount = 0; + for (int i=0; i < 100; ++i ) { + kern_return_t result; + dyld_process_info info = _dyld_process_info_create(task, 0, &result); + //fprintf(stderr, "info=%p, result=%08X\n", info, result); + if ( i == 0 ) + task_resume(task); + if ( info == NULL ) { + failCount++; + //fprintf(stderr, "info=%p, result=%08X\n", info, result); + } + else { + usleep(100); + _dyld_process_info_release(info); + } + } + if ( failCount !=0 ) { + printf("[FAIL] dyld_process_info_unload %d out of 100 calls to _dyld_process_info_create() failed\n", failCount); + return false; + } + return true; +} + + +int main(int argc, const char* argv[]) +{ + printf("[BEGIN] dyld_process_info_unload\n"); + + if ( argc < 2 ) { + printf("[FAIL] dyld_process_info_unload missing argument\n"); + exit(0); + } + const char* testProgPath = argv[1]; + task_t childTask; + + // launch test program suspended + childTask = launchTest(testProgPath, false, true); + if ( ! alwaysGetImages(childTask, true) ) { + task_terminate(childTask); + exit(0); + } + task_terminate(childTask); + + + printf("[PASS] dyld_process_info_unload\n"); + return 0; +} diff --git a/testing/test-cases/dyld_process_info_unload.dtest/target.c b/testing/test-cases/dyld_process_info_unload.dtest/target.c new file mode 100644 index 0000000..be923e7 --- /dev/null +++ b/testing/test-cases/dyld_process_info_unload.dtest/target.c @@ -0,0 +1,24 @@ +#include +#include +#include +#include +#include +#include + + + +int main(int argc, const char* argv[]) +{ + //fprintf(stderr, "target starting\n"); + usleep(1000); + // load and unload in a loop + for (int i=1; i < 10000; ++i) { + void* h = dlopen("./libfoo.dylib", 0); + usleep(100000/(i*100)); + dlclose(h); + } + //fprintf(stderr, "target done\n"); + + return 0; +} + diff --git a/testing/test-cases/interpose-weak.dtest/main.c b/testing/test-cases/interpose-weak.dtest/main.c index f167a6a..5248427 100644 --- a/testing/test-cases/interpose-weak.dtest/main.c +++ b/testing/test-cases/interpose-weak.dtest/main.c @@ -1,10 +1,12 @@ // BUILD: $CC foo.c -dynamiclib -o $BUILD_DIR/libfoo.dylib -install_name $RUN_DIR/libfoo.dylib // BUILD: $CC main.c $BUILD_DIR/libfoo.dylib -o $BUILD_DIR/interpose-weak-present.exe +// BUILD: $DYLD_ENV_VARS_ENABLE $BUILD_DIR/interpose-weak-present.exe // BUILD: $CC interposer.c -dynamiclib $BUILD_DIR/libfoo.dylib -o $BUILD_DIR/libinterposer.dylib -install_name libinterposer.dylib // BUILD: $CC foo.c -dynamiclib -o $TEMP_DIR/libfoo2.dylib -install_name $RUN_DIR/libfoo2.dylib // BUILD: $CC foo.c -DNO_FOO34 -dynamiclib -o $BUILD_DIR/libfoo2.dylib -install_name $RUN_DIR/libfoo2.dylib // BUILD: $CC main.c -DNO_FOO34 $TEMP_DIR/libfoo2.dylib -o $BUILD_DIR/interpose-weak-missing.exe +// BUILD: $DYLD_ENV_VARS_ENABLE $BUILD_DIR/interpose-weak-missing.exe // BUILD: $CC interposer.c -dynamiclib $TEMP_DIR/libfoo2.dylib -o $BUILD_DIR/libinterposer2.dylib -install_name libinterposer.dylib diff --git a/testing/test-cases/shared_cache_range.dtest/main.c b/testing/test-cases/shared_cache_range.dtest/main.c new file mode 100644 index 0000000..693de54 --- /dev/null +++ b/testing/test-cases/shared_cache_range.dtest/main.c @@ -0,0 +1,59 @@ + +// BUILD: $CC main.c -o $BUILD_DIR/shared_cache_range.exe + +// RUN: ./shared_cache_range.exe + +#include +#include +#include +#include +#include + + +int main() +{ + printf("[BEGIN] shared_cache_range\n"); + + // see if image containing malloc is in the dyld cache + Dl_info info; + if ( dladdr(&malloc, &info) == 0 ) { + printf("[FAIL] shared_cache_range: dladdr(&malloc, xx) failed"); + return 0; + } + const struct mach_header* mh = (struct mach_header*)info.dli_fbase; + printf("image with malloc=%p\n", mh); + if ( mh == NULL ) { + printf("[FAIL] shared_cache_range: dladdr(&malloc, xx) => dli_fbase==NULL"); + return 0; + } + bool haveSharedCache = (mh->flags & 0x80000000); + printf("haveSharedCache=%d\n", haveSharedCache); + + size_t cacheLen; + const void* cacheStart = _dyld_get_shared_cache_range(&cacheLen); + + if ( haveSharedCache ) { + if ( cacheStart == NULL ) { + printf("[FAIL] _dyld_get_shared_cache_range() returned NULL even though we have a cache\n"); + return 0; + } + printf("shared cache start=%p, len=0x%0lX\n", cacheStart, cacheLen); + const void* cacheEnd = (char*)cacheStart + cacheLen; + + // verify malloc is in shared cache + if ( ((void*)&malloc < cacheStart) || ((void*)&malloc > cacheEnd) ) { + printf("[FAIL] shared_cache_range: malloc is outside range of cache\n"); + return 0; + } + } + else { + if ( cacheStart != NULL ) { + printf("[FAIL] _dyld_get_shared_cache_range() returned non-NULL even though we don't have a cache\n"); + return 0; + } + } + + printf("[PASS] shared_cache_range\n"); + return 0; +} +