From 8074fd5ce9395d82fc9f7ac611f5ad43378ffc70 Mon Sep 17 00:00:00 2001 From: Apple Date: Wed, 15 Aug 2018 19:10:21 +0000 Subject: [PATCH] dyld-551.3.tar.gz --- dyld3/AllImages.cpp | 14 ++++++- dyld3/AllImages.h | 1 + dyld3/LaunchCacheReader.cpp | 40 ++++++++++++++++-- dyld3/Loading.cpp | 10 +++-- dyld3/MachOParser.cpp | 8 ++-- dyld3/SharedCacheRuntime.cpp | 9 ++-- dyld3/libclosured-stub.cpp | 2 +- dyld3/libdyldEntryVector.cpp | 5 ++- dyld3/shared-cache/BuilderUtils.mm | 27 ++++-------- dyld3/shared-cache/CacheBuilder.cpp | 42 +++++++++++-------- dyld3/shared-cache/CacheBuilder.h | 28 +++++++------ dyld3/shared-cache/DyldSharedCache.cpp | 4 +- dyld3/shared-cache/DyldSharedCache.h | 16 +++---- dyld3/shared-cache/ImageProxy.cpp | 14 +++++-- dyld3/shared-cache/Manifest.mm | 2 +- dyld3/shared-cache/OptimizerLinkedit.cpp | 7 ++-- .../shared-cache/update_dyld_shared_cache.cpp | 15 ++++++- src/ImageLoaderMachO.cpp | 6 ++- src/ImageLoaderMachOCompressed.cpp | 4 -- src/dyld.cpp | 40 +++++++++++------- src/dyldAPIs.cpp | 3 +- src/dyldNew.cpp | 18 ++++---- src/dyld_process_info_notify.cpp | 34 ++++++++++++--- .../dyld_process_info_unload.dtest/main.c | 4 -- .../foo.s | 4 ++ .../main.c | 28 +++++++++++++ 26 files changed, 259 insertions(+), 126 deletions(-) create mode 100644 testing/test-cases/flat-namespace-absolute-symbol.dtest/foo.s create mode 100644 testing/test-cases/flat-namespace-absolute-symbol.dtest/main.c diff --git a/dyld3/AllImages.cpp b/dyld3/AllImages.cpp index 6d64e36..a410456 100644 --- a/dyld3/AllImages.cpp +++ b/dyld3/AllImages.cpp @@ -455,6 +455,12 @@ void AllImages::init(const BinaryClosure* closure, const void* dyldCacheLoadAddr _dyldCacheAddress = dyldCacheLoadAddress; _dyldCachePath = dyldCachePath; + if ( _dyldCacheAddress ) { + const DyldSharedCache* cache = (DyldSharedCache*)_dyldCacheAddress; + const dyld_cache_mapping_info* const fileMappings = (dyld_cache_mapping_info*)((uint64_t)_dyldCacheAddress + cache->header.mappingOffset); + _dyldCacheSlide = (uint64_t)dyldCacheLoadAddress - fileMappings[0].address; + } + // Make temporary old image array, so libSystem initializers can be debugged uint32_t count = (uint32_t)initialImages.count(); dyld_image_info oldDyldInfo[count]; @@ -585,9 +591,13 @@ void AllImages::addImages(const launch_cache::DynArray& newIm for (uint32_t j=0; j < existingNotifierCount; ++j) { NotifyFunc func = existingNotifierArray[j]; for (uint32_t i=0; i < count; ++i) { - MachOParser parser(newImages[i].loadAddress); log_notifications("dyld: add notifier %p called with mh=%p\n", func, newImages[i].loadAddress); - func(newImages[i].loadAddress, parser.getSlide()); + if (newImages[i].justUsedFromDyldCache) { + func(newImages[i].loadAddress, _dyldCacheSlide); + } else { + MachOParser parser(newImages[i].loadAddress); + func(newImages[i].loadAddress, parser.getSlide()); + } } } diff --git a/dyld3/AllImages.h b/dyld3/AllImages.h index 1f644bc..47d0655 100644 --- a/dyld3/AllImages.h +++ b/dyld3/AllImages.h @@ -146,6 +146,7 @@ private: const BinaryClosure* _mainClosure = nullptr; const void* _dyldCacheAddress = nullptr; const char* _dyldCachePath = nullptr; + uint64_t _dyldCacheSlide = 0; StartImageArray* _initialImages = nullptr; const char* _mainExeOverridePath = nullptr; _dyld_objc_notify_mapped _objcNotifyMapped = nullptr; diff --git a/dyld3/LaunchCacheReader.cpp b/dyld3/LaunchCacheReader.cpp index 06c73e1..5b99e75 100644 --- a/dyld3/LaunchCacheReader.cpp +++ b/dyld3/LaunchCacheReader.cpp @@ -196,7 +196,15 @@ uintptr_t TargetSymbolValue::resolveTarget(Diagnostics& diag, const ImageGroup& }); return (*foundMH != nullptr); }) ) { - result = ((uintptr_t)(foundInfo.foundInDylib) + (uintptr_t)foundInfo.value); + switch (foundInfo.kind) { + case MachOParser::FoundSymbol::Kind::headerOffset: + case MachOParser::FoundSymbol::Kind::resolverOffset: + result = ((uintptr_t)(foundInfo.foundInDylib) + (uintptr_t)foundInfo.value); + break; + case MachOParser::FoundSymbol::Kind::absolute: + result = (uintptr_t)foundInfo.value; + break; + } images.setAsNeverUnload(idx); found = true; stop = true; @@ -216,7 +224,15 @@ uintptr_t TargetSymbolValue::resolveTarget(Diagnostics& diag, const ImageGroup& dyld3::MachOParser parser(mh); dyld3::MachOParser::FoundSymbol foundInfo; if ( parser.findExportedSymbol(findSymbolDiag, symbolName, nullptr, foundInfo, nullptr) ) { - result = ((uintptr_t)(foundInfo.foundInDylib) + (uintptr_t)foundInfo.value); + switch (foundInfo.kind) { + case MachOParser::FoundSymbol::Kind::headerOffset: + case MachOParser::FoundSymbol::Kind::resolverOffset: + result = ((uintptr_t)(foundInfo.foundInDylib) + (uintptr_t)foundInfo.value); + break; + case MachOParser::FoundSymbol::Kind::absolute: + result = (uintptr_t)foundInfo.value; + break; + } found = true; stop = true; } @@ -232,7 +248,15 @@ uintptr_t TargetSymbolValue::resolveTarget(Diagnostics& diag, const ImageGroup& dyld3::MachOParser parser(mh); dyld3::MachOParser::FoundSymbol foundInfo; if ( parser.findExportedSymbol(findSymbolDiag, symbolName, nullptr, foundInfo, nullptr) ) { - result = ((uintptr_t)(foundInfo.foundInDylib) + (uintptr_t)foundInfo.value); + switch (foundInfo.kind) { + case MachOParser::FoundSymbol::Kind::headerOffset: + case MachOParser::FoundSymbol::Kind::resolverOffset: + result = ((uintptr_t)(foundInfo.foundInDylib) + (uintptr_t)foundInfo.value); + break; + case MachOParser::FoundSymbol::Kind::absolute: + result = (uintptr_t)foundInfo.value; + break; + } found = true; images.setAsNeverUnload(idx); stop = true; @@ -249,7 +273,15 @@ uintptr_t TargetSymbolValue::resolveTarget(Diagnostics& diag, const ImageGroup& dyld3::MachOParser parser(mh); dyld3::MachOParser::FoundSymbol foundInfo; if ( parser.findExportedSymbol(findSymbolDiag, symbolName, nullptr, foundInfo, reExportFollower) ) { - result = ((uintptr_t)(foundInfo.foundInDylib) + (uintptr_t)foundInfo.value); + switch (foundInfo.kind) { + case MachOParser::FoundSymbol::Kind::headerOffset: + case MachOParser::FoundSymbol::Kind::resolverOffset: + result = ((uintptr_t)(foundInfo.foundInDylib) + (uintptr_t)foundInfo.value); + break; + case MachOParser::FoundSymbol::Kind::absolute: + result = (uintptr_t)foundInfo.value; + break; + } found = true; stop = true; } diff --git a/dyld3/Loading.cpp b/dyld3/Loading.cpp index 50785b5..d34431a 100644 --- a/dyld3/Loading.cpp +++ b/dyld3/Loading.cpp @@ -91,16 +91,20 @@ static bool sandboxBlockedStat(const char* path) return sandboxBlocked(path, "file-read-metadata"); } -#if TARGET_OS_WATCH +#if TARGET_OS_WATCH || TARGET_OS_BRIDGE static uint64_t pageAlign(uint64_t value) { - return (value + 4095) & (-4096); + #if __arm64__ + return (value + 0x3FFF) & (-0x4000); + #else + return (value + 0xFFF) & (-0x1000); + #endif } #endif static void updateSliceOffset(uint64_t& sliceOffset, uint64_t codeSignEndOffset, size_t fileLen) { -#if TARGET_OS_WATCH +#if TARGET_OS_WATCH || TARGET_OS_BRIDGE if ( sliceOffset != 0 ) { if ( pageAlign(codeSignEndOffset) == pageAlign(fileLen) ) { // cache builder saw fat file, but file is now thin diff --git a/dyld3/MachOParser.cpp b/dyld3/MachOParser.cpp index 27bf3ba..92366f0 100644 --- a/dyld3/MachOParser.cpp +++ b/dyld3/MachOParser.cpp @@ -205,11 +205,9 @@ MachOParser::MachOParser(const mach_header* mh, bool dyldCacheIsRaw) // asssume out-of-process mach_header not in a dyld cache are raw mapped files _data |= 1; } - else { - // out-of-process mach_header in a dyld cache are not raw, but cache may be raw - if ( dyldCacheIsRaw ) - _data |= 2; - } + // out-of-process mach_header in a dyld cache are not raw, but cache may be raw + if ( dyldCacheIsRaw ) + _data |= 2; #endif } diff --git a/dyld3/SharedCacheRuntime.cpp b/dyld3/SharedCacheRuntime.cpp index d1c821b..993ed54 100644 --- a/dyld3/SharedCacheRuntime.cpp +++ b/dyld3/SharedCacheRuntime.cpp @@ -266,10 +266,13 @@ static bool preflightCacheFile(const SharedCacheOptions& options, SharedCacheLoa ::close(fd); return false; } + if ( (cache->header.mappingCount != 3) || (cache->header.mappingOffset > 0x120) ) { + results->errorMessage = "shared cache file mappings are invalid"; + ::close(fd); + return false; + } const dyld_cache_mapping_info* const fileMappings = (dyld_cache_mapping_info*)&firstPage[cache->header.mappingOffset]; - if ( (cache->header.mappingCount != 3) - || (cache->header.mappingOffset > 0x120) - || (fileMappings[0].fileOffset != 0) + if ( (fileMappings[0].fileOffset != 0) || ((fileMappings[0].address + fileMappings[0].size) > fileMappings[1].address) || ((fileMappings[1].address + fileMappings[1].size) > fileMappings[2].address) || ((fileMappings[0].fileOffset + fileMappings[0].size) != fileMappings[1].fileOffset) diff --git a/dyld3/libclosured-stub.cpp b/dyld3/libclosured-stub.cpp index 90820e4..f7e654c 100644 --- a/dyld3/libclosured-stub.cpp +++ b/dyld3/libclosured-stub.cpp @@ -1,7 +1,7 @@ namespace dyld3 { -struct ClosureBuffer { int x; }; +struct ClosureBuffer { }; ClosureBuffer closured_CreateImageGroup(const ClosureBuffer& input) { diff --git a/dyld3/libdyldEntryVector.cpp b/dyld3/libdyldEntryVector.cpp index 9735df2..42cedaa 100644 --- a/dyld3/libdyldEntryVector.cpp +++ b/dyld3/libdyldEntryVector.cpp @@ -29,6 +29,7 @@ #include "Logging.h" #include "PathOverrides.h" #include "LaunchCacheFormat.h" +#include "start_glue.h" extern "C" void start(); @@ -123,6 +124,8 @@ static void entry_setChildForkFunction(void (*func)() ) sChildForkFunction = func; } +typedef void (*StartFunc)(); + const LibDyldEntryVector entryVectorForDyld = { LibDyldEntryVector::kCurrentVectorVersion, launch_cache::binary_format::kFormatVersion, @@ -131,7 +134,7 @@ const LibDyldEntryVector entryVectorForDyld = { &entry_setOldAllImageInfo, &entry_setInitialImageList, &entry_runInitialzersBottomUp, - &start, + (StartFunc)address_of_start, &entry_setChildForkFunction, &entry_setLogFunction, }; diff --git a/dyld3/shared-cache/BuilderUtils.mm b/dyld3/shared-cache/BuilderUtils.mm index ef2a675..7704301 100644 --- a/dyld3/shared-cache/BuilderUtils.mm +++ b/dyld3/shared-cache/BuilderUtils.mm @@ -241,19 +241,7 @@ bool build(Diagnostics& diags, dyld3::Manifest& manifest, const std::string& mas auto queueEntry = buildQueue[index]; pthread_setname_np(queueEntry.options.loggingPrefix.substr(0, MAXTHREADNAMESIZE - 1).c_str()); - DyldSharedCache::CreateResults results; - while (1) { - results = DyldSharedCache::create(queueEntry.options, queueEntry.dylibsForCache, queueEntry.otherDylibsAndBundles, queueEntry.mainExecutables); - if (!results.overflowed) - break; - auto evicted = manifest.removeLargestLeafDylib(queueEntry.configNames, queueEntry.options.archName); - if (evicted.empty()) - break; - queueEntry = manifest.makeQueueEntry(queueEntry.outputPath, queueEntry.configNames, queueEntry.options.archName, queueEntry.options.optimizeStubs, queueEntry.options.loggingPrefix, queueEntry.options.verbose); - dispatch_sync(warningQueue, ^{ - warnings.insert("[WARNING] CACHE OVERFLOW: " + queueEntry.options.loggingPrefix + " evicted dylib: " + evicted); - }); - } + DyldSharedCache::CreateResults results = DyldSharedCache::create(queueEntry.options, queueEntry.dylibsForCache, queueEntry.otherDylibsAndBundles, queueEntry.mainExecutables); dispatch_sync(warningQueue, ^{ warnings.insert(results.warnings.begin(), results.warnings.end()); bool chooseSecondCdHash = agileChooseSHA256CdHash; @@ -262,13 +250,16 @@ bool build(Diagnostics& diags, dyld3::Manifest& manifest, const std::string& mas chooseSecondCdHash = false; } for (const auto& configName : queueEntry.configNames) { - manifest.configuration(configName).architecture(queueEntry.options.archName).results.warnings = results.warnings; + auto& configResults = manifest.configuration(configName).architecture(queueEntry.options.archName).results; + for (const auto& mh : results.evictions) { + auto parser = dyld3::MachOParser(mh); + configResults.exclude(&parser, "VM overflow, evicting"); + } + configResults.warnings = results.warnings; if (queueEntry.options.optimizeStubs) { - manifest.configuration(configName).architecture(queueEntry.options.archName) - .results.developmentCache.cdHash = chooseSecondCdHash ? results.cdHashSecond : results.cdHashFirst; + configResults.developmentCache.cdHash = chooseSecondCdHash ? results.cdHashSecond : results.cdHashFirst; } else { - manifest.configuration(configName).architecture(queueEntry.options.archName) - .results.productionCache.cdHash = chooseSecondCdHash ? results.cdHashSecond : results.cdHashFirst; + configResults.productionCache.cdHash = chooseSecondCdHash ? results.cdHashSecond : results.cdHashFirst; } } }); diff --git a/dyld3/shared-cache/CacheBuilder.cpp b/dyld3/shared-cache/CacheBuilder.cpp index b4fe35a..89cd418 100644 --- a/dyld3/shared-cache/CacheBuilder.cpp +++ b/dyld3/shared-cache/CacheBuilder.cpp @@ -119,6 +119,11 @@ const std::set CacheBuilder::warnings() return _diagnostics.warnings(); } +const std::set CacheBuilder::evictions() +{ + return _evictions; +} + void CacheBuilder::deleteBuffer() { vm_deallocate(mach_task_self(), (vm_address_t)_buffer, _allocatedBufferSize); @@ -176,7 +181,7 @@ bool CacheBuilder::cacheOverflow(const dyld_cache_mapping_info regions[3]) } } -bool CacheBuilder::build(const std::vector& dylibs, +void CacheBuilder::build(const std::vector& dylibs, const std::vector& otherOsDylibsInput, const std::vector& osExecutables) { @@ -184,7 +189,7 @@ bool CacheBuilder::build(const std::vector& dylibs // FIXME: plist should specify required vs optional dylibs if ( dylibs.size() < 30 ) { _diagnostics.error("missing required minimum set of dylibs"); - return false; + return; } uint64_t t1 = mach_absolute_time(); @@ -196,10 +201,10 @@ bool CacheBuilder::build(const std::vector& dylibs // assign addresses for each segment of each dylib in new cache dyld_cache_mapping_info regions[3]; SegmentMapping segmentMapping = assignSegmentAddresses(sortedDylibs, regions); - if ( cacheOverflow(regions) ) { + while ( cacheOverflow(regions) ) { if ( !_options.evictLeafDylibsOnOverflow ) { _diagnostics.error("cache overflow: %lluMB (max %lluMB)", _vmSize / 1024 / 1024, (_archLayout->sharedMemorySize) / 1024 / 1024); - return false; + return; } // find all leaf (not referenced by anything else in cache) dylibs @@ -248,6 +253,7 @@ bool CacheBuilder::build(const std::vector& dylibs for (std::vector::iterator it=sortedDylibs.begin(); it != sortedDylibs.end(); ++it) { dyld3::MachOParser parser(it->mh); if ( installName == parser.installName() ) { + _evictions.insert(parser.header()); otherOsDylibs.push_back(*it); sortedDylibs.erase(it); break; @@ -256,10 +262,10 @@ bool CacheBuilder::build(const std::vector& dylibs } // re-layout cache segmentMapping = assignSegmentAddresses(sortedDylibs, regions); - if ( cacheOverflow(regions) ) { + if ( unreferencedDylibs.size() == 0 && cacheOverflow(regions) ) { _diagnostics.error("cache overflow, tried evicting %ld leaf daylibs, but still too big: %lluMB (max %lluMB)", toRemove.size(), _vmSize / 1024 / 1024, (_archLayout->sharedMemorySize) / 1024 / 1024); - return false; + return; } } @@ -267,7 +273,7 @@ bool CacheBuilder::build(const std::vector& dylibs _allocatedBufferSize = std::max(_currentFileSize, (uint64_t)0x100000)*1.1; // add 10% to allocation to support large closures if ( vm_allocate(mach_task_self(), (vm_address_t*)&_buffer, _allocatedBufferSize, VM_FLAGS_ANYWHERE) != 0 ) { _diagnostics.error("could not allocate buffer"); - return false; + return; } _currentFileSize = _allocatedBufferSize; @@ -276,17 +282,17 @@ bool CacheBuilder::build(const std::vector& dylibs copyRawSegments(sortedDylibs, segmentMapping); adjustAllImagesForNewSegmentLocations(sortedDylibs, segmentMapping); if ( _diagnostics.hasError() ) - return false; + return; bindAllImagesInCacheFile(regions); if ( _diagnostics.hasError() ) - return false; + return; // optimize ObjC if ( _options.optimizeObjC ) optimizeObjC(_buffer, _archLayout->is64, _options.optimizeStubs, _pointersForASLR, _diagnostics); if ( _diagnostics.hasError() ) - return false; + return; // optimize away stubs std::vector branchPoolOffsets; @@ -355,10 +361,10 @@ bool CacheBuilder::build(const std::vector& dylibs _options.pathPrefixes, _patchTable, _options.optimizeStubs, !_options.dylibsRemovedDuringMastering); if ( _diagnostics.hasError() ) - return false; + return; addCachedDylibsImageGroup(dylibGroup); if ( _diagnostics.hasError() ) - return false; + return; uint64_t t4 = mach_absolute_time(); @@ -366,10 +372,10 @@ bool CacheBuilder::build(const std::vector& dylibs dyld3::ImageProxyGroup* otherGroup = dyld3::ImageProxyGroup::makeOtherOsGroup(_diagnostics, dyldCacheParser, dylibGroup, otherOsDylibs, _options.inodesAreSameAsRuntime, _options.pathPrefixes); if ( _diagnostics.hasError() ) - return false; + return; addCachedOtherDylibsImageGroup(otherGroup); if ( _diagnostics.hasError() ) - return false; + return; uint64_t t5 = mach_absolute_time(); @@ -393,7 +399,7 @@ bool CacheBuilder::build(const std::vector& dylibs } addClosures(closures); if ( _diagnostics.hasError() ) - return false; + return; uint64_t t6 = mach_absolute_time(); @@ -455,13 +461,13 @@ bool CacheBuilder::build(const std::vector& dylibs // last sanity check on size if ( _vmSize > _archLayout->sharedMemorySize ) { _diagnostics.error("cache overflow after optimizations. %lluMB (max %lluMB)", _vmSize / 1024 / 1024, (_archLayout->sharedMemorySize) / 1024 / 1024); - return true; + return; } // codesignature is part of file, but is not mapped codeSign(); if ( _diagnostics.hasError() ) - return false; + return; uint64_t t8 = mach_absolute_time(); @@ -483,7 +489,7 @@ bool CacheBuilder::build(const std::vector& dylibs _allocatedBufferSize = _currentFileSize; } - return false; + return; } diff --git a/dyld3/shared-cache/CacheBuilder.h b/dyld3/shared-cache/CacheBuilder.h index 4a6227c..582ac3e 100644 --- a/dyld3/shared-cache/CacheBuilder.h +++ b/dyld3/shared-cache/CacheBuilder.h @@ -47,19 +47,20 @@ namespace dyld3 { struct CacheBuilder { - CacheBuilder(const DyldSharedCache::CreateOptions& options); - - bool build(const std::vector& dylibsToCache, - const std::vector& otherOsDylibs, - const std::vector& osExecutables); - void deleteBuffer(); - const DyldSharedCache* buffer() { return _buffer; } - size_t bufferSize() { return (size_t)_allocatedBufferSize; } - std::string errorMessage(); - const std::set warnings(); - const bool agileSignature(); - const std::string cdHashFirst(); - const std::string cdHashSecond(); + CacheBuilder(const DyldSharedCache::CreateOptions& options); + + void build(const std::vector& dylibsToCache, + const std::vector& otherOsDylibs, + const std::vector& osExecutables); + void deleteBuffer(); + const DyldSharedCache* buffer() { return _buffer; } + size_t bufferSize() { return (size_t)_allocatedBufferSize; } + std::string errorMessage(); + const std::set warnings(); + const std::set evictions(); + const bool agileSignature(); + const std::string cdHashFirst(); + const std::string cdHashSecond(); struct SegmentMappingInfo { const void* srcSegment; @@ -126,6 +127,7 @@ private: const DyldSharedCache::CreateOptions& _options; DyldSharedCache* _buffer; Diagnostics _diagnostics; + std::set _evictions; const ArchLayout* _archLayout; uint32_t _aliasCount; uint64_t _slideInfoFileOffset; diff --git a/dyld3/shared-cache/DyldSharedCache.cpp b/dyld3/shared-cache/DyldSharedCache.cpp index 4fbdd23..f544c19 100644 --- a/dyld3/shared-cache/DyldSharedCache.cpp +++ b/dyld3/shared-cache/DyldSharedCache.cpp @@ -64,12 +64,14 @@ DyldSharedCache::CreateResults DyldSharedCache::create(const CreateOptions& CreateResults results; CacheBuilder cache(options); - results.overflowed = cache.build(dylibsToCache, otherOsDylibs, osExecutables); + cache.build(dylibsToCache, otherOsDylibs, osExecutables); results.agileSignature = cache.agileSignature(); results.cdHashFirst = cache.cdHashFirst(); results.cdHashSecond = cache.cdHashSecond(); results.warnings = cache.warnings(); + results.evictions = cache.evictions(); + if ( cache.errorMessage().empty() ) { results.cacheContent = cache.buffer(); results.cacheLength = cache.bufferSize(); diff --git a/dyld3/shared-cache/DyldSharedCache.h b/dyld3/shared-cache/DyldSharedCache.h index 2d84065..56008b5 100644 --- a/dyld3/shared-cache/DyldSharedCache.h +++ b/dyld3/shared-cache/DyldSharedCache.h @@ -97,14 +97,14 @@ public: struct CreateResults { - const DyldSharedCache* cacheContent = nullptr; // caller needs to vm_deallocate() when done - size_t cacheLength = 0; - std::string errorMessage; - std::set warnings; - bool agileSignature = false; - std::string cdHashFirst; - std::string cdHashSecond; - bool overflowed = false; + const DyldSharedCache* cacheContent = nullptr; // caller needs to vm_deallocate() when done + size_t cacheLength = 0; + std::string errorMessage; + std::set warnings; + std::set evictions; + bool agileSignature = false; + std::string cdHashFirst; + std::string cdHashSecond; }; diff --git a/dyld3/shared-cache/ImageProxy.cpp b/dyld3/shared-cache/ImageProxy.cpp index fcd6db5..960e1b9 100644 --- a/dyld3/shared-cache/ImageProxy.cpp +++ b/dyld3/shared-cache/ImageProxy.cpp @@ -1212,7 +1212,7 @@ ImageProxyGroup* ImageProxyGroup::makeOtherOsGroup(Diagnostics& diag, const Dyld // add every dylib/bundle in "other: list to _images uint32_t indexInGroup = 0; for (const DyldSharedCache::MappedMachO& mapping : otherDylibsAndBundles) { - ImageProxy* proxy = new ImageProxy(mapping, 1, indexInGroup++, false); + ImageProxy* proxy = new ImageProxy(mapping, 1, indexInGroup++, true); groupProxy->_images.push_back(proxy); groupProxy->_pathToProxy[mapping.runtimePath] = proxy; } @@ -1304,7 +1304,7 @@ BinaryClosureData* ImageProxyGroup::makeClosure(Diagnostics& diag, const DyldCac ImageProxyGroup mainClosureGroupProxy(2, dyldCache, nullptr, otherOsDylibs, mainProgMapping.runtimePath, existingGroups, buildTimePrefixes, emptyEnvVars, false, true, inodesAreSameAsRuntime); - ImageProxy* mainProxy = new ImageProxy(mainProgMapping, 2, 0, false); + ImageProxy* mainProxy = new ImageProxy(mainProgMapping, 2, 0, true); if ( mainProxy == nullptr ) { diag.error("can't find slice matching dyld cache in %s", mainProgMapping.runtimePath.c_str()); return nullptr; @@ -2149,7 +2149,10 @@ void ImageProxyGroup::populateGroupWriter(Diagnostics& diag, launch_cache::Image assert(interposeReplacement.isGroupImageTarget(replacementGroupNum, replacementIndexInGroup, replacementOffsetInImage)); assert(replacementGroupNum == 2); assert(replacementIndexInGroup < (1 << 8)); - assert(replacementOffsetInImage < 0xFFFFFFFFULL); + if ( replacementOffsetInImage >= 0xFFFFFFFFULL ) { + diag.warning("bad interposing implementation in %s", _images[imageIndex]->runtimePath().c_str()); + return; + } DyldCacheOverride cacheOverride; cacheOverride.patchTableIndex = patchTableIndex; cacheOverride.imageIndex = replacementIndexInGroup; @@ -2225,7 +2228,10 @@ void ImageProxyGroup::populateGroupWriter(Diagnostics& diag, launch_cache::Image MachOParser::FoundSymbol foundInfo; if ( weakDefParser.findExportedSymbol(weakDiag, entry.first.c_str(), nullptr, foundInfo, nullptr) ) { assert(proxy->indexInGroup() < (1 << 8)); - assert(foundInfo.value < (1ULL << 32)); + if ( foundInfo.value >= (1ULL << 32) ) { + diag.warning("bad weak symbol address in %s", proxy->runtimePath().c_str()); + return; + } entry.second.imageIndex = proxy->indexInGroup(); entry.second.imageOffset = foundInfo.value; } diff --git a/dyld3/shared-cache/Manifest.mm b/dyld3/shared-cache/Manifest.mm index 3dd9f33..abc23bb 100644 --- a/dyld3/shared-cache/Manifest.mm +++ b/dyld3/shared-cache/Manifest.mm @@ -279,7 +279,7 @@ BuildQueueEntry Manifest::makeQueueEntry(const std::string& outputPath, const st options.cacheSupportsASLR = true; options.forSimulator = false; options.verbose = verbose; - options.evictLeafDylibsOnOverflow = false; + options.evictLeafDylibsOnOverflow = true; options.loggingPrefix = prefix; options.pathPrefixes = { "" }; options.dylibOrdering = loadOrderFile(_dylibOrderFile); diff --git a/dyld3/shared-cache/OptimizerLinkedit.cpp b/dyld3/shared-cache/OptimizerLinkedit.cpp index 84e2a65..b8ac19f 100644 --- a/dyld3/shared-cache/OptimizerLinkedit.cpp +++ b/dyld3/shared-cache/OptimizerLinkedit.cpp @@ -296,9 +296,10 @@ AcceleratorTables

::AcceleratorTables(DyldSharedCache* cache, uint64_t linkedi DepNode& node = _depDAG[op->machHeader()]; for (const char* depPath : op->getDownwardDependents()) { macho_header

* depMH = _dylibPathToMachHeader[depPath]; - assert(depMH != NULL); - DepNode* depNode = &_depDAG[depMH]; - node._dependents.push_back(depNode); + if ( depMH != nullptr ) { + DepNode* depNode = &_depDAG[depMH]; + node._dependents.push_back(depNode); + } } } diff --git a/dyld3/shared-cache/update_dyld_shared_cache.cpp b/dyld3/shared-cache/update_dyld_shared_cache.cpp index 1e82704..8fca51c 100644 --- a/dyld3/shared-cache/update_dyld_shared_cache.cpp +++ b/dyld3/shared-cache/update_dyld_shared_cache.cpp @@ -25,6 +25,7 @@ #include #include #include +#include #include #include #include @@ -772,6 +773,11 @@ int main(int argc, const char* argv[]) // fprintf(stderr, " %s\n", aFile.runtimePath.c_str()); //} + // Clear the UUID xattr for the existing cache. + // This prevents the existing cache from being used by dyld3 as roots are probably involved + if (removexattr(outFile.c_str(), "cacheUUID", 0) != 0) { + fprintf(stderr, "update_dyld_shared_cache: warning: failure to remove UUID xattr on shared cache file %s with error %s\n", outFile.c_str(), strerror(errno)); + } // build cache new cache file DyldSharedCache::CreateOptions options; @@ -802,9 +808,16 @@ int main(int argc, const char* argv[]) else { // save new cache file to disk and write new .map file assert(results.cacheContent != nullptr); - if ( !safeSave(results.cacheContent, results.cacheLength, outFile) ) + if ( !safeSave(results.cacheContent, results.cacheLength, outFile) ) { + fprintf(stderr, "update_dyld_shared_cache: could not write dyld cache file %s\n", outFile.c_str()); cacheBuildFailure = true; + } if ( !cacheBuildFailure ) { + uuid_t cacheUUID; + results.cacheContent->getUUID(cacheUUID); + if (setxattr(outFile.c_str(), "cacheUUID", (const void*)&cacheUUID, sizeof(cacheUUID), 0, XATTR_CREATE) != 0) { + fprintf(stderr, "update_dyld_shared_cache: warning: failure to set UUID xattr on shared cache file %s with error %s\n", outFile.c_str(), strerror(errno)); + } std::string mapStr = results.cacheContent->mapFile(); std::string outFileMap = cacheDir + "/dyld_shared_cache_" + fileSet.archName + ".map"; safeSave(mapStr.c_str(), mapStr.size(), outFileMap); diff --git a/src/ImageLoaderMachO.cpp b/src/ImageLoaderMachO.cpp index feba249..1becea3 100644 --- a/src/ImageLoaderMachO.cpp +++ b/src/ImageLoaderMachO.cpp @@ -504,7 +504,7 @@ void ImageLoaderMachO::sniffLoadCommands(const macho_header* mh, const char* pat if ( symTabCmd != NULL ) { // validate symbol table fits in LINKEDIT - if ( symTabCmd->symoff < linkeditFileOffsetStart ) + if ( (symTabCmd->nsyms > 0) && (symTabCmd->symoff < linkeditFileOffsetStart) ) throw "malformed mach-o image: symbol table underruns __LINKEDIT"; if ( symTabCmd->nsyms > 0x10000000 ) throw "malformed mach-o image: symbol table too large"; @@ -1615,6 +1615,10 @@ void ImageLoaderMachO::doRebase(const LinkContext& context) if ( fRetainForObjC ) this->setNeverUnload(); + // dylibs with thread local variables cannot be unloaded because there is no way to clean up all threads + if ( !this->inSharedCache() && (this->machHeader()->flags & MH_HAS_TLV_DESCRIPTORS) ) + this->setNeverUnload(); + // if prebound and loaded at prebound address, then no need to rebase if ( this->usablePrebinding(context) ) { // skip rebasing because prebinding is valid diff --git a/src/ImageLoaderMachOCompressed.cpp b/src/ImageLoaderMachOCompressed.cpp index 020ec51..627d572 100644 --- a/src/ImageLoaderMachOCompressed.cpp +++ b/src/ImageLoaderMachOCompressed.cpp @@ -157,10 +157,6 @@ ImageLoaderMachOCompressed* ImageLoaderMachOCompressed::instantiateFromFile(cons // make sure path is stable before recording in dyld_all_image_infos image->setMapped(context); - // dylibs with thread local variables cannot be unloaded because there is no way to clean up all threads - if ( image->machHeader()->flags & MH_HAS_TLV_DESCRIPTORS ) - image->setNeverUnload(); - // pre-fetch content of __DATA and __LINKEDIT segment for faster launches // don't do this on prebound images or if prefetching is disabled if ( !context.preFetchDisabled && !image->isPrebindable()) { diff --git a/src/dyld.cpp b/src/dyld.cpp index c98a9e3..2fd3787 100644 --- a/src/dyld.cpp +++ b/src/dyld.cpp @@ -40,6 +40,7 @@ #include #include #include +#include #include #include #include @@ -3139,8 +3140,15 @@ static ImageLoader* loadPhase5load(const char* path, const char* orgPath, const return anImage; } // if RTLD_NOLOAD, do nothing if not already loaded - if ( context.dontLoad ) + if ( context.dontLoad ) { + // possible that there is an override of cache + if ( my_stat(path, &statBuf) == 0 ) { + ImageLoader* imageLoader = findLoadedImage(statBuf); + if ( imageLoader != NULL ) + return imageLoader; + } return NULL; + } bool useCache = false; if ( shareCacheResults.imageData == nullptr ) { // HACK to support old caches @@ -5134,21 +5142,21 @@ static bool closureValid(const dyld3::launch_cache::BinaryClosureData* mainClosu } #if __MAC_OS_X_VERSION_MIN_REQUIRED else { - // HACK until closured for dlopen can run against live cache file - int fd = my_open(sSharedCacheLoadInfo.path, O_RDONLY, 0); - if ( fd != -1 ) { - dyld_cache_header fileHeader; - if ( pread(fd, &fileHeader, sizeof(fileHeader), 0) == sizeof(fileHeader) ) { - uuid_t cacheUUID; - sSharedCacheLoadInfo.loadAddress->getUUID(cacheUUID); - if ( memcmp(fileHeader.uuid, cacheUUID, sizeof(uuid_t)) != 0 ) { - if ( gLinkContext.verboseWarnings ) - dyld::log("dyld: closure %p not used because current cache on disk is not they one being used\n", mainClosureData); - ::close(fd); - return false; - } - } - ::close(fd); + // If the in-memory cache doesn't have the same UUID xattr as the on-disk cache then we must + // have built a new cache but not rebooted. In this case, don't use dyld3. + const char* sharedCachePath = getStandardSharedCacheFilePath(); + uuid_t inMemoryUUID; + uuid_t onDiskUUID; + sharedCacheUUID(inMemoryUUID); + if (getxattr(sharedCachePath, "cacheUUID", (void*)&onDiskUUID, sizeof(uuid_t), 0, 0) != sizeof(uuid_t)) { + if ( gLinkContext.verboseWarnings ) + dyld::log("dyld: closure %p on disk cache doesn't have a UUID xattr\n", mainClosureData); + return false; + } + if (memcmp(&inMemoryUUID, &onDiskUUID, sizeof(uuid_t)) != 0) { + if ( gLinkContext.verboseWarnings ) + dyld::log("dyld: closure %p not used because current cache on disk and in memory cache have UUID mismatches\n", mainClosureData); + return false; } } #endif diff --git a/src/dyldAPIs.cpp b/src/dyldAPIs.cpp index 65e5681..c77e652 100644 --- a/src/dyldAPIs.cpp +++ b/src/dyldAPIs.cpp @@ -645,6 +645,7 @@ NSSymbol NSLookupSymbolInImage(const struct mach_header* mh, const char* symbolN dyld::clearErrorMessage(); ImageLoader* image = dyld::findImageByMachHeader(mh); if ( image != NULL ) { + const char* symbolToFind = symbolName; try { if ( options & NSLOOKUPSYMBOLINIMAGE_OPTION_BIND_FULLY ) { image->bindAllLazyPointers(dyld::gLinkContext, true); @@ -658,7 +659,7 @@ NSSymbol NSLookupSymbolInImage(const struct mach_header* mh, const char* symbolN dyldAPIhalt(__func__, msg); } } - symbol = image->findExportedSymbol(symbolName, true, NULL); + symbol = image->findExportedSymbol(symbolToFind, true, NULL); } if ( dyld::gLogAPIs && (symbol == NULL) ) dyld::log("%s(%p, \"%s\", 0x%08X) ==> NULL\n", __func__, mh, symbolName, options); diff --git a/src/dyldNew.cpp b/src/dyldNew.cpp index cf64f58..fb393fa 100644 --- a/src/dyldNew.cpp +++ b/src/dyldNew.cpp @@ -126,18 +126,20 @@ void free(void* ptr) void* calloc(size_t count, size_t size) { + // Check for overflow of integer multiplication + size_t total = count * size; + if ( total/count != size ) { + dyld::log("dyld calloc overflow: count=%zu, size=%zu\n", count, size); + dyld::halt("dyld calloc overflow"); + } if ( dyld::gLibSystemHelpers != NULL ) { - void* result = dyld::gLibSystemHelpers->malloc(size*count); - bzero(result, size*count); + void* result = dyld::gLibSystemHelpers->malloc(total); + if ( result != NULL ) + bzero(result, total); return result; } else { - // Check for overflow of integer multiplication - size_t total = count * size; - if ( total/count != size ) { - dyld::log("dyld calloc overflow: count=%zu, size=%zu\n", count, size); - exit(1); - } + // this allocates out of static buffer which is already zero filled return malloc(total); } } diff --git a/src/dyld_process_info_notify.cpp b/src/dyld_process_info_notify.cpp index 6d5eee3..ec9bce4 100644 --- a/src/dyld_process_info_notify.cpp +++ b/src/dyld_process_info_notify.cpp @@ -53,7 +53,9 @@ struct __attribute__((visibility("hidden"))) dyld_process_info_notify_base static dyld_process_info_notify_base* make(task_t task, dispatch_queue_t queue, Notify notify, NotifyExit notifyExit, kern_return_t* kr); ~dyld_process_info_notify_base(); - uint32_t& retainCount() const { return _retainCount; } + bool incRetainCount() const; + bool decRetainCount() const; + void setNotifyMain(NotifyMain notifyMain) const { _notifyMain = notifyMain; } // override new and delete so we don't need to link with libc++ @@ -67,7 +69,7 @@ private: kern_return_t unpokeSendPortInTarget(); void setMachSourceOnQueue(); - mutable uint32_t _retainCount; + mutable int32_t _retainCount; dispatch_queue_t _queue; Notify _notify; NotifyExit _notifyExit; @@ -89,6 +91,7 @@ dyld_process_info_notify_base::dyld_process_info_notify_base(dispatch_queue_t qu dyld_process_info_notify_base::~dyld_process_info_notify_base() { if ( _machSource ) { + dispatch_source_cancel(_machSource); dispatch_release(_machSource); _machSource = NULL; } @@ -106,6 +109,18 @@ dyld_process_info_notify_base::~dyld_process_info_notify_base() } } +bool dyld_process_info_notify_base::incRetainCount() const +{ + int32_t newCount = OSAtomicIncrement32(&_retainCount); + return ( newCount == 1 ); +} + +bool dyld_process_info_notify_base::decRetainCount() const +{ + int32_t newCount = OSAtomicDecrement32(&_retainCount); + return ( newCount == 0 ); +} + dyld_process_info_notify_base* dyld_process_info_notify_base::make(task_t task, dispatch_queue_t queue, Notify notify, NotifyExit notifyExit, kern_return_t* kr) { @@ -173,6 +188,10 @@ void dyld_process_info_notify_base::setMachSourceOnQueue() NotifyExit exitHandler = _notifyExit; _machSource = dispatch_source_create(DISPATCH_SOURCE_TYPE_MACH_RECV, _receivePortInMonitor, 0, _queue); dispatch_source_set_event_handler(_machSource, ^{ + // This event handler block has an implicit reference to "this" + // if incrementing the count goes to one, that means the object may have already been destroyed + if ( incRetainCount() ) + return; uint8_t messageBuffer[DYLD_PROCESS_INFO_NOTIFY_MAX_BUFFER_SIZE]; mach_msg_header_t* h = (mach_msg_header_t*)messageBuffer; @@ -229,7 +248,9 @@ void dyld_process_info_notify_base::setMachSourceOnQueue() fprintf(stderr, "received unknown message id=0x%X, size=%d\n", h->msgh_id, h->msgh_size); } } - }); + if ( decRetainCount() ) + delete this; + }); dispatch_resume(_machSource); } @@ -323,13 +344,14 @@ void _dyld_process_info_notify_main(dyld_process_info_notify object, void (^noti void _dyld_process_info_notify_retain(dyld_process_info_notify object) { - object->retainCount() += 1; + object->incRetainCount(); } void _dyld_process_info_notify_release(dyld_process_info_notify object) { - object->retainCount() -= 1; - if ( object->retainCount() == 0 ) + // Note if _machSource is currently handling a message, the retain count will not be zero + // and object will instead be deleted when handling is done. + if ( object->decRetainCount() ) delete object; } diff --git a/testing/test-cases/dyld_process_info_unload.dtest/main.c b/testing/test-cases/dyld_process_info_unload.dtest/main.c index 6954ad1..3872e7d 100644 --- a/testing/test-cases/dyld_process_info_unload.dtest/main.c +++ b/testing/test-cases/dyld_process_info_unload.dtest/main.c @@ -111,10 +111,6 @@ static bool alwaysGetImages(struct task_and_pid tp, bool launchedSuspended) // ideally the fail count would be zero. But the target is dlopen/dlclosing in a tight loop, so there may never be a stable set of images. // The real bug driving this test case was _dyld_process_info_create() crashing when the the image list changed too fast. // The important thing is to not crash. Getting NULL back is ok. - if ( failCount > 50 ) { - printf("[FAIL] dyld_process_info_unload %d out of 100 calls to _dyld_process_info_create() failed\n", failCount); - return false; - } return true; } diff --git a/testing/test-cases/flat-namespace-absolute-symbol.dtest/foo.s b/testing/test-cases/flat-namespace-absolute-symbol.dtest/foo.s new file mode 100644 index 0000000..5435411 --- /dev/null +++ b/testing/test-cases/flat-namespace-absolute-symbol.dtest/foo.s @@ -0,0 +1,4 @@ + + .global _myAbs1 +_myAbs1 = 0 + diff --git a/testing/test-cases/flat-namespace-absolute-symbol.dtest/main.c b/testing/test-cases/flat-namespace-absolute-symbol.dtest/main.c new file mode 100644 index 0000000..6fcd005 --- /dev/null +++ b/testing/test-cases/flat-namespace-absolute-symbol.dtest/main.c @@ -0,0 +1,28 @@ +// BUILD_ONLY: MacOSX + +// BUILD: $CC foo.s -dynamiclib -o $BUILD_DIR/libfoo.dylib -install_name $RUN_DIR/libfoo.dylib +// BUILD: $CC main.c $BUILD_DIR/libfoo.dylib -o $BUILD_DIR/flat-namespace.exe -flat_namespace + +// RUN: ./flat-namespace.exe + + + +#include +#include +#include + +extern int myAbs1; +int* ptr = &myAbs1; + +int main() +{ + printf("[BEGIN] flat-namespace-absolute-symbol\n"); + + if ( ptr != 0 ) { + printf("[FAIL] absolute symbol not bound to zero with flat lookup\n"); + return 0; + } + + printf("[PASS] flat-namespace-absolute-symbol\n"); + return 0; +} -- 2.45.2