+
+ if ( _options.makeThreadedStartsSection() ) {
+ assert(_threadedRebaseBindIndices.empty());
+
+ std::vector<OutputFile::BindingInfo>& bindInfo = _bindingInfo;
+ std::vector<OutputFile::RebaseInfo>& rebaseInfo = _rebaseInfo;
+
+ std::vector<int64_t>& threadedRebaseBindIndices = _threadedRebaseBindIndices;
+ threadedRebaseBindIndices.reserve(bindInfo.size() + rebaseInfo.size());
+
+ for (int64_t i = 0, e = rebaseInfo.size(); i != e; ++i)
+ threadedRebaseBindIndices.push_back(-i);
+
+ for (int64_t i = 0, e = bindInfo.size(); i != e; ++i)
+ threadedRebaseBindIndices.push_back(i + 1);
+
+ // Now sort the entries by address.
+ std::sort(threadedRebaseBindIndices.begin(), threadedRebaseBindIndices.end(),
+ [&rebaseInfo, &bindInfo](int64_t indexA, int64_t indexB) {
+ if (indexA == indexB)
+ return false;
+ uint64_t addressA = indexA <= 0 ? rebaseInfo[-indexA]._address : bindInfo[indexA - 1]._address;
+ uint64_t addressB = indexB <= 0 ? rebaseInfo[-indexB]._address : bindInfo[indexB - 1]._address;
+ assert(addressA != addressB);
+ return addressA < addressB;
+ });
+ }
+
+ // new rebasing/binding scheme requires making another pass at DATA
+ // segment and building linked list of rebase locations
+ if ( _options.useLinkedListBinding() && !_threadedRebaseBindIndices.empty() ) {
+ uint64_t curSegStart = 0;
+ uint64_t curSegEnd = 0;
+ uint32_t curSegIndex = 0;
+ ld::Internal::FinalSection* curSection = NULL;
+
+ const uint64_t deltaBits = 11;
+ const uint32_t fixupAlignment = _options.makeThreadedStartsSection() ? 4 : 8;
+ const bool allowThreadsToCrossPages = _options.makeThreadedStartsSection();
+ std::vector<uint64_t> threadStarts;
+
+ // Find the thread starts section
+ ld::Internal::FinalSection* threadStartsSection = nullptr;
+ uint64_t threadStartsReservedSpace = 0;
+ if ( _options.makeThreadedStartsSection() ) {
+ for (ld::Internal::FinalSection* sect : state.sections) {
+ if ( sect->type() == ld::Section::typeThreadStarts ) {
+ threadStartsSection = sect;
+ break;
+ }
+ }
+ assert(threadStartsSection);
+ threadStartsReservedSpace = (threadStartsSection->size - 4) / 4;
+ threadStarts.reserve(threadStartsReservedSpace);
+ }
+
+ auto getAddress = [this](int64_t index) {
+ if (index <= 0)
+ return _rebaseInfo[-index]._address;
+ else
+ return _bindingInfo[index - 1]._address;
+ };
+
+ if ( (_bindingInfo.size() > 1)
+ && ! findSegment(state, getAddress(_threadedRebaseBindIndices.front()),
+ &curSegStart, &curSegEnd, &curSegIndex) )
+ throw "binding address outside range of any segment";
+
+ auto applyBind = [&](int64_t currentIndex, int64_t nextIndex) {
+ uint64_t currentAddress = getAddress(currentIndex);
+ uint64_t nextAddress = getAddress(nextIndex);
+
+ // The very first pointer we see must be a new chain
+ if ( _options.makeThreadedStartsSection() && curSection == NULL )
+ threadStarts.push_back(currentAddress);
+
+ if ( (curSection == NULL)
+ || (currentAddress < curSection->address)
+ || (currentAddress >= curSection->address+curSection->size) ) {
+ for (ld::Internal::FinalSection* sect : state.sections) {
+ if ( (sect->address <= currentAddress)
+ && (currentAddress < sect->address+sect->size) ) {
+ curSection = sect;
+ break;
+ }
+ }
+ }
+
+ bool makeChainToNextAddress = true;
+ if ( allowThreadsToCrossPages ) {
+ // Even if we allow threads to cross pages, we still need to have the same section.
+ if ( (nextAddress < curSection->address) || (nextAddress >= curSection->address+curSection->size) )
+ makeChainToNextAddress = false;
+ } else {
+ // If threads can't cross pages then make sure they are on the same page.
+ uint64_t currentPageIndex = ( currentAddress - curSegStart) / 4096;
+ uint64_t nextPageIndex = ( nextAddress - curSegStart) / 4096;
+ if ( currentPageIndex != nextPageIndex )
+ makeChainToNextAddress = false;
+ }
+
+ uint64_t delta = 0;
+ if (makeChainToNextAddress) {
+ delta = nextAddress - currentAddress;
+
+ // The value should already be aligned to 4 or 8, so make sure the low bits are zeroes
+ assert( (delta & (fixupAlignment - 1)) == 0 );
+ delta /= fixupAlignment;
+ if ( delta >= (1 << deltaBits) ) {
+ // Current and next are both in the same segment, so see if they are
+ // on the same page. If so, patch current to point to next.
+ makeChainToNextAddress = false;
+ }
+ }
+
+ if (!makeChainToNextAddress) {
+ delta = 0;
+ if (_options.makeThreadedStartsSection())
+ threadStarts.push_back(nextAddress);
+ }
+
+ uint8_t* lastBindLocation = wholeBuffer + curSection->fileOffset + currentAddress - curSection->address;
+ switch ( _options.architecture() ) {
+ case CPU_TYPE_X86_64:
+ case CPU_TYPE_ARM64:
+ uint64_t value = 0;
+ if (currentIndex <= 0) {
+ // For rebases, bits [0..50] is the mh offset which is already set
+ // Bit 62 is a 0 to say this is a rebase
+ value = get64LE(lastBindLocation);
+#if SUPPORT_ARCH_arm64e
+ auto fixupOffset = (uintptr_t)(lastBindLocation - mhAddress);
+ auto it = _authenticatedFixupData.find(fixupOffset);
+ if (it != _authenticatedFixupData.end()) {
+ // For authenticated data, we zeroed out the location
+ assert(value == 0);
+ const auto &authData = it->second.first;
+ uint64_t accumulator = it->second.second;
+ assert(accumulator >= mhAddress);
+ accumulator -= mhAddress;
+
+ // Make sure the high bits aren't set. The low 32-bits may
+ // be the target value.
+ assert((accumulator & 0xFFFFFFFF00000000ULL) == 0);
+ accumulator |= ((uint64_t)authData.discriminator) << 32;
+ accumulator |= ((uint64_t)authData.hasAddressDiversity) << 48;
+ accumulator |= ((uint64_t)authData.key) << 49;
+ // Set the high bit as we are authenticated
+ accumulator |= 1ULL << 63;
+
+ value = accumulator;
+ } else
+#endif
+ {
+ // Regular pointer which needs to fit in 51-bits of value.
+ // C++ RTTI uses the top bit, so we'll allow the whole top-byte
+ // and the bottom 43-bits with sign-extension to be fit in to 51-bits.
+ uint64_t top8Bits = value & 0xFF00000000000000ULL;
+ uint64_t bottom43Bits = value & 0x000007FFFFFFFFFFULL;
+ // Ensure that the sign-extended bottom 43-bits is equivalent in sign to the gap bits
+ assert( ((value & ~0xFF0003FFFFFFFFFF) == 0) || ((value & ~0xFF0003FFFFFFFFFF) == ~0xFF0003FFFFFFFFFF) );
+ value = ( top8Bits >> 13 ) | bottom43Bits;
+ }
+ } else {
+ // The ordinal in [0..15]
+ // Bit 62 is a 1 to say this is a bind
+ value = get64LE(lastBindLocation);
+#if SUPPORT_ARCH_arm64e
+ auto fixupOffset = (uintptr_t)(lastBindLocation - mhAddress);
+ auto it = _authenticatedFixupData.find(fixupOffset);
+ if (it != _authenticatedFixupData.end()) {
+ // For authenticated data, we zeroed out the location
+ assert(value == 0);
+ const auto &authData = it->second.first;
+ uint64_t accumulator = it->second.second;
+
+ // Make sure the high bits aren't set. The low 32-bits may
+ // be the target value.
+ // Note, this doesn't work for binds to a weak def as we actually
+ // manage to resolve their address to an address in this binary so
+ // its not 0.
+ if (_bindingInfo[currentIndex - 1]._libraryOrdinal == BIND_SPECIAL_DYLIB_WEAK_LOOKUP)
+ accumulator = 0;
+ assert((accumulator & 0xFFFFFFFF00000000ULL) == 0);
+ accumulator |= ((uint64_t)authData.discriminator) << 32;
+ accumulator |= ((uint64_t)authData.hasAddressDiversity) << 48;
+ accumulator |= ((uint64_t)authData.key) << 49;
+ // Set the high bit as we are authenticated
+ accumulator |= 1ULL << 63;
+
+ value = accumulator;
+ } else
+#endif
+ {
+ // Regular pointer
+ // The current data is unused as we get a new address from the bind table.
+ // So zero it out to avoid the bits interfering with the authentication bits.
+ value = 0;
+ }
+ value &= 0xFFFFFFFFFFFF0000;
+ value |= _bindingInfo[currentIndex - 1]._threadedBindOrdinal;
+ value |= 1ULL << 62;
+ }
+
+ // The delta is bits [51..61]
+ value |= ( delta << 51 );
+ set64LE(lastBindLocation, value);
+ break;
+ }
+ };
+
+ // Loop over every value and see if it needs to point to its successor.
+ // Note that on every iteration, info[i] is already known to be in the current
+ // segment.
+ for (int64_t i = 0, e = _threadedRebaseBindIndices.size() - 1; i != e; ++i) {
+ int64_t currentIndex = _threadedRebaseBindIndices[i];
+ int64_t nextIndex = _threadedRebaseBindIndices[i + 1];
+ uint64_t nextAddress = getAddress(nextIndex);
+ if ( (nextAddress < curSegStart) || ( nextAddress >= curSegEnd) ) {
+ // The next pointer is in a new segment.
+ // This means current is the end of a chain, and we need to move
+ // the segment addresses on to be the next ones.
+ if ( ! findSegment(state, nextAddress, &curSegStart, &curSegEnd, &curSegIndex) )
+ throw "binding address outside range of any segment";
+ }
+
+ applyBind(currentIndex, nextIndex);
+ }
+
+ applyBind(_threadedRebaseBindIndices.back(), _threadedRebaseBindIndices.back());
+
+ if ( _options.makeThreadedStartsSection() ) {
+ if ( threadStarts.size() > threadStartsReservedSpace )
+ throw "overflow in thread starts section";
+
+ // Now write over this section content with the new array.
+ const ld::Atom *threadStartsAtom = nullptr;
+ for (const ld::Atom *atom : threadStartsSection->atoms) {
+ if ( (atom->contentType() == ld::Atom::typeSectionStart) || (atom->contentType() == ld::Atom::typeSectionEnd) ) {
+ assert(atom->size() == 0);
+ continue;
+ }
+ assert(threadStartsAtom == nullptr);
+ threadStartsAtom = atom;
+ }
+ uint64_t threadStartsFileOffset = threadStartsAtom->finalAddress() - threadStartsSection->address + threadStartsSection->fileOffset;
+ // Skip the header
+ threadStartsFileOffset += sizeof(uint32_t);
+ for (uint64_t threadStart : threadStarts) {
+ uint64_t offset = threadStart - mhAddress;
+ assert(offset < 0x100000000);
+ set32LE(&wholeBuffer[threadStartsFileOffset], offset);
+ threadStartsFileOffset += sizeof(uint32_t);
+ }
+ }
+ }