src/ld/InputFiles.cpp

   1
   2 /* -*- mode: C++; c-basic-offset: 4; tab-width: 4 -*-*
   3  *
   4  * Copyright (c) 2009-2011 Apple Inc. All rights reserved.
   5  *
   6  * @APPLE_LICENSE_HEADER_START@
   7  *
   8  * This file contains Original Code and/or Modifications of Original Code
   9  * as defined in and that are subject to the Apple Public Source License
  10  * Version 2.0 (the 'License'). You may not use this file except in
  11  * compliance with the License. Please obtain a copy of the License at
  12  * http://www.opensource.apple.com/apsl/ and read it before using this
  13  * file.
  14  *
  15  * The Original Code and all software distributed under the License are
  16  * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  17  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  18  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
  19  * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
  20  * Please see the License for the specific language governing rights and
  21  * limitations under the License.
  22  *
  23  * @APPLE_LICENSE_HEADER_END@
  24  */
  25
  26
  27 #include <stdlib.h>
  28 #include <sys/types.h>
  29 #include <sys/stat.h>
  30 #include <sys/mman.h>
  31 #include <sys/sysctl.h>
  32 #include <fcntl.h>
  33 #include <errno.h>
  34 #include <limits.h>
  35 #include <unistd.h>
  36 #include <mach/mach_time.h>
  37 #include <mach/vm_statistics.h>
  38 #include <mach/mach_init.h>
  39 #include <mach/mach_host.h>
  40 #include <dlfcn.h>
  41 #include <mach-o/dyld.h>
  42 #include <mach-o/fat.h>
  43 #include <sys/sysctl.h>
  44 #include <libkern/OSAtomic.h>
  45
  46 #include <string>
  47 #include <map>
  48 #include <set>
  49 #include <string>
  50 #include <vector>
  51 #include <list>
  52 #include <algorithm>
  53 #include <dlfcn.h>
  54 #include <AvailabilityMacros.h>
  55
  56 #include "Options.h"
  57
  58 #include "InputFiles.h"
  59 #include "macho_relocatable_file.h"
  60 #include "macho_dylib_file.h"
  61 #include "textstub_dylib_file.hpp"
  62 #include "archive_file.h"
  63 #include "lto_file.h"
  64 #include "opaque_section_file.h"
  65 #include "MachOFileAbstraction.hpp"
  66 #include "Snapshot.h"
  67
  68 const bool _s_logPThreads = false;
  69
  70 namespace ld {
  71 namespace tool {
  72
  73 class IgnoredFile : public ld::File {
  74 public:
  75         IgnoredFile(const char* pth, time_t modTime, Ordinal ord, Type type) : ld::File(pth, modTime, ord, type) {};
  76         virtual bool                                            forEachAtom(AtomHandler&) const { return false; };
  77         virtual bool                                            justInTimeforEachAtom(const char* name, AtomHandler&) const { return false; };
  78 };
  79
  80
  81 class DSOHandleAtom : public ld::Atom {
  82 public:
  83                                                                         DSOHandleAtom(const char* nm, ld::Atom::Scope sc,
  84                                                                                                                 ld::Atom::SymbolTableInclusion inc, ld::Section& sect=_s_section)
  85                                                                                 : ld::Atom(sect, ld::Atom::definitionRegular,
  86                                                                                                    (sect == _s_section_text) ? ld::Atom::combineByName : ld::Atom::combineNever,
  87                                                                                                    // make "weak def" so that link succeeds even if app defines __dso_handle
  88                                                                                                         sc, ld::Atom::typeUnclassified, inc, true, false, false,
  89                                                                                                          ld::Atom::Alignment(1)), _name(nm) {}
  90
  91         virtual ld::File*                                               file() const                                    { return NULL; }
  92   virtual const char*                                           name() const                                    { return _name; }
  93         virtual uint64_t                                                size() const                                    { return 0; }
  94         virtual uint64_t                                                objectAddress() const                   { return 0; }
  95         virtual void                                                    copyRawContent(uint8_t buffer[]) const
  96                                                                                                                                                         { }
  97         virtual void                                                    setScope(Scope)                                 { }
  98
  99         virtual                                                                 ~DSOHandleAtom() {}
 100
 101         static ld::Section                                              _s_section;
 102         static ld::Section                                              _s_section_preload;
 103         static ld::Section                                              _s_section_text;
 104         static DSOHandleAtom                                    _s_atomAll;
 105         static DSOHandleAtom                                    _s_atomExecutable;
 106         static DSOHandleAtom                                    _s_atomDylib;
 107         static DSOHandleAtom                                    _s_atomBundle;
 108         static DSOHandleAtom                                    _s_atomDyld;
 109         static DSOHandleAtom                                    _s_atomObjectFile;
 110         static DSOHandleAtom                                    _s_atomPreload;
 111         static DSOHandleAtom                                    _s_atomPreloadDSO;
 112 private:
 113         const char*                                                             _name;
 114 };
 115 ld::Section DSOHandleAtom::_s_section("__TEXT", "__mach_header", ld::Section::typeMachHeader, true);
 116 ld::Section DSOHandleAtom::_s_section_preload("__HEADER", "__mach_header", ld::Section::typeMachHeader, true);
 117 ld::Section DSOHandleAtom::_s_section_text("__TEXT", "__text", ld::Section::typeCode, false);
 118 DSOHandleAtom DSOHandleAtom::_s_atomAll("___dso_handle", ld::Atom::scopeLinkageUnit, ld::Atom::symbolTableNotIn);
 119 DSOHandleAtom DSOHandleAtom::_s_atomExecutable("__mh_execute_header", ld::Atom::scopeGlobal, ld::Atom::symbolTableInAndNeverStrip);
 120 DSOHandleAtom DSOHandleAtom::_s_atomDylib("__mh_dylib_header", ld::Atom::scopeLinkageUnit, ld::Atom::symbolTableNotIn);
 121 DSOHandleAtom DSOHandleAtom::_s_atomBundle("__mh_bundle_header", ld::Atom::scopeLinkageUnit, ld::Atom::symbolTableNotIn);
 122 DSOHandleAtom DSOHandleAtom::_s_atomDyld("__mh_dylinker_header", ld::Atom::scopeLinkageUnit, ld::Atom::symbolTableNotIn);
 123 DSOHandleAtom DSOHandleAtom::_s_atomObjectFile("__mh_object_header", ld::Atom::scopeLinkageUnit, ld::Atom::symbolTableNotIn);
 124 DSOHandleAtom DSOHandleAtom::_s_atomPreload("__mh_preload_header", ld::Atom::scopeLinkageUnit, ld::Atom::symbolTableNotIn, _s_section_preload);
 125 DSOHandleAtom DSOHandleAtom::_s_atomPreloadDSO("___dso_handle", ld::Atom::scopeLinkageUnit, ld::Atom::symbolTableNotIn, _s_section_text);
 126
 127
 128
 129 class PageZeroAtom : public ld::Atom {
 130 public:
 131                                                                         PageZeroAtom(uint64_t sz)
 132                                                                                 : ld::Atom(_s_section, ld::Atom::definitionRegular, ld::Atom::combineNever,
 133                                                                                         ld::Atom::scopeTranslationUnit, ld::Atom::typeZeroFill,
 134                                                                                         symbolTableNotIn, true, false, false, ld::Atom::Alignment(12)),
 135                                                                                         _size(sz) {}
 136
 137         virtual ld::File*                                               file() const                                    { return NULL; }
 138         virtual const char*                                             name() const                                    { return "page zero"; }
 139         virtual uint64_t                                                size() const                                    { return _size; }
 140         virtual uint64_t                                                objectAddress() const                   { return 0; }
 141         virtual void                                                    copyRawContent(uint8_t buffer[]) const
 142                                                                                                                                                         { }
 143         virtual void                                                    setScope(Scope)                                 { }
 144
 145         virtual                                                                 ~PageZeroAtom() {}
 146
 147         static ld::Section                                              _s_section;
 148         static DSOHandleAtom                                    _s_atomAll;
 149 private:
 150         uint64_t                                                                _size;
 151 };
 152 ld::Section PageZeroAtom::_s_section("__PAGEZERO", "__pagezero", ld::Section::typePageZero, true);
 153
 154
 155 class CustomStackAtom : public ld::Atom {
 156 public:
 157                                                                         CustomStackAtom(uint64_t sz)
 158                                                                                 : ld::Atom(_s_section, ld::Atom::definitionRegular, ld::Atom::combineNever,
 159                                                                                         ld::Atom::scopeTranslationUnit, ld::Atom::typeZeroFill,
 160                                                                                         symbolTableNotIn, false, false, false, ld::Atom::Alignment(12)),
 161                                                                                         _size(sz) {}
 162
 163         virtual ld::File*                                               file() const                                    { return NULL; }
 164         virtual const char*                                             name() const                                    { return "custom stack"; }
 165         virtual uint64_t                                                size() const                                    { return _size; }
 166         virtual uint64_t                                                objectAddress() const                   { return 0; }
 167         virtual void                                                    copyRawContent(uint8_t buffer[]) const
 168                                                                                                                                                         { }
 169         virtual void                                                    setScope(Scope)                                 { }
 170
 171         virtual                                                                 ~CustomStackAtom() {}
 172
 173 private:
 174         uint64_t                                                                _size;
 175         static ld::Section                                              _s_section;
 176 };
 177 ld::Section CustomStackAtom::_s_section("__UNIXSTACK", "__stack", ld::Section::typeStack, true);
 178
 179
 180 static bool isCompilerSupportLib(const char* path) {
 181         const char* libName = strrchr(path, '/');
 182         return ( (libName != NULL) && (strncmp(libName, "/libclang_rt", 12) == 0) );
 183 }
 184
 185
 186 const char* InputFiles::fileArch(const uint8_t* p, unsigned len)
 187 {
 188         const char* result = mach_o::relocatable::archName(p);
 189         if ( result != NULL  )
 190                  return result;
 191
 192     result = mach_o::dylib::archName(p);
 193     if ( result != NULL  )
 194                 return result;
 195
 196         result = lto::archName(p, len);
 197         if ( result != NULL  )
 198                  return result;
 199
 200         if ( strncmp((const char*)p, "!<arch>\n", 8) == 0 )
 201                 return "archive";
 202
 203         char *unsupported = (char *)malloc(128);
 204         strcpy(unsupported, "unsupported file format (");
 205         for (unsigned i=0; i<len && i < 16; i++) {
 206                 char buf[8];
 207                 sprintf(buf, " 0x%02X", p[i]);
 208                 strcat(unsupported, buf);
 209         }
 210         strcat(unsupported, " )");
 211         return unsupported;
 212 }
 213
 214
 215 ld::File* InputFiles::makeFile(const Options::FileInfo& info, bool indirectDylib)
 216 {
 217         // handle inlined framework first.
 218         if (info.isInlined) {
 219                 auto interface = _options.findTAPIFile(info.path);
 220                 if (!interface)
 221                         throwf("could not find inlined dylib file: %s", info.path);
 222                 auto file = textstub::dylib::parse(info.path, interface, info.modTime, info.ordinal, _options, indirectDylib);
 223                 if (!file)
 224                         throwf("could not parse inlined dylib file: %s(%s)", interface->getInstallName().c_str(), info.path);
 225                 return file;
 226         }
 227         // map in whole file
 228         struct stat stat_buf;
 229         int fd = ::open(info.path, O_RDONLY, 0);
 230         if ( fd == -1 )
 231                 throwf("can't open file, errno=%d", errno);
 232         if ( ::fstat(fd, &stat_buf) != 0 )
 233                 throwf("fstat(%s) failed, errno=%d\n", info.path, errno);
 234         if ( stat_buf.st_size < 20 )
 235                 throwf("file too small (length=%llu)", stat_buf.st_size);
 236         int64_t len = stat_buf.st_size;
 237         uint8_t* p = (uint8_t*)::mmap(NULL, stat_buf.st_size, PROT_READ, MAP_FILE | MAP_PRIVATE, fd, 0);
 238         if ( p == (uint8_t*)(-1) )
 239                 throwf("can't map file, errno=%d", errno);
 240
 241         // if fat file, skip to architecture we want
 242         // Note: fat header is always big-endian
 243         bool isFatFile = false;
 244         uint32_t sliceToUse, sliceCount;
 245         const fat_header* fh = (fat_header*)p;
 246         if ( fh->magic == OSSwapBigToHostInt32(FAT_MAGIC) ) {
 247                 isFatFile = true;
 248                 const struct fat_arch* archs = (struct fat_arch*)(p + sizeof(struct fat_header));
 249                 bool sliceFound = false;
 250                 sliceCount = OSSwapBigToHostInt32(fh->nfat_arch);
 251                 // first try to find a slice that match cpu-type and cpu-sub-type
 252                 for (uint32_t i=0; i < sliceCount; ++i) {
 253                         if ( (OSSwapBigToHostInt32(archs[i].cputype) == (uint32_t)_options.architecture())
 254                           && ((OSSwapBigToHostInt32(archs[i].cpusubtype) & ~CPU_SUBTYPE_MASK) == (uint32_t)_options.subArchitecture()) ) {
 255                                 sliceToUse = i;
 256                                 sliceFound = true;
 257                                 break;
 258                         }
 259                 }
 260                 if ( !sliceFound && _options.allowSubArchitectureMismatches() ) {
 261                         // look for any slice that matches just cpu-type
 262                         for (uint32_t i=0; i < sliceCount; ++i) {
 263                                 if ( OSSwapBigToHostInt32(archs[i].cputype) == (uint32_t)_options.architecture() ) {
 264                                         sliceToUse = i;
 265                                         sliceFound = true;
 266                                         break;
 267                                 }
 268                         }
 269                 }
 270                 if ( !sliceFound ) {
 271                         // Look for a fallback slice.
 272                         for (uint32_t i = 0; i < sliceCount; ++i) {
 273                                 if ( OSSwapBigToHostInt32(archs[i].cputype) == (uint32_t)_options.fallbackArchitecture() &&
 274                                         OSSwapBigToHostInt32(archs[i].cpusubtype) == (uint32_t)_options.fallbackSubArchitecture() ) {
 275                                         sliceToUse = i;
 276                                         sliceFound = true;
 277                                         break;
 278                                 }
 279                         }
 280                 }
 281                 if ( sliceFound ) {
 282                         uint32_t fileOffset = OSSwapBigToHostInt32(archs[sliceToUse].offset);
 283                         len = OSSwapBigToHostInt32(archs[sliceToUse].size);
 284                         if ( fileOffset+len > stat_buf.st_size ) {
 285                                 // <rdar://problem/17593430> file size was read awhile ago.  If file is being written, wait a second to see if big enough now
 286                                 sleep(1);
 287                                 int64_t newFileLen = stat_buf.st_size;
 288                                 struct stat statBuffer;
 289                                 if ( stat(info.path, &statBuffer) == 0 ) {
 290                                         newFileLen = statBuffer.st_size;
 291                                 }
 292                                 if ( fileOffset+len > newFileLen ) {
 293                                         throwf("truncated fat file. Slice from %u to %llu is past end of file with length %llu",
 294                                                 fileOffset, fileOffset+len, stat_buf.st_size);
 295                                 }
 296                         }
 297                         // if requested architecture is page aligned within fat file, then remap just that portion of file
 298                         if ( (fileOffset & 0x00000FFF) == 0 ) {
 299                                 // unmap whole file
 300                                 munmap((caddr_t)p, stat_buf.st_size);
 301                                 // re-map just part we need
 302                                 p = (uint8_t*)::mmap(NULL, len, PROT_READ, MAP_FILE | MAP_PRIVATE, fd, fileOffset);
 303                                 if ( p == (uint8_t*)(-1) )
 304                                         throwf("can't re-map file, errno=%d", errno);
 305                         }
 306                         else {
 307                                 p = &p[fileOffset];
 308                         }
 309                 }
 310         }
 311         ::close(fd);
 312
 313         // see if it is an object file
 314         mach_o::relocatable::ParserOptions objOpts;
 315         objOpts.architecture            = _options.architecture();
 316         objOpts.objSubtypeMustMatch = !_options.allowSubArchitectureMismatches();
 317         objOpts.logAllFiles                     = _options.logAllFiles();
 318         objOpts.warnUnwindConversionProblems    = _options.needsUnwindInfoSection();
 319         objOpts.keepDwarfUnwind         = _options.keepDwarfUnwind();
 320         objOpts.forceDwarfConversion= (_options.outputKind() == Options::kDyld);
 321         objOpts.neverConvertDwarf   = !_options.needsUnwindInfoSection();
 322         objOpts.verboseOptimizationHints = _options.verboseOptimizationHints();
 323         objOpts.armUsesZeroCostExceptions = _options.armUsesZeroCostExceptions();
 324         objOpts.simulator                       = _options.targetIOSSimulator();
 325         objOpts.ignoreMismatchPlatform = ((_options.outputKind() == Options::kPreload) || (_options.outputKind() == Options::kStaticExecutable));
 326 #if SUPPORT_ARCH_arm64e
 327         objOpts.supportsAuthenticatedPointers = _options.supportsAuthenticatedPointers();
 328 #endif
 329         objOpts.subType                         = _options.subArchitecture();
 330         objOpts.platforms                       = _options.platforms();
 331         objOpts.srcKind                         = ld::relocatable::File::kSourceObj;
 332         objOpts.treateBitcodeAsData     = _options.bitcodeKind() == Options::kBitcodeAsData;
 333         objOpts.usingBitcode            = _options.bundleBitcode();
 334         objOpts.maxDefaultCommonAlignment = _options.maxDefaultCommonAlign();
 335
 336         ld::relocatable::File* objResult = mach_o::relocatable::parse(p, len, info.path, info.modTime, info.ordinal, objOpts);
 337         if ( objResult != NULL ) {
 338                 OSAtomicAdd64(len, &_totalObjectSize);
 339                 OSAtomicIncrement32(&_totalObjectLoaded);
 340                 return objResult;
 341         }
 342
 343         // see if it is an llvm object file
 344         objResult = lto::parse(p, len, info.path, info.modTime, info.ordinal, _options.architecture(), _options.subArchitecture(), _options.logAllFiles(), _options.verboseOptimizationHints());
 345         if ( objResult != NULL ) {
 346                 OSAtomicAdd64(len, &_totalObjectSize);
 347                 OSAtomicIncrement32(&_totalObjectLoaded);
 348                 return objResult;
 349         }
 350
 351         // see if it is a dynamic library (or text-based dynamic library)
 352         ld::dylib::File* dylibResult;
 353         bool dylibsNotAllowed = false;
 354         switch ( _options.outputKind() ) {
 355                 case Options::kDynamicExecutable:
 356                 case Options::kDynamicLibrary:
 357                 case Options::kDynamicBundle:
 358                         dylibResult = mach_o::dylib::parse(p, len, info.path, info.modTime, _options, info.ordinal, info.options.fBundleLoader, indirectDylib);
 359                         if ( dylibResult != NULL ) {
 360                                 return dylibResult;
 361                         }
 362                         dylibResult = textstub::dylib::parse(p, len, info.path, info.modTime, _options, info.ordinal, info.options.fBundleLoader, indirectDylib);
 363                         if ( dylibResult != NULL ) {
 364                                 return dylibResult;
 365                         }
 366                         break;
 367                 case Options::kStaticExecutable:
 368                 case Options::kDyld:
 369                 case Options::kPreload:
 370                 case Options::kObjectFile:
 371                 case Options::kKextBundle:
 372                         dylibsNotAllowed = true;
 373                         break;
 374         }
 375
 376         // see if it is a static library
 377         ::archive::ParserOptions archOpts;
 378         archOpts.objOpts                                = objOpts;
 379         archOpts.forceLoadThisArchive   = info.options.fForceLoad;
 380         archOpts.forceLoadAll                   = _options.fullyLoadArchives();
 381         archOpts.forceLoadObjC                  = _options.loadAllObjcObjectsFromArchives();
 382         archOpts.objcABI2                               = _options.objCABIVersion2POverride();
 383         archOpts.verboseLoad                    = _options.whyLoad();
 384         archOpts.logAllFiles                    = _options.logAllFiles();
 385         // Set ObjSource Kind, libclang_rt is compiler static library
 386         if ( isCompilerSupportLib(info.path) )
 387                 archOpts.objOpts.srcKind = ld::relocatable::File::kSourceCompilerArchive;
 388         else
 389                 archOpts.objOpts.srcKind = ld::relocatable::File::kSourceArchive;
 390         archOpts.objOpts.treateBitcodeAsData = _options.bitcodeKind() == Options::kBitcodeAsData;
 391         archOpts.objOpts.usingBitcode = _options.bundleBitcode();
 392
 393         ld::archive::File* archiveResult = ::archive::parse(p, len, info.path, info.modTime, info.ordinal, archOpts);
 394         if ( archiveResult != NULL ) {
 395
 396                 OSAtomicAdd64(len, &_totalArchiveSize);
 397                 OSAtomicIncrement32(&_totalArchivesLoaded);
 398                 return archiveResult;
 399         }
 400
 401         // does not seem to be any valid linker input file, check LTO misconfiguration problems
 402         if ( lto::archName((uint8_t*)p, len) != NULL ) {
 403                 if ( lto::libLTOisLoaded() ) {
 404                         throwf("lto file was built for %s which is not the architecture being linked (%s): %s", fileArch(p, len), _options.architectureName(), info.path);
 405                 }
 406                 else {
 407                         const char* libLTO = "libLTO.dylib";
 408                         char ldPath[PATH_MAX];
 409                         char tmpPath[PATH_MAX];
 410                         char libLTOPath[PATH_MAX];
 411                         uint32_t bufSize = PATH_MAX;
 412                         if ( _options.overridePathlibLTO() != NULL ) {
 413                                 libLTO = _options.overridePathlibLTO();
 414                         }
 415                         else if ( _NSGetExecutablePath(ldPath, &bufSize) != -1 ) {
 416                                 if ( realpath(ldPath, tmpPath) != NULL ) {
 417                                         char* lastSlash = strrchr(tmpPath, '/');
 418                                         if ( lastSlash != NULL )
 419                                                 strcpy(lastSlash, "/../lib/libLTO.dylib");
 420                                         libLTO = tmpPath;
 421                                         if ( realpath(tmpPath, libLTOPath) != NULL )
 422                                                 libLTO = libLTOPath;
 423                                 }
 424                         }
 425                         throwf("could not process llvm bitcode object file, because %s could not be loaded", libLTO);
 426                 }
 427         }
 428
 429         if ( dylibsNotAllowed ) {
 430                 cpu_type_t dummy1;
 431                 cpu_type_t dummy2;
 432                 if ( mach_o::dylib::isDylibFile(p, &dummy1, &dummy2) )
 433                         throw "ignoring unexpected dylib file";
 434         }
 435
 436         // error handling
 437         if ( ((fat_header*)p)->magic == OSSwapBigToHostInt32(FAT_MAGIC) ) {
 438                 throwf("missing required architecture %s in file %s (%u slices)", _options.architectureName(), info.path, sliceCount);
 439         }
 440         else {
 441                 if ( isFatFile )
 442                         throwf("file is universal (%u slices) but does not contain the %s architecture: %s", sliceCount, _options.architectureName(), info.path);
 443                 else
 444                         throwf("file was built for %s which is not the architecture being linked (%s): %s", fileArch(p, len), _options.architectureName(), info.path);
 445         }
 446 }
 447
 448 void InputFiles::logDylib(ld::File* file, bool indirect, bool speculative)
 449 {
 450         if ( _options.traceDylibs() ) {
 451                 const char* fullPath = file->path();
 452                 char realName[MAXPATHLEN];
 453                 if ( realpath(fullPath, realName) != NULL )
 454                         fullPath = realName;
 455                 const ld::dylib::File* dylib = dynamic_cast<const ld::dylib::File*>(file);
 456                 if ( (dylib != NULL ) && dylib->willBeUpwardDylib() ) {
 457                         // don't log upward dylibs when XBS is computing dependencies
 458                         logTraceInfo("[Logging for XBS] Used upward dynamic library: %s\n", fullPath);
 459                 }
 460                 else if ( (dylib != NULL ) && dylib->speculativelyLoaded() ) {
 461                         logTraceInfo("[Logging for XBS] Speculatively loaded dynamic library: %s\n", fullPath);
 462                 }
 463                 else {
 464                         if ( indirect ) {
 465                                 if ( speculative )
 466                                         logTraceInfo("[Logging for XBS] Speculatively loaded indirect dynamic library: %s\n", fullPath);
 467                                 else
 468                                         logTraceInfo("[Logging for XBS] Used indirect dynamic library: %s\n", fullPath);
 469                         }
 470                         else {
 471                                 logTraceInfo("[Logging for XBS] Used dynamic library: %s\n", fullPath);
 472                         }
 473                 }
 474         }
 475
 476         if ( _options.dumpDependencyInfo() ) {
 477                 const ld::dylib::File* dylib = dynamic_cast<const ld::dylib::File*>(file);
 478                 if ( file == _bundleLoader ) {
 479                         _options.addDependency(Options::depBundleLoader, file->path());
 480                 }
 481                 else if ( (dylib != NULL ) && dylib->willBeUpwardDylib() ) {
 482                         if ( indirect )
 483                                 _options.addDependency(Options::depUpwardIndirectDylib, file->path());
 484                         else
 485                                 _options.addDependency(Options::depUpwardDirectDylib, file->path());
 486                 }
 487                 else {
 488                         if ( indirect )
 489                                 _options.addDependency(Options::depIndirectDylib, file->path());
 490                         else
 491                                 _options.addDependency(Options::depDirectDylib, file->path());
 492                 }
 493         }
 494 }
 495
 496 void InputFiles::logArchive(ld::File* file) const
 497 {
 498         if ( (_options.traceArchives() || _options.traceEmitJSON()) && (_archiveFilesLogged.count(file) == 0) ) {
 499                 // <rdar://problem/4947347> LD_TRACE_ARCHIVES should only print out when a .o is actually used from an archive
 500                 _archiveFilesLogged.insert(file);
 501                 const char* fullPath = file->path();
 502                 char realName[MAXPATHLEN];
 503                 if ( realpath(fullPath, realName) != NULL )
 504                         fullPath = realName;
 505                 logTraceInfo("[Logging for XBS] Used static archive: %s\n", fullPath);
 506
 507                 std::string archivePath(fullPath);
 508                 _archiveFilePaths.push_back(archivePath);
 509         }
 510 }
 511
 512
 513 void InputFiles::logTraceInfo(const char* format, ...) const
 514 {
 515         char trace_buffer[MAXPATHLEN * 2];
 516     va_list ap;
 517         va_start(ap, format);
 518         int length = vsnprintf(trace_buffer, sizeof(trace_buffer), format, ap);
 519         va_end(ap);
 520         _options.writeToTraceFile(trace_buffer, length);
 521 }
 522
 523
 524 ld::dylib::File* InputFiles::findDylib(const char* installPath, const ld::dylib::File* fromDylib, bool speculative)
 525 {
 526         //fprintf(stderr, "findDylib(%s, %s)\n", installPath, fromPath);
 527         InstallNameToDylib::iterator pos = _installPathToDylibs.find(installPath);
 528         if ( pos != _installPathToDylibs.end() ) {
 529                 return pos->second;
 530         }
 531         else {
 532                 // allow -dylib_path option to override indirect library to use
 533                 for (std::vector<Options::DylibOverride>::const_iterator dit = _options.dylibOverrides().begin(); dit != _options.dylibOverrides().end(); ++dit) {
 534                         if ( strcmp(dit->installName,installPath) == 0 ) {
 535                                 try {
 536                                         Options::FileInfo info = _options.findFile(dit->useInstead);
 537                                         _indirectDylibOrdinal = _indirectDylibOrdinal.nextIndirectDylibOrdinal();
 538                                         info.ordinal = _indirectDylibOrdinal;
 539                                         info.options.fIndirectDylib = true;
 540                                         ld::File* reader = this->makeFile(info, true);
 541                                         ld::dylib::File* dylibReader = dynamic_cast<ld::dylib::File*>(reader);
 542                                         if ( dylibReader != NULL ) {
 543                                                 addDylib(dylibReader, info);
 544                                                 //_installPathToDylibs[strdup(installPath)] = dylibReader;
 545                                                 this->logDylib(dylibReader, true, speculative);
 546                                                 return dylibReader;
 547                                         }
 548                                         else
 549                                                 throwf("indirect dylib at %s is not a dylib", dit->useInstead);
 550                                 }
 551                                 catch (const char* msg) {
 552                                         warning("ignoring -dylib_file option, %s", msg);
 553                                 }
 554                         }
 555                 }
 556
 557                 // search for dylib using -F and -L paths and expanding @ paths
 558                 Options::FileInfo info = _options.findIndirectDylib(installPath, fromDylib);
 559                 _indirectDylibOrdinal = _indirectDylibOrdinal.nextIndirectDylibOrdinal();
 560                 info.ordinal = _indirectDylibOrdinal;
 561                 info.options.fIndirectDylib = true;
 562                 try {
 563                         ld::File* reader = this->makeFile(info, true);
 564                         ld::dylib::File* dylibReader = dynamic_cast<ld::dylib::File*>(reader);
 565                         if ( dylibReader != NULL ) {
 566                                 //assert(_installPathToDylibs.find(installPath) !=  _installPathToDylibs.end());
 567                                 //_installPathToDylibs[strdup(installPath)] = dylibReader;
 568                                 addDylib(dylibReader, info);
 569                                 this->logDylib(dylibReader, true, speculative);
 570                                 return dylibReader;
 571                         }
 572                         else
 573                                 throwf("indirect dylib at %s is not a dylib", info.path);
 574                 }
 575                 catch (const char* msg) {
 576                         throwf("in '%s', %s", info.path, msg);
 577                 }
 578         }
 579 }
 580
 581
 582 // mark all dylibs initially specified as required, and check if they can be used
 583 void InputFiles::markExplicitlyLinkedDylibs()
 584 {
 585         for (InstallNameToDylib::iterator it=_installPathToDylibs.begin(); it != _installPathToDylibs.end(); it++) {
 586                 it->second->setExplicitlyLinked();
 587                 this->checkDylibClientRestrictions(it->second);
 588         }
 589 }
 590
 591 bool InputFiles::frameworkAlreadyLoaded(const char* path, const char* frameworkName)
 592 {
 593         for (ld::File* file : _inputFiles) {
 594                 if ( strcmp(path, file->path()) == 0 )
 595                         return true;
 596         }
 597         for (ld::dylib::File* dylibx : _allDylibs) {
 598                 const char* fname = dylibx->frameworkName();
 599                 if ( fname == NULL )
 600                         continue;
 601                 if ( strcmp(frameworkName, fname) == 0 )
 602                         return true;
 603         }
 604         return false;
 605 }
 606
 607 bool InputFiles::libraryAlreadyLoaded(const char* path)
 608 {
 609         for (ld::File* file : _inputFiles) {
 610                 if ( strcmp(path, file->path()) == 0 )
 611                         return true;
 612         }
 613         for (ld::dylib::File* dylib : _allDylibs) {
 614                 if ( strcmp(path, dylib->path()) == 0 )
 615                         return true;
 616         }
 617         for (const LibraryInfo& libInfo : _searchLibraries) {
 618                 if ( strcmp(path, libInfo.archive()->path()) == 0 )
 619                         return true;
 620         }
 621
 622         char realDylibPath[PATH_MAX];
 623         if ( (realpath(path, realDylibPath) != NULL) && (strcmp(path, realDylibPath) != 0) ) {
 624                 return libraryAlreadyLoaded(realDylibPath);
 625         }
 626
 627         return false;
 628 }
 629
 630
 631 void InputFiles::addLinkerOptionLibraries(ld::Internal& state, ld::File::AtomHandler& handler)
 632 {
 633         if ( _options.outputKind() == Options::kObjectFile )
 634                 return;
 635
 636         while (! state.unprocessedLinkerOptionLibraries.empty() || ! state.unprocessedLinkerOptionFrameworks.empty()) {
 637
 638                 // process frameworks specified in .o linker options
 639                 CStringSet newFrameworks = std::move(state.unprocessedLinkerOptionFrameworks);
 640                 state.unprocessedLinkerOptionFrameworks.clear();
 641                 for (const char* frameworkName : newFrameworks) {
 642                         if ( state.linkerOptionFrameworks.count(frameworkName) )
 643                                 continue;
 644                         try {
 645                                 Options::FileInfo info = _options.findFramework(frameworkName);
 646                                 if ( ! this->frameworkAlreadyLoaded(info.path, frameworkName) ) {
 647                                         _linkerOptionOrdinal = _linkerOptionOrdinal.nextLinkerOptionOrdinal();
 648                                         info.ordinal = _linkerOptionOrdinal;
 649                                         ld::File* reader = this->makeFile(info, true);
 650                                         ld::dylib::File* dylibReader = dynamic_cast<ld::dylib::File*>(reader);
 651                                         ld::archive::File* archiveReader = dynamic_cast<ld::archive::File*>(reader);
 652                                         if ( dylibReader != NULL ) {
 653                                                 if ( ! dylibReader->installPathVersionSpecific() ) {
 654                                                         dylibReader->forEachAtom(handler);
 655                                                         dylibReader->setImplicitlyLinked();
 656                                                         dylibReader->setSpeculativelyLoaded();
 657                                                         this->addDylib(dylibReader, info);
 658                                                 }
 659                                         }
 660                                         else if ( archiveReader != NULL ) {
 661                                                 _searchLibraries.push_back(LibraryInfo(archiveReader));
 662                                                 _options.addDependency(Options::depArchive, archiveReader->path());
 663                                                 //<rdar://problem/17787306> -force_load_swift_libs
 664                                                 if (info.options.fForceLoad) {
 665                                                         archiveReader->forEachAtom(handler);
 666                                                 }
 667                                         }
 668                                         else {
 669                                                 throwf("framework linker option at %s is not a dylib and not an archive", info.path);
 670                                         }
 671                                 }
 672                         }
 673                         catch (const char* msg) {
 674                                 // <rdar://problem/40829444> only warn about missing auto-linked framework if some missing symbol error happens later
 675                                 state.missingLinkerOptionFrameworks.insert(frameworkName);
 676                         }
 677                         state.linkerOptionFrameworks.insert(frameworkName);
 678                 }
 679
 680                 // process libraries specified in .o linker options
 681                 // fixme optimize with std::move?
 682                 CStringSet newLibraries = std::move(state.unprocessedLinkerOptionLibraries);
 683                 state.unprocessedLinkerOptionLibraries.clear();
 684                 for (const char* libName : newLibraries) {
 685                         if ( state.linkerOptionLibraries.count(libName) )
 686                                 continue;
 687                         try {
 688                                 Options::FileInfo info = _options.findLibrary(libName);
 689                                 if ( ! this->libraryAlreadyLoaded(info.path) ) {
 690                                         _linkerOptionOrdinal = _linkerOptionOrdinal.nextLinkerOptionOrdinal();
 691                                         info.ordinal = _linkerOptionOrdinal;
 692                                         //<rdar://problem/17787306> -force_load_swift_libs
 693                                         info.options.fForceLoad = _options.forceLoadSwiftLibs() && (strncmp(libName, "swift", 5) == 0);
 694                                         ld::File* reader = this->makeFile(info, true);
 695                                         ld::dylib::File* dylibReader = dynamic_cast<ld::dylib::File*>(reader);
 696                                         ld::archive::File* archiveReader = dynamic_cast<ld::archive::File*>(reader);
 697                                         if ( dylibReader != NULL ) {
 698                                                 dylibReader->forEachAtom(handler);
 699                                                 dylibReader->setImplicitlyLinked();
 700                                                 dylibReader->setSpeculativelyLoaded();
 701                                                 this->addDylib(dylibReader, info);
 702                                         }
 703                                         else if ( archiveReader != NULL ) {
 704                                                 _searchLibraries.push_back(LibraryInfo(archiveReader));
 705                                                 _options.addDependency(Options::depArchive, archiveReader->path());
 706                                                 //<rdar://problem/17787306> -force_load_swift_libs
 707                                                 if (info.options.fForceLoad) {
 708                                                         archiveReader->forEachAtom(handler);
 709                                                 }
 710                                         }
 711                                         else {
 712                                                 throwf("linker option dylib at %s is not a dylib", info.path);
 713                                         }
 714                                 }
 715                         }
 716                         catch (const char* msg) {
 717                                 // <rdar://problem/40829444> only warn about missing auto-linked library if some missing symbol error happens later
 718                                 state.missingLinkerOptionLibraries.insert(libName);
 719                         }
 720                         state.linkerOptionLibraries.insert(libName);
 721                 }
 722         }
 723 }
 724
 725 void InputFiles::createIndirectDylibs()
 726 {
 727         // keep processing dylibs until no more dylibs are added
 728         unsigned long lastMapSize = 0;
 729         std::set<ld::dylib::File*>  dylibsProcessed;
 730         while ( lastMapSize != _allDylibs.size() ) {
 731                 lastMapSize = _allDylibs.size();
 732                 // can't iterator _installPathToDylibs while modifying it, so use temp buffer
 733                 std::vector<ld::dylib::File*> unprocessedDylibs;
 734                 for (std::set<ld::dylib::File*>::iterator it=_allDylibs.begin(); it != _allDylibs.end(); it++) {
 735                         if ( dylibsProcessed.count(*it) == 0 )
 736                                 unprocessedDylibs.push_back(*it);
 737                 }
 738                 for (std::vector<ld::dylib::File*>::iterator it=unprocessedDylibs.begin(); it != unprocessedDylibs.end(); it++) {
 739                         dylibsProcessed.insert(*it);
 740                         (*it)->processIndirectLibraries(this, _options.implicitlyLinkIndirectPublicDylibs());
 741                 }
 742         }
 743
 744         // go back over original dylibs and mark sub frameworks as re-exported
 745         if ( _options.outputKind() == Options::kDynamicLibrary ) {
 746                 const char* myLeaf = strrchr(_options.installPath(), '/');
 747                 if ( myLeaf != NULL ) {
 748                         for (std::vector<class ld::File*>::const_iterator it=_inputFiles.begin(); it != _inputFiles.end(); it++) {
 749                                 ld::dylib::File* dylibReader = dynamic_cast<ld::dylib::File*>(*it);
 750                                 if ( dylibReader != NULL ) {
 751                                         const char* childParent = dylibReader->parentUmbrella();
 752                                         if ( childParent != NULL ) {
 753                                                 if ( strcmp(childParent, &myLeaf[1]) == 0 ) {
 754                                                         // mark that this dylib will be re-exported
 755                                                         dylibReader->setWillBeReExported();
 756                                                 }
 757                                         }
 758                                 }
 759                         }
 760                 }
 761         }
 762
 763 }
 764
 765 void InputFiles::createOpaqueFileSections()
 766 {
 767         // extra command line sections always at end
 768         for (Options::ExtraSection::const_iterator it=_options.extraSectionsBegin(); it != _options.extraSectionsEnd(); ++it) {
 769                 _inputFiles.push_back(opaque_section::parse(it->segmentName, it->sectionName, it->path, it->data, it->dataLen));
 770                 _options.addDependency(Options::depSection, it->path);
 771         }
 772
 773 }
 774
 775
 776 void InputFiles::checkDylibClientRestrictions(ld::dylib::File* dylib)
 777 {
 778         // Check for any restrictions on who can link with this dylib
 779         const char* dylibParentName = dylib->parentUmbrella() ;
 780         const std::vector<const char*>* clients = dylib->allowableClients();
 781         if ( (dylibParentName != NULL) || (clients != NULL) ) {
 782                 // only dylibs that are in an umbrella or have a client list need verification
 783                 const char* installName = _options.installPath();
 784                 const char* installNameLastSlash = strrchr(installName, '/');
 785                 bool isParent = false;
 786                 bool isSibling = false;
 787                 bool isAllowableClient = false;
 788                 // There are three cases:
 789                 if ( (dylibParentName != NULL) && (installNameLastSlash != NULL) ) {
 790                         // starts after last slash
 791                         const char* myName = &installNameLastSlash[1];
 792                         unsigned int myNameLen = strlen(myName);
 793                         if ( strncmp(myName, "lib", 3) == 0 )
 794                                 myName = &myName[3];
 795                         // up to first dot
 796                         const char* firstDot = strchr(myName, '.');
 797                         if ( firstDot != NULL )
 798                                 myNameLen = firstDot - myName;
 799                         // up to first underscore
 800                         const char* firstUnderscore = strchr(myName, '_');
 801                         if ( (firstUnderscore != NULL) && ((firstUnderscore - myName) < (int)myNameLen) )
 802                                 myNameLen = firstUnderscore - myName;
 803
 804                         // case 1) The dylib has a parent umbrella, and we are creating the parent umbrella
 805                         isParent = ( (strlen(dylibParentName) == myNameLen) && (strncmp(myName, dylibParentName, myNameLen) == 0) );
 806
 807                         // case 2) The dylib has a parent umbrella, and we are creating a sibling with the same parent
 808                         isSibling = ( (_options.umbrellaName() != NULL) && (strcmp(_options.umbrellaName(), dylibParentName) == 0) );
 809                 }
 810
 811                 if ( !isParent && !isSibling && (clients != NULL) ) {
 812                         // case 3) the dylib has a list of allowable clients, and we are creating one of them
 813                         const char* clientName = _options.clientName();
 814                         int clientNameLen = 0;
 815                         if ( clientName != NULL ) {
 816                                 // use client name as specified on command line
 817                                 clientNameLen = strlen(clientName);
 818                         }
 819                         else {
 820                                 // infer client name from output path (e.g. xxx/libfoo_variant.A.dylib --> foo, Bar.framework/Bar_variant --> Bar)
 821                                 clientName = installName;
 822                                 clientNameLen = strlen(clientName);
 823                                 // starts after last slash
 824                                 if ( installNameLastSlash != NULL )
 825                                         clientName = &installNameLastSlash[1];
 826                                 if ( strncmp(clientName, "lib", 3) == 0 )
 827                                         clientName = &clientName[3];
 828                                 // up to first dot
 829                                 const char* firstDot = strchr(clientName, '.');
 830                                 if ( firstDot != NULL )
 831                                         clientNameLen = firstDot - clientName;
 832                                 // up to first underscore
 833                                 const char* firstUnderscore = strchr(clientName, '_');
 834                                 if ( (firstUnderscore != NULL) && ((firstUnderscore - clientName) < clientNameLen) )
 835                                         clientNameLen = firstUnderscore - clientName;
 836                         }
 837
 838                         // Use clientName to check if this dylib is able to link against the allowable clients.
 839                         for (std::vector<const char*>::const_iterator it = clients->begin(); it != clients->end(); it++) {
 840                                 if ( strncmp(*it, clientName, clientNameLen) == 0 )
 841                                         isAllowableClient = true;
 842                         }
 843                 }
 844
 845                 if ( !isParent && !isSibling && !isAllowableClient ) {
 846                         if ( dylibParentName != NULL ) {
 847                                 throwf("cannot link directly with %s.  Link against the umbrella framework '%s.framework' instead.",
 848                                         dylib->path(), dylibParentName);
 849                         }
 850                         else {
 851                                 throwf("cannot link directly with %s", dylib->path());
 852                         }
 853                 }
 854         }
 855 }
 856
 857
 858 void InputFiles::inferArchitecture(Options& opts, const char** archName)
 859 {
 860         _inferredArch = true;
 861         // scan all input files, looking for a thin .o file.
 862         // the first one found is presumably the architecture to link
 863         uint8_t buffer[4096];
 864         const std::vector<Options::FileInfo>& files = opts.getInputFiles();
 865         for (std::vector<Options::FileInfo>::const_iterator it = files.begin(); it != files.end(); ++it) {
 866                 int fd = ::open(it->path, O_RDONLY, 0);
 867                 if ( fd != -1 ) {
 868                         struct stat stat_buf;
 869                         if ( fstat(fd, &stat_buf) != -1) {
 870                                 ssize_t readAmount = stat_buf.st_size;
 871                                 if ( 4096 < readAmount )
 872                                         readAmount = 4096;
 873                                 ssize_t amount = read(fd, buffer, readAmount);
 874                                 ::close(fd);
 875                                 if ( amount >= readAmount ) {
 876                                         cpu_type_t type;
 877                                         cpu_subtype_t subtype;
 878                                         ld::Platform platform;
 879                                         uint32_t        minOsVersion;
 880                                         if ( mach_o::relocatable::isObjectFile(buffer, &type, &subtype, &platform, &minOsVersion) ) {
 881                                                 opts.setArchitecture(type, subtype, platform, minOsVersion);
 882                                                 *archName = opts.architectureName();
 883                                                 return;
 884                                         }
 885                                 }
 886                         }
 887                 }
 888         }
 889
 890         // no thin .o files found, so default to same architecture this tool was built as
 891         warning("-arch not specified");
 892 #if __i386__
 893         opts.setArchitecture(CPU_TYPE_I386, CPU_SUBTYPE_X86_ALL, ld::kPlatform_macOS, 0);
 894 #elif __x86_64__
 895         opts.setArchitecture(CPU_TYPE_X86_64, CPU_SUBTYPE_X86_64_ALL, ld::kPlatform_macOS, 0);
 896 #elif __arm__
 897         opts.setArchitecture(CPU_TYPE_ARM, CPU_SUBTYPE_ARM_V6, ld::kPlatform_macOS, 0);
 898 #else
 899         #error unknown default architecture
 900 #endif
 901         *archName = opts.architectureName();
 902 }
 903
 904
 905 InputFiles::InputFiles(Options& opts, const char** archName)
 906  : _totalObjectSize(0), _totalArchiveSize(0),
 907    _totalObjectLoaded(0), _totalArchivesLoaded(0), _totalDylibsLoaded(0),
 908         _options(opts), _bundleLoader(NULL),
 909         _inferredArch(false),
 910         _exception(NULL),
 911         _indirectDylibOrdinal(ld::File::Ordinal::indirectDylibBase()),
 912         _linkerOptionOrdinal(ld::File::Ordinal::linkeOptionBase())
 913 {
 914 //      fStartCreateReadersTime = mach_absolute_time();
 915         if ( opts.architecture() == 0 ) {
 916                 // command line missing -arch, so guess arch
 917                 inferArchitecture(opts, archName);
 918         }
 919 #if HAVE_PTHREADS
 920         pthread_mutex_init(&_parseLock, NULL);
 921         pthread_cond_init(&_parseWorkReady, NULL);
 922         pthread_cond_init(&_newFileAvailable, NULL);
 923         _neededFileSlot = -1;
 924 #endif
 925         const std::vector<Options::FileInfo>& files = _options.getInputFiles();
 926         if ( files.size() == 0 )
 927                 throw "no object files specified";
 928
 929         _inputFiles.reserve(files.size());
 930 #if HAVE_PTHREADS
 931         unsigned int inputFileSlot = 0;
 932         _availableInputFiles = 0;
 933         _parseCursor = 0;
 934 #endif
 935         Options::FileInfo* entry;
 936         for (std::vector<Options::FileInfo>::const_iterator it = files.begin(); it != files.end(); ++it) {
 937                 entry = (Options::FileInfo*)&(*it);
 938 #if HAVE_PTHREADS
 939                 // Assign input file slots to all the FileInfos.
 940                 // Also chain all FileInfos into one big list to set up for worker threads to do parsing.
 941                 entry->inputFileSlot = inputFileSlot;
 942                 entry->readyToParse = !entry->fromFileList || !_options.pipelineEnabled();
 943                 if (entry->readyToParse)
 944                         _availableInputFiles++;
 945                 _inputFiles.push_back(NULL);
 946                 inputFileSlot++;
 947 #else
 948                 // In the non-threaded case just parse the file now.
 949                 _inputFiles.push_back(makeFile(*entry, false));
 950 #endif
 951         }
 952
 953 #if HAVE_PTHREADS
 954         _remainingInputFiles = files.size();
 955
 956         // initialize info for parsing input files on worker threads
 957         unsigned int ncpus;
 958         int mib[2];
 959         size_t len = sizeof(ncpus);
 960         mib[0] = CTL_HW;
 961         mib[1] = HW_NCPU;
 962         if (sysctl(mib, 2, &ncpus, &len, NULL, 0) != 0) {
 963                 ncpus = 1;
 964         }
 965         _availableWorkers = MIN(ncpus, files.size()); // max # workers we permit
 966         _idleWorkers = 0;
 967
 968         if (_options.pipelineEnabled()) {
 969                 // start up a thread to listen for available input files
 970                 startThread(InputFiles::waitForInputFiles);
 971         }
 972
 973         // Start up one parser thread. More start on demand as parsed input files get consumed.
 974         startThread(InputFiles::parseWorkerThread);
 975         _availableWorkers--;
 976 #else
 977         if (_options.pipelineEnabled()) {
 978                 throwf("pipelined linking not supported on this platform");
 979         }
 980 #endif
 981 }
 982
 983
 984 #if HAVE_PTHREADS
 985 void InputFiles::startThread(void (*threadFunc)(InputFiles *)) const {
 986         pthread_t thread;
 987         pthread_attr_t attr;
 988         pthread_attr_init(&attr);
 989         // set a nice big stack (same as main thread) because some code uses potentially large stack buffers
 990         pthread_attr_setstacksize(&attr, 8 * 1024 * 1024);
 991         pthread_create(&thread, &attr, (void *(*)(void*))threadFunc, (void *)this);
 992         pthread_detach(thread);
 993         pthread_attr_destroy(&attr);
 994 }
 995
 996 // Work loop for input file parsing threads
 997 void InputFiles::parseWorkerThread() {
 998         ld::File *file;
 999         const char *exception = NULL;
1000         pthread_mutex_lock(&_parseLock);
1001         const std::vector<Options::FileInfo>& files = _options.getInputFiles();
1002         if (_s_logPThreads) printf("worker starting\n");
1003         do {
1004                 if (_availableInputFiles == 0) {
1005                         _idleWorkers++;
1006                         pthread_cond_wait(&_parseWorkReady, &_parseLock);
1007                         _idleWorkers--;
1008                 } else {
1009                         int slot = _parseCursor;
1010                         while (slot < (int)files.size() && (_inputFiles[slot] != NULL || !files[slot].readyToParse))
1011                                 slot++;
1012                         assert(slot < (int)files.size());
1013                         Options::FileInfo& entry = (Options::FileInfo&)files[slot];
1014                         _parseCursor = slot+1;
1015                         _availableInputFiles--;
1016                         entry.readyToParse = false; // to avoid multiple threads finding this file
1017                         pthread_mutex_unlock(&_parseLock);
1018                         if (_s_logPThreads) printf("parsing index %u\n", slot);
1019                         try {
1020                                 file = makeFile(entry, false);
1021                         }
1022                         catch (const char *msg) {
1023                                 if ( (strstr(msg, "architecture") != NULL) && !_options.errorOnOtherArchFiles() ) {
1024                                         if ( _options.ignoreOtherArchInputFiles() ) {
1025                                                 // ignore, because this is about an architecture not in use
1026                                         }
1027                                         else {
1028                                                 warning("ignoring file %s, %s", entry.path, msg);
1029                                         }
1030                                 }
1031                                 else if ( strstr(msg, "ignoring unexpected") != NULL ) {
1032                                         warning("%s, %s", entry.path, msg);
1033                                 }
1034                                 else {
1035                                         asprintf((char**)&exception, "%s file '%s'", msg, entry.path);
1036                                 }
1037                                 file = new IgnoredFile(entry.path, entry.modTime, entry.ordinal, ld::File::Other);
1038                         }
1039                         pthread_mutex_lock(&_parseLock);
1040                         if (_remainingInputFiles > 0)
1041                                 _remainingInputFiles--;
1042                         if (_s_logPThreads) printf("done with index %u, %d remaining\n", slot, _remainingInputFiles);
1043                         if (exception) {
1044                                 // We are about to die, so set to zero to stop other threads from doing unneeded work.
1045                                 _remainingInputFiles = 0;
1046                                 _exception = exception;
1047                         }
1048                         else {
1049                                 _inputFiles[slot] = file;
1050                                 if (_neededFileSlot == slot)
1051                                         pthread_cond_signal(&_newFileAvailable);
1052                         }
1053                 }
1054         } while (_remainingInputFiles);
1055         if (_s_logPThreads) printf("worker exiting\n");
1056         pthread_cond_broadcast(&_parseWorkReady);
1057         pthread_cond_signal(&_newFileAvailable);
1058         pthread_mutex_unlock(&_parseLock);
1059 }
1060
1061
1062 void InputFiles::parseWorkerThread(InputFiles *inputFiles) {
1063         inputFiles->parseWorkerThread();
1064 }
1065 #endif
1066
1067
1068 ld::File* InputFiles::addDylib(ld::dylib::File* reader, const Options::FileInfo& info)
1069 {
1070         _allDylibs.insert(reader);
1071
1072         if ( (reader->installPath() == NULL) && !info.options.fBundleLoader ) {
1073                 // this is a "blank" stub
1074                 // silently ignore it
1075                 return reader;
1076         }
1077         // store options about how dylib will be used in dylib itself
1078         if ( info.options.fWeakImport )
1079                 reader->setForcedWeakLinked();
1080         if ( info.options.fReExport )
1081                 reader->setWillBeReExported();
1082         if ( info.options.fUpward ) {
1083                 if ( _options.outputKind() == Options::kDynamicLibrary )
1084                         reader->setWillBeUpwardDylib();
1085                 else
1086                         warning("ignoring upward dylib option for %s\n", info.path);
1087         }
1088         if ( info.options.fLazyLoad )
1089                 reader->setWillBeLazyLoadedDylb();
1090
1091         // add to map of loaded dylibs
1092         const char* installPath = reader->installPath();
1093         if ( installPath != NULL ) {
1094                 InstallNameToDylib::iterator pos = _installPathToDylibs.find(installPath);
1095                 if ( pos == _installPathToDylibs.end() ) {
1096                         _installPathToDylibs[strdup(installPath)] = reader;
1097                 }
1098                 else {
1099                         bool dylibOnCommandLineTwice = ( strcmp(pos->second->path(), reader->path()) == 0 );
1100                         bool isSymlink = false;
1101                         // ignore if this is a symlink to a dylib we've already loaded
1102                         if ( !dylibOnCommandLineTwice ) {
1103                                 char existingDylibPath[PATH_MAX];
1104                                 if ( realpath(pos->second->path(), existingDylibPath) != NULL ) {
1105                                         char newDylibPath[PATH_MAX];
1106                                         if ( realpath(reader->path(), newDylibPath) != NULL ) {
1107                                                 isSymlink = ( strcmp(existingDylibPath, newDylibPath) == 0 );
1108                                         }
1109                                 }
1110                         }
1111                         // remove warning for <rdar://problem/10860629> Same install name for CoreServices and CFNetwork?
1112                         //if ( !dylibOnCommandLineTwice && !isSymlink )
1113                         //      warning("dylibs with same install name: %p %s and %p %s", pos->second, pos->second->path(), reader, reader->path());
1114                 }
1115         }
1116         else if ( info.options.fBundleLoader )
1117                 _bundleLoader = reader;
1118
1119         // log direct readers
1120         if ( ! info.options.fIndirectDylib )
1121                 this->logDylib(reader, false, false);
1122
1123         // update stats
1124         _totalDylibsLoaded++;
1125
1126         // just add direct libraries to search-first list
1127         if ( ! info.options.fIndirectDylib )
1128                 _searchLibraries.push_back(LibraryInfo(reader));
1129
1130         return reader;
1131 }
1132
1133
1134 #if HAVE_PTHREADS
1135 // Called during pipelined linking to listen for available input files.
1136 // Available files are enqueued for parsing.
1137 void InputFiles::waitForInputFiles()
1138 {
1139         if (_s_logPThreads) printf("starting pipeline listener\n");
1140         try {
1141                 const char *fifo = _options.pipelineFifo();
1142                 assert(fifo);
1143                 std::map<const char *, const Options::FileInfo*, strcompclass> fileMap;
1144                 const std::vector<Options::FileInfo>& files = _options.getInputFiles();
1145                 for (std::vector<Options::FileInfo>::const_iterator it = files.begin(); it != files.end(); ++it) {
1146                         const Options::FileInfo& entry = *it;
1147                         if (entry.fromFileList) {
1148                                 fileMap[entry.path] = &entry;
1149                         }
1150                 }
1151                 FILE *fileStream = fopen(fifo, "r");
1152                 if (!fileStream)
1153                         throwf("pipelined linking error - failed to open stream. fopen() returns %s for \"%s\"\n", strerror(errno), fifo);
1154                 while (fileMap.size() > 0) {
1155                         char path_buf[PATH_MAX+1];
1156                         if (fgets(path_buf, PATH_MAX, fileStream) == NULL)
1157                                 throwf("pipelined linking error - %lu missing input files", fileMap.size());
1158                         int len = strlen(path_buf);
1159                         if (path_buf[len-1] == '\n')
1160                                 path_buf[len-1] = 0;
1161                         std::map<const char *, const Options::FileInfo*, strcompclass>::iterator it = fileMap.find(path_buf);
1162                         if (it == fileMap.end())
1163                                 throwf("pipelined linking error - not in file list: %s\n", path_buf);
1164                         Options::FileInfo* inputInfo = (Options::FileInfo*)it->second;
1165                         if (!inputInfo->checkFileExists(_options))
1166                                 throwf("pipelined linking error - file does not exist: %s\n", inputInfo->path);
1167                         pthread_mutex_lock(&_parseLock);
1168                         if (_idleWorkers)
1169                                 pthread_cond_signal(&_parseWorkReady);
1170                         inputInfo->readyToParse = true;
1171                         if (_parseCursor > inputInfo->inputFileSlot)
1172                                 _parseCursor = inputInfo->inputFileSlot;
1173                         _availableInputFiles++;
1174                         if (_s_logPThreads) printf("pipeline listener: %s slot=%d, _parseCursor=%d, _availableInputFiles = %d remaining = %ld\n", path_buf, inputInfo->inputFileSlot, _parseCursor, _availableInputFiles, fileMap.size()-1);
1175                         pthread_mutex_unlock(&_parseLock);
1176                         fileMap.erase(it);
1177                 }
1178         } catch (const char *msg) {
1179                 pthread_mutex_lock(&_parseLock);
1180                 _exception = msg;
1181                 pthread_cond_signal(&_newFileAvailable);
1182                 pthread_mutex_unlock(&_parseLock);
1183         }
1184 }
1185
1186
1187 void InputFiles::waitForInputFiles(InputFiles *inputFiles) {
1188         inputFiles->waitForInputFiles();
1189 }
1190 #endif
1191
1192
1193 void InputFiles::forEachInitialAtom(ld::File::AtomHandler& handler, ld::Internal& state)
1194 {
1195         // add all direct object, archives, and dylibs
1196         const std::vector<Options::FileInfo>& files = _options.getInputFiles();
1197         size_t fileIndex;
1198         for (fileIndex=0; fileIndex<_inputFiles.size(); fileIndex++) {
1199                 ld::File *file;
1200 #if HAVE_PTHREADS
1201                 pthread_mutex_lock(&_parseLock);
1202
1203                 // this loop waits for the needed file to be ready (parsed by worker thread)
1204                 while (_inputFiles[fileIndex] == NULL && _exception == NULL) {
1205                         // We are starved for input. If there are still files to parse and we have
1206                         // not maxed out the worker thread count start a new worker thread.
1207                         if (_availableInputFiles > 0 && _availableWorkers > 0) {
1208                                 if (_s_logPThreads) printf("starting worker\n");
1209                                 startThread(InputFiles::parseWorkerThread);
1210                                 _availableWorkers--;
1211                         }
1212                         _neededFileSlot = fileIndex;
1213                         if (_s_logPThreads) printf("consumer blocking for %lu: %s\n", fileIndex, files[fileIndex].path);
1214                         pthread_cond_wait(&_newFileAvailable, &_parseLock);
1215                 }
1216
1217                 if (_exception)
1218                         throw _exception;
1219
1220                 // The input file is parsed. Assimilate it and call its atom iterator.
1221                 if (_s_logPThreads) printf("consuming slot %lu\n", fileIndex);
1222                 file = _inputFiles[fileIndex];
1223                 pthread_mutex_unlock(&_parseLock);
1224 #else
1225                 file = _inputFiles[fileIndex];
1226 #endif
1227                 const Options::FileInfo& info = files[fileIndex];
1228                 switch (file->type()) {
1229                         case ld::File::Reloc:
1230                         {
1231                                 ld::relocatable::File* reloc = (ld::relocatable::File*)file;
1232                                 _options.snapshot().recordObjectFile(reloc->path());
1233                                 _options.addDependency(Options::depObjectFile, reloc->path());
1234                         }
1235                                 break;
1236                         case ld::File::Dylib:
1237                         {
1238                                 ld::dylib::File* dylib = (ld::dylib::File*)file;
1239                                 addDylib(dylib, info);
1240                         }
1241                                 break;
1242                         case ld::File::Archive:
1243                         {
1244                                 ld::archive::File* archive = (ld::archive::File*)file;
1245                                 // <rdar://problem/9740166> force loaded archives should be in LD_TRACE
1246                                 if ( (info.options.fForceLoad || _options.fullyLoadArchives()) && (_options.traceArchives() || _options.traceEmitJSON()) )
1247                                         logArchive(archive);
1248
1249                                 if ( isCompilerSupportLib(info.path) && (info.options.fForceLoad || _options.fullyLoadArchives()) )
1250                                         state.forceLoadCompilerRT = true;
1251
1252                                 _searchLibraries.push_back(LibraryInfo(archive));
1253                                 _options.addDependency(Options::depArchive, archive->path());
1254                         }
1255                                 break;
1256                         case ld::File::Other:
1257                                 break;
1258                         default:
1259                         {
1260                                 throwf("Unknown file type for %s", file->path());
1261                         }
1262                                 break;
1263                 }
1264                 try {
1265                         file->forEachAtom(handler);
1266                 }
1267                 catch (const char* msg) {
1268                         asprintf((char**)&_exception, "%s file '%s'", msg, file->path());
1269                 }
1270         }
1271         if (_exception)
1272                 throw _exception;
1273
1274         markExplicitlyLinkedDylibs();
1275         addLinkerOptionLibraries(state, handler);
1276         createIndirectDylibs();
1277         createOpaqueFileSections();
1278
1279         while (fileIndex < _inputFiles.size()) {
1280                 ld::File *file = _inputFiles[fileIndex];
1281                 file->forEachAtom(handler);
1282                 fileIndex++;
1283         }
1284
1285     switch ( _options.outputKind() ) {
1286         case Options::kStaticExecutable:
1287         case Options::kDynamicExecutable:
1288             // add implicit __dso_handle label
1289             handler.doAtom(DSOHandleAtom::_s_atomExecutable);
1290             handler.doAtom(DSOHandleAtom::_s_atomAll);
1291             if ( _options.pageZeroSize() != 0 )
1292                 handler.doAtom(*new PageZeroAtom(_options.pageZeroSize()));
1293             if ( _options.hasCustomStack() && !_options.needsEntryPointLoadCommand() )
1294                 handler.doAtom(*new CustomStackAtom(_options.customStackSize()));
1295             break;
1296         case Options::kDynamicLibrary:
1297             // add implicit __dso_handle label
1298             handler.doAtom(DSOHandleAtom::_s_atomDylib);
1299             handler.doAtom(DSOHandleAtom::_s_atomAll);
1300             break;
1301         case Options::kDynamicBundle:
1302             // add implicit __dso_handle label
1303             handler.doAtom(DSOHandleAtom::_s_atomBundle);
1304             handler.doAtom(DSOHandleAtom::_s_atomAll);
1305             break;
1306         case Options::kDyld:
1307             // add implicit __dso_handle label
1308             handler.doAtom(DSOHandleAtom::_s_atomDyld);
1309             handler.doAtom(DSOHandleAtom::_s_atomAll);
1310             break;
1311         case Options::kPreload:
1312             // add implicit __mh_preload_header label
1313             handler.doAtom(DSOHandleAtom::_s_atomPreload);
1314             // add implicit __dso_handle label, but put it in __text section because
1315             // with -preload the mach_header is no in the address space.
1316             handler.doAtom(DSOHandleAtom::_s_atomPreloadDSO);
1317             break;
1318         case Options::kObjectFile:
1319             handler.doAtom(DSOHandleAtom::_s_atomObjectFile);
1320             break;
1321         case Options::kKextBundle:
1322             // add implicit __dso_handle label
1323             handler.doAtom(DSOHandleAtom::_s_atomAll);
1324             break;
1325         }
1326 }
1327
1328
1329 bool InputFiles::searchLibraries(const char* name, bool searchDylibs, bool searchArchives, bool dataSymbolOnly, ld::File::AtomHandler& handler) const
1330 {
1331         // Check each input library.
1332     for (std::vector<LibraryInfo>::const_iterator it=_searchLibraries.begin(); it != _searchLibraries.end(); ++it) {
1333         LibraryInfo lib = *it;
1334         if (lib.isDylib()) {
1335             if (searchDylibs) {
1336                 ld::dylib::File *dylibFile = lib.dylib();
1337                 //fprintf(stderr, "searchLibraries(%s), looking in linked %s\n", name, dylibFile->path() );
1338                 if ( dylibFile->justInTimeforEachAtom(name, handler) ) {
1339                     // we found a definition in this dylib
1340                     // done, unless it is a weak definition in which case we keep searching
1341                     _options.snapshot().recordDylibSymbol(dylibFile, name);
1342                     if ( !dylibFile->hasWeakExternals() || !dylibFile->hasWeakDefinition(name)) {
1343                         return true;
1344                     }
1345                     // else continue search for a non-weak definition
1346                 }
1347             }
1348         } else {
1349             if (searchArchives) {
1350                 ld::archive::File *archiveFile = lib.archive();
1351                 if ( dataSymbolOnly ) {
1352                     if ( archiveFile->justInTimeDataOnlyforEachAtom(name, handler) ) {
1353                         if ( _options.traceArchives() || _options.traceEmitJSON())
1354                             logArchive(archiveFile);
1355                         _options.snapshot().recordArchive(archiveFile->path());
1356                                                 // DALLAS _state.archives.push_back(archiveFile);
1357                         // found data definition in static library, done
1358                        return true;
1359                     }
1360                 }
1361                 else {
1362                     if ( archiveFile->justInTimeforEachAtom(name, handler) ) {
1363                         if ( _options.traceArchives() || _options.traceEmitJSON())
1364                             logArchive(archiveFile);
1365                         _options.snapshot().recordArchive(archiveFile->path());
1366                         // found definition in static library, done
1367                         return true;
1368                     }
1369                 }
1370             }
1371         }
1372     }
1373
1374         // search indirect dylibs
1375         if ( searchDylibs ) {
1376                 for (InstallNameToDylib::const_iterator it=_installPathToDylibs.begin(); it != _installPathToDylibs.end(); ++it) {
1377                         ld::dylib::File* dylibFile = it->second;
1378                         bool searchThisDylib = false;
1379                         if ( _options.nameSpace() == Options::kTwoLevelNameSpace ) {
1380                                 // for two level namesapce, just check all implicitly linked dylibs
1381                                 searchThisDylib = dylibFile->implicitlyLinked() && !dylibFile->explicitlyLinked();
1382                         }
1383                         else {
1384                                 // for flat namespace, check all indirect dylibs
1385                                 searchThisDylib = ! dylibFile->explicitlyLinked();
1386                         }
1387                         if ( searchThisDylib ) {
1388                                 //fprintf(stderr, "searchLibraries(%s), looking in implicitly linked %s\n", name, dylibFile->path() );
1389                                 if ( dylibFile->justInTimeforEachAtom(name, handler) ) {
1390                                         // we found a definition in this dylib
1391                                         // done, unless it is a weak definition in which case we keep searching
1392                     _options.snapshot().recordDylibSymbol(dylibFile, name);
1393                                         if ( !dylibFile->hasWeakExternals() || !dylibFile->hasWeakDefinition(name)) {
1394                                                 return true;
1395                     }
1396                                         // else continue search for a non-weak definition
1397                                 }
1398                         }
1399                 }
1400         }
1401
1402         return false;
1403 }
1404
1405
1406 bool InputFiles::searchWeakDefInDylib(const char* name) const
1407 {
1408         // search all relevant dylibs to see if any have a weak-def with this name
1409         for (InstallNameToDylib::const_iterator it=_installPathToDylibs.begin(); it != _installPathToDylibs.end(); ++it) {
1410                 ld::dylib::File* dylibFile = it->second;
1411                 if ( dylibFile->implicitlyLinked() || dylibFile->explicitlyLinked() ) {
1412                         if ( dylibFile->hasWeakExternals() && dylibFile->hasWeakDefinition(name) ) {
1413                                 return true;
1414                         }
1415                 }
1416         }
1417         return false;
1418 }
1419
1420 static bool vectorContains(const std::vector<ld::dylib::File*>& vec, ld::dylib::File* key)
1421 {
1422         return std::find(vec.begin(), vec.end(), key) != vec.end();
1423 }
1424
1425 struct DylibByInstallNameSorter
1426 {
1427          bool operator()(const ld::dylib::File* left, const ld::dylib::File* right)
1428          {
1429           return (strcmp(left->installPath(), right->installPath()) < 0);
1430          }
1431 };
1432
1433 void InputFiles::dylibs(ld::Internal& state)
1434 {
1435         bool dylibsOK = false;
1436         switch ( _options.outputKind() ) {
1437                 case Options::kDynamicExecutable:
1438                 case Options::kDynamicLibrary:
1439                 case Options::kDynamicBundle:
1440                         dylibsOK = true;
1441                         break;
1442                 case Options::kStaticExecutable:
1443                 case Options::kDyld:
1444                 case Options::kPreload:
1445                 case Options::kObjectFile:
1446                 case Options::kKextBundle:
1447                         dylibsOK = false;
1448                         break;
1449         }
1450
1451         // add command line dylibs in order
1452         for (std::vector<ld::File*>::const_iterator it=_inputFiles.begin(); it != _inputFiles.end(); ++it) {
1453                 ld::dylib::File* dylibFile = dynamic_cast<ld::dylib::File*>(*it);
1454                 // only add dylibs that are not "blank" dylib stubs
1455                 if ( (dylibFile != NULL) && ((dylibFile->installPath() != NULL) || (dylibFile == _bundleLoader)) ) {
1456                         if ( dylibsOK ) {
1457                                 if ( ! vectorContains(state.dylibs, dylibFile) ) {
1458                                         state.dylibs.push_back(dylibFile);
1459                                 }
1460                         }
1461                         else
1462                                 warning("unexpected dylib (%s) on link line", dylibFile->path());
1463                 }
1464         }
1465         // add implicitly linked dylibs
1466         if ( _options.nameSpace() == Options::kTwoLevelNameSpace ) {
1467                 std::vector<ld::dylib::File*> implicitDylibs;
1468                 for (InstallNameToDylib::const_iterator it=_installPathToDylibs.begin(); it != _installPathToDylibs.end(); ++it) {
1469                         ld::dylib::File* dylibFile = it->second;
1470                         if ( dylibFile->implicitlyLinked() && dylibsOK ) {
1471                                 if ( ! vectorContains(implicitDylibs, dylibFile) ) {
1472                                         implicitDylibs.push_back(dylibFile);
1473                                 }
1474                         }
1475                 }
1476                 // <rdar://problem/15002251> make implicit dylib order be deterministic by sorting by install_name
1477                 std::sort(implicitDylibs.begin(), implicitDylibs.end(), DylibByInstallNameSorter());
1478
1479                 if ( _options.traceDylibs() ) {
1480                         for (ld::dylib::File* dylib :  implicitDylibs) {
1481                                 if ( dylib->speculativelyLoaded() && !dylib->explicitlyLinked() && dylib->providedExportAtom() ) {
1482                                         const char* fullPath = dylib->path();
1483                                         char realName[MAXPATHLEN];
1484                                         if ( realpath(fullPath, realName) != NULL )
1485                                                 fullPath = realName;
1486                                         logTraceInfo("[Logging for XBS] Used dynamic library: %s\n", fullPath);
1487                                 }
1488                         }
1489                 }
1490                 state.dylibs.insert(state.dylibs.end(), implicitDylibs.begin(), implicitDylibs.end());
1491         }
1492
1493         //fprintf(stderr, "all dylibs:\n");
1494         //for(std::vector<ld::dylib::File*>::iterator it=state.dylibs.begin(); it != state.dylibs.end(); ++it) {
1495         //      const ld::dylib::File* dylib = *it;
1496         //      fprintf(stderr, "    %p impl=%d %s\n", dylib, dylib->implicitlyLinked(), dylib->path());
1497         //}
1498
1499         // and -bundle_loader
1500         state.bundleLoader = _bundleLoader;
1501
1502         // <rdar://problem/10807040> give an error when -nostdlib is used and libSystem is missing
1503         if ( (state.dylibs.size() == 0) && _options.needsEntryPointLoadCommand() )  {
1504                 // HACK until 39514191 is fixed
1505                 bool grandfather = false;
1506                 for (const File* inFile : _inputFiles) {
1507                         if ( strstr(inFile->path(), "exit-asm.o") != NULL )
1508                                 grandfather = true;
1509                 }
1510                 if ( !grandfather )
1511                         throw "dynamic main executables must link with libSystem.dylib";
1512         }
1513 }
1514
1515 void InputFiles::archives(ld::Internal& state)
1516 {
1517         for (const std::string path :  _archiveFilePaths) {
1518
1519                 state.archivePaths.push_back(path);
1520         }
1521 }
1522
1523
1524 } // namespace tool
1525 } // namespace ld
1526