src/ld/InputFiles.cpp

   1
   2 /* -*- mode: C++; c-basic-offset: 4; tab-width: 4 -*-*
   3  *
   4  * Copyright (c) 2009-2011 Apple Inc. All rights reserved.
   5  *
   6  * @APPLE_LICENSE_HEADER_START@
   7  *
   8  * This file contains Original Code and/or Modifications of Original Code
   9  * as defined in and that are subject to the Apple Public Source License
  10  * Version 2.0 (the 'License'). You may not use this file except in
  11  * compliance with the License. Please obtain a copy of the License at
  12  * http://www.opensource.apple.com/apsl/ and read it before using this
  13  * file.
  14  *
  15  * The Original Code and all software distributed under the License are
  16  * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  17  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  18  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
  19  * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
  20  * Please see the License for the specific language governing rights and
  21  * limitations under the License.
  22  *
  23  * @APPLE_LICENSE_HEADER_END@
  24  */
  25
  26
  27 #include <stdlib.h>
  28 #include <sys/types.h>
  29 #include <sys/stat.h>
  30 #include <sys/mman.h>
  31 #include <sys/sysctl.h>
  32 #include <fcntl.h>
  33 #include <errno.h>
  34 #include <limits.h>
  35 #include <unistd.h>
  36 #include <mach/mach_time.h>
  37 #include <mach/vm_statistics.h>
  38 #include <mach/mach_init.h>
  39 #include <mach/mach_host.h>
  40 #include <dlfcn.h>
  41 #include <mach-o/dyld.h>
  42 #include <mach-o/fat.h>
  43 #include <sys/sysctl.h>
  44 #include <libkern/OSAtomic.h>
  45
  46 #include <string>
  47 #include <map>
  48 #include <set>
  49 #include <string>
  50 #include <vector>
  51 #include <list>
  52 #include <algorithm>
  53 #include <dlfcn.h>
  54 #include <AvailabilityMacros.h>
  55
  56 #include "Options.h"
  57
  58 #include "InputFiles.h"
  59 #include "macho_relocatable_file.h"
  60 #include "macho_dylib_file.h"
  61 #include "textstub_dylib_file.hpp"
  62 #include "archive_file.h"
  63 #include "lto_file.h"
  64 #include "opaque_section_file.h"
  65 #include "MachOFileAbstraction.hpp"
  66 #include "Snapshot.h"
  67
  68 const bool _s_logPThreads = false;
  69
  70 namespace ld {
  71 namespace tool {
  72
  73 class IgnoredFile : public ld::File {
  74 public:
  75         IgnoredFile(const char* pth, time_t modTime, Ordinal ord, Type type) : ld::File(pth, modTime, ord, type) {};
  76         virtual bool                                            forEachAtom(AtomHandler&) const { return false; };
  77         virtual bool                                            justInTimeforEachAtom(const char* name, AtomHandler&) const { return false; };
  78 };
  79
  80
  81 class DSOHandleAtom : public ld::Atom {
  82 public:
  83                                                                         DSOHandleAtom(const char* nm, ld::Atom::Scope sc,
  84                                                                                                                 ld::Atom::SymbolTableInclusion inc, ld::Section& sect=_s_section)
  85                                                                                 : ld::Atom(sect, ld::Atom::definitionRegular,
  86                                                                                                    (sect == _s_section_text) ? ld::Atom::combineByName : ld::Atom::combineNever,
  87                                                                                                    // make "weak def" so that link succeeds even if app defines __dso_handle
  88                                                                                                         sc, ld::Atom::typeUnclassified, inc, true, false, false,
  89                                                                                                          ld::Atom::Alignment(1)), _name(nm) {}
  90
  91         virtual ld::File*                                               file() const                                    { return NULL; }
  92   virtual const char*                                           name() const                                    { return _name; }
  93         virtual uint64_t                                                size() const                                    { return 0; }
  94         virtual uint64_t                                                objectAddress() const                   { return 0; }
  95         virtual void                                                    copyRawContent(uint8_t buffer[]) const
  96                                                                                                                                                         { }
  97         virtual void                                                    setScope(Scope)                                 { }
  98
  99         virtual                                                                 ~DSOHandleAtom() {}
 100
 101         static ld::Section                                              _s_section;
 102         static ld::Section                                              _s_section_preload;
 103         static ld::Section                                              _s_section_text;
 104         static DSOHandleAtom                                    _s_atomAll;
 105         static DSOHandleAtom                                    _s_atomExecutable;
 106         static DSOHandleAtom                                    _s_atomDylib;
 107         static DSOHandleAtom                                    _s_atomBundle;
 108         static DSOHandleAtom                                    _s_atomDyld;
 109         static DSOHandleAtom                                    _s_atomObjectFile;
 110         static DSOHandleAtom                                    _s_atomPreload;
 111         static DSOHandleAtom                                    _s_atomPreloadDSO;
 112 private:
 113         const char*                                                             _name;
 114 };
 115 ld::Section DSOHandleAtom::_s_section("__TEXT", "__mach_header", ld::Section::typeMachHeader, true);
 116 ld::Section DSOHandleAtom::_s_section_preload("__HEADER", "__mach_header", ld::Section::typeMachHeader, true);
 117 ld::Section DSOHandleAtom::_s_section_text("__TEXT", "__text", ld::Section::typeCode, false);
 118 DSOHandleAtom DSOHandleAtom::_s_atomAll("___dso_handle", ld::Atom::scopeLinkageUnit, ld::Atom::symbolTableNotIn);
 119 DSOHandleAtom DSOHandleAtom::_s_atomExecutable("__mh_execute_header", ld::Atom::scopeGlobal, ld::Atom::symbolTableInAndNeverStrip);
 120 DSOHandleAtom DSOHandleAtom::_s_atomDylib("__mh_dylib_header", ld::Atom::scopeLinkageUnit, ld::Atom::symbolTableNotIn);
 121 DSOHandleAtom DSOHandleAtom::_s_atomBundle("__mh_bundle_header", ld::Atom::scopeLinkageUnit, ld::Atom::symbolTableNotIn);
 122 DSOHandleAtom DSOHandleAtom::_s_atomDyld("__mh_dylinker_header", ld::Atom::scopeLinkageUnit, ld::Atom::symbolTableNotIn);
 123 DSOHandleAtom DSOHandleAtom::_s_atomObjectFile("__mh_object_header", ld::Atom::scopeLinkageUnit, ld::Atom::symbolTableNotIn);
 124 DSOHandleAtom DSOHandleAtom::_s_atomPreload("__mh_preload_header", ld::Atom::scopeLinkageUnit, ld::Atom::symbolTableNotIn, _s_section_preload);
 125 DSOHandleAtom DSOHandleAtom::_s_atomPreloadDSO("___dso_handle", ld::Atom::scopeLinkageUnit, ld::Atom::symbolTableNotIn, _s_section_text);
 126
 127
 128
 129 class PageZeroAtom : public ld::Atom {
 130 public:
 131                                                                         PageZeroAtom(uint64_t sz)
 132                                                                                 : ld::Atom(_s_section, ld::Atom::definitionRegular, ld::Atom::combineNever,
 133                                                                                         ld::Atom::scopeTranslationUnit, ld::Atom::typeZeroFill,
 134                                                                                         symbolTableNotIn, true, false, false, ld::Atom::Alignment(12)),
 135                                                                                         _size(sz) {}
 136
 137         virtual ld::File*                                               file() const                                    { return NULL; }
 138         virtual const char*                                             name() const                                    { return "page zero"; }
 139         virtual uint64_t                                                size() const                                    { return _size; }
 140         virtual uint64_t                                                objectAddress() const                   { return 0; }
 141         virtual void                                                    copyRawContent(uint8_t buffer[]) const
 142                                                                                                                                                         { }
 143         virtual void                                                    setScope(Scope)                                 { }
 144
 145         virtual                                                                 ~PageZeroAtom() {}
 146
 147         static ld::Section                                              _s_section;
 148         static DSOHandleAtom                                    _s_atomAll;
 149 private:
 150         uint64_t                                                                _size;
 151 };
 152 ld::Section PageZeroAtom::_s_section("__PAGEZERO", "__pagezero", ld::Section::typePageZero, true);
 153
 154
 155 class CustomStackAtom : public ld::Atom {
 156 public:
 157                                                                         CustomStackAtom(uint64_t sz)
 158                                                                                 : ld::Atom(_s_section, ld::Atom::definitionRegular, ld::Atom::combineNever,
 159                                                                                         ld::Atom::scopeTranslationUnit, ld::Atom::typeZeroFill,
 160                                                                                         symbolTableNotIn, false, false, false, ld::Atom::Alignment(12)),
 161                                                                                         _size(sz) {}
 162
 163         virtual ld::File*                                               file() const                                    { return NULL; }
 164         virtual const char*                                             name() const                                    { return "custom stack"; }
 165         virtual uint64_t                                                size() const                                    { return _size; }
 166         virtual uint64_t                                                objectAddress() const                   { return 0; }
 167         virtual void                                                    copyRawContent(uint8_t buffer[]) const
 168                                                                                                                                                         { }
 169         virtual void                                                    setScope(Scope)                                 { }
 170
 171         virtual                                                                 ~CustomStackAtom() {}
 172
 173 private:
 174         uint64_t                                                                _size;
 175         static ld::Section                                              _s_section;
 176 };
 177 ld::Section CustomStackAtom::_s_section("__UNIXSTACK", "__stack", ld::Section::typeStack, true);
 178
 179
 180
 181 const char* InputFiles::fileArch(const uint8_t* p, unsigned len)
 182 {
 183         const char* result = mach_o::relocatable::archName(p);
 184         if ( result != NULL  )
 185                  return result;
 186
 187     result = mach_o::dylib::archName(p);
 188     if ( result != NULL  )
 189                 return result;
 190
 191         result = lto::archName(p, len);
 192         if ( result != NULL  )
 193                  return result;
 194
 195         if ( strncmp((const char*)p, "!<arch>\n", 8) == 0 )
 196                 return "archive";
 197
 198         char *unsupported = (char *)malloc(128);
 199         strcpy(unsupported, "unsupported file format (");
 200         for (unsigned i=0; i<len && i < 16; i++) {
 201                 char buf[8];
 202                 sprintf(buf, " 0x%02X", p[i]);
 203                 strcat(unsupported, buf);
 204         }
 205         strcat(unsupported, " )");
 206         return unsupported;
 207 }
 208
 209
 210 ld::File* InputFiles::makeFile(const Options::FileInfo& info, bool indirectDylib)
 211 {
 212         // map in whole file
 213         uint64_t len = info.fileLen;
 214         int fd = ::open(info.path, O_RDONLY, 0);
 215         if ( fd == -1 )
 216                 throwf("can't open file, errno=%d", errno);
 217         if ( info.fileLen < 20 )
 218                 throwf("file too small (length=%llu)", info.fileLen);
 219
 220         uint8_t* p = (uint8_t*)::mmap(NULL, info.fileLen, PROT_READ, MAP_FILE | MAP_PRIVATE, fd, 0);
 221         if ( p == (uint8_t*)(-1) )
 222                 throwf("can't map file, errno=%d", errno);
 223
 224         // if fat file, skip to architecture we want
 225         // Note: fat header is always big-endian
 226         bool isFatFile = false;
 227         uint32_t sliceToUse, sliceCount;
 228         const fat_header* fh = (fat_header*)p;
 229         if ( fh->magic == OSSwapBigToHostInt32(FAT_MAGIC) ) {
 230                 isFatFile = true;
 231                 const struct fat_arch* archs = (struct fat_arch*)(p + sizeof(struct fat_header));
 232                 bool sliceFound = false;
 233                 sliceCount = OSSwapBigToHostInt32(fh->nfat_arch);
 234                 if ( _options.preferSubArchitecture() ) {
 235                         // first try to find a slice that match cpu-type and cpu-sub-type
 236                         for (uint32_t i=0; i < sliceCount; ++i) {
 237                                 if ( (OSSwapBigToHostInt32(archs[i].cputype) == (uint32_t)_options.architecture())
 238                                   && (OSSwapBigToHostInt32(archs[i].cpusubtype) == (uint32_t)_options.subArchitecture()) ) {
 239                                         sliceToUse = i;
 240                                         sliceFound = true;
 241                                         break;
 242                                 }
 243                         }
 244                 }
 245                 if ( !sliceFound ) {
 246                         // look for any slice that matches just cpu-type
 247                         for (uint32_t i=0; i < sliceCount; ++i) {
 248                                 if ( OSSwapBigToHostInt32(archs[i].cputype) == (uint32_t)_options.architecture() ) {
 249                                         sliceToUse = i;
 250                                         sliceFound = true;
 251                                         break;
 252                                 }
 253                         }
 254                 }
 255                 if ( sliceFound ) {
 256                         uint32_t fileOffset = OSSwapBigToHostInt32(archs[sliceToUse].offset);
 257                         len = OSSwapBigToHostInt32(archs[sliceToUse].size);
 258                         if ( fileOffset+len > info.fileLen ) {
 259                                 // <rdar://problem/17593430> file size was read awhile ago.  If file is being written, wait a second to see if big enough now
 260                                 sleep(1);
 261                                 uint64_t newFileLen = info.fileLen;
 262                                 struct stat statBuffer;
 263                                 if ( stat(info.path, &statBuffer) == 0 ) {
 264                                         newFileLen = statBuffer.st_size;
 265                                 }
 266                                 if ( fileOffset+len > newFileLen ) {
 267                                         throwf("truncated fat file. Slice from %u to %llu is past end of file with length %llu",
 268                                                 fileOffset, fileOffset+len, info.fileLen);
 269                                 }
 270                         }
 271                         // if requested architecture is page aligned within fat file, then remap just that portion of file
 272                         if ( (fileOffset & 0x00000FFF) == 0 ) {
 273                                 // unmap whole file
 274                                 munmap((caddr_t)p, info.fileLen);
 275                                 // re-map just part we need
 276                                 p = (uint8_t*)::mmap(NULL, len, PROT_READ, MAP_FILE | MAP_PRIVATE, fd, fileOffset);
 277                                 if ( p == (uint8_t*)(-1) )
 278                                         throwf("can't re-map file, errno=%d", errno);
 279                         }
 280                         else {
 281                                 p = &p[fileOffset];
 282                         }
 283                 }
 284         }
 285         ::close(fd);
 286
 287         // see if it is an object file
 288         mach_o::relocatable::ParserOptions objOpts;
 289         objOpts.architecture            = _options.architecture();
 290         objOpts.objSubtypeMustMatch = !_options.allowSubArchitectureMismatches();
 291         objOpts.logAllFiles                     = _options.logAllFiles();
 292         objOpts.warnUnwindConversionProblems    = _options.needsUnwindInfoSection();
 293         objOpts.keepDwarfUnwind         = _options.keepDwarfUnwind();
 294         objOpts.forceDwarfConversion= (_options.outputKind() == Options::kDyld);
 295         objOpts.neverConvertDwarf   = !_options.needsUnwindInfoSection();
 296         objOpts.verboseOptimizationHints = _options.verboseOptimizationHints();
 297         objOpts.armUsesZeroCostExceptions = _options.armUsesZeroCostExceptions();
 298         objOpts.simulator                       = _options.targetIOSSimulator();
 299         objOpts.ignoreMismatchPlatform = ((_options.outputKind() == Options::kPreload) || (_options.outputKind() == Options::kStaticExecutable));
 300         objOpts.subType                         = _options.subArchitecture();
 301         objOpts.platform                        = _options.platform();
 302         objOpts.minOSVersion            = _options.minOSversion();
 303         objOpts.srcKind                         = ld::relocatable::File::kSourceObj;
 304         objOpts.treateBitcodeAsData     = _options.bitcodeKind() == Options::kBitcodeAsData;
 305         objOpts.usingBitcode            = _options.bundleBitcode();
 306         objOpts.maxDefaultCommonAlignment = _options.maxDefaultCommonAlign();
 307
 308         ld::relocatable::File* objResult = mach_o::relocatable::parse(p, len, info.path, info.modTime, info.ordinal, objOpts);
 309         if ( objResult != NULL ) {
 310                 OSAtomicAdd64(len, &_totalObjectSize);
 311                 OSAtomicIncrement32(&_totalObjectLoaded);
 312                 return objResult;
 313         }
 314
 315         // see if it is an llvm object file
 316         objResult = lto::parse(p, len, info.path, info.modTime, info.ordinal, _options.architecture(), _options.subArchitecture(), _options.logAllFiles(), _options.verboseOptimizationHints());
 317         if ( objResult != NULL ) {
 318                 OSAtomicAdd64(len, &_totalObjectSize);
 319                 OSAtomicIncrement32(&_totalObjectLoaded);
 320                 return objResult;
 321         }
 322
 323         // see if it is a dynamic library (or text-based dynamic library)
 324         ld::dylib::File* dylibResult;
 325         bool dylibsNotAllowed = false;
 326         switch ( _options.outputKind() ) {
 327                 case Options::kDynamicExecutable:
 328                 case Options::kDynamicLibrary:
 329                 case Options::kDynamicBundle:
 330                         dylibResult = mach_o::dylib::parse(p, len, info.path, info.modTime, _options, info.ordinal, info.options.fBundleLoader, indirectDylib);
 331                         if ( dylibResult != NULL ) {
 332                                 return dylibResult;
 333                         }
 334                         dylibResult = textstub::dylib::parse(p, len, info.path, info.modTime, _options, info.ordinal, info.options.fBundleLoader, indirectDylib);
 335                         if ( dylibResult != NULL ) {
 336                                 return dylibResult;
 337                         }
 338                         break;
 339                 case Options::kStaticExecutable:
 340                 case Options::kDyld:
 341                 case Options::kPreload:
 342                 case Options::kObjectFile:
 343                 case Options::kKextBundle:
 344                         dylibsNotAllowed = true;
 345                         break;
 346         }
 347
 348         // see if it is a static library
 349         ::archive::ParserOptions archOpts;
 350         archOpts.objOpts                                = objOpts;
 351         archOpts.forceLoadThisArchive   = info.options.fForceLoad;
 352         archOpts.forceLoadAll                   = _options.fullyLoadArchives();
 353         archOpts.forceLoadObjC                  = _options.loadAllObjcObjectsFromArchives();
 354         archOpts.objcABI2                               = _options.objCABIVersion2POverride();
 355         archOpts.verboseLoad                    = _options.whyLoad();
 356         archOpts.logAllFiles                    = _options.logAllFiles();
 357         // Set ObjSource Kind, libclang_rt is compiler static library
 358         const char* libName = strrchr(info.path, '/');
 359         if ( (libName != NULL) && (strncmp(libName, "/libclang_rt", 12) == 0) )
 360                 archOpts.objOpts.srcKind = ld::relocatable::File::kSourceCompilerArchive;
 361         else
 362                 archOpts.objOpts.srcKind = ld::relocatable::File::kSourceArchive;
 363         archOpts.objOpts.treateBitcodeAsData = _options.bitcodeKind() == Options::kBitcodeAsData;
 364         archOpts.objOpts.usingBitcode = _options.bundleBitcode();
 365
 366         ld::archive::File* archiveResult = ::archive::parse(p, len, info.path, info.modTime, info.ordinal, archOpts);
 367         if ( archiveResult != NULL ) {
 368                 OSAtomicAdd64(len, &_totalArchiveSize);
 369                 OSAtomicIncrement32(&_totalArchivesLoaded);
 370                 return archiveResult;
 371         }
 372
 373         // does not seem to be any valid linker input file, check LTO misconfiguration problems
 374         if ( lto::archName((uint8_t*)p, len) != NULL ) {
 375                 if ( lto::libLTOisLoaded() ) {
 376                         throwf("lto file was built for %s which is not the architecture being linked (%s): %s", fileArch(p, len), _options.architectureName(), info.path);
 377                 }
 378                 else {
 379                         const char* libLTO = "libLTO.dylib";
 380                         char ldPath[PATH_MAX];
 381                         char tmpPath[PATH_MAX];
 382                         char libLTOPath[PATH_MAX];
 383                         uint32_t bufSize = PATH_MAX;
 384                         if ( _options.overridePathlibLTO() != NULL ) {
 385                                 libLTO = _options.overridePathlibLTO();
 386                         }
 387                         else if ( _NSGetExecutablePath(ldPath, &bufSize) != -1 ) {
 388                                 if ( realpath(ldPath, tmpPath) != NULL ) {
 389                                         char* lastSlash = strrchr(tmpPath, '/');
 390                                         if ( lastSlash != NULL )
 391                                                 strcpy(lastSlash, "/../lib/libLTO.dylib");
 392                                         libLTO = tmpPath;
 393                                         if ( realpath(tmpPath, libLTOPath) != NULL )
 394                                                 libLTO = libLTOPath;
 395                                 }
 396                         }
 397                         throwf("could not process llvm bitcode object file, because %s could not be loaded", libLTO);
 398                 }
 399         }
 400
 401         if ( dylibsNotAllowed ) {
 402                 cpu_type_t dummy1;
 403                 cpu_type_t dummy2;
 404                 if ( mach_o::dylib::isDylibFile(p, &dummy1, &dummy2) )
 405                         throw "ignoring unexpected dylib file";
 406         }
 407
 408         // error handling
 409         if ( ((fat_header*)p)->magic == OSSwapBigToHostInt32(FAT_MAGIC) ) {
 410                 throwf("missing required architecture %s in file %s (%u slices)", _options.architectureName(), info.path, sliceCount);
 411         }
 412         else {
 413                 if ( isFatFile )
 414                         throwf("file is universal (%u slices) but does not contain a(n) %s slice: %s", sliceCount, _options.architectureName(), info.path);
 415                 else
 416                         throwf("file was built for %s which is not the architecture being linked (%s): %s", fileArch(p, len), _options.architectureName(), info.path);
 417         }
 418 }
 419
 420 void InputFiles::logDylib(ld::File* file, bool indirect)
 421 {
 422         if ( _options.traceDylibs() ) {
 423                 const char* fullPath = file->path();
 424                 char realName[MAXPATHLEN];
 425                 if ( realpath(fullPath, realName) != NULL )
 426                         fullPath = realName;
 427                 const ld::dylib::File* dylib = dynamic_cast<const ld::dylib::File*>(file);
 428                 if ( (dylib != NULL ) && dylib->willBeUpwardDylib() ) {
 429                         // don't log upward dylibs when XBS is computing dependencies
 430                         logTraceInfo("[Logging for XBS] Used upward dynamic library: %s\n", fullPath);
 431                 }
 432                 else {
 433                         if ( indirect )
 434                                 logTraceInfo("[Logging for XBS] Used indirect dynamic library: %s\n", fullPath);
 435                         else
 436                                 logTraceInfo("[Logging for XBS] Used dynamic library: %s\n", fullPath);
 437                 }
 438         }
 439
 440         if ( _options.dumpDependencyInfo() ) {
 441                 const ld::dylib::File* dylib = dynamic_cast<const ld::dylib::File*>(file);
 442                 if ( file == _bundleLoader ) {
 443                         _options.dumpDependency(Options::depBundleLoader, file->path());
 444                 }
 445                 else if ( (dylib != NULL ) && dylib->willBeUpwardDylib() ) {
 446                         if ( indirect )
 447                                 _options.dumpDependency(Options::depUpwardIndirectDylib, file->path());
 448                         else
 449                                 _options.dumpDependency(Options::depUpwardDirectDylib, file->path());
 450                 }
 451                 else {
 452                         if ( indirect )
 453                                 _options.dumpDependency(Options::depIndirectDylib, file->path());
 454                         else
 455                                 _options.dumpDependency(Options::depDirectDylib, file->path());
 456                 }
 457         }
 458 }
 459
 460 void InputFiles::logArchive(ld::File* file) const
 461 {
 462         if ( _options.traceArchives() && (_archiveFilesLogged.count(file) == 0) ) {
 463                 // <rdar://problem/4947347> LD_TRACE_ARCHIVES should only print out when a .o is actually used from an archive
 464                 _archiveFilesLogged.insert(file);
 465                 const char* fullPath = file->path();
 466                 char realName[MAXPATHLEN];
 467                 if ( realpath(fullPath, realName) != NULL )
 468                         fullPath = realName;
 469                 logTraceInfo("[Logging for XBS] Used static archive: %s\n", fullPath);
 470         }
 471 }
 472
 473
 474 void InputFiles::logTraceInfo(const char* format, ...) const
 475 {
 476         // one time open() of custom LD_TRACE_FILE
 477         static int trace_file = -1;
 478         if ( trace_file == -1 ) {
 479                 const char *trace_file_path = _options.traceOutputFile();
 480                 if ( trace_file_path != NULL ) {
 481                         trace_file = open(trace_file_path, O_WRONLY | O_APPEND | O_CREAT, 0666);
 482                         if ( trace_file == -1 )
 483                                 throwf("Could not open or create trace file (errno=%d): %s", errno, trace_file_path);
 484                 }
 485                 else {
 486                         trace_file = fileno(stderr);
 487                 }
 488         }
 489
 490         char trace_buffer[MAXPATHLEN * 2];
 491     va_list ap;
 492         va_start(ap, format);
 493         int length = vsnprintf(trace_buffer, sizeof(trace_buffer), format, ap);
 494         va_end(ap);
 495         char* buffer_ptr = trace_buffer;
 496
 497         while (length > 0) {
 498                 ssize_t amount_written = write(trace_file, buffer_ptr, length);
 499                 if(amount_written == -1)
 500                         /* Failure to write shouldn't fail the build. */
 501                         return;
 502                 buffer_ptr += amount_written;
 503                 length -= amount_written;
 504         }
 505 }
 506
 507
 508 ld::dylib::File* InputFiles::findDylib(const char* installPath, const char* fromPath)
 509 {
 510         //fprintf(stderr, "findDylib(%s, %s)\n", installPath, fromPath);
 511         InstallNameToDylib::iterator pos = _installPathToDylibs.find(installPath);
 512         if ( pos != _installPathToDylibs.end() ) {
 513                 return pos->second;
 514         }
 515         else {
 516                 // allow -dylib_path option to override indirect library to use
 517                 for (std::vector<Options::DylibOverride>::const_iterator dit = _options.dylibOverrides().begin(); dit != _options.dylibOverrides().end(); ++dit) {
 518                         if ( strcmp(dit->installName,installPath) == 0 ) {
 519                                 try {
 520                                         Options::FileInfo info = _options.findFile(dit->useInstead);
 521                                         _indirectDylibOrdinal = _indirectDylibOrdinal.nextIndirectDylibOrdinal();
 522                                         info.ordinal = _indirectDylibOrdinal;
 523                                         info.options.fIndirectDylib = true;
 524                                         ld::File* reader = this->makeFile(info, true);
 525                                         ld::dylib::File* dylibReader = dynamic_cast<ld::dylib::File*>(reader);
 526                                         if ( dylibReader != NULL ) {
 527                                                 addDylib(dylibReader, info);
 528                                                 //_installPathToDylibs[strdup(installPath)] = dylibReader;
 529                                                 this->logDylib(dylibReader, true);
 530                                                 return dylibReader;
 531                                         }
 532                                         else
 533                                                 throwf("indirect dylib at %s is not a dylib", dit->useInstead);
 534                                 }
 535                                 catch (const char* msg) {
 536                                         warning("ignoring -dylib_file option, %s", msg);
 537                                 }
 538                         }
 539                 }
 540                 char newPath[MAXPATHLEN];
 541                 // handle @loader_path
 542                 if ( strncmp(installPath, "@loader_path/", 13) == 0 ) {
 543                         strcpy(newPath, fromPath);
 544                         char* addPoint = strrchr(newPath,'/');
 545                         if ( addPoint != NULL )
 546                                 strcpy(&addPoint[1], &installPath[13]);
 547                         else
 548                                 strcpy(newPath, &installPath[13]);
 549                         installPath = newPath;
 550                 }
 551                 // note: @executable_path case is handled inside findFileUsingPaths()
 552                 // search for dylib using -F and -L paths
 553                 Options::FileInfo info = _options.findFileUsingPaths(installPath);
 554                 _indirectDylibOrdinal = _indirectDylibOrdinal.nextIndirectDylibOrdinal();
 555                 info.ordinal = _indirectDylibOrdinal;
 556                 info.options.fIndirectDylib = true;
 557                 try {
 558                         ld::File* reader = this->makeFile(info, true);
 559                         ld::dylib::File* dylibReader = dynamic_cast<ld::dylib::File*>(reader);
 560                         if ( dylibReader != NULL ) {
 561                                 //assert(_installPathToDylibs.find(installPath) !=  _installPathToDylibs.end());
 562                                 //_installPathToDylibs[strdup(installPath)] = dylibReader;
 563                                 addDylib(dylibReader, info);
 564                                 this->logDylib(dylibReader, true);
 565                                 return dylibReader;
 566                         }
 567                         else
 568                                 throwf("indirect dylib at %s is not a dylib", info.path);
 569                 }
 570                 catch (const char* msg) {
 571                         throwf("in '%s', %s", info.path, msg);
 572                 }
 573         }
 574 }
 575
 576
 577 // mark all dylibs initially specified as required, and check if they can be used
 578 void InputFiles::markExplicitlyLinkedDylibs()
 579 {
 580         for (InstallNameToDylib::iterator it=_installPathToDylibs.begin(); it != _installPathToDylibs.end(); it++) {
 581                 it->second->setExplicitlyLinked();
 582                 this->checkDylibClientRestrictions(it->second);
 583         }
 584 }
 585
 586 bool InputFiles::frameworkAlreadyLoaded(const char* path, const char* frameworkName)
 587 {
 588         for (ld::File* file : _inputFiles) {
 589                 if ( strcmp(path, file->path()) == 0 )
 590                         return true;
 591         }
 592         for (ld::dylib::File* dylibx : _allDylibs) {
 593                 const char* fname = dylibx->frameworkName();
 594                 if ( fname == NULL )
 595                         continue;
 596                 if ( strcmp(frameworkName, fname) == 0 )
 597                         return true;
 598         }
 599         return false;
 600 }
 601
 602 bool InputFiles::libraryAlreadyLoaded(const char* path)
 603 {
 604         for (ld::File* file : _inputFiles) {
 605                 if ( strcmp(path, file->path()) == 0 )
 606                         return true;
 607         }
 608         for (ld::dylib::File* dylib : _allDylibs) {
 609                 if ( strcmp(path, dylib->path()) == 0 )
 610                         return true;
 611         }
 612         for (const LibraryInfo& libInfo : _searchLibraries) {
 613                 if ( strcmp(path, libInfo.archive()->path()) == 0 )
 614                         return true;
 615         }
 616
 617         return false;
 618 }
 619
 620
 621 void InputFiles::addLinkerOptionLibraries(ld::Internal& state, ld::File::AtomHandler& handler)
 622 {
 623     if ( _options.outputKind() == Options::kObjectFile )
 624                 return;
 625
 626         // process frameworks specified in .o linker options
 627         for (CStringSet::const_iterator it = state.linkerOptionFrameworks.begin(); it != state.linkerOptionFrameworks.end(); ++it) {
 628                 const char* frameworkName = *it;
 629                 if ( state.linkerOptionFrameworksProcessed.count(frameworkName) )
 630                         continue;
 631                 Options::FileInfo info = _options.findFramework(frameworkName);
 632                 if ( ! this->frameworkAlreadyLoaded(info.path, frameworkName) ) {
 633                         info.ordinal = _linkerOptionOrdinal.nextLinkerOptionOrdinal();
 634                         try {
 635                                 ld::File* reader = this->makeFile(info, true);
 636                                 ld::dylib::File* dylibReader = dynamic_cast<ld::dylib::File*>(reader);
 637                                 if ( dylibReader != NULL ) {
 638                                         if ( ! dylibReader->installPathVersionSpecific() ) {
 639                                                 dylibReader->forEachAtom(handler);
 640                                                 dylibReader->setImplicitlyLinked();
 641                                                 this->addDylib(dylibReader, info);
 642                                         }
 643                                 }
 644                                 else {
 645                                         throwf("framework linker option at %s is not a dylib", info.path);
 646                                 }
 647                         }
 648                         catch (const char* msg) {
 649                                 warning("Auto-Linking supplied '%s', %s", info.path, msg);
 650                         }
 651                 }
 652                 state.linkerOptionFrameworksProcessed.insert(frameworkName);
 653         }
 654         // process libraries specified in .o linker options
 655         for (CStringSet::const_iterator it = state.linkerOptionLibraries.begin(); it != state.linkerOptionLibraries.end(); ++it) {
 656                 const char* libName = *it;
 657                 if ( state.linkerOptionLibrariesProcessed.count(libName) )
 658                         continue;
 659                 Options::FileInfo info = _options.findLibrary(libName);
 660                 if ( ! this->libraryAlreadyLoaded(info.path) ) {
 661                         info.ordinal = _linkerOptionOrdinal.nextLinkerOptionOrdinal();
 662                         try {
 663                                 //<rdar://problem/17787306> -force_load_swift_libs
 664                                 info.options.fForceLoad = _options.forceLoadSwiftLibs() && (strncmp(libName, "swift", 5) == 0);
 665                                 ld::File* reader = this->makeFile(info, true);
 666                                 ld::dylib::File* dylibReader = dynamic_cast<ld::dylib::File*>(reader);
 667                                 ld::archive::File* archiveReader = dynamic_cast<ld::archive::File*>(reader);
 668                                 if ( dylibReader != NULL ) {
 669                                         dylibReader->forEachAtom(handler);
 670                                         dylibReader->setImplicitlyLinked();
 671                                         this->addDylib(dylibReader, info);
 672                                 }
 673                                 else if ( archiveReader != NULL ) {
 674                                         _searchLibraries.push_back(LibraryInfo(archiveReader));
 675                                         if ( _options.dumpDependencyInfo() )
 676                                                 _options.dumpDependency(Options::depArchive, archiveReader->path());
 677                                         //<rdar://problem/17787306> -force_load_swift_libs
 678                                         if (info.options.fForceLoad) {
 679                                                 archiveReader->forEachAtom(handler);
 680                                         }
 681                                 }
 682                                 else {
 683                                         throwf("linker option dylib at %s is not a dylib", info.path);
 684                                 }
 685                         }
 686                         catch (const char* msg) {
 687                                 warning("Auto-Linking supplied '%s', %s", info.path, msg);
 688                         }
 689                 }
 690                 state.linkerOptionLibrariesProcessed.insert(libName);
 691         }
 692 }
 693
 694 void InputFiles::createIndirectDylibs()
 695 {
 696         // keep processing dylibs until no more dylibs are added
 697         unsigned long lastMapSize = 0;
 698         std::set<ld::dylib::File*>  dylibsProcessed;
 699         while ( lastMapSize != _allDylibs.size() ) {
 700                 lastMapSize = _allDylibs.size();
 701                 // can't iterator _installPathToDylibs while modifying it, so use temp buffer
 702                 std::vector<ld::dylib::File*> unprocessedDylibs;
 703                 for (std::set<ld::dylib::File*>::iterator it=_allDylibs.begin(); it != _allDylibs.end(); it++) {
 704                         if ( dylibsProcessed.count(*it) == 0 )
 705                                 unprocessedDylibs.push_back(*it);
 706                 }
 707                 for (std::vector<ld::dylib::File*>::iterator it=unprocessedDylibs.begin(); it != unprocessedDylibs.end(); it++) {
 708                         dylibsProcessed.insert(*it);
 709                         (*it)->processIndirectLibraries(this, _options.implicitlyLinkIndirectPublicDylibs());
 710                 }
 711         }
 712
 713         // go back over original dylibs and mark sub frameworks as re-exported
 714         if ( _options.outputKind() == Options::kDynamicLibrary ) {
 715                 const char* myLeaf = strrchr(_options.installPath(), '/');
 716                 if ( myLeaf != NULL ) {
 717                         for (std::vector<class ld::File*>::const_iterator it=_inputFiles.begin(); it != _inputFiles.end(); it++) {
 718                                 ld::dylib::File* dylibReader = dynamic_cast<ld::dylib::File*>(*it);
 719                                 if ( dylibReader != NULL ) {
 720                                         const char* childParent = dylibReader->parentUmbrella();
 721                                         if ( childParent != NULL ) {
 722                                                 if ( strcmp(childParent, &myLeaf[1]) == 0 ) {
 723                                                         // mark that this dylib will be re-exported
 724                                                         dylibReader->setWillBeReExported();
 725                                                 }
 726                                         }
 727                                 }
 728                         }
 729                 }
 730         }
 731
 732 }
 733
 734 void InputFiles::createOpaqueFileSections()
 735 {
 736         // extra command line sections always at end
 737         for (Options::ExtraSection::const_iterator it=_options.extraSectionsBegin(); it != _options.extraSectionsEnd(); ++it) {
 738                 _inputFiles.push_back(opaque_section::parse(it->segmentName, it->sectionName, it->path, it->data, it->dataLen));
 739                 if ( _options.dumpDependencyInfo() )
 740                         _options.dumpDependency(Options::depSection, it->path);
 741         }
 742
 743 }
 744
 745
 746 void InputFiles::checkDylibClientRestrictions(ld::dylib::File* dylib)
 747 {
 748         // Check for any restrictions on who can link with this dylib
 749         const char* dylibParentName = dylib->parentUmbrella() ;
 750         const std::vector<const char*>* clients = dylib->allowableClients();
 751         if ( (dylibParentName != NULL) || (clients != NULL) ) {
 752                 // only dylibs that are in an umbrella or have a client list need verification
 753                 const char* installName = _options.installPath();
 754                 const char* installNameLastSlash = strrchr(installName, '/');
 755                 bool isParent = false;
 756                 bool isSibling = false;
 757                 bool isAllowableClient = false;
 758                 // There are three cases:
 759                 if ( (dylibParentName != NULL) && (installNameLastSlash != NULL) ) {
 760                         // starts after last slash
 761                         const char* myName = &installNameLastSlash[1];
 762                         unsigned int myNameLen = strlen(myName);
 763                         if ( strncmp(myName, "lib", 3) == 0 )
 764                                 myName = &myName[3];
 765                         // up to first dot
 766                         const char* firstDot = strchr(myName, '.');
 767                         if ( firstDot != NULL )
 768                                 myNameLen = firstDot - myName;
 769                         // up to first underscore
 770                         const char* firstUnderscore = strchr(myName, '_');
 771                         if ( (firstUnderscore != NULL) && ((firstUnderscore - myName) < (int)myNameLen) )
 772                                 myNameLen = firstUnderscore - myName;
 773
 774                         // case 1) The dylib has a parent umbrella, and we are creating the parent umbrella
 775                         isParent = ( (strlen(dylibParentName) == myNameLen) && (strncmp(myName, dylibParentName, myNameLen) == 0) );
 776
 777                         // case 2) The dylib has a parent umbrella, and we are creating a sibling with the same parent
 778                         isSibling = ( (_options.umbrellaName() != NULL) && (strcmp(_options.umbrellaName(), dylibParentName) == 0) );
 779                 }
 780
 781                 if ( !isParent && !isSibling && (clients != NULL) ) {
 782                         // case 3) the dylib has a list of allowable clients, and we are creating one of them
 783                         const char* clientName = _options.clientName();
 784                         int clientNameLen = 0;
 785                         if ( clientName != NULL ) {
 786                                 // use client name as specified on command line
 787                                 clientNameLen = strlen(clientName);
 788                         }
 789                         else {
 790                                 // infer client name from output path (e.g. xxx/libfoo_variant.A.dylib --> foo, Bar.framework/Bar_variant --> Bar)
 791                                 clientName = installName;
 792                                 clientNameLen = strlen(clientName);
 793                                 // starts after last slash
 794                                 if ( installNameLastSlash != NULL )
 795                                         clientName = &installNameLastSlash[1];
 796                                 if ( strncmp(clientName, "lib", 3) == 0 )
 797                                         clientName = &clientName[3];
 798                                 // up to first dot
 799                                 const char* firstDot = strchr(clientName, '.');
 800                                 if ( firstDot != NULL )
 801                                         clientNameLen = firstDot - clientName;
 802                                 // up to first underscore
 803                                 const char* firstUnderscore = strchr(clientName, '_');
 804                                 if ( (firstUnderscore != NULL) && ((firstUnderscore - clientName) < clientNameLen) )
 805                                         clientNameLen = firstUnderscore - clientName;
 806                         }
 807
 808                         // Use clientName to check if this dylib is able to link against the allowable clients.
 809                         for (std::vector<const char*>::const_iterator it = clients->begin(); it != clients->end(); it++) {
 810                                 if ( strncmp(*it, clientName, clientNameLen) == 0 )
 811                                         isAllowableClient = true;
 812                         }
 813                 }
 814
 815                 if ( !isParent && !isSibling && !isAllowableClient ) {
 816                         if ( dylibParentName != NULL ) {
 817                                 throwf("cannot link directly with %s.  Link against the umbrella framework '%s.framework' instead.",
 818                                         dylib->path(), dylibParentName);
 819                         }
 820                         else {
 821                                 throwf("cannot link directly with %s", dylib->path());
 822                         }
 823                 }
 824         }
 825 }
 826
 827
 828 void InputFiles::inferArchitecture(Options& opts, const char** archName)
 829 {
 830         _inferredArch = true;
 831         // scan all input files, looking for a thin .o file.
 832         // the first one found is presumably the architecture to link
 833         uint8_t buffer[4096];
 834         const std::vector<Options::FileInfo>& files = opts.getInputFiles();
 835         for (std::vector<Options::FileInfo>::const_iterator it = files.begin(); it != files.end(); ++it) {
 836                 int fd = ::open(it->path, O_RDONLY, 0);
 837                 if ( fd != -1 ) {
 838                         struct stat stat_buf;
 839                         if ( fstat(fd, &stat_buf) != -1) {
 840                                 ssize_t readAmount = stat_buf.st_size;
 841                                 if ( 4096 < readAmount )
 842                                         readAmount = 4096;
 843                                 ssize_t amount = read(fd, buffer, readAmount);
 844                                 ::close(fd);
 845                                 if ( amount >= readAmount ) {
 846                                         cpu_type_t type;
 847                                         cpu_subtype_t subtype;
 848                                         Options::Platform platform;
 849                                         if ( mach_o::relocatable::isObjectFile(buffer, &type, &subtype, &platform) ) {
 850                                                 opts.setArchitecture(type, subtype, platform);
 851                                                 *archName = opts.architectureName();
 852                                                 return;
 853                                         }
 854                                 }
 855                         }
 856                 }
 857         }
 858
 859         // no thin .o files found, so default to same architecture this tool was built as
 860         warning("-arch not specified");
 861 #if __i386__
 862         opts.setArchitecture(CPU_TYPE_I386, CPU_SUBTYPE_X86_ALL, Options::kPlatformOSX);
 863 #elif __x86_64__
 864         opts.setArchitecture(CPU_TYPE_X86_64, CPU_SUBTYPE_X86_64_ALL, Options::kPlatformOSX);
 865 #elif __arm__
 866         opts.setArchitecture(CPU_TYPE_ARM, CPU_SUBTYPE_ARM_V6, Options::kPlatformOSX);
 867 #else
 868         #error unknown default architecture
 869 #endif
 870         *archName = opts.architectureName();
 871 }
 872
 873
 874 InputFiles::InputFiles(Options& opts, const char** archName)
 875  : _totalObjectSize(0), _totalArchiveSize(0),
 876    _totalObjectLoaded(0), _totalArchivesLoaded(0), _totalDylibsLoaded(0),
 877         _options(opts), _bundleLoader(NULL),
 878         _inferredArch(false),
 879         _exception(NULL),
 880         _indirectDylibOrdinal(ld::File::Ordinal::indirectDylibBase()),
 881         _linkerOptionOrdinal(ld::File::Ordinal::linkeOptionBase())
 882 {
 883 //      fStartCreateReadersTime = mach_absolute_time();
 884         if ( opts.architecture() == 0 ) {
 885                 // command line missing -arch, so guess arch
 886                 inferArchitecture(opts, archName);
 887         }
 888 #if HAVE_PTHREADS
 889         pthread_mutex_init(&_parseLock, NULL);
 890         pthread_cond_init(&_parseWorkReady, NULL);
 891         pthread_cond_init(&_newFileAvailable, NULL);
 892 #endif
 893         const std::vector<Options::FileInfo>& files = _options.getInputFiles();
 894         if ( files.size() == 0 )
 895                 throw "no object files specified";
 896
 897         _inputFiles.reserve(files.size());
 898 #if HAVE_PTHREADS
 899         unsigned int inputFileSlot = 0;
 900         _availableInputFiles = 0;
 901         _parseCursor = 0;
 902 #endif
 903         Options::FileInfo* entry;
 904         for (std::vector<Options::FileInfo>::const_iterator it = files.begin(); it != files.end(); ++it) {
 905                 entry = (Options::FileInfo*)&(*it);
 906 #if HAVE_PTHREADS
 907                 // Assign input file slots to all the FileInfos.
 908                 // Also chain all FileInfos into one big list to set up for worker threads to do parsing.
 909                 entry->inputFileSlot = inputFileSlot;
 910                 entry->readyToParse = !entry->fromFileList || !_options.pipelineEnabled();
 911                 if (entry->readyToParse)
 912                         _availableInputFiles++;
 913                 _inputFiles.push_back(NULL);
 914                 inputFileSlot++;
 915 #else
 916                 // In the non-threaded case just parse the file now.
 917                 _inputFiles.push_back(makeFile(*entry, false));
 918 #endif
 919         }
 920
 921 #if HAVE_PTHREADS
 922         _remainingInputFiles = files.size();
 923
 924         // initialize info for parsing input files on worker threads
 925         unsigned int ncpus;
 926         int mib[2];
 927         size_t len = sizeof(ncpus);
 928         mib[0] = CTL_HW;
 929         mib[1] = HW_NCPU;
 930         if (sysctl(mib, 2, &ncpus, &len, NULL, 0) != 0) {
 931                 ncpus = 1;
 932         }
 933         _availableWorkers = MIN(ncpus, files.size()); // max # workers we permit
 934         _idleWorkers = 0;
 935
 936         if (_options.pipelineEnabled()) {
 937                 // start up a thread to listen for available input files
 938                 startThread(InputFiles::waitForInputFiles);
 939         }
 940
 941         // Start up one parser thread. More start on demand as parsed input files get consumed.
 942         startThread(InputFiles::parseWorkerThread);
 943         _availableWorkers--;
 944 #else
 945         if (_options.pipelineEnabled()) {
 946                 throwf("pipelined linking not supported on this platform");
 947         }
 948 #endif
 949 }
 950
 951
 952 #if HAVE_PTHREADS
 953 void InputFiles::startThread(void (*threadFunc)(InputFiles *)) const {
 954         pthread_t thread;
 955         pthread_attr_t attr;
 956         pthread_attr_init(&attr);
 957         // set a nice big stack (same as main thread) because some code uses potentially large stack buffers
 958         pthread_attr_setstacksize(&attr, 8 * 1024 * 1024);
 959         pthread_create(&thread, &attr, (void *(*)(void*))threadFunc, (void *)this);
 960         pthread_detach(thread);
 961         pthread_attr_destroy(&attr);
 962 }
 963
 964 // Work loop for input file parsing threads
 965 void InputFiles::parseWorkerThread() {
 966         ld::File *file;
 967         const char *exception = NULL;
 968         pthread_mutex_lock(&_parseLock);
 969         const std::vector<Options::FileInfo>& files = _options.getInputFiles();
 970         if (_s_logPThreads) printf("worker starting\n");
 971         do {
 972                 if (_availableInputFiles == 0) {
 973                         _idleWorkers++;
 974                         pthread_cond_wait(&_parseWorkReady, &_parseLock);
 975                         _idleWorkers--;
 976                 } else {
 977                         int slot = _parseCursor;
 978                         while (slot < (int)files.size() && (_inputFiles[slot] != NULL || !files[slot].readyToParse))
 979                                 slot++;
 980                         assert(slot < (int)files.size());
 981                         Options::FileInfo& entry = (Options::FileInfo&)files[slot];
 982                         _parseCursor = slot+1;
 983                         _availableInputFiles--;
 984                         entry.readyToParse = false; // to avoid multiple threads finding this file
 985                         pthread_mutex_unlock(&_parseLock);
 986                         if (_s_logPThreads) printf("parsing index %u\n", slot);
 987                         try {
 988                                 file = makeFile(entry, false);
 989                         }
 990                         catch (const char *msg) {
 991                                 if ( (strstr(msg, "architecture") != NULL) && !_options.errorOnOtherArchFiles() ) {
 992                                         if ( _options.ignoreOtherArchInputFiles() ) {
 993                                                 // ignore, because this is about an architecture not in use
 994                                         }
 995                                         else {
 996                                                 warning("ignoring file %s, %s", entry.path, msg);
 997                                         }
 998                                 }
 999                                 else if ( strstr(msg, "ignoring unexpected") != NULL ) {
1000                                         warning("%s, %s", entry.path, msg);
1001                                 }
1002                                 else {
1003                                         asprintf((char**)&exception, "%s file '%s'", msg, entry.path);
1004                                 }
1005                                 file = new IgnoredFile(entry.path, entry.modTime, entry.ordinal, ld::File::Other);
1006                         }
1007                         pthread_mutex_lock(&_parseLock);
1008                         if (_remainingInputFiles > 0)
1009                                 _remainingInputFiles--;
1010                         if (_s_logPThreads) printf("done with index %u, %d remaining\n", slot, _remainingInputFiles);
1011                         if (exception) {
1012                                 // We are about to die, so set to zero to stop other threads from doing unneeded work.
1013                                 _remainingInputFiles = 0;
1014                                 _exception = exception;
1015                         }
1016                         else {
1017                                 _inputFiles[slot] = file;
1018                                 if (_neededFileSlot == slot)
1019                                         pthread_cond_signal(&_newFileAvailable);
1020                         }
1021                 }
1022         } while (_remainingInputFiles);
1023         if (_s_logPThreads) printf("worker exiting\n");
1024         pthread_cond_broadcast(&_parseWorkReady);
1025         pthread_cond_signal(&_newFileAvailable);
1026         pthread_mutex_unlock(&_parseLock);
1027 }
1028
1029
1030 void InputFiles::parseWorkerThread(InputFiles *inputFiles) {
1031         inputFiles->parseWorkerThread();
1032 }
1033 #endif
1034
1035
1036 ld::File* InputFiles::addDylib(ld::dylib::File* reader, const Options::FileInfo& info)
1037 {
1038         _allDylibs.insert(reader);
1039
1040         if ( (reader->installPath() == NULL) && !info.options.fBundleLoader ) {
1041                 // this is a "blank" stub
1042                 // silently ignore it
1043                 return reader;
1044         }
1045         // store options about how dylib will be used in dylib itself
1046         if ( info.options.fWeakImport )
1047                 reader->setForcedWeakLinked();
1048         if ( info.options.fReExport )
1049                 reader->setWillBeReExported();
1050         if ( info.options.fUpward ) {
1051                 if ( _options.outputKind() == Options::kDynamicLibrary )
1052                         reader->setWillBeUpwardDylib();
1053                 else
1054                         warning("ignoring upward dylib option for %s\n", info.path);
1055         }
1056         if ( info.options.fLazyLoad )
1057                 reader->setWillBeLazyLoadedDylb();
1058
1059         // add to map of loaded dylibs
1060         const char* installPath = reader->installPath();
1061         if ( installPath != NULL ) {
1062                 InstallNameToDylib::iterator pos = _installPathToDylibs.find(installPath);
1063                 if ( pos == _installPathToDylibs.end() ) {
1064                         _installPathToDylibs[strdup(installPath)] = reader;
1065                 }
1066                 else {
1067                         bool dylibOnCommandLineTwice = ( strcmp(pos->second->path(), reader->path()) == 0 );
1068                         bool isSymlink = false;
1069                         // ignore if this is a symlink to a dylib we've already loaded
1070                         if ( !dylibOnCommandLineTwice ) {
1071                                 char existingDylibPath[PATH_MAX];
1072                                 if ( realpath(pos->second->path(), existingDylibPath) != NULL ) {
1073                                         char newDylibPath[PATH_MAX];
1074                                         if ( realpath(reader->path(), newDylibPath) != NULL ) {
1075                                                 isSymlink = ( strcmp(existingDylibPath, newDylibPath) == 0 );
1076                                         }
1077                                 }
1078                         }
1079                         // remove warning for <rdar://problem/10860629> Same install name for CoreServices and CFNetwork?
1080                         //if ( !dylibOnCommandLineTwice && !isSymlink )
1081                         //      warning("dylibs with same install name: %p %s and %p %s", pos->second, pos->second->path(), reader, reader->path());
1082                 }
1083         }
1084         else if ( info.options.fBundleLoader )
1085                 _bundleLoader = reader;
1086
1087         // log direct readers
1088         if ( ! info.options.fIndirectDylib )
1089                 this->logDylib(reader, false);
1090
1091         // update stats
1092         _totalDylibsLoaded++;
1093
1094         // just add direct libraries to search-first list
1095         if ( ! info.options.fIndirectDylib )
1096                 _searchLibraries.push_back(LibraryInfo(reader));
1097
1098         return reader;
1099 }
1100
1101
1102 #if HAVE_PTHREADS
1103 // Called during pipelined linking to listen for available input files.
1104 // Available files are enqueued for parsing.
1105 void InputFiles::waitForInputFiles()
1106 {
1107         if (_s_logPThreads) printf("starting pipeline listener\n");
1108         try {
1109                 const char *fifo = _options.pipelineFifo();
1110                 assert(fifo);
1111                 std::map<const char *, const Options::FileInfo*, strcompclass> fileMap;
1112                 const std::vector<Options::FileInfo>& files = _options.getInputFiles();
1113                 for (std::vector<Options::FileInfo>::const_iterator it = files.begin(); it != files.end(); ++it) {
1114                         const Options::FileInfo& entry = *it;
1115                         if (entry.fromFileList) {
1116                                 fileMap[entry.path] = &entry;
1117                         }
1118                 }
1119                 FILE *fileStream = fopen(fifo, "r");
1120                 if (!fileStream)
1121                         throwf("pipelined linking error - failed to open stream. fopen() returns %s for \"%s\"\n", strerror(errno), fifo);
1122                 while (fileMap.size() > 0) {
1123                         char path_buf[PATH_MAX+1];
1124                         if (fgets(path_buf, PATH_MAX, fileStream) == NULL)
1125                                 throwf("pipelined linking error - %lu missing input files", fileMap.size());
1126                         int len = strlen(path_buf);
1127                         if (path_buf[len-1] == '\n')
1128                                 path_buf[len-1] = 0;
1129                         std::map<const char *, const Options::FileInfo*, strcompclass>::iterator it = fileMap.find(path_buf);
1130                         if (it == fileMap.end())
1131                                 throwf("pipelined linking error - not in file list: %s\n", path_buf);
1132                         Options::FileInfo* inputInfo = (Options::FileInfo*)it->second;
1133                         if (!inputInfo->checkFileExists(_options))
1134                                 throwf("pipelined linking error - file does not exist: %s\n", inputInfo->path);
1135                         pthread_mutex_lock(&_parseLock);
1136                         if (_idleWorkers)
1137                                 pthread_cond_signal(&_parseWorkReady);
1138                         inputInfo->readyToParse = true;
1139                         if (_parseCursor > inputInfo->inputFileSlot)
1140                                 _parseCursor = inputInfo->inputFileSlot;
1141                         _availableInputFiles++;
1142                         if (_s_logPThreads) printf("pipeline listener: %s slot=%d, _parseCursor=%d, _availableInputFiles = %d remaining = %ld\n", path_buf, inputInfo->inputFileSlot, _parseCursor, _availableInputFiles, fileMap.size()-1);
1143                         pthread_mutex_unlock(&_parseLock);
1144                         fileMap.erase(it);
1145                 }
1146         } catch (const char *msg) {
1147                 pthread_mutex_lock(&_parseLock);
1148                 _exception = msg;
1149                 pthread_cond_signal(&_newFileAvailable);
1150                 pthread_mutex_unlock(&_parseLock);
1151         }
1152 }
1153
1154
1155 void InputFiles::waitForInputFiles(InputFiles *inputFiles) {
1156         inputFiles->waitForInputFiles();
1157 }
1158 #endif
1159
1160
1161 void InputFiles::forEachInitialAtom(ld::File::AtomHandler& handler, ld::Internal& state)
1162 {
1163         // add all direct object, archives, and dylibs
1164         const std::vector<Options::FileInfo>& files = _options.getInputFiles();
1165         size_t fileIndex;
1166         for (fileIndex=0; fileIndex<_inputFiles.size(); fileIndex++) {
1167                 ld::File *file;
1168 #if HAVE_PTHREADS
1169                 pthread_mutex_lock(&_parseLock);
1170
1171                 // this loop waits for the needed file to be ready (parsed by worker thread)
1172                 while (_inputFiles[fileIndex] == NULL && _exception == NULL) {
1173                         // We are starved for input. If there are still files to parse and we have
1174                         // not maxed out the worker thread count start a new worker thread.
1175                         if (_availableInputFiles > 0 && _availableWorkers > 0) {
1176                                 if (_s_logPThreads) printf("starting worker\n");
1177                                 startThread(InputFiles::parseWorkerThread);
1178                                 _availableWorkers--;
1179                         }
1180                         _neededFileSlot = fileIndex;
1181                         if (_s_logPThreads) printf("consumer blocking for %lu: %s\n", fileIndex, files[fileIndex].path);
1182                         pthread_cond_wait(&_newFileAvailable, &_parseLock);
1183                 }
1184
1185                 if (_exception)
1186                         throw _exception;
1187
1188                 // The input file is parsed. Assimilate it and call its atom iterator.
1189                 if (_s_logPThreads) printf("consuming slot %lu\n", fileIndex);
1190                 file = _inputFiles[fileIndex];
1191                 pthread_mutex_unlock(&_parseLock);
1192 #else
1193                 file = _inputFiles[fileIndex];
1194 #endif
1195                 const Options::FileInfo& info = files[fileIndex];
1196                 switch (file->type()) {
1197                         case ld::File::Reloc:
1198                         {
1199                                 ld::relocatable::File* reloc = (ld::relocatable::File*)file;
1200                                 _options.snapshot().recordObjectFile(reloc->path());
1201                                 if ( _options.dumpDependencyInfo() )
1202                                         _options.dumpDependency(Options::depObjectFile, reloc->path());
1203                         }
1204                                 break;
1205                         case ld::File::Dylib:
1206                         {
1207                                 ld::dylib::File* dylib = (ld::dylib::File*)file;
1208                                 addDylib(dylib, info);
1209                         }
1210                                 break;
1211                         case ld::File::Archive:
1212                         {
1213                                 ld::archive::File* archive = (ld::archive::File*)file;
1214                                 // <rdar://problem/9740166> force loaded archives should be in LD_TRACE
1215                                 if ( (info.options.fForceLoad || _options.fullyLoadArchives()) && _options.traceArchives() )
1216                                         logArchive(archive);
1217                                 _searchLibraries.push_back(LibraryInfo(archive));
1218                                 if ( _options.dumpDependencyInfo() )
1219                                         _options.dumpDependency(Options::depArchive, archive->path());
1220                         }
1221                                 break;
1222                         case ld::File::Other:
1223                                 break;
1224                         default:
1225                         {
1226                                 throwf("Unknown file type for %s", file->path());
1227                         }
1228                                 break;
1229                 }
1230                 file->forEachAtom(handler);
1231         }
1232
1233         markExplicitlyLinkedDylibs();
1234         addLinkerOptionLibraries(state, handler);
1235         createIndirectDylibs();
1236         createOpaqueFileSections();
1237
1238         while (fileIndex < _inputFiles.size()) {
1239                 ld::File *file = _inputFiles[fileIndex];
1240                 file->forEachAtom(handler);
1241                 fileIndex++;
1242         }
1243
1244     switch ( _options.outputKind() ) {
1245         case Options::kStaticExecutable:
1246         case Options::kDynamicExecutable:
1247             // add implicit __dso_handle label
1248             handler.doAtom(DSOHandleAtom::_s_atomExecutable);
1249             handler.doAtom(DSOHandleAtom::_s_atomAll);
1250             if ( _options.pageZeroSize() != 0 )
1251                 handler.doAtom(*new PageZeroAtom(_options.pageZeroSize()));
1252             if ( _options.hasCustomStack() && !_options.needsEntryPointLoadCommand() )
1253                 handler.doAtom(*new CustomStackAtom(_options.customStackSize()));
1254             break;
1255         case Options::kDynamicLibrary:
1256             // add implicit __dso_handle label
1257             handler.doAtom(DSOHandleAtom::_s_atomDylib);
1258             handler.doAtom(DSOHandleAtom::_s_atomAll);
1259             break;
1260         case Options::kDynamicBundle:
1261             // add implicit __dso_handle label
1262             handler.doAtom(DSOHandleAtom::_s_atomBundle);
1263             handler.doAtom(DSOHandleAtom::_s_atomAll);
1264             break;
1265         case Options::kDyld:
1266             // add implicit __dso_handle label
1267             handler.doAtom(DSOHandleAtom::_s_atomDyld);
1268             handler.doAtom(DSOHandleAtom::_s_atomAll);
1269             break;
1270         case Options::kPreload:
1271             // add implicit __mh_preload_header label
1272             handler.doAtom(DSOHandleAtom::_s_atomPreload);
1273             // add implicit __dso_handle label, but put it in __text section because
1274             // with -preload the mach_header is no in the address space.
1275             handler.doAtom(DSOHandleAtom::_s_atomPreloadDSO);
1276             break;
1277         case Options::kObjectFile:
1278             handler.doAtom(DSOHandleAtom::_s_atomObjectFile);
1279             break;
1280         case Options::kKextBundle:
1281             // add implicit __dso_handle label
1282             handler.doAtom(DSOHandleAtom::_s_atomAll);
1283             break;
1284         }
1285 }
1286
1287
1288 bool InputFiles::searchLibraries(const char* name, bool searchDylibs, bool searchArchives, bool dataSymbolOnly, ld::File::AtomHandler& handler) const
1289 {
1290         // Check each input library.
1291     for (std::vector<LibraryInfo>::const_iterator it=_searchLibraries.begin(); it != _searchLibraries.end(); ++it) {
1292         LibraryInfo lib = *it;
1293         if (lib.isDylib()) {
1294             if (searchDylibs) {
1295                 ld::dylib::File *dylibFile = lib.dylib();
1296                 //fprintf(stderr, "searchLibraries(%s), looking in linked %s\n", name, dylibFile->path() );
1297                 if ( dylibFile->justInTimeforEachAtom(name, handler) ) {
1298                     // we found a definition in this dylib
1299                     // done, unless it is a weak definition in which case we keep searching
1300                     _options.snapshot().recordDylibSymbol(dylibFile, name);
1301                     if ( !dylibFile->hasWeakExternals() || !dylibFile->hasWeakDefinition(name)) {
1302                         return true;
1303                     }
1304                     // else continue search for a non-weak definition
1305                 }
1306             }
1307         } else {
1308             if (searchArchives) {
1309                 ld::archive::File *archiveFile = lib.archive();
1310                 if ( dataSymbolOnly ) {
1311                     if ( archiveFile->justInTimeDataOnlyforEachAtom(name, handler) ) {
1312                         if ( _options.traceArchives() )
1313                             logArchive(archiveFile);
1314                         _options.snapshot().recordArchive(archiveFile->path());
1315                         // found data definition in static library, done
1316                        return true;
1317                     }
1318                 }
1319                 else {
1320                     if ( archiveFile->justInTimeforEachAtom(name, handler) ) {
1321                         if ( _options.traceArchives() )
1322                             logArchive(archiveFile);
1323                         _options.snapshot().recordArchive(archiveFile->path());
1324                         // found definition in static library, done
1325                         return true;
1326                     }
1327                 }
1328             }
1329         }
1330     }
1331
1332         // search indirect dylibs
1333         if ( searchDylibs ) {
1334                 for (InstallNameToDylib::const_iterator it=_installPathToDylibs.begin(); it != _installPathToDylibs.end(); ++it) {
1335                         ld::dylib::File* dylibFile = it->second;
1336                         bool searchThisDylib = false;
1337                         if ( _options.nameSpace() == Options::kTwoLevelNameSpace ) {
1338                                 // for two level namesapce, just check all implicitly linked dylibs
1339                                 searchThisDylib = dylibFile->implicitlyLinked() && !dylibFile->explicitlyLinked();
1340                         }
1341                         else {
1342                                 // for flat namespace, check all indirect dylibs
1343                                 searchThisDylib = ! dylibFile->explicitlyLinked();
1344                         }
1345                         if ( searchThisDylib ) {
1346                                 //fprintf(stderr, "searchLibraries(%s), looking in implicitly linked %s\n", name, dylibFile->path() );
1347                                 if ( dylibFile->justInTimeforEachAtom(name, handler) ) {
1348                                         // we found a definition in this dylib
1349                                         // done, unless it is a weak definition in which case we keep searching
1350                     _options.snapshot().recordDylibSymbol(dylibFile, name);
1351                                         if ( !dylibFile->hasWeakExternals() || !dylibFile->hasWeakDefinition(name)) {
1352                                                 return true;
1353                     }
1354                                         // else continue search for a non-weak definition
1355                                 }
1356                         }
1357                 }
1358         }
1359
1360         return false;
1361 }
1362
1363
1364 bool InputFiles::searchWeakDefInDylib(const char* name) const
1365 {
1366         // search all relevant dylibs to see if any have a weak-def with this name
1367         for (InstallNameToDylib::const_iterator it=_installPathToDylibs.begin(); it != _installPathToDylibs.end(); ++it) {
1368                 ld::dylib::File* dylibFile = it->second;
1369                 if ( dylibFile->implicitlyLinked() || dylibFile->explicitlyLinked() ) {
1370                         if ( dylibFile->hasWeakExternals() && dylibFile->hasWeakDefinition(name) ) {
1371                                 return true;
1372                         }
1373                 }
1374         }
1375         return false;
1376 }
1377
1378 static bool vectorContains(const std::vector<ld::dylib::File*>& vec, ld::dylib::File* key)
1379 {
1380         return std::find(vec.begin(), vec.end(), key) != vec.end();
1381 }
1382
1383 struct DylibByInstallNameSorter
1384 {
1385          bool operator()(const ld::dylib::File* left, const ld::dylib::File* right)
1386          {
1387           return (strcmp(left->installPath(), right->installPath()) < 0);
1388          }
1389 };
1390
1391 void InputFiles::dylibs(ld::Internal& state)
1392 {
1393         bool dylibsOK = false;
1394         switch ( _options.outputKind() ) {
1395                 case Options::kDynamicExecutable:
1396                 case Options::kDynamicLibrary:
1397                 case Options::kDynamicBundle:
1398                         dylibsOK = true;
1399                         break;
1400                 case Options::kStaticExecutable:
1401                 case Options::kDyld:
1402                 case Options::kPreload:
1403                 case Options::kObjectFile:
1404                 case Options::kKextBundle:
1405                         dylibsOK = false;
1406                         break;
1407         }
1408
1409         // add command line dylibs in order
1410         for (std::vector<ld::File*>::const_iterator it=_inputFiles.begin(); it != _inputFiles.end(); ++it) {
1411                 ld::dylib::File* dylibFile = dynamic_cast<ld::dylib::File*>(*it);
1412                 // only add dylibs that are not "blank" dylib stubs
1413                 if ( (dylibFile != NULL) && ((dylibFile->installPath() != NULL) || (dylibFile == _bundleLoader)) ) {
1414                         if ( dylibsOK ) {
1415                                 if ( ! vectorContains(state.dylibs, dylibFile) ) {
1416                                         state.dylibs.push_back(dylibFile);
1417                                 }
1418                         }
1419                         else
1420                                 warning("unexpected dylib (%s) on link line", dylibFile->path());
1421                 }
1422         }
1423         // add implicitly linked dylibs
1424         if ( _options.nameSpace() == Options::kTwoLevelNameSpace ) {
1425                 std::vector<ld::dylib::File*> implicitDylibs;
1426                 for (InstallNameToDylib::const_iterator it=_installPathToDylibs.begin(); it != _installPathToDylibs.end(); ++it) {
1427                         ld::dylib::File* dylibFile = it->second;
1428                         if ( dylibFile->implicitlyLinked() && dylibsOK ) {
1429                                 if ( ! vectorContains(implicitDylibs, dylibFile) ) {
1430                                         implicitDylibs.push_back(dylibFile);
1431                                 }
1432                         }
1433                 }
1434                 // <rdar://problem/15002251> make implicit dylib order be deterministic by sorting by install_name
1435                 std::sort(implicitDylibs.begin(), implicitDylibs.end(), DylibByInstallNameSorter());
1436                 state.dylibs.insert(state.dylibs.end(), implicitDylibs.begin(), implicitDylibs.end());
1437         }
1438
1439         //fprintf(stderr, "all dylibs:\n");
1440         //for(std::vector<ld::dylib::File*>::iterator it=state.dylibs.begin(); it != state.dylibs.end(); ++it) {
1441         //      const ld::dylib::File* dylib = *it;
1442         //      fprintf(stderr, "    %p impl=%d %s\n", dylib, dylib->implicitlyLinked(), dylib->path());
1443         //}
1444
1445         // and -bundle_loader
1446         state.bundleLoader = _bundleLoader;
1447
1448         // <rdar://problem/10807040> give an error when -nostdlib is used and libSystem is missing
1449         if ( (state.dylibs.size() == 0) && _options.needsEntryPointLoadCommand() )
1450                 throw "dynamic main executables must link with libSystem.dylib";
1451 }
1452
1453
1454 } // namespace tool
1455 } // namespace ld
1456