interlinked-dylibs/MachOProxy.cpp

   1 //
   2 //  DylibProxy.cpp
   3 //  dyld
   4 //
   5 //  Created by Louis Gerbarg on 1/27/16.
   6 //
   7 //
   8
   9 #include <mach-o/loader.h>
  10 #include <mach-o/fat.h>
  11
  12 #include "mega-dylib-utils.h"
  13 #include "Logging.h"
  14
  15 #include "Trie.hpp"
  16 #include "MachOProxy.h"
  17
  18 #ifndef EXPORT_SYMBOL_FLAGS_KIND_ABSOLUTE
  19 #define EXPORT_SYMBOL_FLAGS_KIND_ABSOLUTE 0x02
  20 #endif
  21
  22 namespace {
  23 std::vector<MachOProxy*> mapMachOFile(const std::string& buildPath, const std::string& path)
  24 {
  25     std::vector<MachOProxy*> retval;
  26     const uint8_t* p = (uint8_t*)( -1 );
  27     struct stat stat_buf;
  28         bool rootless;
  29
  30     std::tie(p, stat_buf, rootless) = fileCache.cacheLoad(buildPath);
  31
  32     if (p == (uint8_t*)(-1)) {
  33         return retval;
  34     }
  35
  36     // if fat file, process each architecture
  37     const fat_header* fh = (fat_header*)p;
  38     const mach_header* mh = (mach_header*)p;
  39     if ( OSSwapBigToHostInt32( fh->magic ) == FAT_MAGIC ) {
  40         // Fat header is always big-endian
  41         const fat_arch* slices = (const fat_arch*)( (char*)fh + sizeof( fat_header ) );
  42         const uint32_t sliceCount = OSSwapBigToHostInt32( fh->nfat_arch );
  43         for ( uint32_t i = 0; i < sliceCount; ++i ) {
  44             // FIXME Should we validate the fat header matches the slices?
  45             ArchPair arch( OSSwapBigToHostInt32( slices[i].cputype ), OSSwapBigToHostInt32( slices[i].cpusubtype ) );
  46             uint32_t fileOffset = OSSwapBigToHostInt32( slices[i].offset );
  47             const mach_header* th = (mach_header*)(p+fileOffset);
  48             if ( ( OSSwapLittleToHostInt32( th->magic ) == MH_MAGIC ) || ( OSSwapLittleToHostInt32( th->magic ) == MH_MAGIC_64 ) ) {
  49                 uint32_t fileSize = static_cast<uint32_t>( stat_buf.st_size );
  50                 retval.push_back(new MachOProxy(buildPath, path, stringForArch(arch), stat_buf.st_ino, stat_buf.st_mtime, fileOffset, fileSize, rootless));
  51                 //retval[stringForArch( arch )] = new MachOProxy( path, stat_buf.st_ino, stat_buf.st_mtime, fileOffset, fileSize, rootless );
  52             }
  53         }
  54     } else if ( ( OSSwapLittleToHostInt32( mh->magic ) == MH_MAGIC ) || ( OSSwapLittleToHostInt32( mh->magic ) == MH_MAGIC_64 ) ) {
  55         ArchPair arch( OSSwapLittleToHostInt32( mh->cputype ), OSSwapLittleToHostInt32( mh->cpusubtype ) );
  56         uint32_t fileOffset = OSSwapBigToHostInt32( 0 );
  57         uint32_t fileSize = static_cast<uint32_t>( stat_buf.st_size );
  58         retval.push_back(new MachOProxy(buildPath, path, stringForArch(arch), stat_buf.st_ino, stat_buf.st_mtime, fileOffset, fileSize, rootless));
  59         //retval[stringForArch( arch )] = new MachOProxy( path, stat_buf.st_ino, stat_buf.st_mtime, fileOffset, fileSize, rootless );
  60     } else {
  61         //    warning( "file '%s' is not contain requested a MachO", path.c_str() );
  62     }
  63     return retval;
  64 }
  65
  66 } /* Anonymous namespace */
  67
  68 template <typename P>
  69 std::vector<std::string> MachOProxy::dependencies()
  70 {
  71     const uint8_t*                     buffer = getBuffer();
  72     const macho_header<P>*             mh = (const macho_header<P>*)buffer;
  73     const macho_load_command<P>* const cmds = (macho_load_command<P>*)((uint8_t*)mh + sizeof(macho_header<P>));
  74     const uint32_t                     cmd_count = mh->ncmds();
  75     const macho_load_command<P>*       cmd = cmds;
  76     std::vector<std::string>           retval;
  77
  78     for (uint32_t i = 0; i < cmd_count; ++i) {
  79         switch (cmd->cmd()) {
  80             case LC_LOAD_DYLIB:
  81             case LC_LOAD_WEAK_DYLIB:
  82             case LC_REEXPORT_DYLIB:
  83             case LC_LOAD_UPWARD_DYLIB: {
  84                 macho_dylib_command<P>* dylib = (macho_dylib_command<P>*)cmd;
  85                 std::string             depName = dylib->name();
  86
  87                 retval.push_back(depName);
  88             } break;
  89         }
  90         cmd = (const macho_load_command<P>*)(((uint8_t*)cmd) + cmd->cmdsize());
  91     }
  92
  93     return retval;
  94 }
  95
  96 template <typename P>
  97 std::vector<std::string> MachOProxy::reexports()
  98 {
  99     const uint8_t*                     buffer = getBuffer();
 100     const macho_header<P>*             mh = (const macho_header<P>*)buffer;
 101     const macho_load_command<P>* const cmds = (macho_load_command<P>*)((uint8_t*)mh + sizeof(macho_header<P>));
 102     const uint32_t                     cmd_count = mh->ncmds();
 103     const macho_load_command<P>*       cmd = cmds;
 104     std::vector<std::string>           retval;
 105
 106     for (uint32_t i = 0; i < cmd_count; ++i) {
 107         switch (cmd->cmd()) {
 108         case LC_REEXPORT_DYLIB: {
 109             macho_dylib_command<P>* dylib = (macho_dylib_command<P>*)cmd;
 110             std::string             depName = dylib->name();
 111
 112             retval.push_back(depName);
 113         } break;
 114         }
 115         cmd = (const macho_load_command<P>*)(((uint8_t*)cmd) + cmd->cmdsize());
 116     }
 117
 118     return retval;
 119 }
 120
 121 std::vector<std::string> MachOProxy::dependencies()
 122 {
 123     switch (archForString(arch).arch) {
 124         case CPU_TYPE_ARM:
 125         case CPU_TYPE_I386:
 126             return dependencies<Pointer32<LittleEndian>>();
 127         case CPU_TYPE_X86_64:
 128         case CPU_TYPE_ARM64:
 129             return dependencies<Pointer64<LittleEndian>>();
 130             break;
 131         default:
 132             return std::vector<std::string>();
 133         }
 134 }
 135
 136 std::vector<std::string> MachOProxy::reexports()
 137 {
 138     switch (archForString(arch).arch) {
 139         case CPU_TYPE_ARM:
 140         case CPU_TYPE_I386:
 141             return reexports<Pointer32<LittleEndian>>();
 142         case CPU_TYPE_X86_64:
 143         case CPU_TYPE_ARM64:
 144             return reexports<Pointer64<LittleEndian>>();
 145             break;
 146         default:
 147             return std::vector<std::string>();
 148     }
 149 }
 150
 151 template <typename P>
 152 std::string MachOProxy::machoParser(bool ignoreUncacheableDylibsInExecutables)
 153 {
 154     const uint8_t*                     buffer = getBuffer();
 155     bool                               hasSplitSegInfo = false;
 156     bool                               hasDylidInfo = false;
 157     const macho_header<P>*             mh = (const macho_header<P>*)buffer;
 158     const macho_symtab_command<P>*     symTab = nullptr;
 159     const macho_dysymtab_command<P>*   dynSymTab = nullptr;
 160     const macho_load_command<P>* const cmds = (macho_load_command<P>*)((uint8_t*)mh + sizeof(macho_header<P>));
 161     const macho_dyld_info_command<P>*  dyldInfo = nullptr;
 162     const uint32_t                     cmd_count = mh->ncmds();
 163     const macho_load_command<P>*       cmd = cmds;
 164     uint64_t                           baseAddr = 0;
 165     _filetype = mh->filetype();
 166     if (_filetype == MH_DYLIB_STUB) {
 167         return "stub dylib";
 168     }
 169     if (_filetype == MH_DSYM) {
 170         return "DSYM";
 171     }
 172     for (uint32_t i = 0; i < cmd_count; ++i) {
 173         switch (cmd->cmd()) {
 174         case LC_ID_DYLIB: {
 175             macho_dylib_command<P>* dylib = (macho_dylib_command<P>*)cmd;
 176             if (dylib->name()[0] != '/') {
 177                 if (strncmp(dylib->name(), "@rpath", 6) == 0)
 178                     return "@rpath cannot be used in -install_name for OS dylibs";
 179                 else
 180                     return "-install_name is not an absolute path";
 181             }
 182             installName = dylib->name();
 183             installNameOffsetInTEXT = (uint32_t)((uint8_t*)cmd - buffer) + dylib->name_offset();
 184             addAlias(path);
 185         } break;
 186         case LC_UUID: {
 187             const macho_uuid_command<P>* uuidCmd = (macho_uuid_command<P>*)cmd;
 188             uuid = UUID(uuidCmd->uuid());
 189         } break;
 190         case LC_LOAD_DYLIB:
 191         case LC_LOAD_WEAK_DYLIB:
 192         case LC_REEXPORT_DYLIB:
 193         case LC_LOAD_UPWARD_DYLIB: {
 194             macho_dylib_command<P>* dylib = (macho_dylib_command<P>*)cmd;
 195             std::string             depName = dylib->name();
 196             if ( isExecutable() && ignoreUncacheableDylibsInExecutables && !has_prefix(depName, "/usr/lib/") && !has_prefix(depName, "/System/Library/") ) {
 197                 // <rdar://problem/25918268> in update_dyld_shared_cache don't warn if root executable links with something not eligible for shared cache
 198                 break;
 199             }
 200             else if ( depName[0] != '/' ) {
 201                 return "linked against a dylib whose -install_name was non-absolute (e.g. @rpath)";
 202             }
 203         } break;
 204         case macho_segment_command<P>::CMD: {
 205             const macho_segment_command<P>* segCmd = (macho_segment_command<P>*)cmd;
 206             MachOProxySegment               seg;
 207             seg.name = segCmd->segname();
 208             seg.size = align(segCmd->vmsize(), 12);
 209             seg.vmaddr = segCmd->vmaddr();
 210             seg.diskSize = (uint32_t)segCmd->filesize();
 211             seg.fileOffset = (uint32_t)segCmd->fileoff();
 212             seg.protection = segCmd->initprot();
 213             if (segCmd->nsects() > 0) {
 214                 seg.p2align = 0;
 215                 const macho_section<P>* const sectionsStart = (macho_section<P>*)((uint8_t*)segCmd + sizeof(macho_segment_command<P>));
 216                 const macho_section<P>* const sectionsLast = &sectionsStart[segCmd->nsects() - 1];
 217                 const macho_section<P>* const sectionsEnd = &sectionsStart[segCmd->nsects()];
 218                 for (const macho_section<P>* sect = sectionsStart; sect < sectionsEnd; ++sect) {
 219                     if (sect->align() > seg.p2align)
 220                         seg.p2align = sect->align();
 221                 }
 222                 seg.sizeOfSections = sectionsLast->addr() + sectionsLast->size() - segCmd->vmaddr();
 223             } else {
 224                 seg.p2align = 12;
 225             }
 226             segments.push_back(seg);
 227             if (seg.name == "__TEXT") {
 228                 baseAddr =  seg.vmaddr;
 229             }
 230         } break;
 231         case LC_SEGMENT_SPLIT_INFO:
 232             hasSplitSegInfo = true;
 233             break;
 234         case LC_SYMTAB:
 235             symTab = (macho_symtab_command<P>*)cmd;
 236             break;
 237         case LC_DYSYMTAB:
 238             dynSymTab = (macho_dysymtab_command<P>*)cmd;
 239             break;
 240         case LC_DYLD_INFO:
 241         case LC_DYLD_INFO_ONLY:
 242             dyldInfo = (macho_dyld_info_command<P>*)cmd;
 243             hasDylidInfo = true;
 244             break;
 245         }
 246         cmd = (const macho_load_command<P>*)(((uint8_t*)cmd) + cmd->cmdsize());
 247     }
 248
 249     identifier = uuid;
 250
 251     if (!hasDylidInfo) {
 252         return "built for old OS";
 253     }
 254
 255     if (dyldInfo && dyldInfo->bind_size() != 0) {
 256         _bind_offset = dyldInfo->bind_off();
 257         _bind_size = dyldInfo->bind_size();
 258     }
 259
 260     if (dyldInfo && dyldInfo->lazy_bind_size() != 0) {
 261         _lazy_bind_offset = dyldInfo->lazy_bind_off();
 262         _lazy_bind_size = dyldInfo->lazy_bind_size();
 263     }
 264
 265     // if no export info, no _exports map to build
 266     if (dyldInfo && dyldInfo->export_size() != 0) {
 267         std::vector<ExportInfoTrie::Entry> exports;
 268         const uint8_t*                     exportsStart = &buffer[dyldInfo->export_off()];
 269         const uint8_t*                     exportsEnd = &exportsStart[dyldInfo->export_size()];
 270         if (!ExportInfoTrie::parseTrie(exportsStart, exportsEnd, exports)) {
 271             terminate("malformed exports trie in %s", path.c_str());
 272         }
 273
 274         for (const ExportInfoTrie::Entry& entry : exports) {
 275             if (!_exports[entry.name].isAbsolute) {
 276                 for (const auto& seg : segments) {
 277                     if (seg.size > 0 && (seg.vmaddr - baseAddr) <= entry.info.address && entry.info.address < (seg.vmaddr - baseAddr) + seg.size) {
 278                         _exports[entry.name].segmentOffset = entry.info.address - (seg.vmaddr - baseAddr);
 279                         _exports[entry.name].segmentName = seg.name;
 280                         break;
 281                     }
 282                 }
 283             } else {
 284                 _exports[entry.name].segmentOffset = (uint64_t)entry.info.address;
 285                 _exports[entry.name].segmentName = "";
 286             }
 287
 288             switch (entry.info.flags & EXPORT_SYMBOL_FLAGS_KIND_MASK) {
 289                 case EXPORT_SYMBOL_FLAGS_KIND_REGULAR:
 290                     if ((entry.info.flags & EXPORT_SYMBOL_FLAGS_STUB_AND_RESOLVER)) {
 291                         _exports[entry.name].isResolver = true;
 292                     }
 293                     if (entry.info.flags & EXPORT_SYMBOL_FLAGS_REEXPORT) {
 294                         SymbolInfo& info = _exports[entry.name];
 295                         info.isSymbolReExport = true;
 296                         info.reExportDylibIndex = (int)entry.info.other;
 297                         if (!entry.info.importName.empty())
 298                             info.reExportName = entry.info.importName;
 299                     }
 300                     break;
 301                 case EXPORT_SYMBOL_FLAGS_KIND_THREAD_LOCAL:
 302                     _exports[entry.name].isThreadLocal = true;
 303                     break;
 304                 case EXPORT_SYMBOL_FLAGS_KIND_ABSOLUTE:
 305                     _exports[entry.name].isAbsolute = true;
 306                     break;
 307                 default:
 308                     terminate("non-regular symbol binding not supported for %s in %s", entry.name.c_str(), path.c_str());
 309                     break;
 310             }
 311         }
 312     }
 313
 314     if (!isDylib()) {
 315         return "";
 316     }
 317
 318     if ((mh->flags() & MH_TWOLEVEL) == 0) {
 319         return "built with -flat_namespace";
 320     }
 321
 322     if (!hasSplitSegInfo) {
 323         bool inUsrLib = (installName.size() > 9) && (installName.substr(0, 9) == "/usr/lib/");
 324         bool inSystemLibrary = (installName.size() > 16) && (installName.substr(0, 16) == "/System/Library/");
 325         if (!inUsrLib && !inSystemLibrary) {
 326             return "-install_name not /usr/lib/* or /System/Library/*";
 327         }
 328         return "no shared region info";
 329     }
 330
 331     if ((symTab == nullptr) && (dynSymTab == nullptr)) {
 332         return "no symbol table";
 333     }
 334
 335     if (installName.empty()) {
 336         return "dylib missing install name";
 337     }
 338
 339     // scan undefines looking for invalid ordinals
 340     const macho_nlist<P>* symbolTable = (macho_nlist<P>*)((uint8_t*)mh + symTab->symoff());
 341     const uint32_t        startUndefs = dynSymTab->iundefsym();
 342     const uint32_t        endUndefs = startUndefs + dynSymTab->nundefsym();
 343     for (uint32_t i = startUndefs; i < endUndefs; ++i) {
 344         uint8_t ordinal = GET_LIBRARY_ORDINAL(symbolTable[i].n_desc());
 345         if (ordinal == DYNAMIC_LOOKUP_ORDINAL) {
 346             return "built with '-undefined dynamic_lookup'";
 347         } else if (ordinal == EXECUTABLE_ORDINAL) {
 348             return "built with -bundle_loader";
 349         }
 350     }
 351
 352     return "";
 353 }
 354
 355 const bool MachOProxy::isDylib()
 356 {
 357     return (_filetype == MH_DYLIB);
 358 }
 359
 360 const bool MachOProxy::isExecutable()
 361 {
 362     return (_filetype == MH_EXECUTE);
 363 }
 364
 365 static std::map<ImageIdentifier, MachOProxy*> identifierMap;
 366 std::map<std::pair<std::string, std::string>, MachOProxy*> archMap;
 367 static dispatch_queue_t identifierQueue;
 368
 369 MachOProxy* MachOProxy::forIdentifier(const ImageIdentifier& identifier, const std::string preferredArch)
 370 {
 371     auto i = identifierMap.find(identifier);
 372     // We need an identifier
 373     if (i == identifierMap.end())
 374         return nullptr;
 375
 376     // Is the identifier the arch we want?
 377     if (i->second->arch == preferredArch)
 378         return i->second;
 379
 380     // Fallback to a slow path to try to find a best fit
 381     return forInstallnameAndArch(i->second->installName, preferredArch);
 382 }
 383
 384 MachOProxy* MachOProxy::forInstallnameAndArch(const std::string& installname, const std::string& arch)
 385 {
 386     auto i = archMap.find(std::make_pair(installname, arch));
 387     if (i == archMap.end())
 388         i = archMap.find(std::make_pair(installname, fallbackArchStringForArchString(arch)));
 389     if (i != archMap.end())
 390         return i->second;
 391     return nullptr;
 392 }
 393
 394 void MachOProxy::mapDependencies()
 395 {
 396     // Build a complete map of all installname/alias,archs to their proxies
 397     runOnAllProxies(false, [&](MachOProxy* proxy) {
 398         archMap[std::make_pair(proxy->path, proxy->arch)] = proxy;
 399         for (auto& alias : proxy->installNameAliases) {
 400             archMap[std::make_pair(alias, proxy->arch)] = proxy;
 401         }
 402     });
 403
 404     //Wire up the dependencies
 405     runOnAllProxies(false, [&](MachOProxy* proxy) {
 406         auto dependencyInstallnames = proxy->dependencies();
 407         for (auto dependencyInstallname : dependencyInstallnames) {
 408             auto dependencyProxy = forInstallnameAndArch(dependencyInstallname, proxy->arch);
 409             if (dependencyProxy == nullptr) {
 410                 proxy->error = "Missing dependency: " + dependencyInstallname;
 411             } else {
 412                 proxy->requiredIdentifiers.push_back(dependencyProxy->identifier);
 413                 dependencyProxy->dependentIdentifiers.push_back(proxy->identifier);
 414             }
 415         }
 416
 417         auto reexportInstallnames = proxy->reexports();
 418         for (auto reexportInstallname : reexportInstallnames) {
 419             auto reexportProxy = forInstallnameAndArch(reexportInstallname, proxy->arch);
 420             if (reexportProxy == nullptr) {
 421                 proxy->error = "Missing reexport dylib: " + reexportInstallname;
 422             } else {
 423                 proxy->_reexportProxies.push_back(reexportProxy);
 424             }
 425         }
 426
 427     });
 428 }
 429
 430 void MachOProxy::runOnAllProxies(bool concurrently, std::function<void(MachOProxy* proxy)> lambda)
 431 {
 432     dispatch_group_t runGroup = dispatch_group_create();
 433     dispatch_queue_t runQueue = dispatch_get_global_queue(QOS_CLASS_USER_INITIATED, NULL);
 434
 435     for (auto& identifier : identifierMap) {
 436         if (concurrently) {
 437             cacheBuilderDispatchGroupAsync(runGroup, runQueue, [&] {
 438                 lambda(identifier.second);
 439             });
 440         } else {
 441             lambda(identifier.second);
 442         }
 443     }
 444
 445     dispatch_group_wait(runGroup, DISPATCH_TIME_FOREVER);
 446 }
 447
 448 std::map<std::string, MachOProxy*> MachOProxy::loadProxies(const std::string& buildPath, const std::string& path, bool warnOnProblems, bool ignoreUncacheableDylibsInExecutables)
 449 {
 450     std::vector<MachOProxy*> slices = mapMachOFile(buildPath, path);
 451     std::map<std::string, MachOProxy*> retval;
 452
 453     for ( auto& slice : slices ) {
 454         std::string errorMessage;
 455         verboseLog( "analyzing file '%s'", path.c_str() );
 456         switch (archForString(slice->arch).arch) {
 457             case CPU_TYPE_ARM:
 458             case CPU_TYPE_I386:
 459                 errorMessage = slice->machoParser<Pointer32<LittleEndian>>(ignoreUncacheableDylibsInExecutables);
 460                 break;
 461             case CPU_TYPE_X86_64:
 462             case CPU_TYPE_ARM64:
 463                 errorMessage = slice->machoParser<Pointer64<LittleEndian>>(ignoreUncacheableDylibsInExecutables);
 464                 break;
 465             default:
 466                 errorMessage = "unsupported arch '" + slice->arch + "'";
 467                 break;
 468         }
 469
 470         if (errorMessage.empty()) {
 471             static dispatch_once_t onceToken;
 472             dispatch_once(&onceToken, ^{
 473                 identifierQueue = dispatch_queue_create("com.apple.dyld.cache.metabom.ids", DISPATCH_QUEUE_SERIAL);
 474             });
 475             retval[slice->arch] = slice;
 476             dispatch_sync(identifierQueue, ^{
 477                 identifierMap[slice->identifier] = slice;
 478             });
 479         } else {
 480             if (warnOnProblems)
 481                 warning("%s (%s)", errorMessage.c_str(), path.c_str());
 482         }
 483     }
 484
 485     return retval;
 486 }
 487
 488 const uint8_t* MachOProxy::getBuffer() {
 489     const uint8_t* p = (uint8_t*)( -1 );
 490     struct stat stat_buf;
 491         bool rootless;
 492     std::tie(p, stat_buf, rootless) = fileCache.cacheLoad(buildPath);
 493     return p + fatFileOffset;
 494 }
 495
 496 bool MachOProxy::addAlias( const std::string& alias ) {
 497     if (!has_prefix(alias, "/usr/lib/") && !has_prefix(alias, "/System/Library/"))
 498         return false;
 499     if ( alias != installName && installNameAliases.count( alias ) == 0 ) {
 500         installNameAliases.insert( alias );
 501         return true;
 502     }
 503     return false;
 504 }