launch-cache/dsc_extractor.cpp

   1 /* -*- mode: C++; c-basic-offset: 4; tab-width: 4 -*-
   2  *
   3  * Copyright (c) 2011 Apple Inc. All rights reserved.
   4  *
   5  * @APPLE_LICENSE_HEADER_START@
   6  *
   7  * This file contains Original Code and/or Modifications of Original Code
   8  * as defined in and that are subject to the Apple Public Source License
   9  * Version 2.0 (the 'License'). You may not use this file except in
  10  * compliance with the License. Please obtain a copy of the License at
  11  * http://www.opensource.apple.com/apsl/ and read it before using this
  12  * file.
  13  *
  14  * The Original Code and all software distributed under the License are
  15  * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  16  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  17  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
  18  * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
  19  * Please see the License for the specific language governing rights and
  20  * limitations under the License.
  21  *
  22  * @APPLE_LICENSE_HEADER_END@
  23  */
  24
  25 #include <stdlib.h>
  26 #include <stdio.h>
  27 #include <unistd.h>
  28 #include <sys/stat.h>
  29 #include <string.h>
  30 #include <fcntl.h>
  31 #include <stdlib.h>
  32 #include <errno.h>
  33 #include <sys/mman.h>
  34 #include <sys/syslimits.h>
  35 #include <libkern/OSByteOrder.h>
  36 #include <mach-o/fat.h>
  37 #include <mach-o/arch.h>
  38 #include <mach-o/loader.h>
  39 #include <Availability.h>
  40
  41 #define NO_ULEB
  42 #include "Architectures.hpp"
  43 #include "MachOFileAbstraction.hpp"
  44 #include "CacheFileAbstraction.hpp"
  45
  46 #include "dsc_iterator.h"
  47 #include "dsc_extractor.h"
  48 #include "MachOTrie.hpp"
  49
  50 #include <vector>
  51 #include <set>
  52 #include <map>
  53 #include <unordered_map>
  54 #include <algorithm>
  55 #include <dispatch/dispatch.h>
  56
  57 struct seg_info
  58 {
  59                                 seg_info(const char* n, uint64_t o, uint64_t s)
  60                                         : segName(n), offset(o), sizem(s) { }
  61         const char* segName;
  62         uint64_t        offset;
  63         uint64_t        sizem;
  64 };
  65
  66 class CStringHash {
  67 public:
  68         size_t operator()(const char* __s) const {
  69                 size_t __h = 0;
  70                 for ( ; *__s; ++__s)
  71                         __h = 5 * __h + *__s;
  72                 return __h;
  73         };
  74 };
  75 class CStringEquals {
  76 public:
  77         bool operator()(const char* left, const char* right) const { return (strcmp(left, right) == 0); }
  78 };
  79 typedef std::unordered_map<const char*, std::vector<seg_info>, CStringHash, CStringEquals> NameToSegments;
  80
  81 // Filter to find individual symbol re-exports in trie
  82 class NotReExportSymbol {
  83 public:
  84         NotReExportSymbol(const std::set<int> &rd) :_reexportDeps(rd) {}
  85         bool operator()(const mach_o::trie::Entry &entry) const {
  86                 if ( (entry.flags & EXPORT_SYMBOL_FLAGS_KIND_MASK) != EXPORT_SYMBOL_FLAGS_KIND_REGULAR )
  87                         return true;
  88                 if ( (entry.flags & EXPORT_SYMBOL_FLAGS_REEXPORT) == 0 )
  89                         return true;
  90                 // If the symbol comes from a dylib that is re-exported, this is not an individual symbol re-export
  91                 if ( _reexportDeps.count(entry.other) != 0 )
  92                         return true;
  93                 return false;
  94         }
  95 private:
  96         const std::set<int> &_reexportDeps;
  97 };
  98
  99
 100 template <typename A>
 101 int optimize_linkedit(macho_header<typename A::P>* mh, uint64_t textOffsetInCache, const void* mapped_cache, uint64_t* newSize)
 102 {
 103         typedef typename A::P P;
 104         typedef typename A::P::E E;
 105     typedef typename A::P::uint_t pint_t;
 106
 107         // update header flags
 108         mh->set_flags(mh->flags() & 0x7FFFFFFF); // remove in-cache bit
 109
 110         // update load commands
 111         uint64_t cumulativeFileSize = 0;
 112         const macho_load_command<P>* const cmds = (macho_load_command<P>*)((uint8_t*)mh + sizeof(macho_header<P>));
 113         const uint32_t cmd_count = mh->ncmds();
 114         const macho_load_command<P>* cmd = cmds;
 115         macho_segment_command<P>* linkEditSegCmd = NULL;
 116         macho_symtab_command<P>* symtab = NULL;
 117         macho_dysymtab_command<P>*      dynamicSymTab = NULL;
 118         macho_linkedit_data_command<P>* functionStarts = NULL;
 119         macho_linkedit_data_command<P>* dataInCode = NULL;
 120         uint32_t exportsTrieOffset = 0;
 121         uint32_t exportsTrieSize = 0;
 122         std::set<int> reexportDeps;
 123         int depIndex = 0;
 124         for (uint32_t i = 0; i < cmd_count; ++i) {
 125                 switch ( cmd->cmd() ) {
 126                 case macho_segment_command<P>::CMD:
 127                         {
 128                         // update segment/section file offsets
 129                         macho_segment_command<P>* segCmd = (macho_segment_command<P>*)cmd;
 130                         segCmd->set_fileoff(cumulativeFileSize);
 131                         macho_section<P>* const sectionsStart = (macho_section<P>*)((char*)segCmd + sizeof(macho_segment_command<P>));
 132                         macho_section<P>* const sectionsEnd = &sectionsStart[segCmd->nsects()];
 133                         for(macho_section<P>* sect = sectionsStart; sect < sectionsEnd; ++sect) {
 134                                 if ( sect->offset() != 0 )
 135                                         sect->set_offset((uint32_t)(cumulativeFileSize+sect->addr()-segCmd->vmaddr()));
 136                         }
 137                         if ( strcmp(segCmd->segname(), "__LINKEDIT") == 0 ) {
 138                                 linkEditSegCmd = segCmd;
 139                         }
 140                         cumulativeFileSize += segCmd->filesize();
 141                         }
 142                         break;
 143                 case LC_DYLD_INFO_ONLY:
 144                         {
 145                         // zero out all dyld info
 146                         macho_dyld_info_command<P>* dyldInfo = (macho_dyld_info_command<P>*)cmd;
 147                         exportsTrieOffset = dyldInfo->export_off();
 148                         exportsTrieSize = dyldInfo->export_size();
 149                         dyldInfo->set_rebase_off(0);
 150                         dyldInfo->set_rebase_size(0);
 151                         dyldInfo->set_bind_off(0);
 152                         dyldInfo->set_bind_size(0);
 153                         dyldInfo->set_weak_bind_off(0);
 154                         dyldInfo->set_weak_bind_size(0);
 155                         dyldInfo->set_lazy_bind_off(0);
 156                         dyldInfo->set_lazy_bind_size(0);
 157                         dyldInfo->set_export_off(0);
 158                         dyldInfo->set_export_size(0);
 159                         }
 160                         break;
 161                 case LC_SYMTAB:
 162                         symtab = (macho_symtab_command<P>*)cmd;
 163                         break;
 164                 case LC_DYSYMTAB:
 165                         dynamicSymTab = (macho_dysymtab_command<P>*)cmd;
 166                         break;
 167                 case LC_FUNCTION_STARTS:
 168                         functionStarts = (macho_linkedit_data_command<P>*)cmd;
 169                         break;
 170                 case LC_DATA_IN_CODE:
 171                         dataInCode = (macho_linkedit_data_command<P>*)cmd;
 172                         break;
 173                 case LC_LOAD_DYLIB:
 174                 case LC_LOAD_WEAK_DYLIB:
 175                 case LC_REEXPORT_DYLIB:
 176                 case LC_LOAD_UPWARD_DYLIB:
 177                         ++depIndex;
 178                         if ( cmd->cmd() == LC_REEXPORT_DYLIB ) {
 179                                 reexportDeps.insert(depIndex);
 180                         }
 181                         break;
 182                 }
 183                 cmd = (const macho_load_command<P>*)(((uint8_t*)cmd)+cmd->cmdsize());
 184         }
 185
 186         // rebuild symbol table
 187         if ( linkEditSegCmd == NULL ) {
 188                 fprintf(stderr, "__LINKEDIT not found\n");
 189                 return -1;
 190         }
 191         if ( symtab == NULL ) {
 192                 fprintf(stderr, "LC_SYMTAB not found\n");
 193                 return -1;
 194         }
 195         if ( dynamicSymTab == NULL ) {
 196                 fprintf(stderr, "LC_DYSYMTAB not found\n");
 197                 return -1;
 198         }
 199
 200         const uint64_t newFunctionStartsOffset = linkEditSegCmd->fileoff();
 201         uint32_t functionStartsSize = 0;
 202         if ( functionStarts != NULL ) {
 203                 // copy function starts from original cache file to new mapped dylib file
 204                 functionStartsSize = functionStarts->datasize();
 205                 memcpy((char*)mh + newFunctionStartsOffset, (char*)mapped_cache + functionStarts->dataoff(), functionStartsSize);
 206         }
 207         const uint64_t newDataInCodeOffset = (newFunctionStartsOffset + functionStartsSize + sizeof(pint_t) - 1) & (-sizeof(pint_t)); // pointer align
 208         uint32_t dataInCodeSize = 0;
 209         if ( dataInCode != NULL ) {
 210                 // copy data-in-code info from original cache file to new mapped dylib file
 211                 dataInCodeSize = dataInCode->datasize();
 212                 memcpy((char*)mh + newDataInCodeOffset, (char*)mapped_cache + dataInCode->dataoff(), dataInCodeSize);
 213         }
 214
 215         std::vector<mach_o::trie::Entry> exports;
 216         if ( exportsTrieSize != 0 ) {
 217                 const uint8_t* exportsStart = ((uint8_t*)mapped_cache) + exportsTrieOffset;
 218                 const uint8_t* exportsEnd = &exportsStart[exportsTrieSize];
 219                 mach_o::trie::parseTrie(exportsStart, exportsEnd, exports);
 220                 exports.erase(std::remove_if(exports.begin(), exports.end(), NotReExportSymbol(reexportDeps)), exports.end());
 221         }
 222
 223         // look for local symbol info in unmapped part of shared cache
 224         dyldCacheHeader<E>* header = (dyldCacheHeader<E>*)mapped_cache;
 225         macho_nlist<P>* localNlists = NULL;
 226         uint32_t localNlistCount = 0;
 227         const char* localStrings = NULL;
 228         const char* localStringsEnd = NULL;
 229         if ( header->mappingOffset() > offsetof(dyld_cache_header,localSymbolsSize) ) {
 230                 dyldCacheLocalSymbolsInfo<E>* localInfo = (dyldCacheLocalSymbolsInfo<E>*)(((uint8_t*)mapped_cache) + header->localSymbolsOffset());
 231                 dyldCacheLocalSymbolEntry<E>* entries = (dyldCacheLocalSymbolEntry<E>*)(((uint8_t*)mapped_cache) + header->localSymbolsOffset() + localInfo->entriesOffset());
 232                 macho_nlist<P>* allLocalNlists = (macho_nlist<P>*)(((uint8_t*)localInfo) + localInfo->nlistOffset());
 233                 const uint32_t entriesCount = localInfo->entriesCount();
 234                 for (uint32_t i=0; i < entriesCount; ++i) {
 235                         if ( entries[i].dylibOffset() == textOffsetInCache ) {
 236                                 uint32_t localNlistStart = entries[i].nlistStartIndex();
 237                                 localNlistCount = entries[i].nlistCount();
 238                                 localNlists = &allLocalNlists[localNlistStart];
 239                                 localStrings = ((char*)localInfo) + localInfo->stringsOffset();
 240                                 localStringsEnd = &localStrings[localInfo->stringsSize()];
 241                                 break;
 242                         }
 243                 }
 244         }
 245         // compute number of symbols in new symbol table
 246         const macho_nlist<P>* const mergedSymTabStart = (macho_nlist<P>*)(((uint8_t*)mapped_cache) + symtab->symoff());
 247         const macho_nlist<P>* const mergedSymTabend = &mergedSymTabStart[symtab->nsyms()];
 248         uint32_t newSymCount = symtab->nsyms();
 249         if ( localNlists != NULL ) {
 250                 newSymCount = localNlistCount;
 251                 for (const macho_nlist<P>* s = mergedSymTabStart; s != mergedSymTabend; ++s) {
 252                         // skip any locals in cache
 253                         if ( (s->n_type() & (N_TYPE|N_EXT)) == N_SECT )
 254                                 continue;
 255                         ++newSymCount;
 256                 }
 257         }
 258
 259         // add room for N_INDR symbols for re-exported symbols
 260         newSymCount += exports.size();
 261
 262         // copy symbol entries and strings from original cache file to new mapped dylib file
 263         const uint64_t newSymTabOffset = (newDataInCodeOffset + dataInCodeSize + sizeof(pint_t) - 1) & (-sizeof(pint_t)); // pointer align
 264         const uint64_t newIndSymTabOffset = newSymTabOffset + newSymCount*sizeof(macho_nlist<P>);
 265         const uint64_t newStringPoolOffset = newIndSymTabOffset + dynamicSymTab->nindirectsyms()*sizeof(uint32_t);
 266         macho_nlist<P>* const newSymTabStart = (macho_nlist<P>*)(((uint8_t*)mh) + newSymTabOffset);
 267         char* const newStringPoolStart = (char*)mh + newStringPoolOffset;
 268         const uint32_t* mergedIndSymTab = (uint32_t*)((char*)mapped_cache + dynamicSymTab->indirectsymoff());
 269         const char* mergedStringPoolStart = (char*)mapped_cache + symtab->stroff();
 270         const char* mergedStringPoolEnd = &mergedStringPoolStart[symtab->strsize()];
 271         macho_nlist<P>* t = newSymTabStart;
 272         int poolOffset = 0;
 273         uint32_t symbolsCopied = 0;
 274         newStringPoolStart[poolOffset++] = '\0'; // first pool entry is always empty string
 275         for (const macho_nlist<P>* s = mergedSymTabStart; s != mergedSymTabend; ++s) {
 276                 // if we have better local symbol info, skip any locals here
 277                 if ( (localNlists != NULL) && ((s->n_type() & (N_TYPE|N_EXT)) == N_SECT) )
 278                         continue;
 279                 *t = *s;
 280                 t->set_n_strx(poolOffset);
 281                 const char* symName = &mergedStringPoolStart[s->n_strx()];
 282                 if ( symName > mergedStringPoolEnd )
 283                         symName = "<corrupt symbol name>";
 284                 strcpy(&newStringPoolStart[poolOffset], symName);
 285                 poolOffset += (strlen(symName) + 1);
 286                 ++t;
 287                 ++symbolsCopied;
 288         }
 289         // <rdar://problem/16529213> recreate N_INDR symbols in extracted dylibs for debugger
 290         for (std::vector<mach_o::trie::Entry>::iterator it = exports.begin(); it != exports.end(); ++it) {
 291                 strcpy(&newStringPoolStart[poolOffset], it->name);
 292                 t->set_n_strx(poolOffset);
 293                 poolOffset += (strlen(it->name) + 1);
 294                 t->set_n_type(N_INDR | N_EXT);
 295                 t->set_n_sect(0);
 296                 t->set_n_desc(0);
 297                 const char* importName = it->importName;
 298                 if ( *importName == '\0' )
 299                         importName = it->name;
 300                 strcpy(&newStringPoolStart[poolOffset], importName);
 301                 t->set_n_value(poolOffset);
 302                 poolOffset += (strlen(importName) + 1);
 303                 ++t;
 304                 ++symbolsCopied;
 305         }
 306         if ( localNlists != NULL ) {
 307                 // update load command to reflect new count of locals
 308                 dynamicSymTab->set_ilocalsym(symbolsCopied);
 309                 dynamicSymTab->set_nlocalsym(localNlistCount);
 310                 // copy local symbols
 311                 for (uint32_t i=0; i < localNlistCount; ++i) {
 312                         const char* localName = &localStrings[localNlists[i].n_strx()];
 313                         if ( localName > localStringsEnd )
 314                                 localName = "<corrupt local symbol name>";
 315                         *t = localNlists[i];
 316                         t->set_n_strx(poolOffset);
 317                         strcpy(&newStringPoolStart[poolOffset], localName);
 318                         poolOffset += (strlen(localName) + 1);
 319                         ++t;
 320                         ++symbolsCopied;
 321                 }
 322         }
 323
 324         if ( newSymCount != symbolsCopied ) {
 325                 fprintf(stderr, "symbol count miscalculation\n");
 326                 return -1;
 327         }
 328
 329         // pointer align string pool size
 330         while ( (poolOffset % sizeof(pint_t)) != 0 )
 331                 ++poolOffset;
 332         // copy indirect symbol table
 333         uint32_t* newIndSymTab = (uint32_t*)((char*)mh + newIndSymTabOffset);
 334         memcpy(newIndSymTab, mergedIndSymTab, dynamicSymTab->nindirectsyms()*sizeof(uint32_t));
 335
 336         // update load commands
 337         if ( functionStarts != NULL ) {
 338                 functionStarts->set_dataoff((uint32_t)newFunctionStartsOffset);
 339                 functionStarts->set_datasize(functionStartsSize);
 340         }
 341         if ( dataInCode != NULL ) {
 342                 dataInCode->set_dataoff((uint32_t)newDataInCodeOffset);
 343                 dataInCode->set_datasize(dataInCodeSize);
 344         }
 345         symtab->set_nsyms(symbolsCopied);
 346         symtab->set_symoff((uint32_t)newSymTabOffset);
 347         symtab->set_stroff((uint32_t)newStringPoolOffset);
 348         symtab->set_strsize(poolOffset);
 349         dynamicSymTab->set_extreloff(0);
 350         dynamicSymTab->set_nextrel(0);
 351         dynamicSymTab->set_locreloff(0);
 352         dynamicSymTab->set_nlocrel(0);
 353         dynamicSymTab->set_indirectsymoff((uint32_t)newIndSymTabOffset);
 354         linkEditSegCmd->set_filesize(symtab->stroff()+symtab->strsize() - linkEditSegCmd->fileoff());
 355         linkEditSegCmd->set_vmsize( (linkEditSegCmd->filesize()+4095) & (-4096) );
 356
 357         // return new size
 358         *newSize = (symtab->stroff()+symtab->strsize()+4095) & (-4096);
 359
 360         return 0;
 361 }
 362
 363
 364
 365 static void make_dirs(const char* file_path)
 366 {
 367         //printf("make_dirs(%s)\n", file_path);
 368         char dirs[strlen(file_path)+1];
 369         strcpy(dirs, file_path);
 370         char* lastSlash = strrchr(dirs, '/');
 371         if ( lastSlash == NULL )
 372                 return;
 373         lastSlash[1] = '\0';
 374         struct stat stat_buf;
 375         if ( stat(dirs, &stat_buf) != 0 ) {
 376                 char* afterSlash = &dirs[1];
 377                 char* slash;
 378                 while ( (slash = strchr(afterSlash, '/')) != NULL ) {
 379                         *slash = '\0';
 380                         ::mkdir(dirs, S_IRWXU | S_IRGRP|S_IXGRP | S_IROTH|S_IXOTH);
 381                         //printf("mkdir(%s)\n", dirs);
 382                         *slash = '/';
 383                         afterSlash = slash+1;
 384                 }
 385         }
 386 }
 387
 388
 389
 390 template <typename A>
 391 size_t dylib_maker(const void* mapped_cache, std::vector<uint8_t> &dylib_data, const std::vector<seg_info>& segments) {
 392         typedef typename A::P P;
 393
 394     size_t  additionalSize  = 0;
 395         for(std::vector<seg_info>::const_iterator it=segments.begin(); it != segments.end(); ++it) {
 396                 additionalSize                          += it->sizem;
 397         }
 398
 399     dylib_data.reserve(dylib_data.size() + additionalSize);
 400
 401     uint32_t                nfat_archs          = 0;
 402         uint32_t                offsetInFatFile     = 4096;
 403     uint8_t                 *base_ptr           = &dylib_data.front();
 404
 405 #define FH reinterpret_cast<fat_header*>(base_ptr)
 406 #define FA reinterpret_cast<fat_arch*>(base_ptr + (8 + (nfat_archs - 1) * sizeof(fat_arch)))
 407
 408     if(dylib_data.size() >= 4096 && OSSwapBigToHostInt32(FH->magic) == FAT_MAGIC) {
 409                 // have fat header, append new arch to end
 410         nfat_archs                              = OSSwapBigToHostInt32(FH->nfat_arch);
 411                 offsetInFatFile                         = OSSwapBigToHostInt32(FA->offset) + OSSwapBigToHostInt32(FA->size);
 412     }
 413
 414     dylib_data.resize(offsetInFatFile);
 415     base_ptr                                    = &dylib_data.front();
 416
 417     FH->magic                                   = OSSwapHostToBigInt32(FAT_MAGIC);
 418     FH->nfat_arch                               = OSSwapHostToBigInt32(++nfat_archs);
 419
 420     FA->cputype                                 = 0; // filled in later
 421     FA->cpusubtype                              = 0; // filled in later
 422     FA->offset                                  = OSSwapHostToBigInt32(offsetInFatFile);
 423     FA->size                                    = 0; // filled in later
 424     FA->align                                   = OSSwapHostToBigInt32(12);
 425
 426         // Write regular segments into the buffer
 427         uint64_t                totalSize           = 0;
 428     uint64_t                            textOffsetInCache       = 0;
 429         for( std::vector<seg_info>::const_iterator it=segments.begin(); it != segments.end(); ++it) {
 430
 431         if(strcmp(it->segName, "__TEXT") == 0 ) {
 432                         textOffsetInCache                                       = it->offset;
 433             const macho_header<P>   *textMH     = reinterpret_cast<macho_header<P>*>((uint8_t*)mapped_cache+textOffsetInCache);
 434             FA->cputype                         = OSSwapHostToBigInt32(textMH->cputype());
 435             FA->cpusubtype                      = OSSwapHostToBigInt32(textMH->cpusubtype());
 436
 437             // if this cputype/subtype already exist in fat header, then return immediately
 438             for(uint32_t i=0; i < nfat_archs-1; ++i) {
 439                 fat_arch            *afa        = reinterpret_cast<fat_arch*>(base_ptr+8)+i;
 440
 441                 if(   afa->cputype == FA->cputype
 442                    && afa->cpusubtype == FA->cpusubtype) {
 443                     //fprintf(stderr, "arch already exists in fat dylib\n");
 444                     dylib_data.resize(offsetInFatFile);
 445                     return offsetInFatFile;
 446                 }
 447             }
 448                 }
 449
 450                 //printf("segName=%s, offset=0x%llX, size=0x%0llX\n", it->segName, it->offset, it->sizem);
 451         std::copy(((uint8_t*)mapped_cache)+it->offset, ((uint8_t*)mapped_cache)+it->offset+it->sizem, std::back_inserter(dylib_data));
 452         base_ptr                                = &dylib_data.front();
 453         totalSize                               += it->sizem;
 454         }
 455
 456         FA->size                                    = OSSwapHostToBigInt32(totalSize);
 457
 458         // optimize linkedit
 459         uint64_t                newSize             = dylib_data.size();
 460         optimize_linkedit<A>(((macho_header<P>*)(base_ptr+offsetInFatFile)), textOffsetInCache, mapped_cache, &newSize);
 461
 462         // update fat header with new file size
 463     dylib_data.resize(offsetInFatFile+newSize);
 464     base_ptr                                    = &dylib_data.front();
 465         FA->size                                    = OSSwapHostToBigInt32(newSize);
 466 #undef FH
 467 #undef FA
 468         return offsetInFatFile;
 469 }
 470
 471
 472 int dyld_shared_cache_extract_dylibs_progress(const char* shared_cache_file_path, const char* extraction_root_path,
 473                                                                                                         void (^progress)(unsigned current, unsigned total))
 474 {
 475         struct stat statbuf;
 476         if (stat(shared_cache_file_path, &statbuf)) {
 477                 fprintf(stderr, "Error: stat failed for dyld shared cache at %s\n", shared_cache_file_path);
 478                 return -1;
 479         }
 480
 481         int cache_fd = open(shared_cache_file_path, O_RDONLY);
 482         if (cache_fd < 0) {
 483                 fprintf(stderr, "Error: failed to open shared cache file at %s\n", shared_cache_file_path);
 484                 return -1;
 485         }
 486
 487         void* mapped_cache = mmap(NULL, statbuf.st_size, PROT_READ, MAP_PRIVATE, cache_fd, 0);
 488         if (mapped_cache == MAP_FAILED) {
 489                 fprintf(stderr, "Error: mmap() for shared cache at %s failed, errno=%d\n", shared_cache_file_path, errno);
 490                 return -1;
 491         }
 492
 493     close(cache_fd);
 494
 495         // instantiate arch specific dylib maker
 496     size_t (*dylib_create_func)(const void*, std::vector<uint8_t>&, const std::vector<seg_info>&) = NULL;
 497              if ( strcmp((char*)mapped_cache, "dyld_v1    i386") == 0 )
 498                 dylib_create_func = dylib_maker<x86>;
 499         else if ( strcmp((char*)mapped_cache, "dyld_v1  x86_64") == 0 )
 500                 dylib_create_func = dylib_maker<x86_64>;
 501         else if ( strcmp((char*)mapped_cache, "dyld_v1 x86_64h") == 0 )
 502                 dylib_create_func = dylib_maker<x86_64>;
 503         else if ( strcmp((char*)mapped_cache, "dyld_v1   armv5") == 0 )
 504                 dylib_create_func = dylib_maker<arm>;
 505         else if ( strcmp((char*)mapped_cache, "dyld_v1   armv6") == 0 )
 506                 dylib_create_func = dylib_maker<arm>;
 507         else if ( strcmp((char*)mapped_cache, "dyld_v1   armv7") == 0 )
 508                 dylib_create_func = dylib_maker<arm>;
 509         else if ( strncmp((char*)mapped_cache, "dyld_v1  armv7", 14) == 0 )
 510                 dylib_create_func = dylib_maker<arm>;
 511         else if ( strcmp((char*)mapped_cache, "dyld_v1   arm64") == 0 )
 512                 dylib_create_func = dylib_maker<arm64>;
 513         else {
 514                 fprintf(stderr, "Error: unrecognized dyld shared cache magic.\n");
 515         munmap(mapped_cache, statbuf.st_size);
 516                 return -1;
 517         }
 518
 519         // iterate through all images in cache and build map of dylibs and segments
 520         __block NameToSegments  map;
 521         __block int                             result              = dyld_shared_cache_iterate(mapped_cache, (uint32_t)statbuf.st_size, ^(const dyld_shared_cache_dylib_info* dylibInfo, const dyld_shared_cache_segment_info* segInfo) {
 522         map[dylibInfo->path].push_back(seg_info(segInfo->name, segInfo->fileOffset, segInfo->fileSize));
 523     });
 524
 525     if(result != 0) {
 526                 fprintf(stderr, "Error: dyld_shared_cache_iterate_segments_with_slide failed.\n");
 527         munmap(mapped_cache, statbuf.st_size);
 528                 return result;
 529     }
 530
 531         // for each dylib instantiate a dylib file
 532     dispatch_group_t        group               = dispatch_group_create();
 533     dispatch_semaphore_t    sema                = dispatch_semaphore_create(2);
 534     dispatch_queue_t        process_queue       = dispatch_get_global_queue(DISPATCH_QUEUE_PRIORITY_LOW, 0);
 535     dispatch_queue_t        writer_queue        = dispatch_queue_create("dyld writer queue", 0);
 536
 537         __block unsigned        count               = 0;
 538
 539         for ( NameToSegments::iterator it = map.begin(); it != map.end(); ++it) {
 540                 dispatch_semaphore_wait(sema, DISPATCH_TIME_FOREVER);
 541         dispatch_group_async(group, process_queue, ^{
 542
 543             char    dylib_path[PATH_MAX];
 544             strcpy(dylib_path, extraction_root_path);
 545             strcat(dylib_path, "/");
 546             strcat(dylib_path, it->first);
 547
 548             //printf("%s with %lu segments\n", dylib_path, it->second.size());
 549             // make sure all directories in this path exist
 550             make_dirs(dylib_path);
 551
 552             // open file, create if does not already exist
 553             int fd = ::open(dylib_path, O_CREAT | O_EXLOCK | O_RDWR, 0644);
 554             if ( fd == -1 ) {
 555                 fprintf(stderr, "can't open or create dylib file %s, errnor=%d\n", dylib_path, errno);
 556                 result    = -1;
 557                 return;
 558             }
 559
 560             struct stat statbuf;
 561             if (fstat(fd, &statbuf)) {
 562                 fprintf(stderr, "Error: stat failed for dyld file %s, errnor=%d\n", dylib_path, errno);
 563                 close(fd);
 564                 result    = -1;
 565                 return;
 566             }
 567
 568             std::vector<uint8_t> *vec   = new std::vector<uint8_t>(statbuf.st_size);
 569             if(pread(fd, &vec->front(), vec->size(), 0) != (long)vec->size()) {
 570                 fprintf(stderr, "can't read dylib file %s, errnor=%d\n", dylib_path, errno);
 571                 close(fd);
 572                 result    = -1;
 573                 return;
 574             }
 575
 576             const size_t    offset  = dylib_create_func(mapped_cache, *vec, it->second);
 577
 578             dispatch_group_async(group, writer_queue, ^{
 579                 progress(count++, (unsigned)map.size());
 580
 581                 if(offset != vec->size()) {
 582                     //Write out the first page, and everything after offset
 583                     if(   pwrite(fd, &vec->front(), 4096, 0) == -1
 584                        || pwrite(fd, &vec->front() + offset, vec->size() - offset, offset) == -1) {
 585                         fprintf(stderr, "error writing, errnor=%d\n", errno);
 586                         result    = -1;
 587                     }
 588                 }
 589
 590                 delete vec;
 591                 close(fd);
 592                 dispatch_semaphore_signal(sema);
 593             });
 594         });
 595         }
 596
 597     dispatch_group_wait(group, DISPATCH_TIME_FOREVER);
 598     dispatch_release(group);
 599     dispatch_release(writer_queue);
 600
 601     munmap(mapped_cache, statbuf.st_size);
 602         return result;
 603 }
 604
 605
 606
 607 int dyld_shared_cache_extract_dylibs(const char* shared_cache_file_path, const char* extraction_root_path)
 608 {
 609         return dyld_shared_cache_extract_dylibs_progress(shared_cache_file_path, extraction_root_path,
 610                                                                                                         ^(unsigned , unsigned) {} );
 611 }
 612
 613
 614 #if 0
 615 // test program
 616 #include <stdio.h>
 617 #include <stddef.h>
 618 #include <dlfcn.h>
 619
 620
 621 typedef int (*extractor_proc)(const char* shared_cache_file_path, const char* extraction_root_path,
 622                                                                                                         void (^progress)(unsigned current, unsigned total));
 623
 624 int main(int argc, const char* argv[])
 625 {
 626         if ( argc != 3 ) {
 627                 fprintf(stderr, "usage: dsc_extractor <path-to-cache-file> <path-to-device-dir>\n");
 628                 return 1;
 629         }
 630
 631         //void* handle = dlopen("/Volumes/my/src/dyld/build/Debug/dsc_extractor.bundle", RTLD_LAZY);
 632         void* handle = dlopen("/Applications/Xcode.app/Contents/Developer/Platforms/iPhoneOS.platform/usr/lib/dsc_extractor.bundle", RTLD_LAZY);
 633         if ( handle == NULL ) {
 634                 fprintf(stderr, "dsc_extractor.bundle could not be loaded\n");
 635                 return 1;
 636         }
 637
 638         extractor_proc proc = (extractor_proc)dlsym(handle, "dyld_shared_cache_extract_dylibs_progress");
 639         if ( proc == NULL ) {
 640                 fprintf(stderr, "dsc_extractor.bundle did not have dyld_shared_cache_extract_dylibs_progress symbol\n");
 641                 return 1;
 642         }
 643
 644         int result = (*proc)(argv[1], argv[2], ^(unsigned c, unsigned total) { printf("%d/%d\n", c, total); } );
 645         fprintf(stderr, "dyld_shared_cache_extract_dylibs_progress() => %d\n", result);
 646         return 0;
 647 }
 648
 649
 650 #endif
 651
 652
 653
 654