interlinked-dylibs/MultiCacheBuilder.mm

   1 //
   2 //  SharedCacheBuilder.m
   3 //  dyld
   4 //
   5 //  Created by Louis Gerbarg on 6/15/15.
   6 //
   7 //
   8
   9 #include <CommonCrypto/CommonCrypto.h>
  10 #include <Bom/Bom.h>
  11
  12 #include <sys/types.h>
  13 #include <sys/sysctl.h>
  14 #include <pthread.h>
  15 #include <mach/mach.h>
  16 #include <unistd.h>
  17
  18 #include <cstring>
  19
  20 #include <array>
  21 #include <sstream>
  22 #include <iomanip>      // std::setfill, std::setw
  23 #include "mega-dylib-utils.h"
  24 #include "Logging.h"
  25
  26 #include "MultiCacheBuilder.h"
  27
  28
  29 namespace {
  30 #if BOM_SUPPORT
  31 void insertDirInBom( const std::string& path, const std::string& name, BOMBom bom ) {
  32     std::string fullPath = path + "/" + name;
  33     BOMFSObject fso = BOMFSObjectNew( BOMDirectoryType );
  34     BOMFSObjectSetFlags( fso, B_PATHONLY );
  35     BOMFSObjectSetPathName( fso, fullPath.c_str(), true );
  36     BOMFSObjectSetShortName( fso, name.c_str(), true );
  37     (void)BOMBomInsertFSObject( bom, fso, false );
  38     BOMFSObjectFree( fso );
  39 }
  40
  41 void insertFileInBom( const std::string& path, const std::string& name, BOMBom bom ) {
  42     std::string fullPath = path + "/" + name;
  43     BOMFSObject fso = BOMFSObjectNew( BOMFileType );
  44     BOMFSObjectSetFlags( fso, B_PATHONLY );
  45     BOMFSObjectSetPathName( fso, fullPath.c_str(), true );
  46     BOMFSObjectSetShortName( fso, name.c_str(), true );
  47     (void)BOMBomInsertFSObject( bom, fso, false );
  48     BOMFSObjectFree( fso );
  49 }
  50
  51 void insertCacheDirInBom( BOMBom bom ) {
  52     BOMFSObject fso = BOMFSObjectNew( BOMDirectoryType );
  53     BOMFSObjectSetFlags( fso, B_PATHONLY );
  54     BOMFSObjectSetPathName( fso, ".", true );
  55     BOMFSObjectSetShortName( fso, ".", true );
  56     (void)BOMBomInsertFSObject( bom, fso, false );
  57     BOMFSObjectFree( fso );
  58     insertDirInBom( ".", "System", bom );
  59     insertDirInBom( "./System", "Library", bom );
  60     insertDirInBom( "./System/Library", "Caches", bom );
  61     insertDirInBom( "./System/Library/Caches", "com.apple.dyld", bom );
  62 }
  63 #endif /* BOM_SUPPORT */
  64 }
  65
  66 MultiCacheBuilder::MultiCacheBuilder(Manifest& manifest, bool BNI, bool SW, bool buildRoot, bool skipBuilds, bool enforceRootles)
  67         : _manifest(manifest), _bniMode(BNI), _skipWrites(SW), _buildRoot(buildRoot), _skipBuilds(skipBuilds), _enforceRootless(enforceRootles),
  68         _writeQueue(dispatch_queue_create("com.apple.dyld.cache.writeout",
  69                                                                           dispatch_queue_attr_make_with_qos_class(DISPATCH_QUEUE_SERIAL,
  70                                                                                                                                                   QOS_CLASS_USER_INITIATED, 0))),
  71         _writeGroup(dispatch_group_create()),
  72         _buildQueue(dispatch_queue_create("com.apple.dyld.cache.multi-build", DISPATCH_QUEUE_CONCURRENT)) {
  73                 uint64_t thread_count;
  74                 uint64_t ram_size;
  75
  76                 size_t len = sizeof(thread_count);
  77                 sysctlbyname ("hw.logicalcpu",&thread_count,&len,NULL,0);
  78                 len = sizeof(ram_size);
  79                 sysctlbyname ("hw.memsize",&ram_size,&len,NULL,0);
  80
  81                 uint64_t buildCount = MIN((ram_size/(1024*1024*1024)/2), thread_count);
  82                 uint64_t writerCount = MAX((ram_size/((uint64_t)2*1024*1024*1024)) - buildCount, 1);
  83
  84                 _buildQueue = dispatch_queue_create("com.apple.dyld.cache.build", DISPATCH_QUEUE_CONCURRENT);
  85                 _concurrencyLimitingSemaphore = dispatch_semaphore_create(buildCount);
  86                 _writeLimitingSemaphore = dispatch_semaphore_create(writerCount);
  87
  88                 if ( _bniMode ) {
  89                         log("Running: %llu threads", buildCount);
  90                         log("Queuing: %llu writers", writerCount);
  91                 }
  92 }
  93
  94 void MultiCacheBuilder::write_cache(std::string cachePath, const std::set<std::string>& configurations, const std::string& architecture, std::shared_ptr<SharedCache> cache, bool developmentCache)
  95 {
  96     //FIXME
  97     dispatch_semaphore_wait(_writeLimitingSemaphore, DISPATCH_TIME_FOREVER);
  98     dispatch_group_enter(_writeGroup);
  99     cacheBuilderDispatchAsync(_writeQueue, [=] {
 100         if (!_skipWrites) {
 101             verboseLog("Queuing write out: %s", cachePath.c_str());
 102
 103             //Turn off file caching since we won't read it back
 104             //(void)fcntl(fd, F_NOCACHE, 1);
 105             // We should do this after the cache write, but that would involve copying the path string
 106             std::string tempPath = cachePath;
 107             cache->writeCacheMapFile(cachePath + ".map");
 108             char tempEXT[] = ".XXXXXX";
 109             mktemp(tempEXT);
 110             tempPath += tempEXT;
 111
 112             int fd = ::open(tempPath.c_str(), O_CREAT | O_RDWR | O_TRUNC, 0644);
 113             if (fd == -1) {
 114                 dispatch_group_leave(_writeGroup);
 115                 dispatch_semaphore_signal(_writeLimitingSemaphore);
 116                 terminate("can't create temp file for %s, errnor=%d (%s)", cachePath.c_str(), errno, strerror(errno));
 117             }
 118
 119             if (isProtectedBySIP(tempPath, fd) != _enforceRootless) {
 120                 ::close(fd);
 121                 ::unlink(tempPath.c_str());
 122                 dispatch_group_leave(_writeGroup);
 123                 dispatch_semaphore_signal(_writeLimitingSemaphore);
 124                 terminate("SIP protection of output cache file changed (%s)", cachePath.c_str());
 125             }
 126
 127             ssize_t writtenSize = pwrite(fd, cache->buffer().get(), cache->fileSize(), 0);
 128             if (writtenSize != cache->fileSize()) {
 129                 ::close(fd);
 130                 ::unlink(tempPath.c_str());
 131                 dispatch_group_leave(_writeGroup);
 132                 dispatch_semaphore_signal(_writeLimitingSemaphore);
 133                 terminate("write() failure creating cache file, requested %lld, wrote %ld, errno=%d (%s)", cache->fileSize(), writtenSize, errno, strerror(errno));
 134             }
 135
 136             ::close(fd);
 137
 138             if (rename(tempPath.c_str(), cachePath.c_str()) != 0) {
 139                 dispatch_group_leave(_writeGroup);
 140                 dispatch_semaphore_signal(_writeLimitingSemaphore);
 141                 terminate("move() failure creating cache file, errno=%d (%s)", errno, strerror(errno));
 142             }
 143             if (_bniMode)
 144                 log("Wrote out: %s", cachePath.c_str());
 145         } else {
 146             log("Skipped: %s", cachePath.c_str());
 147         }
 148         _filesWritten++;
 149         _bytesWritten += cache->fileSize();
 150         dispatch_group_leave(_writeGroup);
 151         dispatch_semaphore_signal(_writeLimitingSemaphore);
 152     });
 153 }
 154
 155 //FIXME (make development a type)
 156 void MultiCacheBuilder::buildCache(const std::string cachePath, const std::set<std::string> configurations, const std::string architecture, bool development)
 157 {
 158     auto& configResults = _manifest.configurations[*configurations.begin()].architectures[architecture].results.dylibs;
 159
 160     if ( _skipBuilds ) {
 161         log( "Build Skipped" );
 162
 163         for ( auto& config : configurations ) {
 164             for ( auto& dylib : configResults ) {
 165                 _manifest.configurations[config].architectures[architecture].results.dylibs[dylib.first].exclude(
 166                     "All dylibs excluded" );
 167             }
 168         }
 169         return;
 170         }
 171
 172         Manifest::Architecture arch;
 173         std::vector<std::unique_ptr<MachOProxy>> dylibs;
 174         std::vector<std::string> emptyList;
 175         std::shared_ptr<SharedCache> cache = std::make_shared<SharedCache>(_manifest, *configurations.begin(), architecture);
 176
 177         for (auto& config : configurations) {
 178                 auto& results = _manifest.configurations[config].architectures[architecture].results.dylibs;
 179
 180                 for (auto& dylib : configResults) {
 181                         if (dylib.second.included == false
 182                                 && results.count(dylib.first)
 183                                 && results[dylib.first].included == true) {
 184                                 results[dylib.first].exclude(dylib.second.exclusionInfo);
 185                         }
 186                 }
 187         }
 188
 189         if (development) {
 190                 cache->buildForDevelopment(cachePath);
 191         } else {
 192                 cache->buildForProduction(cachePath);
 193         }
 194
 195         std::vector<uint64_t> regionStartAddresses;
 196         std::vector<uint64_t> regionSizes;
 197         std::vector<uint64_t> regionFileOffsets;
 198
 199         cache->forEachRegion([&] (void* content, uint64_t vmAddr, uint64_t size, uint32_t permissions) {
 200                 regionStartAddresses.push_back(vmAddr);
 201                 regionSizes.push_back(size);
 202                 regionFileOffsets.push_back((uint8_t*)content - (uint8_t*)cache->buffer().get());
 203                 const char* prot = "RW";
 204                 if ( permissions == (VM_PROT_EXECUTE|VM_PROT_READ) )
 205                         prot = "EX";
 206                 else if ( permissions == VM_PROT_READ )
 207                         prot = "RO";
 208                 for (auto& config : configurations) {
 209                         if (development) {
 210                                 _manifest.configurations[config].architectures[architecture].results.developmentCache.regions.push_back({prot, vmAddr,vmAddr+size });
 211                         } else {
 212                                 _manifest.configurations[config].architectures[architecture].results.productionCache.regions.push_back({prot, vmAddr,vmAddr+size });
 213                         }
 214                 }
 215         });
 216
 217         cache->forEachImage([&](const void* machHeader, const char* installName, time_t mtime,
 218                                                                 ino_t inode, const std::vector<MachOProxy::Segment>& segments) {
 219                 for (auto& seg : segments) {
 220                         uint64_t vmAddr = 0;
 221                         for (int i=0; i < regionSizes.size(); ++i) {
 222                                 if ( (seg.fileOffset >= regionFileOffsets[i]) && (seg.fileOffset < (regionFileOffsets[i]+regionSizes[i])) ) {
 223                                         vmAddr = regionStartAddresses[i] + seg.fileOffset - regionFileOffsets[i];
 224                                 }
 225                         }
 226                         for (auto& config : configurations) {
 227                                 _manifest.configurations[config].architectures[architecture].results.dylibs[installName].segments.push_back({seg.name, vmAddr, vmAddr+seg.size});
 228                                 if (_manifest.configurations[config].architectures[architecture].results.dylibs[installName].segments.size() == 0) {
 229                                         warning("Attempting to write info for excluded dylib");
 230                                         _manifest.configurations[config].architectures[architecture].results.dylibs[installName].exclude("Internal Error");
 231                                 }
 232                         }
 233                 }
 234         });
 235         if (development) {
 236                 verboseLog("developement cache size = %llu", cache->fileSize());
 237         } else {
 238                 verboseLog("production cache size = %llu", cache->fileSize());
 239         }
 240         if ( cache->vmSize()+align(cache->vmSize()/200, sharedRegionRegionAlignment(archForString(architecture))) > sharedRegionRegionSize(archForString(architecture))) {
 241             warning("shared cache will not fit in shared regions address space.  Overflow amount: %llu",
 242                 cache->vmSize() + align(cache->vmSize() / 200, sharedRegionRegionAlignment(archForString(architecture))) - sharedRegionRegionSize(archForString(architecture)));
 243             return;
 244         }
 245         write_cache(cachePath, configurations, architecture, cache, development);
 246         for (auto& config : configurations) {
 247                 if (development) {
 248                         _manifest.configurations[config].architectures[architecture].results.developmentCache.cdHash = cache->cdHashString();
 249                 }
 250                 else {
 251                         _manifest.configurations[config].architectures[architecture].results.productionCache.cdHash = cache->cdHashString();
 252                 }
 253         }
 254 }
 255
 256 void MultiCacheBuilder::runOnManifestConcurrently(std::function<void(const std::string configuration, const std::string architecture)> lambda)
 257 {
 258     dispatch_group_t runGroup = dispatch_group_create();
 259     for (auto& config : _manifest.configurations) {
 260         for (auto& architecture : config.second.architectures) {
 261             dispatch_semaphore_wait(_concurrencyLimitingSemaphore, DISPATCH_TIME_FOREVER);
 262             cacheBuilderDispatchGroupAsync(runGroup, _buildQueue, [&] {
 263                 WarningTargets targets;
 264                 targets.first = &_manifest;
 265                 targets.second.insert(std::make_pair(config.first, architecture.first));
 266                 auto ctx = std::make_shared<LoggingContext>(config.first + "/" + architecture.first, targets);
 267                 setLoggingContext(ctx);
 268                 lambda(config.first, architecture.first);
 269                 dispatch_semaphore_signal(_concurrencyLimitingSemaphore);
 270             });
 271         }
 272     }
 273
 274     dispatch_group_wait(runGroup, DISPATCH_TIME_FOREVER);
 275 }
 276
 277 void MultiCacheBuilder::buildCaches(std::string masterDstRoot) {
 278         if (_bniMode) {
 279                 std::vector<std::set<std::string>> dedupedCacheSets;
 280                 for (auto& config : _manifest.configurations) {
 281                         bool dupeFound = false;
 282
 283                         for (auto& cacheSet : dedupedCacheSets) {
 284                                 if (config.second.equivalent(_manifest.configurations[*cacheSet.begin()])) {
 285                                         cacheSet.insert(config.first);
 286                                         dupeFound = true;
 287                                         break;
 288                                 }
 289                         }
 290
 291                         if (!dupeFound) {
 292                                 std::set<std::string> temp;
 293                                 temp.insert(config.first);
 294                                 dedupedCacheSets.push_back(temp);
 295                         }
 296                 }
 297
 298                 for (auto& cacheSet : dedupedCacheSets) {
 299                         //FIXME we may want to consider moving to hashes of UUID sets
 300                         std::string setName;
 301
 302                         for (auto &archName : cacheSet) {
 303                                 if (!setName.empty()) {
 304                                         setName += "|";
 305                                 }
 306                                 setName += archName;
 307                         }
 308
 309                         std::stringstream fileNameStream;
 310                         std::array<uint8_t, CC_SHA1_DIGEST_LENGTH> digest = {0};
 311                         CC_SHA1(setName.c_str(), (unsigned int)setName.length(), &digest[0]);
 312
 313                         fileNameStream << std::hex << std::uppercase << std::setfill( '0' );
 314                         for( int c : digest ) {
 315                                 fileNameStream << std::setw( 2 ) << c;
 316                         }
 317
 318                         std::string fileName(fileNameStream.str());
 319
 320                         for (auto& config : cacheSet) {
 321                                 if (!_skipWrites) {
 322                                         int err = symlink(("DedupedConfigs/" + fileName).c_str(), (masterDstRoot + "/" + config).c_str());
 323                                         if (err) {
 324                                                 warning("Could not create symlink '%s' -> 'DedupedConfigs/%s' (%d)", config.c_str(), fileName.c_str(), err);
 325                                         }
 326                                 }
 327                         }
 328
 329                         for (auto& arch : _manifest.configurations[*cacheSet.begin()].architectures) {
 330                                 dispatch_semaphore_wait(_concurrencyLimitingSemaphore, DISPATCH_TIME_FOREVER);
 331                 cacheBuilderDispatchGroupAsync(_writeGroup, _buildQueue, [=] {
 332                     WarningTargets targets;
 333                     targets.first = &_manifest;
 334                     for (auto& config : cacheSet) {
 335                         targets.second.insert(std::make_pair(config, arch.first));
 336                     }
 337                     auto ctx = std::make_shared<LoggingContext>(setName + "/" + arch.first, targets);
 338                     setLoggingContext(ctx);
 339
 340                     std::string configPath = masterDstRoot + "/DedupedConfigs/" + fileName + "/System/Library/Caches/com.apple.dyld/";
 341
 342                     if (!_skipWrites) {
 343                         int err = mkpath_np(configPath.c_str(), 0755);
 344
 345                         if (err != 0 && err != EEXIST) {
 346                             dispatch_semaphore_signal(_concurrencyLimitingSemaphore);
 347                             terminate("mkpath_np fail: %d", err);
 348                         }
 349                     }
 350
 351                     buildCache(configPath + "dyld_shared_cache_" + arch.first + ".development", cacheSet, arch.first, true);
 352                     buildCache(configPath + "dyld_shared_cache_" + arch.first, cacheSet, arch.first, false);
 353                     dispatch_semaphore_signal(_concurrencyLimitingSemaphore);
 354                 });
 355             }
 356                 }
 357
 358                 dispatch_group_wait(_writeGroup, DISPATCH_TIME_FOREVER);
 359
 360 #if BOM_SUPPORT
 361                 if ( !_skipWrites ) {
 362                     for ( auto& configuration : _manifest.configurations ) {
 363                                 std::vector<std::string> prodBomPaths;
 364                                 std::vector<std::string> devBomPaths;
 365
 366                                 for (auto& arch : configuration.second.architectures) {
 367                                         std::string cachePath = "dyld_shared_cache_" + arch.first;
 368                                         prodBomPaths.push_back(cachePath);
 369                                         cachePath += ".development";
 370                                         devBomPaths.push_back(cachePath);
 371                                         dispatch_group_enter(_writeGroup);
 372                     cacheBuilderDispatchAsync(_writeQueue, [=] {
 373                         char buffer[MAXPATHLEN];
 374                         sprintf(buffer, "%s/Boms/%s.prod.bom", masterDstRoot.c_str(), configuration.first.c_str());
 375                         BOMBom bom = BOMBomNew(buffer);
 376                         insertCacheDirInBom(bom);
 377                         for (auto& path : prodBomPaths) {
 378                             insertFileInBom("./System/Library/Caches/com.apple.dyld", path, bom);
 379                         }
 380                         BOMBomFree(bom);
 381
 382                         sprintf(buffer, "%s/Boms/%s.dev.bom", masterDstRoot.c_str(), configuration.first.c_str());
 383                         bom = BOMBomNew(buffer);
 384                         insertCacheDirInBom(bom);
 385                         for (auto& path : devBomPaths) {
 386                             insertFileInBom("./System/Library/Caches/com.apple.dyld", path, bom);
 387                         }
 388                         BOMBomFree(bom);
 389
 390                         sprintf(buffer, "%s/Boms/%s.full.bom", masterDstRoot.c_str(), configuration.first.c_str());
 391                         bom = BOMBomNew(buffer);
 392                         insertCacheDirInBom(bom);
 393                         for (auto& path : prodBomPaths) {
 394                             insertFileInBom("./System/Library/Caches/com.apple.dyld", path, bom);
 395                         }
 396                         for (auto& path : devBomPaths) {
 397                             insertFileInBom("./System/Library/Caches/com.apple.dyld", path, bom);
 398                         }
 399                         BOMBomFree(bom);
 400                         dispatch_group_leave(_writeGroup);
 401                     });
 402                 }
 403                         }
 404                 }
 405 #endif /* BOM_SUPPORT */
 406         } else {
 407             runOnManifestConcurrently(
 408                 [&](const std::string configuration, const std::string architecture) {
 409                     cacheBuilderDispatchGroupAsync(_writeGroup, _buildQueue, [=] {
 410                         std::set<std::string> configurations;
 411                         configurations.insert( configuration );
 412                         // FIXME hacky, we make implicit assumptions about dev vs non-dev and layout depending on the flags
 413                         if ( _buildRoot ) {
 414                             int err = mkpath_np( ( masterDstRoot + "/System/Library/Caches/com.apple.dyld/" ).c_str(), 0755 );
 415
 416                             if ( err != 0 && err != EEXIST ) {
 417                                 terminate( "mkpath_np fail: %d", err );
 418                             }
 419                             buildCache(masterDstRoot + "/System/Library/Caches/com.apple.dyld/dyld_shared_cache_" + architecture,
 420                                 configurations, architecture, false);
 421                             buildCache(masterDstRoot + "/System/Library/Caches/com.apple.dyld/dyld_shared_cache_" + architecture + ".development",
 422                                 configurations, architecture, true);
 423                         } else {
 424                             buildCache(masterDstRoot + "/dyld_shared_cache_" + architecture, configurations, architecture, true);
 425                         }
 426                     });
 427                 });
 428             dispatch_group_wait(_writeGroup, DISPATCH_TIME_FOREVER);
 429         }
 430
 431         int err = sync_volume_np(masterDstRoot.c_str(), SYNC_VOLUME_FULLSYNC | SYNC_VOLUME_WAIT);
 432         if (err) {
 433                 warning("Volume sync failed errnor=%d (%s)", err, strerror(err));
 434         }
 435 }
 436
 437 void MultiCacheBuilder::logStats(void) {
 438         if ( _bniMode )
 439                 log("Processed %llu caches (%.2fGB)", _filesWritten, ((float)_bytesWritten)/(1024*1024*1024));
 440 }
 441