dyld3/shared-cache/FileUtils.cpp

   1 /*
   2  * Copyright (c) 2017 Apple Inc. All rights reserved.
   3  *
   4  * @APPLE_LICENSE_HEADER_START@
   5  *
   6  * This file contains Original Code and/or Modifications of Original Code
   7  * as defined in and that are subject to the Apple Public Source License
   8  * Version 2.0 (the 'License'). You may not use this file except in
   9  * compliance with the License. Please obtain a copy of the License at
  10  * http://www.opensource.apple.com/apsl/ and read it before using this
  11  * file.
  12  *
  13  * The Original Code and all software distributed under the License are
  14  * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  15  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  16  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
  17  * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
  18  * Please see the License for the specific language governing rights and
  19  * limitations under the License.
  20  *
  21  * @APPLE_LICENSE_HEADER_END@
  22  */
  23
  24
  25 #include <string.h>
  26 #include <stdint.h>
  27 #include <unistd.h>
  28 #include <dirent.h>
  29 #include <stdlib.h>
  30 #include <fcntl.h>
  31 #include <limits.h>
  32 #include <sys/types.h>
  33 #include <sys/stat.h>
  34 #include <sys/types.h>
  35 #include <sys/param.h>
  36 #include <sys/mount.h>
  37 #include <sys/mman.h>
  38 #include <dispatch/dispatch.h>
  39 #include <mach-o/dyld.h>
  40 #include <System/sys/csr.h>
  41 #include <rootless.h>
  42
  43 #include <string>
  44 #include <fstream>
  45 #include <sstream>
  46
  47 #include "FileUtils.h"
  48 #include "StringUtils.h"
  49 #include "Diagnostics.h"
  50 #include "JSONReader.h"
  51
  52
  53 void iterateDirectoryTree(const std::string& pathPrefix, const std::string& path, bool (^dirFilter)(const std::string& path), void (^fileCallback)(const std::string& path, const struct stat&), bool processFiles, bool recurse)
  54 {
  55     std::string fullDirPath = pathPrefix + path;
  56     DIR* dir = ::opendir(fullDirPath.c_str());
  57     if ( dir == nullptr ) {
  58         //fprintf(stderr, "can't read 'dir '%s', errno=%d\n", inputPath.c_str(), errno);
  59         return;
  60     }
  61     while (dirent* entry = readdir(dir)) {
  62         struct stat statBuf;
  63         std::string dirAndFile = path + (path.back() != '/' ? "/" : "") + entry->d_name;
  64         std::string fullDirAndFile = pathPrefix + dirAndFile;
  65          switch ( entry->d_type ) {
  66             case DT_REG:
  67                 if ( processFiles ) {
  68                     if ( ::lstat(fullDirAndFile.c_str(), &statBuf) == -1 )
  69                         break;
  70                     if ( ! S_ISREG(statBuf.st_mode)  )
  71                         break;
  72                     fileCallback(dirAndFile, statBuf);
  73                 }
  74                 break;
  75             case DT_DIR:
  76                 if ( strcmp(entry->d_name, ".") == 0 )
  77                     break;
  78                 if ( strcmp(entry->d_name, "..") == 0 )
  79                     break;
  80                 if ( dirFilter(dirAndFile) )
  81                     break;
  82                 if (recurse)
  83                     iterateDirectoryTree(pathPrefix, dirAndFile, dirFilter, fileCallback, processFiles, true);
  84                 break;
  85             case DT_LNK:
  86                 // don't follow symlinks, dylib will be found through absolute path
  87                 break;
  88         }
  89     }
  90     ::closedir(dir);
  91 }
  92
  93
  94 bool safeSave(const void* buffer, size_t bufferLen, const std::string& path)
  95 {
  96     std::string pathTemplate = path + "-XXXXXX";
  97     size_t templateLen = strlen(pathTemplate.c_str())+2;
  98     char pathTemplateSpace[templateLen];
  99     strlcpy(pathTemplateSpace, pathTemplate.c_str(), templateLen);
 100     int fd = mkstemp(pathTemplateSpace);
 101     if ( fd != -1 ) {
 102         ssize_t writtenSize = pwrite(fd, buffer, bufferLen, 0);
 103         if ( (size_t)writtenSize == bufferLen ) {
 104             ::fchmod(fd, S_IRUSR|S_IWUSR|S_IRGRP|S_IROTH); // mkstemp() makes file "rw-------", switch it to "rw-r--r--"
 105             if ( ::rename(pathTemplateSpace, path.c_str()) == 0) {
 106                 ::close(fd);
 107                 return true; // success
 108             }
 109         }
 110         ::close(fd);
 111         ::unlink(pathTemplateSpace);
 112     }
 113     return false; // failure
 114 }
 115
 116 const void* mapFileReadOnly(const char* path, size_t& mappedSize)
 117 {
 118     struct stat statBuf;
 119     if ( ::stat(path, &statBuf) != 0 )
 120         return nullptr;
 121
 122     int fd = ::open(path, O_RDONLY);
 123     if ( fd < 0 )
 124         return nullptr;
 125
 126     const void *p = ::mmap(NULL, (size_t)statBuf.st_size, PROT_READ, MAP_PRIVATE, fd, 0);
 127     ::close(fd);
 128     if ( p != MAP_FAILED ) {
 129         mappedSize = (size_t)statBuf.st_size;
 130         return p;
 131     }
 132
 133     return nullptr;
 134 }
 135
 136 bool fileExists(const std::string& path)
 137 {
 138     struct stat statBuf;
 139     return ( ::stat(path.c_str(), &statBuf) == 0 );
 140 }
 141
 142 // There is an order file specifying the order in which dylibs are laid out in
 143 // general, as well as an order file specifying the order in which __DATA_DIRTY
 144 // segments are laid out in particular.
 145 //
 146 // The syntax is one dylib (install name) per line.  Blank lines are ignored.
 147 // Comments start with the # character.
 148 std::unordered_map<std::string, uint32_t> parseOrderFile(const std::string& orderFileData) {
 149     std::unordered_map<std::string, uint32_t> order;
 150
 151     if (orderFileData.empty())
 152         return order;
 153
 154     std::stringstream myData(orderFileData);
 155
 156     uint32_t count = 0;
 157     std::string line;
 158     while ( std::getline(myData, line) ) {
 159         size_t pos = line.find('#');
 160         if ( pos != std::string::npos )
 161             line.resize(pos);
 162         while ( !line.empty() && isspace(line.back()) ) {
 163             line.pop_back();
 164         }
 165         if ( !line.empty() )
 166             order[line] = count++;
 167     }
 168     return order;
 169 }
 170
 171 std::string loadOrderFile(const std::string& orderFilePath) {
 172     std::string order;
 173
 174     size_t size = 0;
 175     char* data = (char*)mapFileReadOnly(orderFilePath.c_str(), size);
 176     if (data) {
 177         order = std::string(data, size);
 178         ::munmap((void*)data, size);
 179     }
 180
 181     return order;
 182 }
 183
 184
 185 std::string toolDir()
 186 {
 187     char buffer[PATH_MAX];
 188     uint32_t bufsize = PATH_MAX;
 189     int result = _NSGetExecutablePath(buffer, &bufsize);
 190     if ( result == 0 ) {
 191         std::string path = buffer;
 192         size_t pos = path.rfind('/');
 193         if ( pos != std::string::npos )
 194             return path.substr(0,pos+1);
 195     }
 196     //warning("tool directory not found");
 197     return "/tmp/";
 198 }
 199
 200 std::string basePath(const std::string& path)
 201 {
 202     std::string::size_type slash_pos = path.rfind("/");
 203     if (slash_pos != std::string::npos) {
 204         slash_pos++;
 205         return path.substr(slash_pos);
 206     } else {
 207         return path;
 208     }
 209 }
 210
 211 std::string dirPath(const std::string& path)
 212 {
 213     std::string::size_type slash_pos = path.rfind("/");
 214     if (slash_pos != std::string::npos) {
 215         slash_pos++;
 216         return path.substr(0, slash_pos);
 217     } else {
 218         char cwd[MAXPATHLEN];
 219         (void)getcwd(cwd, MAXPATHLEN);
 220         return cwd;
 221     }
 222 }
 223
 224 std::string realPath(const std::string& path)
 225 {
 226     char resolvedPath[PATH_MAX];
 227     if (realpath(dirPath(path).c_str(), &resolvedPath[0]) != nullptr) {
 228         return std::string(resolvedPath) + "/" + basePath(path);
 229     } else {
 230         return "";
 231     }
 232 }
 233
 234 std::string realFilePath(const std::string& path)
 235 {
 236     char resolvedPath[PATH_MAX];
 237     if ( realpath(path.c_str(), resolvedPath) != nullptr )
 238         return std::string(resolvedPath);
 239     else
 240         return "";
 241 }
 242
 243
 244 std::string normalize_absolute_file_path(std::string path) {
 245     std::vector<std::string> components;
 246     std::vector<std::string> processed_components;
 247     std::stringstream ss(path);
 248     std::string retval;
 249     std::string item;
 250
 251     while (std::getline(ss, item, '/')) {
 252         components.push_back(item);
 253     }
 254
 255     if (components[0] == ".") {
 256         retval = ".";
 257     }
 258
 259     for (auto& component : components) {
 260         if (component.empty() || component == ".")
 261             continue;
 262         else if (component == ".." && processed_components.size())
 263             processed_components.pop_back();
 264         else
 265             processed_components.push_back(component);
 266     }
 267
 268     for (auto & component : processed_components) {
 269         retval = retval + "/" + component;
 270     }
 271
 272     return retval;
 273 }
 274
 275
 276 #if BUILDING_CACHE_BUILDER
 277
 278 FileCache fileCache;
 279
 280 FileCache::FileCache(void)
 281 {
 282     cache_queue = dispatch_queue_create("com.apple.dyld.cache.cache", dispatch_queue_attr_make_with_qos_class(DISPATCH_QUEUE_SERIAL, QOS_CLASS_USER_INITIATED, 0));
 283 }
 284
 285 std::pair<uint8_t*, struct stat> FileCache::cacheLoad(Diagnostics& diags, const std::string path)
 286 {
 287     __block bool found = false;
 288     __block std::pair<uint8_t*, struct stat> retval;
 289     std::string normalizedPath = normalize_absolute_file_path(path);
 290     dispatch_sync(cache_queue, ^{
 291         auto entry = entries.find(normalizedPath);
 292         if (entry != entries.end()) {
 293             retval = entry->second;
 294             found = true;
 295         }
 296     });
 297
 298     if (!found) {
 299         auto info = fill(diags, normalizedPath);
 300         dispatch_sync(cache_queue, ^{
 301             auto entry = entries.find(normalizedPath);
 302             if (entry != entries.end()) {
 303                 retval = entry->second;
 304             } else {
 305                 retval = entries[normalizedPath] = info;
 306                 retval = info;
 307             }
 308         });
 309     }
 310
 311     return retval;
 312 }
 313
 314 //FIXME error handling
 315 std::pair<uint8_t*, struct stat> FileCache::fill(Diagnostics& diags, const std::string& path)
 316 {
 317     void* buffer_ptr = nullptr;
 318     struct stat stat_buf;
 319     struct statfs statfs_buf;
 320     bool localcopy = true;
 321
 322     int fd = ::open(path.c_str(), O_RDONLY, 0);
 323     if (fd == -1) {
 324         diags.verbose("can't open file '%s', errno=%d\n", path.c_str(), errno);
 325         return std::make_pair((uint8_t*)(-1), stat_buf);
 326     }
 327
 328     if (fstat(fd, &stat_buf) == -1) {
 329         diags.verbose("can't stat open file '%s', errno=%d\n", path.c_str(), errno);
 330         ::close(fd);
 331         return std::make_pair((uint8_t*)(-1), stat_buf);
 332     }
 333
 334     if (stat_buf.st_size < 4096) {
 335         diags.verbose("file too small '%s'\n", path.c_str());
 336         ::close(fd);
 337         return std::make_pair((uint8_t*)(-1), stat_buf);
 338     }
 339
 340     if(fstatfs(fd, &statfs_buf) == 0) {
 341         std::string fsName = statfs_buf.f_fstypename;
 342         if (fsName == "hfs" || fsName == "apfs") {
 343             localcopy = false;
 344         }
 345     }
 346
 347     if (!localcopy) {
 348         buffer_ptr = mmap(NULL, (size_t)stat_buf.st_size, PROT_READ, MAP_PRIVATE, fd, 0);
 349         if (buffer_ptr == MAP_FAILED) {
 350             diags.verbose("mmap() for file at %s failed, errno=%d\n", path.c_str(), errno);
 351             ::close(fd);
 352             return std::make_pair((uint8_t*)(-1), stat_buf);
 353         }
 354     } else {
 355         buffer_ptr = malloc((size_t)stat_buf.st_size);
 356         ssize_t readBytes = pread(fd, buffer_ptr, (size_t)stat_buf.st_size, 0);
 357         if (readBytes == -1) {
 358             diags.verbose("Network read for file at %s failed, errno=%d\n", path.c_str(), errno);
 359             ::close(fd);
 360             return std::make_pair((uint8_t*)(-1), stat_buf);
 361         } else if (readBytes != stat_buf.st_size) {
 362             diags.verbose("Network read udnerrun for file at %s, expected %lld bytes, got  %zd bytes\n", path.c_str(), stat_buf.st_size, readBytes);
 363             ::close(fd);
 364             return std::make_pair((uint8_t*)(-1), stat_buf);
 365         }
 366     }
 367
 368     ::close(fd);
 369
 370     return std::make_pair((uint8_t*)buffer_ptr, stat_buf);
 371 }
 372
 373 static void normalizePath(std::string& path) {
 374     // Remove a bunch of stuff we don't need, like trailing slashes.
 375     while ( !path.empty() && (path.back() == '/'))
 376         path.pop_back();
 377 }
 378
 379 void SymlinkResolver::addFile(Diagnostics& diags, std::string path) {
 380     if (path.front() != '/') {
 381         diags.error("Path must start with '/'");
 382         return;
 383     }
 384     if (symlinks.find(path) != symlinks.end()) {
 385         diags.error("Cannot add regular file as it is already a symlink");
 386         return;
 387     }
 388     filePaths.insert(path);
 389 }
 390
 391 void SymlinkResolver::addSymlink(Diagnostics& diags, std::string fromPath, std::string toPath) {
 392     normalizePath(fromPath);
 393     normalizePath(toPath);
 394     if (fromPath.front() != '/') {
 395         diags.error("Path must start with '/'");
 396         return;
 397     }
 398     if (filePaths.find(fromPath) != filePaths.end()) {
 399         diags.error("Cannot add symlink from '%s' as it is already a regular path", fromPath.c_str());
 400         return;
 401     }
 402     auto itAndInserted = symlinks.insert({ fromPath, toPath });
 403     if (!itAndInserted.second) {
 404         // The path is already a symlink.  Make sure its a dupe.
 405         if (toPath != itAndInserted.first->second) {
 406             diags.error("Duplicate symlink for path '%s'", fromPath.c_str());
 407             return;
 408         }
 409     }
 410 }
 411
 412 std::string SymlinkResolver::realPath(Diagnostics& diags, const std::string& originalPath) const {
 413     // First make sure the path doesn't have any magic in it.
 414     std::string path = originalPath;
 415     normalizePath(path);
 416
 417     std::set<std::string> seenSymlinks;
 418
 419     // Now see if any prefix is a symlink
 420     if (path.front() != '/')
 421         return path;
 422
 423     std::string::size_type prev_pos = 0;
 424     while (prev_pos != std::string::npos) {
 425         std::string::size_type pos = path.find("/", prev_pos + 1);
 426
 427         // First look to see if this path component is special, eg, ., .., etc.
 428         std::string component = path.substr(prev_pos, pos - prev_pos);
 429         if (component == "/..") {
 430             // Fold with the previous path component.
 431             if (prev_pos == 0) {
 432                 // This is the root path, and .. applied to / is just /
 433                 path = path.substr(3);
 434                 prev_pos = 0;
 435             } else {
 436                 std::string::size_type lastSlashPos = path.rfind("/", prev_pos - 1);
 437                 path = path.substr(0, lastSlashPos) + path.substr(pos);
 438                 prev_pos = lastSlashPos;
 439             }
 440             continue;
 441         } else if (component == "/.") {
 442             if (prev_pos == 0) {
 443                 // Path starts with /./ so just remove the first one.
 444                 path = path.substr(2);
 445             } else {
 446                 if (pos == std::string::npos) {
 447                     // Trailing . on the path
 448                     path = path.substr(0, prev_pos );
 449                 } else {
 450                     path = path.substr(0, prev_pos) + path.substr(pos);
 451                 }
 452             }
 453             continue;
 454         } else if (component == "/") {
 455             // Path must contain // somewhere so strip out the duplicates.
 456             if (prev_pos == 0) {
 457                 // Path starts with // so just remove the first one.
 458                 path = path.substr(1);
 459             } else {
 460                 if (pos == std::string::npos) {
 461                     // Trailing / on the path
 462                     path = path.substr(0, prev_pos);
 463                     prev_pos = pos;
 464                 } else {
 465                     path = path.substr(0, pos) + path.substr(pos + 1);
 466                 }
 467             }
 468             continue;
 469         }
 470
 471         // Path is not special, so see if it is a symlink to something.
 472         std::string prefix = path.substr(0, pos);
 473         //printf("%s\n", prefix.c_str());
 474         auto it = symlinks.find(prefix);
 475         if (it == symlinks.end()) {
 476             // This is not a symlink so move to the next prefix.
 477             prev_pos = pos;
 478             continue;
 479         }
 480
 481         // If we've already done this prefix then error out.
 482         if (seenSymlinks.count(prefix)) {
 483             diags.error("Loop in symlink processing for '%s'", originalPath.c_str());
 484             return std::string();
 485         }
 486
 487         seenSymlinks.insert(prefix);
 488
 489         // This is a symlink, so resolve the new path.
 490         std::string toPath = it->second;
 491         if (toPath.front() == '/') {
 492             // Symlink points to an absolute address so substitute the whole prefix for the new path
 493             // If we didn't substitute the last component of the path then there is also a path suffix.
 494             std::string pathSuffix = "";
 495             if (pos != std::string::npos) {
 496                 std::string::size_type nextSlashPos = path.find("/", pos + 1);
 497                 if (nextSlashPos != std::string::npos)
 498                     pathSuffix = path.substr(nextSlashPos);
 499             }
 500             path = toPath + pathSuffix;
 501             prev_pos = 0;
 502             continue;
 503         }
 504
 505         // Symlink points to a relative path so we need to do more processing to get the real path.
 506
 507         // First calculate which part of the previous prefix we'll keep.  Eg, in /a/b/c where "b -> blah", we want to keep /a here.
 508         std::string prevPrefix = path.substr(0, prev_pos);
 509         //printf("prevPrefix %s\n", prevPrefix.c_str());
 510
 511         // If we didn't substitute the last component of the path then there is also a path suffix.
 512         std::string pathSuffix = "";
 513         if (prefix.size() != path.size())
 514             pathSuffix = path.substr(pos);
 515
 516         // The new path is the remaining prefix, plus the symlink target, plus any remaining suffix from the original path.
 517         path = prevPrefix + "/" + toPath + pathSuffix;
 518         prev_pos = 0;
 519     }
 520     return path;
 521 }
 522
 523 std::vector<DyldSharedCache::FileAlias> SymlinkResolver::getResolvedSymlinks(Diagnostics& diags) {
 524     diags.assertNoError();
 525     std::vector<DyldSharedCache::FileAlias> aliases;
 526     for (auto& fromPathAndToPath : symlinks) {
 527         std::string newPath = realPath(diags, fromPathAndToPath.first);
 528         if (diags.hasError()) {
 529             aliases.clear();
 530             return aliases;
 531         }
 532
 533         if (filePaths.count(newPath)) {
 534             aliases.push_back({ newPath, fromPathAndToPath.first });
 535             // printf("symlink ('%s' -> '%s') resolved to '%s'\n", fromPathAndToPath.first.c_str(), fromPathAndToPath.second.c_str(), newPath.c_str());
 536         }
 537     }
 538     return aliases;
 539 }
 540
 541 #endif // BUILDING_CACHE_BUILDER
 542