dyld3/shared-cache/FileUtils.cpp

   1 /*
   2  * Copyright (c) 2017 Apple Inc. All rights reserved.
   3  *
   4  * @APPLE_LICENSE_HEADER_START@
   5  *
   6  * This file contains Original Code and/or Modifications of Original Code
   7  * as defined in and that are subject to the Apple Public Source License
   8  * Version 2.0 (the 'License'). You may not use this file except in
   9  * compliance with the License. Please obtain a copy of the License at
  10  * http://www.opensource.apple.com/apsl/ and read it before using this
  11  * file.
  12  *
  13  * The Original Code and all software distributed under the License are
  14  * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  15  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  16  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
  17  * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
  18  * Please see the License for the specific language governing rights and
  19  * limitations under the License.
  20  *
  21  * @APPLE_LICENSE_HEADER_END@
  22  */
  23
  24
  25 #include <string.h>
  26 #include <stdint.h>
  27 #include <unistd.h>
  28 #include <dirent.h>
  29 #include <stdlib.h>
  30 #include <fcntl.h>
  31 #include <limits.h>
  32 #include <sys/types.h>
  33 #include <sys/stat.h>
  34 #include <sys/types.h>
  35 #include <sys/param.h>
  36 #include <sys/mount.h>
  37 #include <sys/mman.h>
  38 #include <dispatch/dispatch.h>
  39 #include <mach-o/dyld.h>
  40 #include <System/sys/csr.h>
  41 #include <rootless.h>
  42
  43 #include <string>
  44 #include <fstream>
  45 #include <sstream>
  46
  47 #include "FileUtils.h"
  48 #include "StringUtils.h"
  49 #include "Diagnostics.h"
  50
  51 #if __MAC_OS_X_VERSION_MIN_REQUIRED < 101200
  52 extern "C" int rootless_check_trusted_fd(int fd) __attribute__((weak_import));
  53 #endif
  54
  55
  56 void iterateDirectoryTree(const std::string& pathPrefix, const std::string& path, bool (^dirFilter)(const std::string& path), void (^fileCallback)(const std::string& path, const struct stat&), bool processFiles, bool recurse)
  57 {
  58     std::string fullDirPath = pathPrefix + path;
  59     DIR* dir = ::opendir(fullDirPath.c_str());
  60     if ( dir == nullptr ) {
  61         //fprintf(stderr, "can't read 'dir '%s', errno=%d\n", inputPath.c_str(), errno);
  62         return;
  63     }
  64     while (dirent* entry = readdir(dir)) {
  65         struct stat statBuf;
  66         std::string dirAndFile = path + "/" + entry->d_name;
  67         std::string fullDirAndFile = pathPrefix + dirAndFile;
  68          switch ( entry->d_type ) {
  69             case DT_REG:
  70                 if ( processFiles ) {
  71                     if ( ::lstat(fullDirAndFile.c_str(), &statBuf) == -1 )
  72                         break;
  73                     if ( ! S_ISREG(statBuf.st_mode)  )
  74                         break;
  75                     fileCallback(dirAndFile, statBuf);
  76                 }
  77                 break;
  78             case DT_DIR:
  79                 if ( strcmp(entry->d_name, ".") == 0 )
  80                     break;
  81                 if ( strcmp(entry->d_name, "..") == 0 )
  82                     break;
  83                 if ( dirFilter(dirAndFile) )
  84                     break;
  85                 if (recurse)
  86                     iterateDirectoryTree(pathPrefix, dirAndFile, dirFilter, fileCallback, processFiles, true);
  87                 break;
  88             case DT_LNK:
  89                 // don't follow symlinks, dylib will be found through absolute path
  90                 break;
  91         }
  92     }
  93     ::closedir(dir);
  94 }
  95
  96
  97 bool safeSave(const void* buffer, size_t bufferLen, const std::string& path)
  98 {
  99     std::string pathTemplate = path + "-XXXXXX";
 100     size_t templateLen = strlen(pathTemplate.c_str())+2;
 101     char pathTemplateSpace[templateLen];
 102     strlcpy(pathTemplateSpace, pathTemplate.c_str(), templateLen);
 103     int fd = mkstemp(pathTemplateSpace);
 104     if ( fd != -1 ) {
 105         ssize_t writtenSize = pwrite(fd, buffer, bufferLen, 0);
 106         if ( (size_t)writtenSize == bufferLen ) {
 107             ::fchmod(fd, S_IRUSR|S_IWUSR|S_IRGRP|S_IROTH); // mkstemp() makes file "rw-------", switch it to "rw-r--r--"
 108             if ( ::rename(pathTemplateSpace, path.c_str()) == 0) {
 109                 ::close(fd);
 110                 return true; // success
 111             }
 112         }
 113         ::close(fd);
 114         ::unlink(pathTemplateSpace);
 115     }
 116     return false; // failure
 117 }
 118
 119 const void* mapFileReadOnly(const char* path, size_t& mappedSize)
 120 {
 121     struct stat statBuf;
 122     if ( ::stat(path, &statBuf) != 0 )
 123         return nullptr;
 124
 125     int fd = ::open(path, O_RDONLY);
 126     if ( fd < 0 )
 127         return nullptr;
 128
 129     const void *p = ::mmap(NULL, (size_t)statBuf.st_size, PROT_READ, MAP_PRIVATE, fd, 0);
 130     ::close(fd);
 131     if ( p != MAP_FAILED ) {
 132         mappedSize = (size_t)statBuf.st_size;
 133         return p;
 134     }
 135
 136     return nullptr;
 137 }
 138
 139 static bool sipIsEnabled()
 140 {
 141     static bool             rootlessEnabled;
 142     static dispatch_once_t  onceToken;
 143     // Check to make sure file system protections are on at all
 144     dispatch_once(&onceToken, ^{
 145         rootlessEnabled = (csr_check(CSR_ALLOW_UNRESTRICTED_FS) != 0);
 146     });
 147     return rootlessEnabled;
 148 }
 149
 150 bool isProtectedBySIP(const std::string& path)
 151 {
 152     if ( !sipIsEnabled() )
 153         return false;
 154
 155     return (rootless_check_trusted(path.c_str()) == 0);
 156 }
 157
 158 bool isProtectedBySIP(int fd)
 159 {
 160     if ( !sipIsEnabled() )
 161         return false;
 162
 163 #if __MAC_OS_X_VERSION_MIN_REQUIRED >= 101200
 164     return (rootless_check_trusted_fd(fd) == 0);
 165 #else
 166     // fallback to using rootless_check_trusted
 167     char realPath[MAXPATHLEN];
 168     if ( fcntl(fd, F_GETPATH, realPath) == 0 )
 169         return (rootless_check_trusted(realPath) == 0);
 170     return false;
 171 #endif
 172 }
 173
 174 bool fileExists(const std::string& path)
 175 {
 176     struct stat statBuf;
 177     return ( ::stat(path.c_str(), &statBuf) == 0 );
 178 }
 179
 180 // There is an order file specifying the order in which dylibs are laid out in
 181 // general, as well as an order file specifying the order in which __DATA_DIRTY
 182 // segments are laid out in particular.
 183 //
 184 // The syntax is one dylib (install name) per line.  Blank lines are ignored.
 185 // Comments start with the # character.
 186 std::unordered_map<std::string, uint32_t> parseOrderFile(const std::string& orderFileData) {
 187     std::unordered_map<std::string, uint32_t> order;
 188
 189     if (orderFileData.empty())
 190         return order;
 191
 192     std::stringstream myData(orderFileData);
 193
 194     uint32_t count = 0;
 195     std::string line;
 196     while ( std::getline(myData, line) ) {
 197         size_t pos = line.find('#');
 198         if ( pos != std::string::npos )
 199             line.resize(pos);
 200         while ( !line.empty() && isspace(line.back()) ) {
 201             line.pop_back();
 202         }
 203         if ( !line.empty() )
 204             order[line] = count++;
 205     }
 206     return order;
 207 }
 208
 209 std::string loadOrderFile(const std::string& orderFilePath) {
 210     std::string order;
 211
 212     size_t size = 0;
 213     char* data = (char*)mapFileReadOnly(orderFilePath.c_str(), size);
 214     if (data) {
 215         order = std::string(data, size);
 216         ::munmap((void*)data, size);
 217     }
 218
 219     return order;
 220 }
 221
 222
 223 std::string toolDir()
 224 {
 225     char buffer[PATH_MAX];
 226     uint32_t bufsize = PATH_MAX;
 227     int result = _NSGetExecutablePath(buffer, &bufsize);
 228     if ( result == 0 ) {
 229         std::string path = buffer;
 230         size_t pos = path.rfind('/');
 231         if ( pos != std::string::npos )
 232             return path.substr(0,pos+1);
 233     }
 234     //warning("tool directory not found");
 235     return "/tmp/";
 236 }
 237
 238 std::string basePath(const std::string& path)
 239 {
 240     std::string::size_type slash_pos = path.rfind("/");
 241     if (slash_pos != std::string::npos) {
 242         slash_pos++;
 243         return path.substr(slash_pos);
 244     } else {
 245         return path;
 246     }
 247 }
 248
 249 std::string dirPath(const std::string& path)
 250 {
 251     std::string::size_type slash_pos = path.rfind("/");
 252     if (slash_pos != std::string::npos) {
 253         slash_pos++;
 254         return path.substr(0, slash_pos);
 255     } else {
 256         char cwd[MAXPATHLEN];
 257         (void)getcwd(cwd, MAXPATHLEN);
 258         return cwd;
 259     }
 260 }
 261
 262 std::string realPath(const std::string& path)
 263 {
 264     char resolvedPath[PATH_MAX];
 265     if (realpath(dirPath(path).c_str(), &resolvedPath[0]) != nullptr) {
 266         return std::string(resolvedPath) + "/" + basePath(path);
 267     } else {
 268         return "";
 269     }
 270 }
 271
 272 std::string realFilePath(const std::string& path)
 273 {
 274     char resolvedPath[PATH_MAX];
 275     if ( realpath(path.c_str(), resolvedPath) != nullptr )
 276         return std::string(resolvedPath);
 277     else
 278         return "";
 279 }
 280
 281
 282 std::string normalize_absolute_file_path(std::string path) {
 283     std::vector<std::string> components;
 284     std::vector<std::string> processed_components;
 285     std::stringstream ss(path);
 286     std::string retval;
 287     std::string item;
 288
 289     while (std::getline(ss, item, '/')) {
 290         components.push_back(item);
 291     }
 292
 293     if (components[0] == ".") {
 294         retval = ".";
 295     }
 296
 297     for (auto& component : components) {
 298         if (component.empty() || component == ".")
 299             continue;
 300         else if (component == ".." && processed_components.size())
 301             processed_components.pop_back();
 302         else
 303             processed_components.push_back(component);
 304     }
 305
 306     for (auto & component : processed_components) {
 307         retval = retval + "/" + component;
 308     }
 309
 310     return retval;
 311 }
 312
 313
 314 #if BUILDING_CACHE_BUILDER
 315
 316 FileCache fileCache;
 317
 318 FileCache::FileCache(void)
 319 {
 320     cache_queue = dispatch_queue_create("com.apple.dyld.cache.cache", dispatch_queue_attr_make_with_qos_class(DISPATCH_QUEUE_SERIAL, QOS_CLASS_USER_INITIATED, 0));
 321 }
 322
 323 std::pair<uint8_t*, struct stat> FileCache::cacheLoad(Diagnostics& diags, const std::string path)
 324 {
 325     __block bool found = false;
 326     __block std::pair<uint8_t*, struct stat> retval;
 327     std::string normalizedPath = normalize_absolute_file_path(path);
 328     dispatch_sync(cache_queue, ^{
 329         auto entry = entries.find(normalizedPath);
 330         if (entry != entries.end()) {
 331             retval = entry->second;
 332             found = true;
 333         }
 334     });
 335
 336     if (!found) {
 337         auto info = fill(diags, normalizedPath);
 338         dispatch_sync(cache_queue, ^{
 339             auto entry = entries.find(normalizedPath);
 340             if (entry != entries.end()) {
 341                 retval = entry->second;
 342             } else {
 343                 retval = entries[normalizedPath] = info;
 344                 retval = info;
 345             }
 346         });
 347     }
 348
 349     return retval;
 350 }
 351
 352 //FIXME error handling
 353 std::pair<uint8_t*, struct stat> FileCache::fill(Diagnostics& diags, const std::string& path)
 354 {
 355     void* buffer_ptr = nullptr;
 356     struct stat stat_buf;
 357     struct statfs statfs_buf;
 358     bool localcopy = true;
 359
 360     int fd = ::open(path.c_str(), O_RDONLY, 0);
 361     if (fd == -1) {
 362         diags.verbose("can't open file '%s', errno=%d\n", path.c_str(), errno);
 363         return std::make_pair((uint8_t*)(-1), stat_buf);
 364     }
 365
 366     if (fstat(fd, &stat_buf) == -1) {
 367         diags.verbose("can't stat open file '%s', errno=%d\n", path.c_str(), errno);
 368         ::close(fd);
 369         return std::make_pair((uint8_t*)(-1), stat_buf);
 370     }
 371
 372     if (stat_buf.st_size < 4096) {
 373         diags.verbose("file too small '%s'\n", path.c_str());
 374         ::close(fd);
 375         return std::make_pair((uint8_t*)(-1), stat_buf);
 376     }
 377
 378     if(fstatfs(fd, &statfs_buf) == 0) {
 379         std::string fsName = statfs_buf.f_fstypename;
 380         if (fsName == "hfs" || fsName == "apfs") {
 381             localcopy = false;
 382         }
 383     }
 384
 385     if (!localcopy) {
 386         buffer_ptr = mmap(NULL, (size_t)stat_buf.st_size, PROT_READ, MAP_PRIVATE, fd, 0);
 387         if (buffer_ptr == MAP_FAILED) {
 388             diags.verbose("mmap() for file at %s failed, errno=%d\n", path.c_str(), errno);
 389             ::close(fd);
 390             return std::make_pair((uint8_t*)(-1), stat_buf);
 391         }
 392     } else {
 393         buffer_ptr = malloc((size_t)stat_buf.st_size);
 394         ssize_t readBytes = pread(fd, buffer_ptr, (size_t)stat_buf.st_size, 0);
 395         if (readBytes == -1) {
 396             diags.verbose("Network read for file at %s failed, errno=%d\n", path.c_str(), errno);
 397             ::close(fd);
 398             return std::make_pair((uint8_t*)(-1), stat_buf);
 399         } else if (readBytes != stat_buf.st_size) {
 400             diags.verbose("Network read udnerrun for file at %s, expected %lld bytes, got  %zd bytes\n", path.c_str(), stat_buf.st_size, readBytes);
 401             ::close(fd);
 402             return std::make_pair((uint8_t*)(-1), stat_buf);
 403         }
 404     }
 405
 406     ::close(fd);
 407
 408     return std::make_pair((uint8_t*)buffer_ptr, stat_buf);
 409 }
 410
 411 static void normalizePath(std::string& path) {
 412     // Remove a bunch of stuff we don't need, like trailing slashes.
 413     while ( !path.empty() && (path.back() == '/'))
 414         path.pop_back();
 415 }
 416
 417 void SymlinkResolver::addFile(Diagnostics& diags, std::string path) {
 418     if (path.front() != '/') {
 419         diags.error("Path must start with '/'");
 420         return;
 421     }
 422     if (symlinks.find(path) != symlinks.end()) {
 423         diags.error("Cannot add regular file as it is already a symlink");
 424         return;
 425     }
 426     filePaths.insert(path);
 427 }
 428
 429 void SymlinkResolver::addSymlink(Diagnostics& diags, std::string fromPath, std::string toPath) {
 430     normalizePath(fromPath);
 431     normalizePath(toPath);
 432     if (fromPath.front() != '/') {
 433         diags.error("Path must start with '/'");
 434         return;
 435     }
 436     if (filePaths.find(fromPath) != filePaths.end()) {
 437         diags.error("Cannot add symlink from '%s' as it is already a regular path", fromPath.c_str());
 438         return;
 439     }
 440     auto itAndInserted = symlinks.insert({ fromPath, toPath });
 441     if (!itAndInserted.second) {
 442         // The path is already a symlink.  Make sure its a dupe.
 443         if (toPath != itAndInserted.first->second) {
 444             diags.error("Duplicate symlink for path '%s'", fromPath.c_str());
 445             return;
 446         }
 447     }
 448 }
 449
 450 std::string SymlinkResolver::realPath(Diagnostics& diags, const std::string& originalPath) const {
 451     // First make sure the path doesn't have any magic in it.
 452     std::string path = originalPath;
 453     normalizePath(path);
 454
 455     std::set<std::string> seenSymlinks;
 456
 457     // Now see if any prefix is a symlink
 458     if (path.front() != '/')
 459         return path;
 460
 461     std::string::size_type prev_pos = 0;
 462     while (prev_pos != std::string::npos) {
 463         std::string::size_type pos = path.find("/", prev_pos + 1);
 464
 465         // First look to see if this path component is special, eg, ., .., etc.
 466         std::string component = path.substr(prev_pos, pos - prev_pos);
 467         if (component == "/..") {
 468             // Fold with the previous path component.
 469             if (prev_pos == 0) {
 470                 // This is the root path, and .. applied to / is just /
 471                 path = path.substr(3);
 472                 prev_pos = 0;
 473             } else {
 474                 std::string::size_type lastSlashPos = path.rfind("/", prev_pos - 1);
 475                 path = path.substr(0, lastSlashPos) + path.substr(pos);
 476                 prev_pos = lastSlashPos;
 477             }
 478             continue;
 479         } else if (component == "/.") {
 480             if (prev_pos == 0) {
 481                 // Path starts with /./ so just remove the first one.
 482                 path = path.substr(2);
 483             } else {
 484                 if (pos == std::string::npos) {
 485                     // Trailing . on the path
 486                     path = path.substr(0, prev_pos );
 487                 } else {
 488                     path = path.substr(0, prev_pos) + path.substr(pos);
 489                 }
 490             }
 491             continue;
 492         } else if (component == "/") {
 493             // Path must contain // somewhere so strip out the duplicates.
 494             if (prev_pos == 0) {
 495                 // Path starts with // so just remove the first one.
 496                 path = path.substr(1);
 497             } else {
 498                 if (pos == std::string::npos) {
 499                     // Trailing / on the path
 500                     path = path.substr(0, prev_pos);
 501                     prev_pos = pos;
 502                 } else {
 503                     path = path.substr(0, pos) + path.substr(pos + 1);
 504                 }
 505             }
 506             continue;
 507         }
 508
 509         // Path is not special, so see if it is a symlink to something.
 510         std::string prefix = path.substr(0, pos);
 511         //printf("%s\n", prefix.c_str());
 512         auto it = symlinks.find(prefix);
 513         if (it == symlinks.end()) {
 514             // This is not a symlink so move to the next prefix.
 515             prev_pos = pos;
 516             continue;
 517         }
 518
 519         // If we've already done this prefix then error out.
 520         if (seenSymlinks.count(prefix)) {
 521             diags.error("Loop in symlink processing for '%s'", originalPath.c_str());
 522             return std::string();
 523         }
 524
 525         seenSymlinks.insert(prefix);
 526
 527         // This is a symlink, so resolve the new path.
 528         std::string toPath = it->second;
 529         if (toPath.front() == '/') {
 530             // Symlink points to an absolute address so substitute the whole prefix for the new path
 531             // If we didn't substitute the last component of the path then there is also a path suffix.
 532             std::string pathSuffix = "";
 533             if (pos != std::string::npos) {
 534                 std::string::size_type nextSlashPos = path.find("/", pos + 1);
 535                 if (nextSlashPos != std::string::npos)
 536                     pathSuffix = path.substr(nextSlashPos);
 537             }
 538             path = toPath + pathSuffix;
 539             prev_pos = 0;
 540             continue;
 541         }
 542
 543         // Symlink points to a relative path so we need to do more processing to get the real path.
 544
 545         // First calculate which part of the previous prefix we'll keep.  Eg, in /a/b/c where "b -> blah", we want to keep /a here.
 546         std::string prevPrefix = path.substr(0, prev_pos);
 547         //printf("prevPrefix %s\n", prevPrefix.c_str());
 548
 549         // If we didn't substitute the last component of the path then there is also a path suffix.
 550         std::string pathSuffix = "";
 551         if (prefix.size() != path.size())
 552             pathSuffix = path.substr(pos);
 553
 554         // The new path is the remaining prefix, plus the symlink target, plus any remaining suffix from the original path.
 555         path = prevPrefix + "/" + toPath + pathSuffix;
 556         prev_pos = 0;
 557     }
 558     return path;
 559 }
 560
 561 std::vector<DyldSharedCache::FileAlias> SymlinkResolver::getResolvedSymlinks(Diagnostics& diags) {
 562     diags.assertNoError();
 563     std::vector<DyldSharedCache::FileAlias> aliases;
 564     for (auto& fromPathAndToPath : symlinks) {
 565         std::string newPath = realPath(diags, fromPathAndToPath.first);
 566         if (diags.hasError()) {
 567             aliases.clear();
 568             return aliases;
 569         }
 570
 571         if (filePaths.count(newPath)) {
 572             aliases.push_back({ newPath, fromPathAndToPath.first });
 573             // printf("symlink ('%s' -> '%s') resolved to '%s'\n", fromPathAndToPath.first.c_str(), fromPathAndToPath.second.c_str(), newPath.c_str());
 574         }
 575     }
 576     return aliases;
 577 }
 578
 579 #endif // BUILDING_CACHE_BUILDER
 580