]> git.saurik.com Git - apple/dyld.git/blob - interlinked-dylibs/multi_dyld_shared_cache_builder.mm
dyld-421.2.tar.gz
[apple/dyld.git] / interlinked-dylibs / multi_dyld_shared_cache_builder.mm
1 /* -*- mode: C++; c-basic-offset: 4; indent-tabs-mode: nil -*-
2 *
3 * Copyright (c) 2016 Apple Inc. All rights reserved.
4 *
5 * @APPLE_LICENSE_HEADER_START@
6 *
7 * This file contains Original Code and/or Modifications of Original Code
8 * as defined in and that are subject to the Apple Public Source License
9 * Version 2.0 (the 'License'). You may not use this file except in
10 * compliance with the License. Please obtain a copy of the License at
11 * http://www.opensource.apple.com/apsl/ and read it before using this
12 * file.
13 *
14 * The Original Code and all software distributed under the License are
15 * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
16 * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
17 * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
18 * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
19 * Please see the License for the specific language governing rights and
20 * limitations under the License.
21 *
22 * @APPLE_LICENSE_HEADER_END@
23 */
24
25 #include <dispatch/dispatch.h>
26 #include <Security/Security.h>
27 #include <Security/SecCodeSigner.h>
28
29 #include <sys/types.h>
30 #include <sys/stat.h>
31 #include <sys/mman.h>
32 #include <sys/resource.h>
33 #include <mach/mach.h>
34 #include <mach/mach_time.h>
35 #include <limits.h>
36 #include <stdarg.h>
37 #include <stdio.h>
38 #include <stdlib.h>
39 #include <math.h>
40 #include <fcntl.h>
41 #include <dlfcn.h>
42 #include <signal.h>
43 #include <errno.h>
44 #include <sys/uio.h>
45 #include <unistd.h>
46 #include <sys/param.h>
47 #include <sys/sysctl.h>
48 #include <sys/resource.h>
49 #include <dirent.h>
50 #include <libgen.h>
51 #include <pthread.h>
52
53 #include <vector>
54 #include <array>
55 #include <set>
56 #include <map>
57 #include <algorithm>
58
59 #include "MultiCacheBuilder.h"
60 #include "Manifest.h"
61
62 #include "mega-dylib-utils.h"
63 #include "Logging.h"
64
65 #if !__has_feature(objc_arc)
66 #error The use of libdispatch in this files requires it to be compiled with ARC in order to avoid leaks
67 #endif
68
69 static dispatch_queue_t build_queue = dispatch_queue_create("com.apple.dyld.build", DISPATCH_QUEUE_CONCURRENT);
70 static dispatch_group_t build_group = dispatch_group_create();
71 static dispatch_semaphore_t writeLimitingSemaphore = dispatch_semaphore_create(1);
72
73 bool copyFile(const std::string& from, const std::string& to)
74 {
75 const uint8_t* p = (uint8_t*)(-1);
76 struct stat stat_buf;
77 bool rp;
78
79 std::tie(p, stat_buf, rp) = fileCache.cacheLoad(from);
80 if (p == (uint8_t*)(-1))
81 return false;
82
83 dispatch_group_enter(build_group);
84 mkpath_np(dirpath(to).c_str(), 0755);
85
86 dispatch_semaphore_wait(writeLimitingSemaphore, DISPATCH_TIME_FOREVER);
87 int fd = open(to.c_str(), O_WRONLY | O_CREAT | O_TRUNC, 0644);
88 if (fd > 0) {
89 ssize_t writtenSize = pwrite(fd, p, stat_buf.st_size, 0);
90 if (writtenSize != stat_buf.st_size)
91 terminate("write() failure creating cache file, errno=%d (%s)", errno, strerror(errno));
92
93 ::close(fd);
94 verboseLog("Wrote out: %s", to.c_str());
95 } else {
96 terminate("can't create file '%s', errnor=%d (%s)", to.c_str(), errno, strerror(errno));
97 }
98 dispatch_semaphore_signal(writeLimitingSemaphore);
99 dispatch_group_leave(build_group);
100
101 return true;
102 }
103
104 #define kDylibCachePrefix "/AppleInternal/Developer/DylibCaches/"
105
106 void createArtifact(Manifest& manifest, const std::string& dylibCachePath, bool includeExecutables)
107 {
108 mkpath_np(dylibCachePath.c_str(), 0755);
109 (void)copyFile(manifest.dylibOrderFile, dylibCachePath + "Metadata/dylibOrderFile.txt");
110 (void)copyFile(manifest.dirtyDataOrderFile, dylibCachePath + "Metadata/dirtyDataOrderFile.txt");
111 (void)copyFile(manifest.metabomFile, dylibCachePath + "Metadata/metabom.bom");
112
113 std::set<std::string> copied;
114
115 for (auto archFiles : manifest.architectureFiles) {
116 for (auto& file : archFiles.second.dylibs) {
117 std::string installname = file.first;
118 if (copied.count(installname) > 0) {
119 continue;
120 }
121 (void)copyFile(file.second.proxy->path, normalize_absolute_file_path(dylibCachePath + "/Root/" + file.first));
122 copied.insert(installname);
123 }
124 if (includeExecutables) {
125 for (auto& file : archFiles.second.executables) {
126 std::string installname = file.first;
127 if (copied.count(installname) > 0) {
128 continue;
129 }
130 (void)copyFile(file.second.proxy->path, normalize_absolute_file_path(dylibCachePath + "/Root/" + file.first));
131 copied.insert(installname);
132 }
133 }
134 }
135
136 log("Artifact dylibs copied");
137 }
138
139 void addArtifactPaths(Manifest &manifest) {
140 manifest.dylibOrderFile = "./Metadata/dylibOrderFile.txt";
141 manifest.dirtyDataOrderFile = "./Metadata/dirtyDataOrderFile.txt";
142 manifest.metabomFile = "./Metadata/metabom.bom";
143
144 for ( auto& projects : manifest.projects ) {
145 if ( projects.second.sources[0] != "./Root/" ) {
146 projects.second.sources.insert( projects.second.sources.begin(), "./Root/" );
147 }
148 }
149 }
150
151 int main (int argc, const char * argv[]) {
152 @autoreleasepool {
153 auto defaultCtx = std::make_shared<LoggingContext>("");
154 setLoggingContext(defaultCtx);
155
156 Manifest manifest;
157 std::string masterDstRoot;
158 std::string dylibCacheDir;
159 std::string resultPath;
160 bool skipWrites = false;
161 bool skipBuilds = false;
162 bool preflight = false;
163 std::string manifestPath;
164
165 time_t mytime = time(0);
166 log("Started: %s", asctime(localtime(&mytime)));
167
168 // parse command line options
169 for (int i = 1; i < argc; ++i) {
170 const char* arg = argv[i];
171 if (arg[0] == '-') {
172 if (strcmp(arg, "-debug") == 0) {
173 setVerbose(true);
174 } else if (strcmp(arg, "-skip_writes") == 0) {
175 skipWrites = true;
176 } else if (strcmp(arg, "-skip_builds") == 0) {
177 skipBuilds = true;
178 } else if (strcmp(arg, "-delete_writes") == 0) {
179 skipWrites = true;
180 } else if (strcmp(arg, "-dylib_cache") == 0) {
181 dylibCacheDir = argv[++i];
182 } else if (strcmp(arg, "-preflight") == 0) {
183 preflight = true;
184 skipWrites = true;
185 } else if (strcmp(arg, "-master_dst_root") == 0) {
186 masterDstRoot = argv[++i];
187 if (masterDstRoot.empty()) {
188 terminate("-master_dst_root missing path argument");
189 }
190 } else if (strcmp(arg, "-results") == 0) {
191 resultPath = argv[++i];
192 } else if (strcmp(arg, "-plist") == 0) {
193 manifestPath = argv[++i];
194 (void)chdir(dirname(strdup(manifestPath.c_str())));
195 manifest = Manifest(manifestPath);
196 } else {
197 // usage();
198 terminate("unknown option: %s", arg);
199 }
200 } else {
201 manifestPath = argv[i];
202
203 (void)chdir(dirname(strdup(argv[i])));
204 }
205 }
206
207 if ( getenv( "LGG_SKIP_CACHE_FUN" ) != nullptr ) {
208 skipBuilds = true;
209 }
210
211 if (manifest.build.empty()) {
212 terminate("No version found in manifest");
213 }
214 log("Building Caches for %s", manifest.build.c_str());
215
216 if ( masterDstRoot.empty() ) {
217 terminate("-master_dst_root required path argument");
218 }
219
220 if (manifest.manifest_version < 4) {
221 terminate("must specify valid manifest file");
222 }
223
224 struct rlimit rl = {OPEN_MAX, OPEN_MAX};
225 (void)setrlimit(RLIMIT_NOFILE, &rl);
226
227 if (!skipWrites) {
228 (void)mkpath_np((masterDstRoot + "/Boms/").c_str(), 0755);
229 }
230
231 if (manifest.dylibOrderFile.empty()) {
232 manifest.dylibOrderFile = toolDir() + "/dylib-order.txt";
233 }
234
235 if (manifest.dirtyDataOrderFile.empty()) {
236 manifest.dirtyDataOrderFile = toolDir() + "/dirty-data-segments-order.txt";
237 }
238
239 auto dylibCacheCtx = std::make_shared<LoggingContext>("DylibCache");
240 setLoggingContext(dylibCacheCtx);
241
242 if (!skipWrites) {
243 cacheBuilderDispatchGroupAsync(build_group, build_queue, [&] {
244 createArtifact(manifest, masterDstRoot + "/Artifact.dlc/", true);
245 });
246
247 if (!dylibCacheDir.empty()) {
248 cacheBuilderDispatchGroupAsync(build_group, build_queue, [&] {
249 createArtifact(manifest, dylibCacheDir + "/AppleInternal/Developer/DylibCaches/" + manifest.build + ".dlc/", false);
250 });
251 }
252 }
253 setLoggingContext(defaultCtx);
254
255 manifest.calculateClosure(false);
256 std::shared_ptr<MultiCacheBuilder> builder = std::make_shared<MultiCacheBuilder>(manifest, true, skipWrites, false, skipBuilds);
257 dispatch_group_async(build_group, build_queue, [&] { builder->buildCaches(masterDstRoot); });
258 dispatch_group_wait(build_group, DISPATCH_TIME_FOREVER);
259
260 if (!preflight) {
261 setLoggingContext(dylibCacheCtx);
262 addArtifactPaths(manifest);
263 setLoggingContext(defaultCtx);
264
265 if (!resultPath.empty()) {
266 manifest.write(resultPath);
267 }
268
269 setLoggingContext(dylibCacheCtx);
270 copyFile(manifestPath, masterDstRoot + "/Artifact.dlc/BNIManifest.plist");
271 manifest.write(masterDstRoot + "/Artifact.dlc/Manifest.plist");
272
273 if (!dylibCacheDir.empty()) {
274 manifest.write(dylibCacheDir + kDylibCachePrefix + manifest.build + ".dlc/Manifest.plist");
275 }
276 setLoggingContext(defaultCtx);
277 }
278
279 builder->logStats();
280
281 dumpLogAndExit();
282 }
283
284 dispatch_main();
285
286 return 0;
287 }
288
289