dyld-551.3.tar.gz
[apple/dyld.git] / dyld3 / shared-cache / BuilderUtils.mm
1 /*
2 * Copyright (c) 2017 Apple Inc. All rights reserved.
3 *
4 * @APPLE_LICENSE_HEADER_START@
5 *
6 * This file contains Original Code and/or Modifications of Original Code
7 * as defined in and that are subject to the Apple Public Source License
8 * Version 2.0 (the 'License'). You may not use this file except in
9 * compliance with the License. Please obtain a copy of the License at
10 * http://www.opensource.apple.com/apsl/ and read it before using this
11 * file.
12 *
13 * The Original Code and all software distributed under the License are
14 * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
15 * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
16 * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
17 * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
18 * Please see the License for the specific language governing rights and
19 * limitations under the License.
20 *
21 * @APPLE_LICENSE_HEADER_END@
22 */
23
24
25 #include <set>
26 #include <string>
27 #include <sstream>
28 #include <iomanip> // std::setfill, std::setw
29 #include <pthread.h>
30 #include <mach/mach.h>
31 #include <dispatch/dispatch.h>
32
33 #include <Bom/Bom.h>
34 #include <Security/Security.h>
35 #include <Security/SecCodeSigner.h>
36 #include <CommonCrypto/CommonCrypto.h>
37
38 #include "Manifest.h"
39 #include "Diagnostics.h"
40 #include "FileUtils.h"
41
42 #include "BuilderUtils.h"
43
44 static dispatch_queue_t write_queue = dispatch_queue_create("com.apple.dyld.cache-builder.write", DISPATCH_QUEUE_CONCURRENT);
45 static dispatch_group_t build_group = dispatch_group_create();
46
47 dispatch_group_t buildGroup() {
48 return build_group;
49 }
50
51 void insertFileInBom(const std::string& path, BOMBom bom)
52 {
53 std::vector<std::string> components;
54 std::vector<std::string> processed_components;
55 std::stringstream ss(path);
56 std::string item;
57
58 while (std::getline(ss, item, '/')) {
59 if (!item.empty()) {
60 components.push_back(item);
61 }
62 }
63
64 std::string partialPath = ".";
65 std::string lastComponent = components.back();
66 components.pop_back();
67 BOMFSObject fso = BOMFSObjectNew(BOMDirectoryType);
68 BOMFSObjectSetFlags(fso, B_PATHONLY);
69 BOMFSObjectSetPathName(fso, ".", true);
70 BOMFSObjectSetShortName(fso, ".", true);
71 (void)BOMBomInsertFSObject(bom, fso, false);
72 BOMFSObjectFree(fso);
73
74 for (const auto& component : components) {
75 partialPath = partialPath + "/" + component;
76 fso = BOMFSObjectNew(BOMDirectoryType);
77 BOMFSObjectSetFlags(fso, B_PATHONLY);
78 BOMFSObjectSetPathName(fso, partialPath.c_str(), true);
79 BOMFSObjectSetShortName(fso, component.c_str(), true);
80 (void)BOMBomInsertFSObject(bom, fso, false);
81 BOMFSObjectFree(fso);
82 }
83
84 partialPath = partialPath + "/" + lastComponent;
85 fso = BOMFSObjectNew(BOMFileType);
86 BOMFSObjectSetFlags(fso, B_PATHONLY);
87 BOMFSObjectSetPathName(fso, partialPath.c_str(), true);
88 BOMFSObjectSetShortName(fso, lastComponent.c_str(), true);
89 (void)BOMBomInsertFSObject(bom, fso, false);
90 BOMFSObjectFree(fso);
91 }
92
93 void makeBoms(dyld3::Manifest& manifest, const std::string& masterDstRoot)
94 {
95 mkpath_np((masterDstRoot + "/Boms/").c_str(), 0755);
96
97 manifest.forEachConfiguration([&manifest, &masterDstRoot](const std::string& configName) {
98 auto config = manifest.configuration(configName);
99 std::vector<std::string> prodBomPaths;
100 std::vector<std::string> devBomPaths;
101
102 std::string runtimePath = "/System/Library/Caches/com.apple.dyld/";
103 if (manifest.platform() == dyld3::Platform::macOS) {
104 runtimePath = "/private/var/db/dyld/";
105 }
106
107 for (auto& arch : config.architectures) {
108 std::string cachePath = "dyld_shared_cache_" + arch.first;
109 prodBomPaths.push_back(cachePath);
110 if (manifest.platform() != dyld3::Platform::macOS) {
111 cachePath += ".development";
112 }
113 devBomPaths.push_back(cachePath);
114 char buffer[MAXPATHLEN];
115 sprintf(buffer, "%s/Boms/%s.prod.bom", masterDstRoot.c_str(), configName.c_str());
116 BOMBom bom = BOMBomNew(buffer);
117 for (auto& path : prodBomPaths) {
118 insertFileInBom(runtimePath + path, bom);
119 }
120 BOMBomFree(bom);
121
122 sprintf(buffer, "%s/Boms/%s.dev.bom", masterDstRoot.c_str(), configName.c_str());
123 bom = BOMBomNew(buffer);
124 for (auto& path : devBomPaths) {
125 insertFileInBom(runtimePath + path, bom);
126 }
127 BOMBomFree(bom);
128
129 sprintf(buffer, "%s/Boms/%s.full.bom", masterDstRoot.c_str(), configName.c_str());
130 bom = BOMBomNew(buffer);
131 for (auto& path : prodBomPaths) {
132 insertFileInBom(runtimePath + path, bom);
133 }
134 for (auto& path : devBomPaths) {
135 insertFileInBom(runtimePath + path, bom);
136 }
137 BOMBomFree(bom);
138 }
139 });
140 }
141
142 bool build(Diagnostics& diags, dyld3::Manifest& manifest, const std::string& masterDstRoot, bool dedupe, bool verbose,
143 bool skipWrites, bool agileChooseSHA256CdHash)
144 {
145 dispatch_queue_t queue = dispatch_get_global_queue(DISPATCH_QUEUE_PRIORITY_DEFAULT, 0);
146 dispatch_queue_t warningQueue = dispatch_queue_create("com.apple.dyld.cache-builder.warnings", DISPATCH_QUEUE_SERIAL);
147 std::vector<std::set<std::string>> dedupedCacheSets;
148 if (dedupe) {
149 manifest.forEachConfiguration([&manifest, &dedupedCacheSets](const std::string& configName) {
150 auto config = manifest.configuration(configName);
151 bool dupeFound = false;
152
153 for (auto& cacheSet : dedupedCacheSets) {
154 if (config == manifest.configuration(*cacheSet.begin())) {
155 cacheSet.insert(configName);
156 dupeFound = true;
157 break;
158 }
159 }
160
161 if (!dupeFound) {
162 std::set<std::string> temp;
163 temp.insert(configName);
164 dedupedCacheSets.push_back(temp);
165 }
166 });
167 } else {
168 manifest.forEachConfiguration([&manifest, &dedupedCacheSets](const std::string& configName) {
169 std::set<std::string> temp;
170 temp.insert(configName);
171 dedupedCacheSets.push_back(temp);
172 });
173 }
174
175 std::vector<dyld3::BuildQueueEntry> buildQueue;
176
177 for (auto& cacheSet : dedupedCacheSets) {
178 //FIXME we may want to consider moving to hashes of UUID sets
179 std::string setName;
180
181 for (auto& archName : cacheSet) {
182 if (!setName.empty()) {
183 setName += "|";
184 }
185 setName += archName;
186 }
187
188 std::stringstream fileNameStream;
189 std::array<uint8_t, CC_SHA1_DIGEST_LENGTH> digest = { 0 };
190 CC_SHA1(setName.c_str(), (unsigned int)setName.length(), &digest[0]);
191
192 fileNameStream << std::hex << std::uppercase << std::setfill('0');
193 for (int c : digest) {
194 fileNameStream << std::setw(2) << c;
195 }
196
197 std::string fileName(fileNameStream.str());
198
199 if (dedupe) {
200 for (auto& config : cacheSet) {
201 if (!skipWrites) {
202 int err = symlink(("DedupedConfigs/" + fileName).c_str(), (masterDstRoot + "/" + config).c_str());
203 if (err) {
204 diags.warning("Could not create symlink '%s' -> 'DedupedConfigs/%s' (%d)", config.c_str(), fileName.c_str(), err);
205 }
206 }
207 }
208 }
209
210 manifest.configuration(*cacheSet.begin()).forEachArchitecture([&masterDstRoot, &dedupe, &fileName, &setName, &manifest, &buildQueue, &cacheSet, verbose](const std::string& arch) {
211 std::string configPath;
212 std::string runtimePath = "/System/Library/Caches/com.apple.dyld/";
213 if (manifest.platform() == dyld3::Platform::macOS) {
214 runtimePath = "/private/var/db/dyld/";
215 }
216 if (dedupe) {
217 configPath = masterDstRoot + "/DedupedConfigs/" + fileName + runtimePath;
218 } else {
219 configPath = masterDstRoot + runtimePath;
220 }
221
222 if (manifest.platform() == dyld3::Platform::macOS) {
223 buildQueue.push_back(manifest.makeQueueEntry(configPath + "dyld_shared_cache_" + arch, cacheSet, arch, false, setName + "/" + arch, verbose));
224 } else {
225 buildQueue.push_back(manifest.makeQueueEntry(configPath + "dyld_shared_cache_" + arch + ".development", cacheSet, arch, false, setName + "/" + arch, verbose));
226 buildQueue.push_back(manifest.makeQueueEntry(configPath + "dyld_shared_cache_" + arch, cacheSet, arch, true, setName + "/" + arch, verbose));
227 }
228 });
229 }
230
231 __block bool cacheBuildFailure = false;
232 __block std::set<std::string> warnings;
233 __block std::set<std::string> errors;
234
235 dispatch_sync(warningQueue, ^{
236 auto manifestWarnings = diags.warnings();
237 warnings.insert(manifestWarnings.begin(), manifestWarnings.end());
238 });
239
240 dispatch_apply(buildQueue.size(), queue, ^(size_t index) {
241 auto queueEntry = buildQueue[index];
242 pthread_setname_np(queueEntry.options.loggingPrefix.substr(0, MAXTHREADNAMESIZE - 1).c_str());
243
244 DyldSharedCache::CreateResults results = DyldSharedCache::create(queueEntry.options, queueEntry.dylibsForCache, queueEntry.otherDylibsAndBundles, queueEntry.mainExecutables);
245 dispatch_sync(warningQueue, ^{
246 warnings.insert(results.warnings.begin(), results.warnings.end());
247 bool chooseSecondCdHash = agileChooseSHA256CdHash;
248 if (agileChooseSHA256CdHash && !results.agileSignature) {
249 // Ignore this option for caches that are not signed agile (which is the majority).
250 chooseSecondCdHash = false;
251 }
252 for (const auto& configName : queueEntry.configNames) {
253 auto& configResults = manifest.configuration(configName).architecture(queueEntry.options.archName).results;
254 for (const auto& mh : results.evictions) {
255 auto parser = dyld3::MachOParser(mh);
256 configResults.exclude(&parser, "VM overflow, evicting");
257 }
258 configResults.warnings = results.warnings;
259 if (queueEntry.options.optimizeStubs) {
260 configResults.developmentCache.cdHash = chooseSecondCdHash ? results.cdHashSecond : results.cdHashFirst;
261 } else {
262 configResults.productionCache.cdHash = chooseSecondCdHash ? results.cdHashSecond : results.cdHashFirst;
263 }
264 }
265 });
266 if (!results.errorMessage.empty()) {
267 fprintf(stderr, "[%s] ERROR: %s\n", queueEntry.options.loggingPrefix.c_str(), results.errorMessage.c_str());
268 } else if (!skipWrites) {
269 dispatch_sync(write_queue, ^{
270 // save new cache file to disk and write new .map file
271 assert(results.cacheContent != nullptr);
272 mkpath_np(dirPath(queueEntry.outputPath).c_str(), 0755);
273 if (!safeSave(results.cacheContent, results.cacheLength, queueEntry.outputPath)) {
274 cacheBuildFailure = true;
275 fprintf(stderr, "[%s] ERROR: Could not write cache to: %s\n", queueEntry.options.loggingPrefix.c_str(), queueEntry.outputPath.c_str());
276 } else {
277 fprintf(stderr, "[%s] Wrote cache to: %s\n", queueEntry.options.loggingPrefix.c_str(), queueEntry.outputPath.c_str());
278 std::string mapStr = results.cacheContent->mapFile();
279 std::string outFileMap = queueEntry.outputPath + ".map";
280 safeSave(mapStr.c_str(), mapStr.size(), outFileMap);
281 }
282 // free created cache buffer
283 vm_deallocate(mach_task_self(), (vm_address_t)results.cacheContent, results.cacheLength);
284 });
285 } else {
286 fprintf(stderr, "[%s] Skipped writing cache to: %s\n", queueEntry.options.loggingPrefix.c_str(), queueEntry.outputPath.c_str());
287 vm_deallocate(mach_task_self(), (vm_address_t)results.cacheContent, results.cacheLength);
288 }
289 });
290
291 // print any warnings
292 for (const std::string& warn : warnings) {
293 fprintf(stderr, "[WARNING] %s\n", warn.c_str());
294 }
295
296 int err = sync_volume_np(masterDstRoot.c_str(), SYNC_VOLUME_FULLSYNC | SYNC_VOLUME_WAIT);
297 if (err) {
298 fprintf(stderr, "Volume sync failed errnor=%d (%s)\n", err, strerror(err));
299 }
300
301 return !cacheBuildFailure;
302 }