]>
Commit | Line | Data |
---|---|---|
2d21ac55 A |
1 | /* |
2 | * Copyright (c) 2007 Apple Inc. All rights reserved. | |
3 | * | |
4 | * @APPLE_LICENSE_HEADER_START@ | |
5 | * | |
6 | * This file contains Original Code and/or Modifications of Original Code | |
7 | * as defined in and that are subject to the Apple Public Source License | |
8 | * Version 2.0 (the 'License'). You may not use this file except in | |
9 | * compliance with the License. Please obtain a copy of the License at | |
10 | * http://www.opensource.apple.com/apsl/ and read it before using this | |
11 | * file. | |
12 | * | |
13 | * The Original Code and all software distributed under the License are | |
14 | * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER | |
15 | * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES, | |
16 | * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY, | |
17 | * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT. | |
18 | * Please see the License for the specific language governing rights and | |
19 | * limitations under the License. | |
20 | * | |
21 | * @APPLE_LICENSE_HEADER_END@ | |
22 | */ | |
23 | ||
24 | /* | |
25 | * Shared region (... and comm page) | |
26 | * | |
27 | * This file handles the VM shared region and comm page. | |
28 | * | |
29 | */ | |
30 | /* | |
31 | * SHARED REGIONS | |
32 | * -------------- | |
33 | * | |
34 | * A shared region is a submap that contains the most common system shared | |
35 | * libraries for a given environment. | |
36 | * An environment is defined by (cpu-type, 64-bitness, root directory). | |
37 | * | |
38 | * The point of a shared region is to reduce the setup overhead when exec'ing | |
39 | * a new process. | |
40 | * A shared region uses a shared VM submap that gets mapped automatically | |
41 | * at exec() time (see vm_map_exec()). The first process of a given | |
42 | * environment sets up the shared region and all further processes in that | |
43 | * environment can re-use that shared region without having to re-create | |
44 | * the same mappings in their VM map. All they need is contained in the shared | |
45 | * region. | |
46 | * It can also shared a pmap (mostly for read-only parts but also for the | |
47 | * initial version of some writable parts), which gets "nested" into the | |
48 | * process's pmap. This reduces the number of soft faults: once one process | |
49 | * brings in a page in the shared region, all the other processes can access | |
50 | * it without having to enter it in their own pmap. | |
51 | * | |
52 | * | |
53 | * When a process is being exec'ed, vm_map_exec() calls vm_shared_region_enter() | |
54 | * to map the appropriate shared region in the process's address space. | |
55 | * We look up the appropriate shared region for the process's environment. | |
56 | * If we can't find one, we create a new (empty) one and add it to the list. | |
57 | * Otherwise, we just take an extra reference on the shared region we found. | |
58 | * | |
59 | * The "dyld" runtime (mapped into the process's address space at exec() time) | |
60 | * will then use the shared_region_check_np() and shared_region_map_np() | |
61 | * system call to validate and/or populate the shared region with the | |
62 | * appropriate dyld_shared_cache file. | |
63 | * | |
64 | * The shared region is inherited on fork() and the child simply takes an | |
65 | * extra reference on its parent's shared region. | |
66 | * | |
67 | * When the task terminates, we release a reference on its shared region. | |
68 | * When the last reference is released, we destroy the shared region. | |
69 | * | |
70 | * After a chroot(), the calling process keeps using its original shared region, | |
71 | * since that's what was mapped when it was started. But its children | |
72 | * will use a different shared region, because they need to use the shared | |
73 | * cache that's relative to the new root directory. | |
74 | */ | |
75 | /* | |
76 | * COMM PAGE | |
77 | * | |
78 | * A "comm page" is an area of memory that is populated by the kernel with | |
79 | * the appropriate platform-specific version of some commonly used code. | |
80 | * There is one "comm page" per platform (cpu-type, 64-bitness) but only | |
81 | * for the native cpu-type. No need to overly optimize translated code | |
82 | * for hardware that is not really there ! | |
83 | * | |
84 | * The comm pages are created and populated at boot time. | |
85 | * | |
86 | * The appropriate comm page is mapped into a process's address space | |
87 | * at exec() time, in vm_map_exec(). | |
88 | * It is then inherited on fork(). | |
89 | * | |
90 | * The comm page is shared between the kernel and all applications of | |
91 | * a given platform. Only the kernel can modify it. | |
92 | * | |
93 | * Applications just branch to fixed addresses in the comm page and find | |
94 | * the right version of the code for the platform. There is also some | |
95 | * data provided and updated by the kernel for processes to retrieve easily | |
96 | * without having to do a system call. | |
97 | */ | |
98 | ||
99 | #include <debug.h> | |
100 | ||
101 | #include <kern/ipc_tt.h> | |
102 | #include <kern/kalloc.h> | |
103 | ||
104 | #include <vm/vm_map.h> | |
105 | #include <vm/vm_shared_region.h> | |
106 | ||
107 | #include <vm/vm_protos.h> | |
108 | ||
109 | #include <machine/commpage.h> | |
110 | #include <machine/cpu_capabilities.h> | |
111 | ||
112 | /* "dyld" uses this to figure out what the kernel supports */ | |
113 | int shared_region_version = 3; | |
114 | ||
115 | /* should local (non-chroot) shared regions persist when no task uses them ? */ | |
116 | int shared_region_persistence = 1; /* yes by default */ | |
117 | ||
118 | /* trace level, output is sent to the system log file */ | |
119 | int shared_region_trace_level = SHARED_REGION_TRACE_ERROR_LVL; | |
120 | ||
121 | /* this lock protects all the shared region data structures */ | |
122 | lck_grp_t *vm_shared_region_lck_grp; | |
123 | lck_mtx_t vm_shared_region_lock; | |
124 | ||
125 | #define vm_shared_region_lock() lck_mtx_lock(&vm_shared_region_lock) | |
126 | #define vm_shared_region_unlock() lck_mtx_unlock(&vm_shared_region_lock) | |
127 | #define vm_shared_region_sleep(event, interruptible) \ | |
128 | lck_mtx_sleep(&vm_shared_region_lock, \ | |
129 | LCK_SLEEP_DEFAULT, \ | |
130 | (event_t) (event), \ | |
131 | (interruptible)) | |
132 | ||
133 | /* the list of currently available shared regions (one per environment) */ | |
134 | queue_head_t vm_shared_region_queue; | |
135 | ||
136 | static void vm_shared_region_reference_locked(vm_shared_region_t shared_region); | |
137 | static vm_shared_region_t vm_shared_region_create( | |
138 | void *root_dir, | |
139 | cpu_type_t cputype, | |
140 | boolean_t is_64bit); | |
141 | static void vm_shared_region_destroy(vm_shared_region_t shared_region); | |
142 | ||
143 | /* | |
144 | * Initialize the module... | |
145 | */ | |
146 | void | |
147 | vm_shared_region_init(void) | |
148 | { | |
149 | SHARED_REGION_TRACE_DEBUG( | |
150 | ("shared_region: -> init\n")); | |
151 | ||
152 | vm_shared_region_lck_grp = lck_grp_alloc_init("vm shared region", | |
153 | LCK_GRP_ATTR_NULL); | |
154 | lck_mtx_init(&vm_shared_region_lock, | |
155 | vm_shared_region_lck_grp, | |
156 | LCK_ATTR_NULL); | |
157 | ||
158 | queue_init(&vm_shared_region_queue); | |
159 | ||
160 | SHARED_REGION_TRACE_DEBUG( | |
161 | ("shared_region: <- init\n")); | |
162 | } | |
163 | ||
164 | /* | |
165 | * Retrieve a task's shared region and grab an extra reference to | |
166 | * make sure it doesn't disappear while the caller is using it. | |
167 | * The caller is responsible for consuming that extra reference if | |
168 | * necessary. | |
169 | */ | |
170 | vm_shared_region_t | |
171 | vm_shared_region_get( | |
172 | task_t task) | |
173 | { | |
174 | vm_shared_region_t shared_region; | |
175 | ||
176 | SHARED_REGION_TRACE_DEBUG( | |
177 | ("shared_region: -> get(%p)\n", | |
178 | task)); | |
179 | ||
180 | task_lock(task); | |
181 | vm_shared_region_lock(); | |
182 | shared_region = task->shared_region; | |
183 | if (shared_region) { | |
184 | assert(shared_region->sr_ref_count > 0); | |
185 | vm_shared_region_reference_locked(shared_region); | |
186 | } | |
187 | vm_shared_region_unlock(); | |
188 | task_unlock(task); | |
189 | ||
190 | SHARED_REGION_TRACE_DEBUG( | |
191 | ("shared_region: get(%p) <- %p\n", | |
192 | task, shared_region)); | |
193 | ||
194 | return shared_region; | |
195 | } | |
196 | ||
197 | /* | |
198 | * Get the base address of the shared region. | |
199 | * That's the address at which it needs to be mapped in the process's address | |
200 | * space. | |
201 | * No need to lock since this data is set when the shared region is | |
202 | * created and is never modified after that. The caller must hold an extra | |
203 | * reference on the shared region to prevent it from being destroyed. | |
204 | */ | |
205 | mach_vm_offset_t | |
206 | vm_shared_region_base_address( | |
207 | vm_shared_region_t shared_region) | |
208 | { | |
209 | SHARED_REGION_TRACE_DEBUG( | |
210 | ("shared_region: -> base_address(%p)\n", | |
211 | shared_region)); | |
212 | assert(shared_region->sr_ref_count > 1); | |
213 | SHARED_REGION_TRACE_DEBUG( | |
214 | ("shared_region: base_address(%p) <- 0x%llx\n", | |
215 | shared_region, (long long)shared_region->sr_base_address)); | |
216 | return shared_region->sr_base_address; | |
217 | } | |
218 | ||
219 | /* | |
220 | * Get the size of the shared region. | |
221 | * That's the size that needs to be mapped in the process's address | |
222 | * space. | |
223 | * No need to lock since this data is set when the shared region is | |
224 | * created and is never modified after that. The caller must hold an extra | |
225 | * reference on the shared region to prevent it from being destroyed. | |
226 | */ | |
227 | mach_vm_size_t | |
228 | vm_shared_region_size( | |
229 | vm_shared_region_t shared_region) | |
230 | { | |
231 | SHARED_REGION_TRACE_DEBUG( | |
232 | ("shared_region: -> size(%p)\n", | |
233 | shared_region)); | |
234 | assert(shared_region->sr_ref_count > 1); | |
235 | SHARED_REGION_TRACE_DEBUG( | |
236 | ("shared_region: size(%p) <- 0x%llx\n", | |
237 | shared_region, (long long)shared_region->sr_size)); | |
238 | return shared_region->sr_size; | |
239 | } | |
240 | ||
241 | /* | |
242 | * Get the memory entry of the shared region. | |
243 | * That's the "memory object" that needs to be mapped in the process's address | |
244 | * space. | |
245 | * No need to lock since this data is set when the shared region is | |
246 | * created and is never modified after that. The caller must hold an extra | |
247 | * reference on the shared region to prevent it from being destroyed. | |
248 | */ | |
249 | ipc_port_t | |
250 | vm_shared_region_mem_entry( | |
251 | vm_shared_region_t shared_region) | |
252 | { | |
253 | SHARED_REGION_TRACE_DEBUG( | |
254 | ("shared_region: -> mem_entry(%p)\n", | |
255 | shared_region)); | |
256 | assert(shared_region->sr_ref_count > 1); | |
257 | SHARED_REGION_TRACE_DEBUG( | |
258 | ("shared_region: mem_entry(%p) <- %p\n", | |
259 | shared_region, shared_region->sr_mem_entry)); | |
260 | return shared_region->sr_mem_entry; | |
261 | } | |
262 | ||
263 | /* | |
264 | * Set the shared region the process should use. | |
265 | * A NULL new shared region means that we just want to release the old | |
266 | * shared region. | |
267 | * The caller should already have an extra reference on the new shared region | |
268 | * (if any). We release a reference on the old shared region (if any). | |
269 | */ | |
270 | void | |
271 | vm_shared_region_set( | |
272 | task_t task, | |
273 | vm_shared_region_t new_shared_region) | |
274 | { | |
275 | vm_shared_region_t old_shared_region; | |
276 | ||
277 | SHARED_REGION_TRACE_DEBUG( | |
278 | ("shared_region: -> set(%p, %p)\n", | |
279 | task, new_shared_region)); | |
280 | ||
281 | task_lock(task); | |
282 | vm_shared_region_lock(); | |
283 | ||
284 | old_shared_region = task->shared_region; | |
285 | if (new_shared_region) { | |
286 | assert(new_shared_region->sr_ref_count > 0); | |
287 | } | |
288 | ||
289 | task->shared_region = new_shared_region; | |
290 | ||
291 | vm_shared_region_unlock(); | |
292 | task_unlock(task); | |
293 | ||
294 | if (old_shared_region) { | |
295 | assert(old_shared_region->sr_ref_count > 0); | |
296 | vm_shared_region_deallocate(old_shared_region); | |
297 | } | |
298 | ||
299 | SHARED_REGION_TRACE_DEBUG( | |
300 | ("shared_region: set(%p) <- old=%p new=%p\n", | |
301 | task, old_shared_region, new_shared_region)); | |
302 | } | |
303 | ||
304 | /* | |
305 | * Lookup up the shared region for the desired environment. | |
306 | * If none is found, create a new (empty) one. | |
307 | * Grab an extra reference on the returned shared region, to make sure | |
308 | * it doesn't get destroyed before the caller is done with it. The caller | |
309 | * is responsible for consuming that extra reference if necessary. | |
310 | */ | |
311 | vm_shared_region_t | |
312 | vm_shared_region_lookup( | |
313 | void *root_dir, | |
314 | cpu_type_t cputype, | |
315 | boolean_t is_64bit) | |
316 | { | |
317 | vm_shared_region_t shared_region; | |
318 | vm_shared_region_t new_shared_region; | |
319 | ||
320 | SHARED_REGION_TRACE_DEBUG( | |
321 | ("shared_region: -> lookup(root=%p,cpu=%d,64bit=%d)\n", | |
322 | root_dir, cputype, is_64bit)); | |
323 | ||
324 | shared_region = NULL; | |
325 | new_shared_region = NULL; | |
326 | ||
327 | vm_shared_region_lock(); | |
328 | for (;;) { | |
329 | queue_iterate(&vm_shared_region_queue, | |
330 | shared_region, | |
331 | vm_shared_region_t, | |
332 | sr_q) { | |
333 | assert(shared_region->sr_ref_count > 0); | |
334 | if (shared_region->sr_cpu_type == cputype && | |
335 | shared_region->sr_root_dir == root_dir && | |
336 | shared_region->sr_64bit == is_64bit) { | |
337 | /* found a match ! */ | |
338 | vm_shared_region_reference_locked(shared_region); | |
339 | goto done; | |
340 | } | |
341 | } | |
342 | if (new_shared_region == NULL) { | |
343 | /* no match: create a new one */ | |
344 | vm_shared_region_unlock(); | |
345 | new_shared_region = vm_shared_region_create(root_dir, | |
346 | cputype, | |
347 | is_64bit); | |
348 | /* do the lookup again, in case we lost a race */ | |
349 | vm_shared_region_lock(); | |
350 | continue; | |
351 | } | |
352 | /* still no match: use our new one */ | |
353 | shared_region = new_shared_region; | |
354 | new_shared_region = NULL; | |
355 | queue_enter(&vm_shared_region_queue, | |
356 | shared_region, | |
357 | vm_shared_region_t, | |
358 | sr_q); | |
359 | break; | |
360 | } | |
361 | ||
362 | done: | |
363 | vm_shared_region_unlock(); | |
364 | ||
365 | if (new_shared_region) { | |
366 | /* | |
367 | * We lost a race with someone else to create a new shared | |
368 | * region for that environment. Get rid of our unused one. | |
369 | */ | |
370 | assert(new_shared_region->sr_ref_count == 1); | |
371 | new_shared_region->sr_ref_count--; | |
372 | vm_shared_region_destroy(new_shared_region); | |
373 | new_shared_region = NULL; | |
374 | } | |
375 | ||
376 | SHARED_REGION_TRACE_DEBUG( | |
377 | ("shared_region: lookup(root=%p,cpu=%d,64bit=%d) <- %p\n", | |
378 | root_dir, cputype, is_64bit, shared_region)); | |
379 | ||
380 | assert(shared_region->sr_ref_count > 0); | |
381 | return shared_region; | |
382 | } | |
383 | ||
384 | /* | |
385 | * Take an extra reference on a shared region. | |
386 | * The vm_shared_region_lock should already be held by the caller. | |
387 | */ | |
388 | static void | |
389 | vm_shared_region_reference_locked( | |
390 | vm_shared_region_t shared_region) | |
391 | { | |
392 | #if DEBUG | |
393 | lck_mtx_assert(&vm_shared_region_lock, LCK_MTX_ASSERT_OWNED); | |
394 | #endif | |
395 | ||
396 | SHARED_REGION_TRACE_DEBUG( | |
397 | ("shared_region: -> reference_locked(%p)\n", | |
398 | shared_region)); | |
399 | assert(shared_region->sr_ref_count > 0); | |
400 | shared_region->sr_ref_count++; | |
401 | SHARED_REGION_TRACE_DEBUG( | |
402 | ("shared_region: reference_locked(%p) <- %d\n", | |
403 | shared_region, shared_region->sr_ref_count)); | |
404 | } | |
405 | ||
406 | /* | |
407 | * Release a reference on the shared region. | |
408 | * Destroy it if there are no references left. | |
409 | */ | |
410 | void | |
411 | vm_shared_region_deallocate( | |
412 | vm_shared_region_t shared_region) | |
413 | { | |
414 | SHARED_REGION_TRACE_DEBUG( | |
415 | ("shared_region: -> deallocate(%p)\n", | |
416 | shared_region)); | |
417 | ||
418 | vm_shared_region_lock(); | |
419 | ||
420 | assert(shared_region->sr_ref_count > 0); | |
421 | ||
422 | if (shared_region->sr_root_dir == NULL) { | |
423 | /* | |
424 | * Local (i.e. based on the boot volume) shared regions | |
425 | * can persist or not based on the "shared_region_persistence" | |
426 | * sysctl. | |
427 | * Make sure that this one complies. | |
428 | */ | |
429 | if (shared_region_persistence && | |
430 | !shared_region->sr_persists) { | |
431 | /* make this one persistent */ | |
432 | shared_region->sr_ref_count++; | |
433 | shared_region->sr_persists = TRUE; | |
434 | } else if (!shared_region_persistence && | |
435 | shared_region->sr_persists) { | |
436 | /* make this one no longer persistent */ | |
437 | assert(shared_region->sr_ref_count > 1); | |
438 | shared_region->sr_ref_count--; | |
439 | shared_region->sr_persists = FALSE; | |
440 | } | |
441 | } | |
442 | ||
443 | assert(shared_region->sr_ref_count > 0); | |
444 | shared_region->sr_ref_count--; | |
445 | SHARED_REGION_TRACE_DEBUG( | |
446 | ("shared_region: deallocate(%p): ref now %d\n", | |
447 | shared_region, shared_region->sr_ref_count)); | |
448 | ||
449 | if (shared_region->sr_ref_count == 0) { | |
450 | assert(! shared_region->sr_mapping_in_progress); | |
451 | /* remove it from the queue first, so no one can find it... */ | |
452 | queue_remove(&vm_shared_region_queue, | |
453 | shared_region, | |
454 | vm_shared_region_t, | |
455 | sr_q); | |
456 | vm_shared_region_unlock(); | |
457 | /* ... and destroy it */ | |
458 | vm_shared_region_destroy(shared_region); | |
459 | shared_region = NULL; | |
460 | } else { | |
461 | vm_shared_region_unlock(); | |
462 | } | |
463 | ||
464 | SHARED_REGION_TRACE_DEBUG( | |
465 | ("shared_region: deallocate(%p) <-\n", | |
466 | shared_region)); | |
467 | } | |
468 | ||
469 | /* | |
470 | * Create a new (empty) shared region for a new environment. | |
471 | */ | |
472 | static vm_shared_region_t | |
473 | vm_shared_region_create( | |
474 | void *root_dir, | |
475 | cpu_type_t cputype, | |
476 | boolean_t is_64bit) | |
477 | { | |
478 | kern_return_t kr; | |
479 | vm_named_entry_t mem_entry; | |
480 | ipc_port_t mem_entry_port; | |
481 | vm_shared_region_t shared_region; | |
482 | vm_map_t sub_map; | |
483 | mach_vm_offset_t base_address, pmap_nesting_start; | |
484 | mach_vm_size_t size, pmap_nesting_size; | |
485 | ||
486 | SHARED_REGION_TRACE_DEBUG( | |
487 | ("shared_region: -> create(root=%p,cpu=%d,64bit=%d)\n", | |
488 | root_dir, cputype, is_64bit)); | |
489 | ||
490 | base_address = 0; | |
491 | size = 0; | |
492 | mem_entry = NULL; | |
493 | mem_entry_port = IPC_PORT_NULL; | |
494 | sub_map = VM_MAP_NULL; | |
495 | ||
496 | /* create a new shared region structure... */ | |
497 | shared_region = kalloc(sizeof (*shared_region)); | |
498 | if (shared_region == NULL) { | |
499 | SHARED_REGION_TRACE_ERROR( | |
500 | ("shared_region: create: couldn't allocate\n")); | |
501 | goto done; | |
502 | } | |
503 | ||
504 | /* figure out the correct settings for the desired environment */ | |
505 | if (is_64bit) { | |
506 | switch (cputype) { | |
507 | case CPU_TYPE_I386: | |
508 | base_address = SHARED_REGION_BASE_X86_64; | |
509 | size = SHARED_REGION_SIZE_X86_64; | |
510 | pmap_nesting_start = SHARED_REGION_NESTING_BASE_X86_64; | |
511 | pmap_nesting_size = SHARED_REGION_NESTING_SIZE_X86_64; | |
512 | break; | |
513 | case CPU_TYPE_POWERPC: | |
514 | base_address = SHARED_REGION_BASE_PPC64; | |
515 | size = SHARED_REGION_SIZE_PPC64; | |
516 | pmap_nesting_start = SHARED_REGION_NESTING_BASE_PPC64; | |
517 | pmap_nesting_size = SHARED_REGION_NESTING_SIZE_PPC64; | |
518 | break; | |
519 | default: | |
520 | SHARED_REGION_TRACE_ERROR( | |
521 | ("shared_region: create: unknown cpu type %d\n", | |
522 | cputype)); | |
523 | kfree(shared_region, sizeof (*shared_region)); | |
524 | shared_region = NULL; | |
525 | goto done; | |
526 | } | |
527 | } else { | |
528 | switch (cputype) { | |
529 | case CPU_TYPE_I386: | |
530 | base_address = SHARED_REGION_BASE_I386; | |
531 | size = SHARED_REGION_SIZE_I386; | |
532 | pmap_nesting_start = SHARED_REGION_NESTING_BASE_I386; | |
533 | pmap_nesting_size = SHARED_REGION_NESTING_SIZE_I386; | |
534 | break; | |
535 | case CPU_TYPE_POWERPC: | |
536 | base_address = SHARED_REGION_BASE_PPC; | |
537 | size = SHARED_REGION_SIZE_PPC; | |
538 | pmap_nesting_start = SHARED_REGION_NESTING_BASE_PPC; | |
539 | pmap_nesting_size = SHARED_REGION_NESTING_SIZE_PPC; | |
540 | break; | |
541 | #ifdef CPU_TYPE_ARM | |
542 | case CPU_TYPE_ARM: | |
543 | base_address = SHARED_REGION_BASE_ARM; | |
544 | size = SHARED_REGION_SIZE_ARM; | |
545 | pmap_nesting_start = SHARED_REGION_NESTING_BASE_ARM; | |
546 | pmap_nesting_size = SHARED_REGION_NESTING_SIZE_ARM; | |
547 | break; | |
548 | #endif /* CPU_TYPE_ARM */ | |
549 | default: | |
550 | SHARED_REGION_TRACE_ERROR( | |
551 | ("shared_region: create: unknown cpu type %d\n", | |
552 | cputype)); | |
553 | kfree(shared_region, sizeof (*shared_region)); | |
554 | shared_region = NULL; | |
555 | goto done; | |
556 | ||
557 | } | |
558 | } | |
559 | ||
560 | /* create a memory entry structure and a Mach port handle */ | |
561 | kr = mach_memory_entry_allocate(&mem_entry, | |
562 | &mem_entry_port); | |
563 | if (kr != KERN_SUCCESS) { | |
564 | kfree(shared_region, sizeof (*shared_region)); | |
565 | shared_region = NULL; | |
566 | SHARED_REGION_TRACE_ERROR( | |
567 | ("shared_region: create: " | |
568 | "couldn't allocate mem_entry\n")); | |
569 | goto done; | |
570 | } | |
571 | ||
572 | /* create a VM sub map and its pmap */ | |
573 | sub_map = vm_map_create(pmap_create(0, is_64bit), | |
574 | 0, size, | |
575 | TRUE); | |
576 | if (sub_map == VM_MAP_NULL) { | |
577 | ipc_port_release_send(mem_entry_port); | |
578 | kfree(shared_region, sizeof (*shared_region)); | |
579 | shared_region = NULL; | |
580 | SHARED_REGION_TRACE_ERROR( | |
581 | ("shared_region: create: " | |
582 | "couldn't allocate map\n")); | |
583 | goto done; | |
584 | } | |
585 | ||
586 | /* make the memory entry point to the VM sub map */ | |
587 | mem_entry->is_sub_map = TRUE; | |
588 | mem_entry->backing.map = sub_map; | |
589 | mem_entry->size = size; | |
590 | mem_entry->protection = VM_PROT_ALL; | |
591 | ||
592 | /* make the shared region point at the memory entry */ | |
593 | shared_region->sr_mem_entry = mem_entry_port; | |
594 | ||
595 | /* fill in the shared region's environment and settings */ | |
596 | shared_region->sr_base_address = base_address; | |
597 | shared_region->sr_size = size; | |
598 | shared_region->sr_pmap_nesting_start = pmap_nesting_start; | |
599 | shared_region->sr_pmap_nesting_size = pmap_nesting_size; | |
600 | shared_region->sr_cpu_type = cputype; | |
601 | shared_region->sr_64bit = is_64bit; | |
602 | shared_region->sr_root_dir = root_dir; | |
603 | ||
604 | queue_init(&shared_region->sr_q); | |
605 | shared_region->sr_mapping_in_progress = FALSE; | |
606 | shared_region->sr_persists = FALSE; | |
607 | shared_region->sr_first_mapping = (mach_vm_offset_t) -1; | |
608 | ||
609 | /* grab a reference for the caller */ | |
610 | shared_region->sr_ref_count = 1; | |
611 | ||
612 | done: | |
613 | if (shared_region) { | |
614 | SHARED_REGION_TRACE_INFO( | |
615 | ("shared_region: create(root=%p,cpu=%d,64bit=%d," | |
616 | "base=0x%llx,size=0x%llx) <- " | |
617 | "%p mem=(%p,%p) map=%p pmap=%p\n", | |
618 | root_dir, cputype, is_64bit, (long long)base_address, | |
619 | (long long)size, shared_region, | |
620 | mem_entry_port, mem_entry, sub_map, sub_map->pmap)); | |
621 | } else { | |
622 | SHARED_REGION_TRACE_INFO( | |
623 | ("shared_region: create(root=%p,cpu=%d,64bit=%d," | |
624 | "base=0x%llx,size=0x%llx) <- NULL", | |
625 | root_dir, cputype, is_64bit, (long long)base_address, | |
626 | (long long)size)); | |
627 | } | |
628 | return shared_region; | |
629 | } | |
630 | ||
631 | /* | |
632 | * Destroy a now-unused shared region. | |
633 | * The shared region is no longer in the queue and can not be looked up. | |
634 | */ | |
635 | static void | |
636 | vm_shared_region_destroy( | |
637 | vm_shared_region_t shared_region) | |
638 | { | |
639 | vm_named_entry_t mem_entry; | |
640 | vm_map_t map; | |
641 | ||
642 | SHARED_REGION_TRACE_INFO( | |
643 | ("shared_region: -> destroy(%p) (root=%p,cpu=%d,64bit=%d)\n", | |
644 | shared_region, | |
645 | shared_region->sr_root_dir, | |
646 | shared_region->sr_cpu_type, | |
647 | shared_region->sr_64bit)); | |
648 | ||
649 | assert(shared_region->sr_ref_count == 0); | |
650 | assert(!shared_region->sr_persists); | |
651 | ||
652 | mem_entry = (vm_named_entry_t) shared_region->sr_mem_entry->ip_kobject; | |
653 | assert(mem_entry->is_sub_map); | |
654 | assert(!mem_entry->internal); | |
655 | assert(!mem_entry->is_pager); | |
656 | map = mem_entry->backing.map; | |
657 | ||
658 | /* | |
659 | * Clean up the pmap first. The virtual addresses that were | |
660 | * entered in this possibly "nested" pmap may have different values | |
661 | * than the VM map's min and max offsets, if the VM sub map was | |
662 | * mapped at a non-zero offset in the processes' main VM maps, which | |
663 | * is usually the case, so the clean-up we do in vm_map_destroy() would | |
664 | * not be enough. | |
665 | */ | |
666 | if (map->pmap) { | |
667 | pmap_remove(map->pmap, | |
668 | shared_region->sr_base_address, | |
669 | (shared_region->sr_base_address + | |
670 | shared_region->sr_size)); | |
671 | } | |
672 | ||
673 | /* | |
674 | * Release our (one and only) handle on the memory entry. | |
675 | * This will generate a no-senders notification, which will be processed | |
676 | * by ipc_kobject_notify(), which will release the one and only | |
677 | * reference on the memory entry and cause it to be destroyed, along | |
678 | * with the VM sub map and its pmap. | |
679 | */ | |
680 | mach_memory_entry_port_release(shared_region->sr_mem_entry); | |
681 | mem_entry = NULL; | |
682 | shared_region->sr_mem_entry = IPC_PORT_NULL; | |
683 | ||
684 | /* release the shared region structure... */ | |
685 | kfree(shared_region, sizeof (*shared_region)); | |
686 | SHARED_REGION_TRACE_DEBUG( | |
687 | ("shared_region: destroy(%p) <-\n", | |
688 | shared_region)); | |
689 | shared_region = NULL; | |
690 | ||
691 | } | |
692 | ||
693 | /* | |
694 | * Gets the address of the first (in time) mapping in the shared region. | |
695 | */ | |
696 | kern_return_t | |
697 | vm_shared_region_start_address( | |
698 | vm_shared_region_t shared_region, | |
699 | mach_vm_offset_t *start_address) | |
700 | { | |
701 | kern_return_t kr; | |
702 | mach_vm_offset_t sr_base_address; | |
703 | mach_vm_offset_t sr_first_mapping; | |
704 | ||
705 | SHARED_REGION_TRACE_DEBUG( | |
706 | ("shared_region: -> start_address(%p)\n", | |
707 | shared_region)); | |
708 | assert(shared_region->sr_ref_count > 1); | |
709 | ||
710 | vm_shared_region_lock(); | |
711 | ||
712 | /* | |
713 | * Wait if there's another thread establishing a mapping | |
714 | * in this shared region right when we're looking at it. | |
715 | * We want a consistent view of the map... | |
716 | */ | |
717 | while (shared_region->sr_mapping_in_progress) { | |
718 | /* wait for our turn... */ | |
719 | assert(shared_region->sr_ref_count > 1); | |
720 | vm_shared_region_sleep(&shared_region->sr_mapping_in_progress, | |
721 | THREAD_UNINT); | |
722 | } | |
723 | assert(! shared_region->sr_mapping_in_progress); | |
724 | assert(shared_region->sr_ref_count > 1); | |
725 | ||
726 | sr_base_address = shared_region->sr_base_address; | |
727 | sr_first_mapping = shared_region->sr_first_mapping; | |
728 | ||
729 | if (sr_first_mapping == (mach_vm_offset_t) -1) { | |
730 | /* shared region is empty */ | |
731 | kr = KERN_INVALID_ADDRESS; | |
732 | } else { | |
733 | kr = KERN_SUCCESS; | |
734 | *start_address = sr_base_address + sr_first_mapping; | |
735 | } | |
736 | ||
737 | vm_shared_region_unlock(); | |
738 | ||
739 | SHARED_REGION_TRACE_DEBUG( | |
740 | ("shared_region: start_address(%p) <- 0x%llx\n", | |
741 | shared_region, (long long)shared_region->sr_base_address)); | |
742 | ||
743 | return kr; | |
744 | } | |
745 | /* | |
746 | * Establish some mappings of a file in the shared region. | |
747 | * This is used by "dyld" via the shared_region_map_np() system call | |
748 | * to populate the shared region with the appropriate shared cache. | |
749 | * | |
750 | * One could also call it several times to incrementally load several | |
751 | * libraries, as long as they do not overlap. | |
752 | * It will return KERN_SUCCESS if the mappings were successfully established | |
753 | * or if they were already established identically by another process. | |
754 | */ | |
755 | kern_return_t | |
756 | vm_shared_region_map_file( | |
757 | vm_shared_region_t shared_region, | |
758 | unsigned int mappings_count, | |
759 | struct shared_file_mapping_np *mappings, | |
760 | memory_object_control_t file_control, | |
761 | memory_object_size_t file_size, | |
762 | void *root_dir) | |
763 | { | |
764 | kern_return_t kr; | |
765 | vm_object_t file_object; | |
766 | ipc_port_t sr_handle; | |
767 | vm_named_entry_t sr_mem_entry; | |
768 | vm_map_t sr_map; | |
769 | mach_vm_offset_t sr_base_address; | |
770 | unsigned int i; | |
771 | mach_port_t map_port; | |
772 | mach_vm_offset_t target_address; | |
773 | ||
774 | kr = KERN_SUCCESS; | |
775 | ||
776 | vm_shared_region_lock(); | |
777 | assert(shared_region->sr_ref_count > 1); | |
778 | ||
779 | if (shared_region->sr_root_dir != root_dir) { | |
780 | /* | |
781 | * This shared region doesn't match the current root | |
782 | * directory of this process. Deny the mapping to | |
783 | * avoid tainting the shared region with something that | |
784 | * doesn't quite belong into it. | |
785 | */ | |
786 | vm_shared_region_unlock(); | |
787 | kr = KERN_PROTECTION_FAILURE; | |
788 | goto done; | |
789 | } | |
790 | ||
791 | /* | |
792 | * Make sure we handle only one mapping at a time in a given | |
793 | * shared region, to avoid race conditions. This should not | |
794 | * happen frequently... | |
795 | */ | |
796 | while (shared_region->sr_mapping_in_progress) { | |
797 | /* wait for our turn... */ | |
798 | vm_shared_region_sleep(&shared_region->sr_mapping_in_progress, | |
799 | THREAD_UNINT); | |
800 | } | |
801 | assert(! shared_region->sr_mapping_in_progress); | |
802 | assert(shared_region->sr_ref_count > 1); | |
803 | /* let others know we're working in this shared region */ | |
804 | shared_region->sr_mapping_in_progress = TRUE; | |
805 | ||
806 | vm_shared_region_unlock(); | |
807 | ||
808 | /* no need to lock because this data is never modified... */ | |
809 | sr_handle = shared_region->sr_mem_entry; | |
810 | sr_mem_entry = (vm_named_entry_t) sr_handle->ip_kobject; | |
811 | sr_map = sr_mem_entry->backing.map; | |
812 | sr_base_address = shared_region->sr_base_address; | |
813 | ||
814 | SHARED_REGION_TRACE_DEBUG( | |
815 | ("shared_region: -> map(%p,%d,%p,%p,0x%llx)\n", | |
816 | shared_region, mappings_count, mappings, | |
817 | file_control, file_size)); | |
818 | ||
819 | /* get the VM object associated with the file to be mapped */ | |
820 | file_object = memory_object_control_to_vm_object(file_control); | |
821 | ||
822 | /* establish the mappings */ | |
823 | for (i = 0; i < mappings_count; i++) { | |
824 | SHARED_REGION_TRACE_INFO( | |
825 | ("shared_region: mapping[%d]: " | |
826 | "address:0x%016llx size:0x%016llx offset:0x%016llx " | |
827 | "maxprot:0x%x prot:0x%x\n", | |
828 | i, | |
829 | (long long)mappings[i].sfm_address, | |
830 | (long long)mappings[i].sfm_size, | |
831 | (long long)mappings[i].sfm_file_offset, | |
832 | mappings[i].sfm_max_prot, | |
833 | mappings[i].sfm_init_prot)); | |
834 | ||
835 | if (mappings[i].sfm_init_prot & VM_PROT_ZF) { | |
836 | /* zero-filled memory */ | |
837 | map_port = MACH_PORT_NULL; | |
838 | } else { | |
839 | /* file-backed memory */ | |
840 | map_port = (ipc_port_t) file_object->pager; | |
841 | } | |
842 | ||
843 | /* mapping's address is relative to the shared region base */ | |
844 | target_address = | |
845 | mappings[i].sfm_address - sr_base_address; | |
846 | ||
847 | /* establish that mapping, OK if it's to "already" there */ | |
848 | kr = vm_map_enter_mem_object( | |
849 | sr_map, | |
850 | &target_address, | |
851 | vm_map_round_page(mappings[i].sfm_size), | |
852 | 0, | |
853 | VM_FLAGS_FIXED | VM_FLAGS_ALREADY, | |
854 | map_port, | |
855 | mappings[i].sfm_file_offset, | |
856 | TRUE, | |
857 | mappings[i].sfm_init_prot & VM_PROT_ALL, | |
858 | mappings[i].sfm_max_prot & VM_PROT_ALL, | |
859 | VM_INHERIT_DEFAULT); | |
860 | if (kr == KERN_MEMORY_PRESENT) { | |
861 | /* this exact mapping was already there: that's fine */ | |
862 | SHARED_REGION_TRACE_INFO( | |
863 | ("shared_region: mapping[%d]: " | |
864 | "address:0x%016llx size:0x%016llx " | |
865 | "offset:0x%016llx " | |
866 | "maxprot:0x%x prot:0x%x already mapped...\n", | |
867 | i, | |
868 | (long long)mappings[i].sfm_address, | |
869 | (long long)mappings[i].sfm_size, | |
870 | (long long)mappings[i].sfm_file_offset, | |
871 | mappings[i].sfm_max_prot, | |
872 | mappings[i].sfm_init_prot)); | |
873 | kr = KERN_SUCCESS; | |
874 | } else if (kr != KERN_SUCCESS) { | |
875 | /* this mapping failed ! */ | |
876 | SHARED_REGION_TRACE_ERROR( | |
877 | ("shared_region: mapping[%d]: " | |
878 | "address:0x%016llx size:0x%016llx " | |
879 | "offset:0x%016llx " | |
880 | "maxprot:0x%x prot:0x%x failed 0x%x\n", | |
881 | i, | |
882 | (long long)mappings[i].sfm_address, | |
883 | (long long)mappings[i].sfm_size, | |
884 | (long long)mappings[i].sfm_file_offset, | |
885 | mappings[i].sfm_max_prot, | |
886 | mappings[i].sfm_init_prot, | |
887 | kr)); | |
888 | break; | |
889 | } | |
890 | ||
891 | /* we're protected by "sr_mapping_in_progress" */ | |
892 | if (shared_region->sr_first_mapping == (mach_vm_offset_t) -1) { | |
893 | shared_region->sr_first_mapping = target_address; | |
894 | } | |
895 | } | |
896 | ||
897 | vm_shared_region_lock(); | |
898 | assert(shared_region->sr_ref_count > 1); | |
899 | assert(shared_region->sr_mapping_in_progress); | |
900 | /* we're done working on that shared region */ | |
901 | shared_region->sr_mapping_in_progress = FALSE; | |
902 | thread_wakeup((event_t) &shared_region->sr_mapping_in_progress); | |
903 | vm_shared_region_unlock(); | |
904 | ||
905 | done: | |
906 | SHARED_REGION_TRACE_DEBUG( | |
907 | ("shared_region: map(%p,%d,%p,%p,0x%llx) <- 0x%x \n", | |
908 | shared_region, mappings_count, mappings, | |
909 | file_control, file_size, kr)); | |
910 | return kr; | |
911 | } | |
912 | ||
913 | /* | |
914 | * Enter the appropriate shared region into "map" for "task". | |
915 | * This involves looking up the shared region (and possibly creating a new | |
916 | * one) for the desired environment, then mapping the VM sub map into the | |
917 | * task's VM "map", with the appropriate level of pmap-nesting. | |
918 | */ | |
919 | kern_return_t | |
920 | vm_shared_region_enter( | |
921 | struct _vm_map *map, | |
922 | struct task *task, | |
923 | void *fsroot, | |
924 | cpu_type_t cpu) | |
925 | { | |
926 | kern_return_t kr; | |
927 | vm_shared_region_t shared_region; | |
928 | vm_map_offset_t sr_address, sr_offset, target_address; | |
929 | vm_map_size_t sr_size, mapping_size; | |
930 | vm_map_offset_t sr_pmap_nesting_start; | |
931 | vm_map_size_t sr_pmap_nesting_size; | |
932 | ipc_port_t sr_handle; | |
933 | boolean_t is_64bit; | |
934 | ||
935 | is_64bit = task_has_64BitAddr(task); | |
936 | ||
937 | SHARED_REGION_TRACE_DEBUG( | |
938 | ("shared_region: -> " | |
939 | "enter(map=%p,task=%p,root=%p,cpu=%d,64bit=%d)\n", | |
940 | map, task, fsroot, cpu, is_64bit)); | |
941 | ||
942 | /* lookup (create if needed) the shared region for this environment */ | |
943 | shared_region = vm_shared_region_lookup(fsroot, cpu, is_64bit); | |
944 | if (shared_region == NULL) { | |
945 | /* this should not happen ! */ | |
946 | SHARED_REGION_TRACE_ERROR( | |
947 | ("shared_region: -> " | |
948 | "enter(map=%p,task=%p,root=%p,cpu=%d,64bit=%d): " | |
949 | "lookup failed !\n", | |
950 | map, task, fsroot, cpu, is_64bit)); | |
951 | //panic("shared_region_enter: lookup failed\n"); | |
952 | return KERN_FAILURE; | |
953 | } | |
954 | ||
955 | /* let the task use that shared region */ | |
956 | vm_shared_region_set(task, shared_region); | |
957 | ||
958 | kr = KERN_SUCCESS; | |
959 | /* no need to lock since this data is never modified */ | |
960 | sr_address = shared_region->sr_base_address; | |
961 | sr_size = shared_region->sr_size; | |
962 | sr_handle = shared_region->sr_mem_entry; | |
963 | sr_pmap_nesting_start = shared_region->sr_pmap_nesting_start; | |
964 | sr_pmap_nesting_size = shared_region->sr_pmap_nesting_size; | |
965 | ||
966 | /* | |
967 | * Start mapping the shared region's VM sub map into the task's VM map. | |
968 | */ | |
969 | sr_offset = 0; | |
970 | ||
971 | if (sr_pmap_nesting_start > sr_address) { | |
972 | /* we need to map a range without pmap-nesting first */ | |
973 | target_address = sr_address; | |
974 | mapping_size = sr_pmap_nesting_start - sr_address; | |
975 | kr = vm_map_enter_mem_object( | |
976 | map, | |
977 | &target_address, | |
978 | mapping_size, | |
979 | 0, | |
980 | VM_FLAGS_FIXED, | |
981 | sr_handle, | |
982 | sr_offset, | |
983 | TRUE, | |
984 | VM_PROT_READ, | |
985 | VM_PROT_ALL, | |
986 | VM_INHERIT_SHARE); | |
987 | if (kr != KERN_SUCCESS) { | |
988 | SHARED_REGION_TRACE_ERROR( | |
989 | ("shared_region: enter(%p,%p,%p,%d,%d): " | |
990 | "vm_map_enter(0x%llx,0x%llx,%p) error 0x%x\n", | |
991 | map, task, fsroot, cpu, is_64bit, | |
992 | (long long)target_address, | |
993 | (long long)mapping_size, sr_handle, kr)); | |
994 | goto done; | |
995 | } | |
996 | SHARED_REGION_TRACE_DEBUG( | |
997 | ("shared_region: enter(%p,%p,%p,%d,%d): " | |
998 | "vm_map_enter(0x%llx,0x%llx,%p) error 0x%x\n", | |
999 | map, task, fsroot, cpu, is_64bit, | |
1000 | (long long)target_address, (long long)mapping_size, | |
1001 | sr_handle, kr)); | |
1002 | sr_offset += mapping_size; | |
1003 | sr_size -= mapping_size; | |
1004 | } | |
1005 | /* | |
1006 | * We may need to map several pmap-nested portions, due to platform | |
1007 | * specific restrictions on pmap nesting. | |
1008 | * The pmap-nesting is triggered by the "VM_MEMORY_SHARED_PMAP" alias... | |
1009 | */ | |
1010 | for (; | |
1011 | sr_pmap_nesting_size > 0; | |
1012 | sr_offset += mapping_size, | |
1013 | sr_size -= mapping_size, | |
1014 | sr_pmap_nesting_size -= mapping_size) { | |
1015 | target_address = sr_address + sr_offset; | |
1016 | mapping_size = sr_pmap_nesting_size; | |
1017 | if (mapping_size > pmap_nesting_size_max) { | |
1018 | mapping_size = (vm_map_offset_t) pmap_nesting_size_max; | |
1019 | } | |
1020 | kr = vm_map_enter_mem_object( | |
1021 | map, | |
1022 | &target_address, | |
1023 | mapping_size, | |
1024 | 0, | |
1025 | (VM_FLAGS_FIXED | VM_MAKE_TAG(VM_MEMORY_SHARED_PMAP)), | |
1026 | sr_handle, | |
1027 | sr_offset, | |
1028 | TRUE, | |
1029 | VM_PROT_READ, | |
1030 | VM_PROT_ALL, | |
1031 | VM_INHERIT_SHARE); | |
1032 | if (kr != KERN_SUCCESS) { | |
1033 | SHARED_REGION_TRACE_ERROR( | |
1034 | ("shared_region: enter(%p,%p,%p,%d,%d): " | |
1035 | "vm_map_enter(0x%llx,0x%llx,%p) error 0x%x\n", | |
1036 | map, task, fsroot, cpu, is_64bit, | |
1037 | (long long)target_address, | |
1038 | (long long)mapping_size, sr_handle, kr)); | |
1039 | goto done; | |
1040 | } | |
1041 | SHARED_REGION_TRACE_DEBUG( | |
1042 | ("shared_region: enter(%p,%p,%p,%d,%d): " | |
1043 | "nested vm_map_enter(0x%llx,0x%llx,%p) error 0x%x\n", | |
1044 | map, task, fsroot, cpu, is_64bit, | |
1045 | (long long)target_address, (long long)mapping_size, | |
1046 | sr_handle, kr)); | |
1047 | } | |
1048 | if (sr_size > 0) { | |
1049 | /* and there's some left to be mapped without pmap-nesting */ | |
1050 | target_address = sr_address + sr_offset; | |
1051 | mapping_size = sr_size; | |
1052 | kr = vm_map_enter_mem_object( | |
1053 | map, | |
1054 | &target_address, | |
1055 | mapping_size, | |
1056 | 0, | |
1057 | VM_FLAGS_FIXED, | |
1058 | sr_handle, | |
1059 | sr_offset, | |
1060 | TRUE, | |
1061 | VM_PROT_READ, | |
1062 | VM_PROT_ALL, | |
1063 | VM_INHERIT_SHARE); | |
1064 | if (kr != KERN_SUCCESS) { | |
1065 | SHARED_REGION_TRACE_ERROR( | |
1066 | ("shared_region: enter(%p,%p,%p,%d,%d): " | |
1067 | "vm_map_enter(0x%llx,0x%llx,%p) error 0x%x\n", | |
1068 | map, task, fsroot, cpu, is_64bit, | |
1069 | (long long)target_address, | |
1070 | (long long)mapping_size, sr_handle, kr)); | |
1071 | goto done; | |
1072 | } | |
1073 | SHARED_REGION_TRACE_DEBUG( | |
1074 | ("shared_region: enter(%p,%p,%p,%d,%d): " | |
1075 | "vm_map_enter(0x%llx,0x%llx,%p) error 0x%x\n", | |
1076 | map, task, fsroot, cpu, is_64bit, | |
1077 | (long long)target_address, (long long)mapping_size, | |
1078 | sr_handle, kr)); | |
1079 | sr_offset += mapping_size; | |
1080 | sr_size -= mapping_size; | |
1081 | } | |
1082 | assert(sr_size == 0); | |
1083 | ||
1084 | done: | |
1085 | SHARED_REGION_TRACE_DEBUG( | |
1086 | ("shared_region: enter(%p,%p,%p,%d,%d) <- 0x%x\n", | |
1087 | map, task, fsroot, cpu, is_64bit, kr)); | |
1088 | return kr; | |
1089 | } | |
1090 | ||
1091 | /******************************************************************************/ | |
1092 | /* Comm page support */ | |
1093 | /******************************************************************************/ | |
1094 | ||
1095 | ipc_port_t commpage32_handle = IPC_PORT_NULL; | |
1096 | ipc_port_t commpage64_handle = IPC_PORT_NULL; | |
1097 | vm_named_entry_t commpage32_entry = NULL; | |
1098 | vm_named_entry_t commpage64_entry = NULL; | |
1099 | vm_map_t commpage32_map = VM_MAP_NULL; | |
1100 | vm_map_t commpage64_map = VM_MAP_NULL; | |
1101 | ||
1102 | /* | |
1103 | * Create a memory entry, VM submap and pmap for one commpage. | |
1104 | */ | |
1105 | static void | |
1106 | _vm_commpage_init( | |
1107 | ipc_port_t *handlep, | |
1108 | vm_map_size_t size) | |
1109 | { | |
1110 | kern_return_t kr; | |
1111 | vm_named_entry_t mem_entry; | |
1112 | vm_map_t new_map; | |
1113 | ||
1114 | SHARED_REGION_TRACE_DEBUG( | |
1115 | ("commpage: -> _init(0x%llx)\n", | |
1116 | (long long)size)); | |
1117 | ||
1118 | kr = mach_memory_entry_allocate(&mem_entry, | |
1119 | handlep); | |
1120 | if (kr != KERN_SUCCESS) { | |
1121 | panic("_vm_commpage_init: could not allocate mem_entry"); | |
1122 | } | |
1123 | new_map = vm_map_create(pmap_create(0, FALSE), 0, size, TRUE); | |
1124 | if (new_map == VM_MAP_NULL) { | |
1125 | panic("_vm_commpage_init: could not allocate VM map"); | |
1126 | } | |
1127 | mem_entry->backing.map = new_map; | |
1128 | mem_entry->internal = TRUE; | |
1129 | mem_entry->is_sub_map = TRUE; | |
1130 | mem_entry->offset = 0; | |
1131 | mem_entry->protection = VM_PROT_ALL; | |
1132 | mem_entry->size = size; | |
1133 | ||
1134 | SHARED_REGION_TRACE_DEBUG( | |
1135 | ("commpage: _init(0x%llx) <- %p\n", | |
1136 | (long long)size, *handlep)); | |
1137 | } | |
1138 | ||
1139 | /* | |
1140 | * Initialize the comm pages at boot time. | |
1141 | */ | |
1142 | void | |
1143 | vm_commpage_init(void) | |
1144 | { | |
1145 | SHARED_REGION_TRACE_DEBUG( | |
1146 | ("commpage: -> init()\n")); | |
1147 | ||
1148 | /* create the 32-bit comm page */ | |
1149 | _vm_commpage_init(&commpage32_handle, _COMM_PAGE32_AREA_LENGTH); | |
1150 | commpage32_entry = (vm_named_entry_t) commpage32_handle->ip_kobject; | |
1151 | commpage32_map = commpage32_entry->backing.map; | |
1152 | ||
1153 | /* XXX if (cpu_is_64bit_capable()) ? */ | |
1154 | /* create the 64-bit comm page */ | |
1155 | _vm_commpage_init(&commpage64_handle, _COMM_PAGE64_AREA_LENGTH); | |
1156 | commpage64_entry = (vm_named_entry_t) commpage64_handle->ip_kobject; | |
1157 | commpage64_map = commpage64_entry->backing.map; | |
1158 | ||
1159 | /* populate them according to this specific platform */ | |
1160 | commpage_populate(); | |
1161 | ||
1162 | SHARED_REGION_TRACE_DEBUG( | |
1163 | ("commpage: init() <-\n")); | |
1164 | } | |
1165 | ||
1166 | /* | |
1167 | * Enter the appropriate comm page into the task's address space. | |
1168 | * This is called at exec() time via vm_map_exec(). | |
1169 | */ | |
1170 | kern_return_t | |
1171 | vm_commpage_enter( | |
1172 | vm_map_t map, | |
1173 | task_t task) | |
1174 | { | |
1175 | ipc_port_t commpage_handle; | |
1176 | vm_map_offset_t commpage_address, objc_address; | |
1177 | vm_map_size_t commpage_size, objc_size; | |
1178 | int vm_flags; | |
1179 | kern_return_t kr; | |
1180 | ||
1181 | SHARED_REGION_TRACE_DEBUG( | |
1182 | ("commpage: -> enter(%p,%p)\n", | |
1183 | map, task)); | |
1184 | ||
1185 | /* the comm page is likely to be beyond the actual end of the VM map */ | |
1186 | vm_flags = VM_FLAGS_FIXED | VM_FLAGS_BEYOND_MAX; | |
1187 | ||
1188 | /* select the appropriate comm page for this task */ | |
1189 | assert(! (task_has_64BitAddr(task) ^ vm_map_is_64bit(map))); | |
1190 | if (task_has_64BitAddr(task)) { | |
1191 | #ifdef __ppc__ | |
1192 | /* | |
1193 | * PPC51: ppc64 is limited to 51-bit addresses. | |
1194 | * Memory above that limit is handled specially at the | |
1195 | * pmap level, so do not interfere. | |
1196 | */ | |
1197 | vm_flags |= VM_FLAGS_NO_PMAP_CHECK; | |
1198 | #endif /* __ppc__ */ | |
1199 | commpage_handle = commpage64_handle; | |
1200 | commpage_address = (vm_map_offset_t) _COMM_PAGE64_BASE_ADDRESS; | |
1201 | commpage_size = _COMM_PAGE64_AREA_LENGTH; | |
1202 | objc_size = _COMM_PAGE64_OBJC_SIZE; | |
1203 | objc_address = _COMM_PAGE64_OBJC_BASE; | |
1204 | } else { | |
1205 | commpage_handle = commpage32_handle; | |
1206 | commpage_address = | |
1207 | (vm_map_offset_t)(unsigned) _COMM_PAGE32_BASE_ADDRESS; | |
1208 | commpage_size = _COMM_PAGE32_AREA_LENGTH; | |
1209 | objc_size = _COMM_PAGE32_OBJC_SIZE; | |
1210 | objc_address = _COMM_PAGE32_OBJC_BASE; | |
1211 | } | |
1212 | ||
1213 | if ((commpage_address & (pmap_nesting_size_min - 1)) == 0 && | |
1214 | (commpage_size & (pmap_nesting_size_min - 1)) == 0) { | |
1215 | /* the commpage is properly aligned or sized for pmap-nesting */ | |
1216 | vm_flags |= VM_MAKE_TAG(VM_MEMORY_SHARED_PMAP); | |
1217 | } | |
1218 | ||
1219 | /* map the comm page in the task's address space */ | |
1220 | assert(commpage_handle != IPC_PORT_NULL); | |
1221 | kr = vm_map_enter_mem_object( | |
1222 | map, | |
1223 | &commpage_address, | |
1224 | commpage_size, | |
1225 | 0, | |
1226 | vm_flags, | |
1227 | commpage_handle, | |
1228 | 0, | |
1229 | FALSE, | |
1230 | VM_PROT_READ|VM_PROT_EXECUTE, | |
1231 | VM_PROT_READ|VM_PROT_EXECUTE, | |
1232 | VM_INHERIT_SHARE); | |
1233 | if (kr != KERN_SUCCESS) { | |
1234 | SHARED_REGION_TRACE_ERROR( | |
1235 | ("commpage: enter(%p,0x%llx,0x%llx) " | |
1236 | "commpage %p mapping failed 0x%x\n", | |
1237 | map, (long long)commpage_address, | |
1238 | (long long)commpage_size, commpage_handle, kr)); | |
1239 | } | |
1240 | ||
1241 | /* | |
1242 | * Since we're here, we also pre-allocate some virtual space for the | |
1243 | * Objective-C run-time, if needed... | |
1244 | */ | |
1245 | if (objc_size != 0) { | |
1246 | kr = vm_map_enter_mem_object( | |
1247 | map, | |
1248 | &objc_address, | |
1249 | objc_size, | |
1250 | 0, | |
1251 | VM_FLAGS_FIXED | VM_FLAGS_BEYOND_MAX, | |
1252 | IPC_PORT_NULL, | |
1253 | 0, | |
1254 | FALSE, | |
1255 | VM_PROT_ALL, | |
1256 | VM_PROT_ALL, | |
1257 | VM_INHERIT_DEFAULT); | |
1258 | if (kr != KERN_SUCCESS) { | |
1259 | SHARED_REGION_TRACE_ERROR( | |
1260 | ("commpage: enter(%p,0x%llx,0x%llx) " | |
1261 | "objc mapping failed 0x%x\n", | |
1262 | map, (long long)objc_address, | |
1263 | (long long)objc_size, kr)); | |
1264 | } | |
1265 | } | |
1266 | ||
1267 | SHARED_REGION_TRACE_DEBUG( | |
1268 | ("commpage: enter(%p,%p) <- 0x%x\n", | |
1269 | map, task, kr)); | |
1270 | return kr; | |
1271 | } |