]> git.saurik.com Git - apple/xnu.git/blame - osfmk/vm/vm_shared_region.c
xnu-1228.tar.gz
[apple/xnu.git] / osfmk / vm / vm_shared_region.c
CommitLineData
2d21ac55
A
1/*
2 * Copyright (c) 2007 Apple Inc. All rights reserved.
3 *
4 * @APPLE_LICENSE_HEADER_START@
5 *
6 * This file contains Original Code and/or Modifications of Original Code
7 * as defined in and that are subject to the Apple Public Source License
8 * Version 2.0 (the 'License'). You may not use this file except in
9 * compliance with the License. Please obtain a copy of the License at
10 * http://www.opensource.apple.com/apsl/ and read it before using this
11 * file.
12 *
13 * The Original Code and all software distributed under the License are
14 * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
15 * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
16 * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
17 * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
18 * Please see the License for the specific language governing rights and
19 * limitations under the License.
20 *
21 * @APPLE_LICENSE_HEADER_END@
22 */
23
24/*
25 * Shared region (... and comm page)
26 *
27 * This file handles the VM shared region and comm page.
28 *
29 */
30/*
31 * SHARED REGIONS
32 * --------------
33 *
34 * A shared region is a submap that contains the most common system shared
35 * libraries for a given environment.
36 * An environment is defined by (cpu-type, 64-bitness, root directory).
37 *
38 * The point of a shared region is to reduce the setup overhead when exec'ing
39 * a new process.
40 * A shared region uses a shared VM submap that gets mapped automatically
41 * at exec() time (see vm_map_exec()). The first process of a given
42 * environment sets up the shared region and all further processes in that
43 * environment can re-use that shared region without having to re-create
44 * the same mappings in their VM map. All they need is contained in the shared
45 * region.
46 * It can also shared a pmap (mostly for read-only parts but also for the
47 * initial version of some writable parts), which gets "nested" into the
48 * process's pmap. This reduces the number of soft faults: once one process
49 * brings in a page in the shared region, all the other processes can access
50 * it without having to enter it in their own pmap.
51 *
52 *
53 * When a process is being exec'ed, vm_map_exec() calls vm_shared_region_enter()
54 * to map the appropriate shared region in the process's address space.
55 * We look up the appropriate shared region for the process's environment.
56 * If we can't find one, we create a new (empty) one and add it to the list.
57 * Otherwise, we just take an extra reference on the shared region we found.
58 *
59 * The "dyld" runtime (mapped into the process's address space at exec() time)
60 * will then use the shared_region_check_np() and shared_region_map_np()
61 * system call to validate and/or populate the shared region with the
62 * appropriate dyld_shared_cache file.
63 *
64 * The shared region is inherited on fork() and the child simply takes an
65 * extra reference on its parent's shared region.
66 *
67 * When the task terminates, we release a reference on its shared region.
68 * When the last reference is released, we destroy the shared region.
69 *
70 * After a chroot(), the calling process keeps using its original shared region,
71 * since that's what was mapped when it was started. But its children
72 * will use a different shared region, because they need to use the shared
73 * cache that's relative to the new root directory.
74 */
75/*
76 * COMM PAGE
77 *
78 * A "comm page" is an area of memory that is populated by the kernel with
79 * the appropriate platform-specific version of some commonly used code.
80 * There is one "comm page" per platform (cpu-type, 64-bitness) but only
81 * for the native cpu-type. No need to overly optimize translated code
82 * for hardware that is not really there !
83 *
84 * The comm pages are created and populated at boot time.
85 *
86 * The appropriate comm page is mapped into a process's address space
87 * at exec() time, in vm_map_exec().
88 * It is then inherited on fork().
89 *
90 * The comm page is shared between the kernel and all applications of
91 * a given platform. Only the kernel can modify it.
92 *
93 * Applications just branch to fixed addresses in the comm page and find
94 * the right version of the code for the platform. There is also some
95 * data provided and updated by the kernel for processes to retrieve easily
96 * without having to do a system call.
97 */
98
99#include <debug.h>
100
101#include <kern/ipc_tt.h>
102#include <kern/kalloc.h>
103
104#include <vm/vm_map.h>
105#include <vm/vm_shared_region.h>
106
107#include <vm/vm_protos.h>
108
109#include <machine/commpage.h>
110#include <machine/cpu_capabilities.h>
111
112/* "dyld" uses this to figure out what the kernel supports */
113int shared_region_version = 3;
114
115/* should local (non-chroot) shared regions persist when no task uses them ? */
116int shared_region_persistence = 1; /* yes by default */
117
118/* trace level, output is sent to the system log file */
119int shared_region_trace_level = SHARED_REGION_TRACE_ERROR_LVL;
120
121/* this lock protects all the shared region data structures */
122lck_grp_t *vm_shared_region_lck_grp;
123lck_mtx_t vm_shared_region_lock;
124
125#define vm_shared_region_lock() lck_mtx_lock(&vm_shared_region_lock)
126#define vm_shared_region_unlock() lck_mtx_unlock(&vm_shared_region_lock)
127#define vm_shared_region_sleep(event, interruptible) \
128 lck_mtx_sleep(&vm_shared_region_lock, \
129 LCK_SLEEP_DEFAULT, \
130 (event_t) (event), \
131 (interruptible))
132
133/* the list of currently available shared regions (one per environment) */
134queue_head_t vm_shared_region_queue;
135
136static void vm_shared_region_reference_locked(vm_shared_region_t shared_region);
137static vm_shared_region_t vm_shared_region_create(
138 void *root_dir,
139 cpu_type_t cputype,
140 boolean_t is_64bit);
141static void vm_shared_region_destroy(vm_shared_region_t shared_region);
142
143/*
144 * Initialize the module...
145 */
146void
147vm_shared_region_init(void)
148{
149 SHARED_REGION_TRACE_DEBUG(
150 ("shared_region: -> init\n"));
151
152 vm_shared_region_lck_grp = lck_grp_alloc_init("vm shared region",
153 LCK_GRP_ATTR_NULL);
154 lck_mtx_init(&vm_shared_region_lock,
155 vm_shared_region_lck_grp,
156 LCK_ATTR_NULL);
157
158 queue_init(&vm_shared_region_queue);
159
160 SHARED_REGION_TRACE_DEBUG(
161 ("shared_region: <- init\n"));
162}
163
164/*
165 * Retrieve a task's shared region and grab an extra reference to
166 * make sure it doesn't disappear while the caller is using it.
167 * The caller is responsible for consuming that extra reference if
168 * necessary.
169 */
170vm_shared_region_t
171vm_shared_region_get(
172 task_t task)
173{
174 vm_shared_region_t shared_region;
175
176 SHARED_REGION_TRACE_DEBUG(
177 ("shared_region: -> get(%p)\n",
178 task));
179
180 task_lock(task);
181 vm_shared_region_lock();
182 shared_region = task->shared_region;
183 if (shared_region) {
184 assert(shared_region->sr_ref_count > 0);
185 vm_shared_region_reference_locked(shared_region);
186 }
187 vm_shared_region_unlock();
188 task_unlock(task);
189
190 SHARED_REGION_TRACE_DEBUG(
191 ("shared_region: get(%p) <- %p\n",
192 task, shared_region));
193
194 return shared_region;
195}
196
197/*
198 * Get the base address of the shared region.
199 * That's the address at which it needs to be mapped in the process's address
200 * space.
201 * No need to lock since this data is set when the shared region is
202 * created and is never modified after that. The caller must hold an extra
203 * reference on the shared region to prevent it from being destroyed.
204 */
205mach_vm_offset_t
206vm_shared_region_base_address(
207 vm_shared_region_t shared_region)
208{
209 SHARED_REGION_TRACE_DEBUG(
210 ("shared_region: -> base_address(%p)\n",
211 shared_region));
212 assert(shared_region->sr_ref_count > 1);
213 SHARED_REGION_TRACE_DEBUG(
214 ("shared_region: base_address(%p) <- 0x%llx\n",
215 shared_region, (long long)shared_region->sr_base_address));
216 return shared_region->sr_base_address;
217}
218
219/*
220 * Get the size of the shared region.
221 * That's the size that needs to be mapped in the process's address
222 * space.
223 * No need to lock since this data is set when the shared region is
224 * created and is never modified after that. The caller must hold an extra
225 * reference on the shared region to prevent it from being destroyed.
226 */
227mach_vm_size_t
228vm_shared_region_size(
229 vm_shared_region_t shared_region)
230{
231 SHARED_REGION_TRACE_DEBUG(
232 ("shared_region: -> size(%p)\n",
233 shared_region));
234 assert(shared_region->sr_ref_count > 1);
235 SHARED_REGION_TRACE_DEBUG(
236 ("shared_region: size(%p) <- 0x%llx\n",
237 shared_region, (long long)shared_region->sr_size));
238 return shared_region->sr_size;
239}
240
241/*
242 * Get the memory entry of the shared region.
243 * That's the "memory object" that needs to be mapped in the process's address
244 * space.
245 * No need to lock since this data is set when the shared region is
246 * created and is never modified after that. The caller must hold an extra
247 * reference on the shared region to prevent it from being destroyed.
248 */
249ipc_port_t
250vm_shared_region_mem_entry(
251 vm_shared_region_t shared_region)
252{
253 SHARED_REGION_TRACE_DEBUG(
254 ("shared_region: -> mem_entry(%p)\n",
255 shared_region));
256 assert(shared_region->sr_ref_count > 1);
257 SHARED_REGION_TRACE_DEBUG(
258 ("shared_region: mem_entry(%p) <- %p\n",
259 shared_region, shared_region->sr_mem_entry));
260 return shared_region->sr_mem_entry;
261}
262
263/*
264 * Set the shared region the process should use.
265 * A NULL new shared region means that we just want to release the old
266 * shared region.
267 * The caller should already have an extra reference on the new shared region
268 * (if any). We release a reference on the old shared region (if any).
269 */
270void
271vm_shared_region_set(
272 task_t task,
273 vm_shared_region_t new_shared_region)
274{
275 vm_shared_region_t old_shared_region;
276
277 SHARED_REGION_TRACE_DEBUG(
278 ("shared_region: -> set(%p, %p)\n",
279 task, new_shared_region));
280
281 task_lock(task);
282 vm_shared_region_lock();
283
284 old_shared_region = task->shared_region;
285 if (new_shared_region) {
286 assert(new_shared_region->sr_ref_count > 0);
287 }
288
289 task->shared_region = new_shared_region;
290
291 vm_shared_region_unlock();
292 task_unlock(task);
293
294 if (old_shared_region) {
295 assert(old_shared_region->sr_ref_count > 0);
296 vm_shared_region_deallocate(old_shared_region);
297 }
298
299 SHARED_REGION_TRACE_DEBUG(
300 ("shared_region: set(%p) <- old=%p new=%p\n",
301 task, old_shared_region, new_shared_region));
302}
303
304/*
305 * Lookup up the shared region for the desired environment.
306 * If none is found, create a new (empty) one.
307 * Grab an extra reference on the returned shared region, to make sure
308 * it doesn't get destroyed before the caller is done with it. The caller
309 * is responsible for consuming that extra reference if necessary.
310 */
311vm_shared_region_t
312vm_shared_region_lookup(
313 void *root_dir,
314 cpu_type_t cputype,
315 boolean_t is_64bit)
316{
317 vm_shared_region_t shared_region;
318 vm_shared_region_t new_shared_region;
319
320 SHARED_REGION_TRACE_DEBUG(
321 ("shared_region: -> lookup(root=%p,cpu=%d,64bit=%d)\n",
322 root_dir, cputype, is_64bit));
323
324 shared_region = NULL;
325 new_shared_region = NULL;
326
327 vm_shared_region_lock();
328 for (;;) {
329 queue_iterate(&vm_shared_region_queue,
330 shared_region,
331 vm_shared_region_t,
332 sr_q) {
333 assert(shared_region->sr_ref_count > 0);
334 if (shared_region->sr_cpu_type == cputype &&
335 shared_region->sr_root_dir == root_dir &&
336 shared_region->sr_64bit == is_64bit) {
337 /* found a match ! */
338 vm_shared_region_reference_locked(shared_region);
339 goto done;
340 }
341 }
342 if (new_shared_region == NULL) {
343 /* no match: create a new one */
344 vm_shared_region_unlock();
345 new_shared_region = vm_shared_region_create(root_dir,
346 cputype,
347 is_64bit);
348 /* do the lookup again, in case we lost a race */
349 vm_shared_region_lock();
350 continue;
351 }
352 /* still no match: use our new one */
353 shared_region = new_shared_region;
354 new_shared_region = NULL;
355 queue_enter(&vm_shared_region_queue,
356 shared_region,
357 vm_shared_region_t,
358 sr_q);
359 break;
360 }
361
362done:
363 vm_shared_region_unlock();
364
365 if (new_shared_region) {
366 /*
367 * We lost a race with someone else to create a new shared
368 * region for that environment. Get rid of our unused one.
369 */
370 assert(new_shared_region->sr_ref_count == 1);
371 new_shared_region->sr_ref_count--;
372 vm_shared_region_destroy(new_shared_region);
373 new_shared_region = NULL;
374 }
375
376 SHARED_REGION_TRACE_DEBUG(
377 ("shared_region: lookup(root=%p,cpu=%d,64bit=%d) <- %p\n",
378 root_dir, cputype, is_64bit, shared_region));
379
380 assert(shared_region->sr_ref_count > 0);
381 return shared_region;
382}
383
384/*
385 * Take an extra reference on a shared region.
386 * The vm_shared_region_lock should already be held by the caller.
387 */
388static void
389vm_shared_region_reference_locked(
390 vm_shared_region_t shared_region)
391{
392#if DEBUG
393 lck_mtx_assert(&vm_shared_region_lock, LCK_MTX_ASSERT_OWNED);
394#endif
395
396 SHARED_REGION_TRACE_DEBUG(
397 ("shared_region: -> reference_locked(%p)\n",
398 shared_region));
399 assert(shared_region->sr_ref_count > 0);
400 shared_region->sr_ref_count++;
401 SHARED_REGION_TRACE_DEBUG(
402 ("shared_region: reference_locked(%p) <- %d\n",
403 shared_region, shared_region->sr_ref_count));
404}
405
406/*
407 * Release a reference on the shared region.
408 * Destroy it if there are no references left.
409 */
410void
411vm_shared_region_deallocate(
412 vm_shared_region_t shared_region)
413{
414 SHARED_REGION_TRACE_DEBUG(
415 ("shared_region: -> deallocate(%p)\n",
416 shared_region));
417
418 vm_shared_region_lock();
419
420 assert(shared_region->sr_ref_count > 0);
421
422 if (shared_region->sr_root_dir == NULL) {
423 /*
424 * Local (i.e. based on the boot volume) shared regions
425 * can persist or not based on the "shared_region_persistence"
426 * sysctl.
427 * Make sure that this one complies.
428 */
429 if (shared_region_persistence &&
430 !shared_region->sr_persists) {
431 /* make this one persistent */
432 shared_region->sr_ref_count++;
433 shared_region->sr_persists = TRUE;
434 } else if (!shared_region_persistence &&
435 shared_region->sr_persists) {
436 /* make this one no longer persistent */
437 assert(shared_region->sr_ref_count > 1);
438 shared_region->sr_ref_count--;
439 shared_region->sr_persists = FALSE;
440 }
441 }
442
443 assert(shared_region->sr_ref_count > 0);
444 shared_region->sr_ref_count--;
445 SHARED_REGION_TRACE_DEBUG(
446 ("shared_region: deallocate(%p): ref now %d\n",
447 shared_region, shared_region->sr_ref_count));
448
449 if (shared_region->sr_ref_count == 0) {
450 assert(! shared_region->sr_mapping_in_progress);
451 /* remove it from the queue first, so no one can find it... */
452 queue_remove(&vm_shared_region_queue,
453 shared_region,
454 vm_shared_region_t,
455 sr_q);
456 vm_shared_region_unlock();
457 /* ... and destroy it */
458 vm_shared_region_destroy(shared_region);
459 shared_region = NULL;
460 } else {
461 vm_shared_region_unlock();
462 }
463
464 SHARED_REGION_TRACE_DEBUG(
465 ("shared_region: deallocate(%p) <-\n",
466 shared_region));
467}
468
469/*
470 * Create a new (empty) shared region for a new environment.
471 */
472static vm_shared_region_t
473vm_shared_region_create(
474 void *root_dir,
475 cpu_type_t cputype,
476 boolean_t is_64bit)
477{
478 kern_return_t kr;
479 vm_named_entry_t mem_entry;
480 ipc_port_t mem_entry_port;
481 vm_shared_region_t shared_region;
482 vm_map_t sub_map;
483 mach_vm_offset_t base_address, pmap_nesting_start;
484 mach_vm_size_t size, pmap_nesting_size;
485
486 SHARED_REGION_TRACE_DEBUG(
487 ("shared_region: -> create(root=%p,cpu=%d,64bit=%d)\n",
488 root_dir, cputype, is_64bit));
489
490 base_address = 0;
491 size = 0;
492 mem_entry = NULL;
493 mem_entry_port = IPC_PORT_NULL;
494 sub_map = VM_MAP_NULL;
495
496 /* create a new shared region structure... */
497 shared_region = kalloc(sizeof (*shared_region));
498 if (shared_region == NULL) {
499 SHARED_REGION_TRACE_ERROR(
500 ("shared_region: create: couldn't allocate\n"));
501 goto done;
502 }
503
504 /* figure out the correct settings for the desired environment */
505 if (is_64bit) {
506 switch (cputype) {
507 case CPU_TYPE_I386:
508 base_address = SHARED_REGION_BASE_X86_64;
509 size = SHARED_REGION_SIZE_X86_64;
510 pmap_nesting_start = SHARED_REGION_NESTING_BASE_X86_64;
511 pmap_nesting_size = SHARED_REGION_NESTING_SIZE_X86_64;
512 break;
513 case CPU_TYPE_POWERPC:
514 base_address = SHARED_REGION_BASE_PPC64;
515 size = SHARED_REGION_SIZE_PPC64;
516 pmap_nesting_start = SHARED_REGION_NESTING_BASE_PPC64;
517 pmap_nesting_size = SHARED_REGION_NESTING_SIZE_PPC64;
518 break;
519 default:
520 SHARED_REGION_TRACE_ERROR(
521 ("shared_region: create: unknown cpu type %d\n",
522 cputype));
523 kfree(shared_region, sizeof (*shared_region));
524 shared_region = NULL;
525 goto done;
526 }
527 } else {
528 switch (cputype) {
529 case CPU_TYPE_I386:
530 base_address = SHARED_REGION_BASE_I386;
531 size = SHARED_REGION_SIZE_I386;
532 pmap_nesting_start = SHARED_REGION_NESTING_BASE_I386;
533 pmap_nesting_size = SHARED_REGION_NESTING_SIZE_I386;
534 break;
535 case CPU_TYPE_POWERPC:
536 base_address = SHARED_REGION_BASE_PPC;
537 size = SHARED_REGION_SIZE_PPC;
538 pmap_nesting_start = SHARED_REGION_NESTING_BASE_PPC;
539 pmap_nesting_size = SHARED_REGION_NESTING_SIZE_PPC;
540 break;
541#ifdef CPU_TYPE_ARM
542 case CPU_TYPE_ARM:
543 base_address = SHARED_REGION_BASE_ARM;
544 size = SHARED_REGION_SIZE_ARM;
545 pmap_nesting_start = SHARED_REGION_NESTING_BASE_ARM;
546 pmap_nesting_size = SHARED_REGION_NESTING_SIZE_ARM;
547 break;
548#endif /* CPU_TYPE_ARM */
549 default:
550 SHARED_REGION_TRACE_ERROR(
551 ("shared_region: create: unknown cpu type %d\n",
552 cputype));
553 kfree(shared_region, sizeof (*shared_region));
554 shared_region = NULL;
555 goto done;
556
557 }
558 }
559
560 /* create a memory entry structure and a Mach port handle */
561 kr = mach_memory_entry_allocate(&mem_entry,
562 &mem_entry_port);
563 if (kr != KERN_SUCCESS) {
564 kfree(shared_region, sizeof (*shared_region));
565 shared_region = NULL;
566 SHARED_REGION_TRACE_ERROR(
567 ("shared_region: create: "
568 "couldn't allocate mem_entry\n"));
569 goto done;
570 }
571
572 /* create a VM sub map and its pmap */
573 sub_map = vm_map_create(pmap_create(0, is_64bit),
574 0, size,
575 TRUE);
576 if (sub_map == VM_MAP_NULL) {
577 ipc_port_release_send(mem_entry_port);
578 kfree(shared_region, sizeof (*shared_region));
579 shared_region = NULL;
580 SHARED_REGION_TRACE_ERROR(
581 ("shared_region: create: "
582 "couldn't allocate map\n"));
583 goto done;
584 }
585
586 /* make the memory entry point to the VM sub map */
587 mem_entry->is_sub_map = TRUE;
588 mem_entry->backing.map = sub_map;
589 mem_entry->size = size;
590 mem_entry->protection = VM_PROT_ALL;
591
592 /* make the shared region point at the memory entry */
593 shared_region->sr_mem_entry = mem_entry_port;
594
595 /* fill in the shared region's environment and settings */
596 shared_region->sr_base_address = base_address;
597 shared_region->sr_size = size;
598 shared_region->sr_pmap_nesting_start = pmap_nesting_start;
599 shared_region->sr_pmap_nesting_size = pmap_nesting_size;
600 shared_region->sr_cpu_type = cputype;
601 shared_region->sr_64bit = is_64bit;
602 shared_region->sr_root_dir = root_dir;
603
604 queue_init(&shared_region->sr_q);
605 shared_region->sr_mapping_in_progress = FALSE;
606 shared_region->sr_persists = FALSE;
607 shared_region->sr_first_mapping = (mach_vm_offset_t) -1;
608
609 /* grab a reference for the caller */
610 shared_region->sr_ref_count = 1;
611
612done:
613 if (shared_region) {
614 SHARED_REGION_TRACE_INFO(
615 ("shared_region: create(root=%p,cpu=%d,64bit=%d,"
616 "base=0x%llx,size=0x%llx) <- "
617 "%p mem=(%p,%p) map=%p pmap=%p\n",
618 root_dir, cputype, is_64bit, (long long)base_address,
619 (long long)size, shared_region,
620 mem_entry_port, mem_entry, sub_map, sub_map->pmap));
621 } else {
622 SHARED_REGION_TRACE_INFO(
623 ("shared_region: create(root=%p,cpu=%d,64bit=%d,"
624 "base=0x%llx,size=0x%llx) <- NULL",
625 root_dir, cputype, is_64bit, (long long)base_address,
626 (long long)size));
627 }
628 return shared_region;
629}
630
631/*
632 * Destroy a now-unused shared region.
633 * The shared region is no longer in the queue and can not be looked up.
634 */
635static void
636vm_shared_region_destroy(
637 vm_shared_region_t shared_region)
638{
639 vm_named_entry_t mem_entry;
640 vm_map_t map;
641
642 SHARED_REGION_TRACE_INFO(
643 ("shared_region: -> destroy(%p) (root=%p,cpu=%d,64bit=%d)\n",
644 shared_region,
645 shared_region->sr_root_dir,
646 shared_region->sr_cpu_type,
647 shared_region->sr_64bit));
648
649 assert(shared_region->sr_ref_count == 0);
650 assert(!shared_region->sr_persists);
651
652 mem_entry = (vm_named_entry_t) shared_region->sr_mem_entry->ip_kobject;
653 assert(mem_entry->is_sub_map);
654 assert(!mem_entry->internal);
655 assert(!mem_entry->is_pager);
656 map = mem_entry->backing.map;
657
658 /*
659 * Clean up the pmap first. The virtual addresses that were
660 * entered in this possibly "nested" pmap may have different values
661 * than the VM map's min and max offsets, if the VM sub map was
662 * mapped at a non-zero offset in the processes' main VM maps, which
663 * is usually the case, so the clean-up we do in vm_map_destroy() would
664 * not be enough.
665 */
666 if (map->pmap) {
667 pmap_remove(map->pmap,
668 shared_region->sr_base_address,
669 (shared_region->sr_base_address +
670 shared_region->sr_size));
671 }
672
673 /*
674 * Release our (one and only) handle on the memory entry.
675 * This will generate a no-senders notification, which will be processed
676 * by ipc_kobject_notify(), which will release the one and only
677 * reference on the memory entry and cause it to be destroyed, along
678 * with the VM sub map and its pmap.
679 */
680 mach_memory_entry_port_release(shared_region->sr_mem_entry);
681 mem_entry = NULL;
682 shared_region->sr_mem_entry = IPC_PORT_NULL;
683
684 /* release the shared region structure... */
685 kfree(shared_region, sizeof (*shared_region));
686 SHARED_REGION_TRACE_DEBUG(
687 ("shared_region: destroy(%p) <-\n",
688 shared_region));
689 shared_region = NULL;
690
691}
692
693/*
694 * Gets the address of the first (in time) mapping in the shared region.
695 */
696kern_return_t
697vm_shared_region_start_address(
698 vm_shared_region_t shared_region,
699 mach_vm_offset_t *start_address)
700{
701 kern_return_t kr;
702 mach_vm_offset_t sr_base_address;
703 mach_vm_offset_t sr_first_mapping;
704
705 SHARED_REGION_TRACE_DEBUG(
706 ("shared_region: -> start_address(%p)\n",
707 shared_region));
708 assert(shared_region->sr_ref_count > 1);
709
710 vm_shared_region_lock();
711
712 /*
713 * Wait if there's another thread establishing a mapping
714 * in this shared region right when we're looking at it.
715 * We want a consistent view of the map...
716 */
717 while (shared_region->sr_mapping_in_progress) {
718 /* wait for our turn... */
719 assert(shared_region->sr_ref_count > 1);
720 vm_shared_region_sleep(&shared_region->sr_mapping_in_progress,
721 THREAD_UNINT);
722 }
723 assert(! shared_region->sr_mapping_in_progress);
724 assert(shared_region->sr_ref_count > 1);
725
726 sr_base_address = shared_region->sr_base_address;
727 sr_first_mapping = shared_region->sr_first_mapping;
728
729 if (sr_first_mapping == (mach_vm_offset_t) -1) {
730 /* shared region is empty */
731 kr = KERN_INVALID_ADDRESS;
732 } else {
733 kr = KERN_SUCCESS;
734 *start_address = sr_base_address + sr_first_mapping;
735 }
736
737 vm_shared_region_unlock();
738
739 SHARED_REGION_TRACE_DEBUG(
740 ("shared_region: start_address(%p) <- 0x%llx\n",
741 shared_region, (long long)shared_region->sr_base_address));
742
743 return kr;
744}
745/*
746 * Establish some mappings of a file in the shared region.
747 * This is used by "dyld" via the shared_region_map_np() system call
748 * to populate the shared region with the appropriate shared cache.
749 *
750 * One could also call it several times to incrementally load several
751 * libraries, as long as they do not overlap.
752 * It will return KERN_SUCCESS if the mappings were successfully established
753 * or if they were already established identically by another process.
754 */
755kern_return_t
756vm_shared_region_map_file(
757 vm_shared_region_t shared_region,
758 unsigned int mappings_count,
759 struct shared_file_mapping_np *mappings,
760 memory_object_control_t file_control,
761 memory_object_size_t file_size,
762 void *root_dir)
763{
764 kern_return_t kr;
765 vm_object_t file_object;
766 ipc_port_t sr_handle;
767 vm_named_entry_t sr_mem_entry;
768 vm_map_t sr_map;
769 mach_vm_offset_t sr_base_address;
770 unsigned int i;
771 mach_port_t map_port;
772 mach_vm_offset_t target_address;
773
774 kr = KERN_SUCCESS;
775
776 vm_shared_region_lock();
777 assert(shared_region->sr_ref_count > 1);
778
779 if (shared_region->sr_root_dir != root_dir) {
780 /*
781 * This shared region doesn't match the current root
782 * directory of this process. Deny the mapping to
783 * avoid tainting the shared region with something that
784 * doesn't quite belong into it.
785 */
786 vm_shared_region_unlock();
787 kr = KERN_PROTECTION_FAILURE;
788 goto done;
789 }
790
791 /*
792 * Make sure we handle only one mapping at a time in a given
793 * shared region, to avoid race conditions. This should not
794 * happen frequently...
795 */
796 while (shared_region->sr_mapping_in_progress) {
797 /* wait for our turn... */
798 vm_shared_region_sleep(&shared_region->sr_mapping_in_progress,
799 THREAD_UNINT);
800 }
801 assert(! shared_region->sr_mapping_in_progress);
802 assert(shared_region->sr_ref_count > 1);
803 /* let others know we're working in this shared region */
804 shared_region->sr_mapping_in_progress = TRUE;
805
806 vm_shared_region_unlock();
807
808 /* no need to lock because this data is never modified... */
809 sr_handle = shared_region->sr_mem_entry;
810 sr_mem_entry = (vm_named_entry_t) sr_handle->ip_kobject;
811 sr_map = sr_mem_entry->backing.map;
812 sr_base_address = shared_region->sr_base_address;
813
814 SHARED_REGION_TRACE_DEBUG(
815 ("shared_region: -> map(%p,%d,%p,%p,0x%llx)\n",
816 shared_region, mappings_count, mappings,
817 file_control, file_size));
818
819 /* get the VM object associated with the file to be mapped */
820 file_object = memory_object_control_to_vm_object(file_control);
821
822 /* establish the mappings */
823 for (i = 0; i < mappings_count; i++) {
824 SHARED_REGION_TRACE_INFO(
825 ("shared_region: mapping[%d]: "
826 "address:0x%016llx size:0x%016llx offset:0x%016llx "
827 "maxprot:0x%x prot:0x%x\n",
828 i,
829 (long long)mappings[i].sfm_address,
830 (long long)mappings[i].sfm_size,
831 (long long)mappings[i].sfm_file_offset,
832 mappings[i].sfm_max_prot,
833 mappings[i].sfm_init_prot));
834
835 if (mappings[i].sfm_init_prot & VM_PROT_ZF) {
836 /* zero-filled memory */
837 map_port = MACH_PORT_NULL;
838 } else {
839 /* file-backed memory */
840 map_port = (ipc_port_t) file_object->pager;
841 }
842
843 /* mapping's address is relative to the shared region base */
844 target_address =
845 mappings[i].sfm_address - sr_base_address;
846
847 /* establish that mapping, OK if it's to "already" there */
848 kr = vm_map_enter_mem_object(
849 sr_map,
850 &target_address,
851 vm_map_round_page(mappings[i].sfm_size),
852 0,
853 VM_FLAGS_FIXED | VM_FLAGS_ALREADY,
854 map_port,
855 mappings[i].sfm_file_offset,
856 TRUE,
857 mappings[i].sfm_init_prot & VM_PROT_ALL,
858 mappings[i].sfm_max_prot & VM_PROT_ALL,
859 VM_INHERIT_DEFAULT);
860 if (kr == KERN_MEMORY_PRESENT) {
861 /* this exact mapping was already there: that's fine */
862 SHARED_REGION_TRACE_INFO(
863 ("shared_region: mapping[%d]: "
864 "address:0x%016llx size:0x%016llx "
865 "offset:0x%016llx "
866 "maxprot:0x%x prot:0x%x already mapped...\n",
867 i,
868 (long long)mappings[i].sfm_address,
869 (long long)mappings[i].sfm_size,
870 (long long)mappings[i].sfm_file_offset,
871 mappings[i].sfm_max_prot,
872 mappings[i].sfm_init_prot));
873 kr = KERN_SUCCESS;
874 } else if (kr != KERN_SUCCESS) {
875 /* this mapping failed ! */
876 SHARED_REGION_TRACE_ERROR(
877 ("shared_region: mapping[%d]: "
878 "address:0x%016llx size:0x%016llx "
879 "offset:0x%016llx "
880 "maxprot:0x%x prot:0x%x failed 0x%x\n",
881 i,
882 (long long)mappings[i].sfm_address,
883 (long long)mappings[i].sfm_size,
884 (long long)mappings[i].sfm_file_offset,
885 mappings[i].sfm_max_prot,
886 mappings[i].sfm_init_prot,
887 kr));
888 break;
889 }
890
891 /* we're protected by "sr_mapping_in_progress" */
892 if (shared_region->sr_first_mapping == (mach_vm_offset_t) -1) {
893 shared_region->sr_first_mapping = target_address;
894 }
895 }
896
897 vm_shared_region_lock();
898 assert(shared_region->sr_ref_count > 1);
899 assert(shared_region->sr_mapping_in_progress);
900 /* we're done working on that shared region */
901 shared_region->sr_mapping_in_progress = FALSE;
902 thread_wakeup((event_t) &shared_region->sr_mapping_in_progress);
903 vm_shared_region_unlock();
904
905done:
906 SHARED_REGION_TRACE_DEBUG(
907 ("shared_region: map(%p,%d,%p,%p,0x%llx) <- 0x%x \n",
908 shared_region, mappings_count, mappings,
909 file_control, file_size, kr));
910 return kr;
911}
912
913/*
914 * Enter the appropriate shared region into "map" for "task".
915 * This involves looking up the shared region (and possibly creating a new
916 * one) for the desired environment, then mapping the VM sub map into the
917 * task's VM "map", with the appropriate level of pmap-nesting.
918 */
919kern_return_t
920vm_shared_region_enter(
921 struct _vm_map *map,
922 struct task *task,
923 void *fsroot,
924 cpu_type_t cpu)
925{
926 kern_return_t kr;
927 vm_shared_region_t shared_region;
928 vm_map_offset_t sr_address, sr_offset, target_address;
929 vm_map_size_t sr_size, mapping_size;
930 vm_map_offset_t sr_pmap_nesting_start;
931 vm_map_size_t sr_pmap_nesting_size;
932 ipc_port_t sr_handle;
933 boolean_t is_64bit;
934
935 is_64bit = task_has_64BitAddr(task);
936
937 SHARED_REGION_TRACE_DEBUG(
938 ("shared_region: -> "
939 "enter(map=%p,task=%p,root=%p,cpu=%d,64bit=%d)\n",
940 map, task, fsroot, cpu, is_64bit));
941
942 /* lookup (create if needed) the shared region for this environment */
943 shared_region = vm_shared_region_lookup(fsroot, cpu, is_64bit);
944 if (shared_region == NULL) {
945 /* this should not happen ! */
946 SHARED_REGION_TRACE_ERROR(
947 ("shared_region: -> "
948 "enter(map=%p,task=%p,root=%p,cpu=%d,64bit=%d): "
949 "lookup failed !\n",
950 map, task, fsroot, cpu, is_64bit));
951 //panic("shared_region_enter: lookup failed\n");
952 return KERN_FAILURE;
953 }
954
955 /* let the task use that shared region */
956 vm_shared_region_set(task, shared_region);
957
958 kr = KERN_SUCCESS;
959 /* no need to lock since this data is never modified */
960 sr_address = shared_region->sr_base_address;
961 sr_size = shared_region->sr_size;
962 sr_handle = shared_region->sr_mem_entry;
963 sr_pmap_nesting_start = shared_region->sr_pmap_nesting_start;
964 sr_pmap_nesting_size = shared_region->sr_pmap_nesting_size;
965
966 /*
967 * Start mapping the shared region's VM sub map into the task's VM map.
968 */
969 sr_offset = 0;
970
971 if (sr_pmap_nesting_start > sr_address) {
972 /* we need to map a range without pmap-nesting first */
973 target_address = sr_address;
974 mapping_size = sr_pmap_nesting_start - sr_address;
975 kr = vm_map_enter_mem_object(
976 map,
977 &target_address,
978 mapping_size,
979 0,
980 VM_FLAGS_FIXED,
981 sr_handle,
982 sr_offset,
983 TRUE,
984 VM_PROT_READ,
985 VM_PROT_ALL,
986 VM_INHERIT_SHARE);
987 if (kr != KERN_SUCCESS) {
988 SHARED_REGION_TRACE_ERROR(
989 ("shared_region: enter(%p,%p,%p,%d,%d): "
990 "vm_map_enter(0x%llx,0x%llx,%p) error 0x%x\n",
991 map, task, fsroot, cpu, is_64bit,
992 (long long)target_address,
993 (long long)mapping_size, sr_handle, kr));
994 goto done;
995 }
996 SHARED_REGION_TRACE_DEBUG(
997 ("shared_region: enter(%p,%p,%p,%d,%d): "
998 "vm_map_enter(0x%llx,0x%llx,%p) error 0x%x\n",
999 map, task, fsroot, cpu, is_64bit,
1000 (long long)target_address, (long long)mapping_size,
1001 sr_handle, kr));
1002 sr_offset += mapping_size;
1003 sr_size -= mapping_size;
1004 }
1005 /*
1006 * We may need to map several pmap-nested portions, due to platform
1007 * specific restrictions on pmap nesting.
1008 * The pmap-nesting is triggered by the "VM_MEMORY_SHARED_PMAP" alias...
1009 */
1010 for (;
1011 sr_pmap_nesting_size > 0;
1012 sr_offset += mapping_size,
1013 sr_size -= mapping_size,
1014 sr_pmap_nesting_size -= mapping_size) {
1015 target_address = sr_address + sr_offset;
1016 mapping_size = sr_pmap_nesting_size;
1017 if (mapping_size > pmap_nesting_size_max) {
1018 mapping_size = (vm_map_offset_t) pmap_nesting_size_max;
1019 }
1020 kr = vm_map_enter_mem_object(
1021 map,
1022 &target_address,
1023 mapping_size,
1024 0,
1025 (VM_FLAGS_FIXED | VM_MAKE_TAG(VM_MEMORY_SHARED_PMAP)),
1026 sr_handle,
1027 sr_offset,
1028 TRUE,
1029 VM_PROT_READ,
1030 VM_PROT_ALL,
1031 VM_INHERIT_SHARE);
1032 if (kr != KERN_SUCCESS) {
1033 SHARED_REGION_TRACE_ERROR(
1034 ("shared_region: enter(%p,%p,%p,%d,%d): "
1035 "vm_map_enter(0x%llx,0x%llx,%p) error 0x%x\n",
1036 map, task, fsroot, cpu, is_64bit,
1037 (long long)target_address,
1038 (long long)mapping_size, sr_handle, kr));
1039 goto done;
1040 }
1041 SHARED_REGION_TRACE_DEBUG(
1042 ("shared_region: enter(%p,%p,%p,%d,%d): "
1043 "nested vm_map_enter(0x%llx,0x%llx,%p) error 0x%x\n",
1044 map, task, fsroot, cpu, is_64bit,
1045 (long long)target_address, (long long)mapping_size,
1046 sr_handle, kr));
1047 }
1048 if (sr_size > 0) {
1049 /* and there's some left to be mapped without pmap-nesting */
1050 target_address = sr_address + sr_offset;
1051 mapping_size = sr_size;
1052 kr = vm_map_enter_mem_object(
1053 map,
1054 &target_address,
1055 mapping_size,
1056 0,
1057 VM_FLAGS_FIXED,
1058 sr_handle,
1059 sr_offset,
1060 TRUE,
1061 VM_PROT_READ,
1062 VM_PROT_ALL,
1063 VM_INHERIT_SHARE);
1064 if (kr != KERN_SUCCESS) {
1065 SHARED_REGION_TRACE_ERROR(
1066 ("shared_region: enter(%p,%p,%p,%d,%d): "
1067 "vm_map_enter(0x%llx,0x%llx,%p) error 0x%x\n",
1068 map, task, fsroot, cpu, is_64bit,
1069 (long long)target_address,
1070 (long long)mapping_size, sr_handle, kr));
1071 goto done;
1072 }
1073 SHARED_REGION_TRACE_DEBUG(
1074 ("shared_region: enter(%p,%p,%p,%d,%d): "
1075 "vm_map_enter(0x%llx,0x%llx,%p) error 0x%x\n",
1076 map, task, fsroot, cpu, is_64bit,
1077 (long long)target_address, (long long)mapping_size,
1078 sr_handle, kr));
1079 sr_offset += mapping_size;
1080 sr_size -= mapping_size;
1081 }
1082 assert(sr_size == 0);
1083
1084done:
1085 SHARED_REGION_TRACE_DEBUG(
1086 ("shared_region: enter(%p,%p,%p,%d,%d) <- 0x%x\n",
1087 map, task, fsroot, cpu, is_64bit, kr));
1088 return kr;
1089}
1090
1091/******************************************************************************/
1092/* Comm page support */
1093/******************************************************************************/
1094
1095ipc_port_t commpage32_handle = IPC_PORT_NULL;
1096ipc_port_t commpage64_handle = IPC_PORT_NULL;
1097vm_named_entry_t commpage32_entry = NULL;
1098vm_named_entry_t commpage64_entry = NULL;
1099vm_map_t commpage32_map = VM_MAP_NULL;
1100vm_map_t commpage64_map = VM_MAP_NULL;
1101
1102/*
1103 * Create a memory entry, VM submap and pmap for one commpage.
1104 */
1105static void
1106_vm_commpage_init(
1107 ipc_port_t *handlep,
1108 vm_map_size_t size)
1109{
1110 kern_return_t kr;
1111 vm_named_entry_t mem_entry;
1112 vm_map_t new_map;
1113
1114 SHARED_REGION_TRACE_DEBUG(
1115 ("commpage: -> _init(0x%llx)\n",
1116 (long long)size));
1117
1118 kr = mach_memory_entry_allocate(&mem_entry,
1119 handlep);
1120 if (kr != KERN_SUCCESS) {
1121 panic("_vm_commpage_init: could not allocate mem_entry");
1122 }
1123 new_map = vm_map_create(pmap_create(0, FALSE), 0, size, TRUE);
1124 if (new_map == VM_MAP_NULL) {
1125 panic("_vm_commpage_init: could not allocate VM map");
1126 }
1127 mem_entry->backing.map = new_map;
1128 mem_entry->internal = TRUE;
1129 mem_entry->is_sub_map = TRUE;
1130 mem_entry->offset = 0;
1131 mem_entry->protection = VM_PROT_ALL;
1132 mem_entry->size = size;
1133
1134 SHARED_REGION_TRACE_DEBUG(
1135 ("commpage: _init(0x%llx) <- %p\n",
1136 (long long)size, *handlep));
1137}
1138
1139/*
1140 * Initialize the comm pages at boot time.
1141 */
1142void
1143vm_commpage_init(void)
1144{
1145 SHARED_REGION_TRACE_DEBUG(
1146 ("commpage: -> init()\n"));
1147
1148 /* create the 32-bit comm page */
1149 _vm_commpage_init(&commpage32_handle, _COMM_PAGE32_AREA_LENGTH);
1150 commpage32_entry = (vm_named_entry_t) commpage32_handle->ip_kobject;
1151 commpage32_map = commpage32_entry->backing.map;
1152
1153 /* XXX if (cpu_is_64bit_capable()) ? */
1154 /* create the 64-bit comm page */
1155 _vm_commpage_init(&commpage64_handle, _COMM_PAGE64_AREA_LENGTH);
1156 commpage64_entry = (vm_named_entry_t) commpage64_handle->ip_kobject;
1157 commpage64_map = commpage64_entry->backing.map;
1158
1159 /* populate them according to this specific platform */
1160 commpage_populate();
1161
1162 SHARED_REGION_TRACE_DEBUG(
1163 ("commpage: init() <-\n"));
1164}
1165
1166/*
1167 * Enter the appropriate comm page into the task's address space.
1168 * This is called at exec() time via vm_map_exec().
1169 */
1170kern_return_t
1171vm_commpage_enter(
1172 vm_map_t map,
1173 task_t task)
1174{
1175 ipc_port_t commpage_handle;
1176 vm_map_offset_t commpage_address, objc_address;
1177 vm_map_size_t commpage_size, objc_size;
1178 int vm_flags;
1179 kern_return_t kr;
1180
1181 SHARED_REGION_TRACE_DEBUG(
1182 ("commpage: -> enter(%p,%p)\n",
1183 map, task));
1184
1185 /* the comm page is likely to be beyond the actual end of the VM map */
1186 vm_flags = VM_FLAGS_FIXED | VM_FLAGS_BEYOND_MAX;
1187
1188 /* select the appropriate comm page for this task */
1189 assert(! (task_has_64BitAddr(task) ^ vm_map_is_64bit(map)));
1190 if (task_has_64BitAddr(task)) {
1191#ifdef __ppc__
1192 /*
1193 * PPC51: ppc64 is limited to 51-bit addresses.
1194 * Memory above that limit is handled specially at the
1195 * pmap level, so do not interfere.
1196 */
1197 vm_flags |= VM_FLAGS_NO_PMAP_CHECK;
1198#endif /* __ppc__ */
1199 commpage_handle = commpage64_handle;
1200 commpage_address = (vm_map_offset_t) _COMM_PAGE64_BASE_ADDRESS;
1201 commpage_size = _COMM_PAGE64_AREA_LENGTH;
1202 objc_size = _COMM_PAGE64_OBJC_SIZE;
1203 objc_address = _COMM_PAGE64_OBJC_BASE;
1204 } else {
1205 commpage_handle = commpage32_handle;
1206 commpage_address =
1207 (vm_map_offset_t)(unsigned) _COMM_PAGE32_BASE_ADDRESS;
1208 commpage_size = _COMM_PAGE32_AREA_LENGTH;
1209 objc_size = _COMM_PAGE32_OBJC_SIZE;
1210 objc_address = _COMM_PAGE32_OBJC_BASE;
1211 }
1212
1213 if ((commpage_address & (pmap_nesting_size_min - 1)) == 0 &&
1214 (commpage_size & (pmap_nesting_size_min - 1)) == 0) {
1215 /* the commpage is properly aligned or sized for pmap-nesting */
1216 vm_flags |= VM_MAKE_TAG(VM_MEMORY_SHARED_PMAP);
1217 }
1218
1219 /* map the comm page in the task's address space */
1220 assert(commpage_handle != IPC_PORT_NULL);
1221 kr = vm_map_enter_mem_object(
1222 map,
1223 &commpage_address,
1224 commpage_size,
1225 0,
1226 vm_flags,
1227 commpage_handle,
1228 0,
1229 FALSE,
1230 VM_PROT_READ|VM_PROT_EXECUTE,
1231 VM_PROT_READ|VM_PROT_EXECUTE,
1232 VM_INHERIT_SHARE);
1233 if (kr != KERN_SUCCESS) {
1234 SHARED_REGION_TRACE_ERROR(
1235 ("commpage: enter(%p,0x%llx,0x%llx) "
1236 "commpage %p mapping failed 0x%x\n",
1237 map, (long long)commpage_address,
1238 (long long)commpage_size, commpage_handle, kr));
1239 }
1240
1241 /*
1242 * Since we're here, we also pre-allocate some virtual space for the
1243 * Objective-C run-time, if needed...
1244 */
1245 if (objc_size != 0) {
1246 kr = vm_map_enter_mem_object(
1247 map,
1248 &objc_address,
1249 objc_size,
1250 0,
1251 VM_FLAGS_FIXED | VM_FLAGS_BEYOND_MAX,
1252 IPC_PORT_NULL,
1253 0,
1254 FALSE,
1255 VM_PROT_ALL,
1256 VM_PROT_ALL,
1257 VM_INHERIT_DEFAULT);
1258 if (kr != KERN_SUCCESS) {
1259 SHARED_REGION_TRACE_ERROR(
1260 ("commpage: enter(%p,0x%llx,0x%llx) "
1261 "objc mapping failed 0x%x\n",
1262 map, (long long)objc_address,
1263 (long long)objc_size, kr));
1264 }
1265 }
1266
1267 SHARED_REGION_TRACE_DEBUG(
1268 ("commpage: enter(%p,%p) <- 0x%x\n",
1269 map, task, kr));
1270 return kr;
1271}