/*
- * Copyright (c) 2000 Apple Computer, Inc. All rights reserved.
+ * Copyright (c) 2000-2005 Apple Computer, Inc. All rights reserved.
*
* @APPLE_LICENSE_HEADER_START@
*
* Virtual memory mapping module.
*/
-#include <cpus.h>
#include <task_swapper.h>
#include <mach_assert.h>
+#include <libkern/OSAtomic.h>
#include <mach/kern_return.h>
#include <mach/port.h>
#include <mach/vm_attributes.h>
#include <mach/vm_param.h>
#include <mach/vm_behavior.h>
+#include <mach/vm_statistics.h>
+#include <mach/memory_object.h>
+#include <machine/cpu_capabilities.h>
+
#include <kern/assert.h>
#include <kern/counters.h>
+#include <kern/kalloc.h>
#include <kern/zalloc.h>
+
+#include <vm/cpm.h>
#include <vm/vm_init.h>
#include <vm/vm_fault.h>
#include <vm/vm_map.h>
#include <ipc/ipc_port.h>
#include <kern/sched_prim.h>
#include <kern/misc_protos.h>
-#include <mach/vm_map_server.h>
-#include <mach/mach_host_server.h>
#include <ddb/tr.h>
+#include <machine/db_machdep.h>
#include <kern/xpr.h>
+#include <mach/vm_map_server.h>
+#include <mach/mach_host_server.h>
+#include <vm/vm_shared_memory_server.h>
+#include <vm/vm_protos.h> // for vm_map_commpage64 and vm_map_remove_compage64
+
+#ifdef ppc
+#include <ppc/mappings.h>
+#endif /* ppc */
+
+#include <vm/vm_protos.h>
+
/* Internal prototypes
*/
-extern boolean_t vm_map_range_check(
+
+static void vm_map_simplify_range(
+ vm_map_t map,
+ vm_map_offset_t start,
+ vm_map_offset_t end); /* forward */
+
+static boolean_t vm_map_range_check(
vm_map_t map,
- vm_offset_t start,
- vm_offset_t end,
+ vm_map_offset_t start,
+ vm_map_offset_t end,
vm_map_entry_t *entry);
-extern vm_map_entry_t _vm_map_entry_create(
+static vm_map_entry_t _vm_map_entry_create(
struct vm_map_header *map_header);
-extern void _vm_map_entry_dispose(
+static void _vm_map_entry_dispose(
struct vm_map_header *map_header,
vm_map_entry_t entry);
-extern void vm_map_pmap_enter(
+static void vm_map_pmap_enter(
vm_map_t map,
- vm_offset_t addr,
- vm_offset_t end_addr,
+ vm_map_offset_t addr,
+ vm_map_offset_t end_addr,
vm_object_t object,
vm_object_offset_t offset,
vm_prot_t protection);
-extern void _vm_map_clip_end(
+static void _vm_map_clip_end(
struct vm_map_header *map_header,
vm_map_entry_t entry,
- vm_offset_t end);
+ vm_map_offset_t end);
-extern void vm_map_entry_delete(
+static void _vm_map_clip_start(
+ struct vm_map_header *map_header,
+ vm_map_entry_t entry,
+ vm_map_offset_t start);
+
+static void vm_map_entry_delete(
vm_map_t map,
vm_map_entry_t entry);
-extern kern_return_t vm_map_delete(
+static kern_return_t vm_map_delete(
vm_map_t map,
- vm_offset_t start,
- vm_offset_t end,
- int flags);
-
-extern void vm_map_copy_steal_pages(
- vm_map_copy_t copy);
+ vm_map_offset_t start,
+ vm_map_offset_t end,
+ int flags,
+ vm_map_t zap_map);
-extern kern_return_t vm_map_copy_overwrite_unaligned(
+static kern_return_t vm_map_copy_overwrite_unaligned(
vm_map_t dst_map,
vm_map_entry_t entry,
vm_map_copy_t copy,
- vm_offset_t start);
+ vm_map_address_t start);
-extern kern_return_t vm_map_copy_overwrite_aligned(
+static kern_return_t vm_map_copy_overwrite_aligned(
vm_map_t dst_map,
vm_map_entry_t tmp_entry,
vm_map_copy_t copy,
- vm_offset_t start,
+ vm_map_offset_t start,
pmap_t pmap);
-extern kern_return_t vm_map_copyin_kernel_buffer(
+static kern_return_t vm_map_copyin_kernel_buffer(
vm_map_t src_map,
- vm_offset_t src_addr,
- vm_size_t len,
+ vm_map_address_t src_addr,
+ vm_map_size_t len,
boolean_t src_destroy,
vm_map_copy_t *copy_result); /* OUT */
-extern kern_return_t vm_map_copyout_kernel_buffer(
+static kern_return_t vm_map_copyout_kernel_buffer(
vm_map_t map,
- vm_offset_t *addr, /* IN/OUT */
+ vm_map_address_t *addr, /* IN/OUT */
vm_map_copy_t copy,
boolean_t overwrite);
-extern kern_return_t vm_map_copyin_page_list_cont(
- vm_map_copyin_args_t cont_args,
- vm_map_copy_t *copy_result); /* OUT */
-
-extern void vm_map_fork_share(
+static void vm_map_fork_share(
vm_map_t old_map,
vm_map_entry_t old_entry,
vm_map_t new_map);
-extern boolean_t vm_map_fork_copy(
+static boolean_t vm_map_fork_copy(
vm_map_t old_map,
vm_map_entry_t *old_entry_p,
vm_map_t new_map);
-extern kern_return_t vm_remap_range_allocate(
- vm_map_t map,
- vm_offset_t *address, /* IN/OUT */
- vm_size_t size,
- vm_offset_t mask,
- boolean_t anywhere,
- vm_map_entry_t *map_entry); /* OUT */
-
-extern void _vm_map_clip_start(
- struct vm_map_header *map_header,
- vm_map_entry_t entry,
- vm_offset_t start);
-
-void vm_region_top_walk(
+static void vm_map_region_top_walk(
vm_map_entry_t entry,
vm_region_top_info_t top);
-void vm_region_walk(
+static void vm_map_region_walk(
+ vm_map_t map,
+ vm_map_offset_t va,
vm_map_entry_t entry,
- vm_region_extended_info_t extended,
vm_object_offset_t offset,
- vm_offset_t range,
+ vm_object_size_t range,
+ vm_region_extended_info_t extended);
+
+static kern_return_t vm_map_wire_nested(
vm_map_t map,
- vm_offset_t va);
+ vm_map_offset_t start,
+ vm_map_offset_t end,
+ vm_prot_t access_type,
+ boolean_t user_wire,
+ pmap_t map_pmap,
+ vm_map_offset_t pmap_addr);
+
+static kern_return_t vm_map_unwire_nested(
+ vm_map_t map,
+ vm_map_offset_t start,
+ vm_map_offset_t end,
+ boolean_t user_wire,
+ pmap_t map_pmap,
+ vm_map_offset_t pmap_addr);
+
+static kern_return_t vm_map_overwrite_submap_recurse(
+ vm_map_t dst_map,
+ vm_map_offset_t dst_addr,
+ vm_map_size_t dst_size);
+
+static kern_return_t vm_map_copy_overwrite_nested(
+ vm_map_t dst_map,
+ vm_map_offset_t dst_addr,
+ vm_map_copy_t copy,
+ boolean_t interruptible,
+ pmap_t pmap);
+
+static kern_return_t vm_map_remap_extract(
+ vm_map_t map,
+ vm_map_offset_t addr,
+ vm_map_size_t size,
+ boolean_t copy,
+ struct vm_map_header *map_header,
+ vm_prot_t *cur_protection,
+ vm_prot_t *max_protection,
+ vm_inherit_t inheritance,
+ boolean_t pageable);
+
+static kern_return_t vm_map_remap_range_allocate(
+ vm_map_t map,
+ vm_map_address_t *address,
+ vm_map_size_t size,
+ vm_map_offset_t mask,
+ boolean_t anywhere,
+ vm_map_entry_t *map_entry);
+
+static void vm_map_region_look_for_page(
+ vm_map_t map,
+ vm_map_offset_t va,
+ vm_object_t object,
+ vm_object_offset_t offset,
+ int max_refcnt,
+ int depth,
+ vm_region_extended_info_t extended);
+
+static int vm_map_region_count_obj_refs(
+ vm_map_entry_t entry,
+ vm_object_t object);
/*
* Macros to copy a vm_map_entry. We must be careful to correctly
* vm_object_copy_strategically() in vm_object.c.
*/
-zone_t vm_map_zone; /* zone for vm_map structures */
-zone_t vm_map_entry_zone; /* zone for vm_map_entry structures */
-zone_t vm_map_kentry_zone; /* zone for kernel entry structures */
-zone_t vm_map_copy_zone; /* zone for vm_map_copy structures */
+static zone_t vm_map_zone; /* zone for vm_map structures */
+static zone_t vm_map_entry_zone; /* zone for vm_map_entry structures */
+static zone_t vm_map_kentry_zone; /* zone for kernel entry structures */
+static zone_t vm_map_copy_zone; /* zone for vm_map_copy structures */
/*
* of a new entry.
*/
-vm_offset_t map_data;
-vm_size_t map_data_size;
-vm_offset_t kentry_data;
-vm_size_t kentry_data_size;
-int kentry_count = 2048; /* to init kentry_data_size */
+static void *map_data;
+static vm_map_size_t map_data_size;
+static void *kentry_data;
+static vm_map_size_t kentry_data_size;
+static int kentry_count = 2048; /* to init kentry_data_size */
+
+#define NO_COALESCE_LIMIT (1024 * 128)
/*
* Threshold for aggressive (eager) page map entering for vm copyout
* operations. Any copyout larger will NOT be aggressively entered.
*/
-vm_size_t vm_map_aggressive_enter_max; /* set by bootstrap */
+static vm_map_size_t vm_map_aggressive_enter_max; /* set by bootstrap */
+
+/* Skip acquiring locks if we're in the midst of a kernel core dump */
+extern unsigned int not_in_kdp;
void
vm_map_init(
void)
{
- vm_map_zone = zinit((vm_size_t) sizeof(struct vm_map), 40*1024,
+ vm_map_zone = zinit((vm_map_size_t) sizeof(struct vm_map), 40*1024,
PAGE_SIZE, "maps");
- vm_map_entry_zone = zinit((vm_size_t) sizeof(struct vm_map_entry),
+ vm_map_entry_zone = zinit((vm_map_size_t) sizeof(struct vm_map_entry),
1024*1024, PAGE_SIZE*5,
"non-kernel map entries");
- vm_map_kentry_zone = zinit((vm_size_t) sizeof(struct vm_map_entry),
+ vm_map_kentry_zone = zinit((vm_map_size_t) sizeof(struct vm_map_entry),
kentry_data_size, kentry_data_size,
"kernel map entries");
- vm_map_copy_zone = zinit((vm_size_t) sizeof(struct vm_map_copy),
+ vm_map_copy_zone = zinit((vm_map_size_t) sizeof(struct vm_map_copy),
16*1024, PAGE_SIZE, "map copies");
/*
vm_map_steal_memory(
void)
{
- map_data_size = round_page(10 * sizeof(struct vm_map));
+ map_data_size = vm_map_round_page(10 * sizeof(struct vm_map));
map_data = pmap_steal_memory(map_data_size);
#if 0
kentry_data_size =
- round_page(kentry_count * sizeof(struct vm_map_entry));
+ vm_map_round_page(kentry_count * sizeof(struct vm_map_entry));
kentry_data = pmap_steal_memory(kentry_data_size);
}
*/
vm_map_t
vm_map_create(
- pmap_t pmap,
- vm_offset_t min,
- vm_offset_t max,
- boolean_t pageable)
+ pmap_t pmap,
+ vm_map_offset_t min,
+ vm_map_offset_t max,
+ boolean_t pageable)
{
register vm_map_t result;
result->max_offset = max;
result->wiring_required = FALSE;
result->no_zero_fill = FALSE;
+ result->mapped = FALSE;
result->wait_for_space = FALSE;
result->first_free = vm_map_to_entry(result);
result->hint = vm_map_to_entry(result);
vm_map_lock_init(result);
- mutex_init(&result->s_lock, ETAP_VM_RESULT);
+ mutex_init(&result->s_lock, 0);
return(result);
}
#define vm_map_copy_entry_create(copy) \
_vm_map_entry_create(&(copy)->cpy_hdr)
-vm_map_entry_t
+static vm_map_entry_t
_vm_map_entry_create(
register struct vm_map_header *map_header)
{
#define vm_map_copy_entry_dispose(map, entry) \
_vm_map_entry_dispose(&(copy)->cpy_hdr, (entry))
-void
+static void
_vm_map_entry_dispose(
register struct vm_map_header *map_header,
register vm_map_entry_t entry)
else
zone = vm_map_kentry_zone;
- zfree(zone, (vm_offset_t) entry);
+ zfree(zone, entry);
}
-boolean_t first_free_is_valid(vm_map_t map); /* forward */
-boolean_t first_free_check = FALSE;
-boolean_t
+#if MACH_ASSERT
+static boolean_t first_free_is_valid(vm_map_t map); /* forward */
+static boolean_t first_free_check = FALSE;
+static boolean_t
first_free_is_valid(
vm_map_t map)
{
entry = vm_map_to_entry(map);
next = entry->vme_next;
- while (trunc_page(next->vme_start) == trunc_page(entry->vme_end) ||
- (trunc_page(next->vme_start) == trunc_page(entry->vme_start) &&
+ while (vm_map_trunc_page(next->vme_start) == vm_map_trunc_page(entry->vme_end) ||
+ (vm_map_trunc_page(next->vme_start) == vm_map_trunc_page(entry->vme_start) &&
next != vm_map_to_entry(map))) {
entry = next;
next = entry->vme_next;
}
return TRUE;
}
+#endif /* MACH_ASSERT */
/*
* UPDATE_FIRST_FREE:
UFF_map = (map); \
UFF_first_free = (new_first_free); \
UFF_next_entry = UFF_first_free->vme_next; \
- while (trunc_page(UFF_next_entry->vme_start) == \
- trunc_page(UFF_first_free->vme_end) || \
- (trunc_page(UFF_next_entry->vme_start) == \
- trunc_page(UFF_first_free->vme_start) && \
+ while (vm_map_trunc_page(UFF_next_entry->vme_start) == \
+ vm_map_trunc_page(UFF_first_free->vme_end) || \
+ (vm_map_trunc_page(UFF_next_entry->vme_start) == \
+ vm_map_trunc_page(UFF_first_free->vme_start) && \
UFF_next_entry != vm_map_to_entry(UFF_map))) { \
UFF_first_free = UFF_next_entry; \
UFF_next_entry = UFF_first_free->vme_next; \
(entry)->vme_prev->vme_next = (entry)->vme_next; \
MACRO_END
-/*
- * kernel_vm_map_reference:
- *
- * kernel internal export version for iokit and bsd components
- * in lieu of component interface semantics.
- *
- */
-void
-kernel_vm_map_reference(
- register vm_map_t map)
-{
- if (map == VM_MAP_NULL)
- return;
-
- mutex_lock(&map->s_lock);
-#if TASK_SWAPPER
- assert(map->res_count > 0);
- assert(map->ref_count >= map->res_count);
- map->res_count++;
-#endif
- map->ref_count++;
- mutex_unlock(&map->s_lock);
-}
-
#if MACH_ASSERT && TASK_SWAPPER
-/*
- * vm_map_reference:
- *
- * Adds valid reference and residence counts to the given map.
- * The map must be in memory (i.e. non-zero residence count).
- *
- */
-void
-vm_map_reference(
- register vm_map_t map)
-{
- if (map == VM_MAP_NULL)
- return;
-
- mutex_lock(&map->s_lock);
- assert(map->res_count > 0);
- assert(map->ref_count >= map->res_count);
- map->ref_count++;
- map->res_count++;
- mutex_unlock(&map->s_lock);
-}
-
/*
* vm_map_res_reference:
*
}
#endif /* MACH_ASSERT && TASK_SWAPPER */
-/*
- * vm_map_deallocate:
- *
- * Removes a reference from the specified map,
- * destroying it if no references remain.
- * The map should not be locked.
- */
-void
-vm_map_deallocate(
- register vm_map_t map)
-{
- unsigned int ref;
-
- if (map == VM_MAP_NULL)
- return;
-
- mutex_lock(&map->s_lock);
- ref = --map->ref_count;
- if (ref > 0) {
- vm_map_res_deallocate(map);
- mutex_unlock(&map->s_lock);
- return;
- }
- assert(map->ref_count == 0);
- mutex_unlock(&map->s_lock);
-
-#if TASK_SWAPPER
- /*
- * The map residence count isn't decremented here because
- * the vm_map_delete below will traverse the entire map,
- * deleting entries, and the residence counts on objects
- * and sharing maps will go away then.
- */
-#endif
-
- vm_map_destroy(map);
-}
-
/*
* vm_map_destroy:
*
void
vm_map_destroy(
register vm_map_t map)
-{
+{
vm_map_lock(map);
(void) vm_map_delete(map, map->min_offset,
- map->max_offset, VM_MAP_NO_FLAGS);
+ map->max_offset, VM_MAP_NO_FLAGS,
+ VM_MAP_NULL);
vm_map_unlock(map);
+
+#ifdef __PPC__
+ if (map->hdr.nentries!=0)
+ vm_map_remove_commpage64(map);
+#endif /* __PPC__ */
+
+ assert(map->hdr.nentries==0);
- pmap_destroy(map->pmap);
+ if(map->pmap)
+ pmap_destroy(map->pmap);
- zfree(vm_map_zone, (vm_offset_t) map);
+ zfree(vm_map_zone, map);
}
#if TASK_SWAPPER
* future lookups. Performs necessary interlocks.
*/
#define SAVE_HINT(map,value) \
+MACRO_BEGIN \
mutex_lock(&(map)->s_lock); \
(map)->hint = (value); \
- mutex_unlock(&(map)->s_lock);
+ mutex_unlock(&(map)->s_lock); \
+MACRO_END
/*
* vm_map_lookup_entry: [ internal use only ]
*/
boolean_t
vm_map_lookup_entry(
- register vm_map_t map,
- register vm_offset_t address,
+ register vm_map_t map,
+ register vm_map_offset_t address,
vm_map_entry_t *entry) /* OUT */
{
register vm_map_entry_t cur;
* Start looking either from the head of the
* list, or from the hint.
*/
-
- mutex_lock(&map->s_lock);
+ if (not_in_kdp)
+ mutex_lock(&map->s_lock);
cur = map->hint;
- mutex_unlock(&map->s_lock);
+ if (not_in_kdp)
+ mutex_unlock(&map->s_lock);
if (cur == vm_map_to_entry(map))
cur = cur->vme_next;
*/
*entry = cur;
- SAVE_HINT(map, cur);
+ if (not_in_kdp)
+ SAVE_HINT(map, cur);
return(TRUE);
}
break;
cur = cur->vme_next;
}
*entry = cur->vme_prev;
- SAVE_HINT(map, *entry);
+ if (not_in_kdp)
+ SAVE_HINT(map, *entry);
return(FALSE);
}
kern_return_t
vm_map_find_space(
register vm_map_t map,
- vm_offset_t *address, /* OUT */
- vm_size_t size,
- vm_offset_t mask,
+ vm_map_offset_t *address, /* OUT */
+ vm_map_size_t size,
+ vm_map_offset_t mask,
vm_map_entry_t *o_entry) /* OUT */
{
register vm_map_entry_t entry, new_entry;
- register vm_offset_t start;
- register vm_offset_t end;
+ register vm_map_offset_t start;
+ register vm_map_offset_t end;
+
+ if (size == 0) {
+ *address = 0;
+ return KERN_INVALID_ARGUMENT;
+ }
new_entry = vm_map_entry_create(map);
int vm_map_pmap_enter_enable = FALSE;
/*
- * Routine: vm_map_pmap_enter
+ * Routine: vm_map_pmap_enter [internal only]
*
* Description:
* Force pages from the specified object to be entered into
* In/out conditions:
* The source map should not be locked on entry.
*/
-void
+static void
vm_map_pmap_enter(
vm_map_t map,
- register vm_offset_t addr,
- register vm_offset_t end_addr,
+ register vm_map_offset_t addr,
+ register vm_map_offset_t end_addr,
register vm_object_t object,
vm_object_offset_t offset,
vm_prot_t protection)
{
+ unsigned int cache_attr;
+
+ if(map->pmap == 0)
+ return;
+
while (addr < end_addr) {
register vm_page_t m;
vm_object_paging_begin(object);
m = vm_page_lookup(object, offset);
- if (m == VM_PAGE_NULL || m->busy ||
+ /*
+ * ENCRYPTED SWAP:
+ * The user should never see encrypted data, so do not
+ * enter an encrypted page in the page table.
+ */
+ if (m == VM_PAGE_NULL || m->busy || m->encrypted ||
(m->unusual && ( m->error || m->restart || m->absent ||
protection & m->page_lock))) {
if (vm_map_pmap_enter_print) {
printf("vm_map_pmap_enter:");
- printf("map: %x, addr: %x, object: %x, offset: %x\n",
- map, addr, object, offset);
+ printf("map: %x, addr: %llx, object: %x, offset: %llx\n",
+ map, (unsigned long long)addr, object, (unsigned long long)offset);
}
-
m->busy = TRUE;
+
+ if (m->no_isync == TRUE) {
+ pmap_sync_page_data_phys(m->phys_page);
+ m->no_isync = FALSE;
+ }
+
+ cache_attr = ((unsigned int)object->wimg_bits) & VM_WIMG_MASK;
vm_object_unlock(object);
- PMAP_ENTER(map->pmap, addr, m,
- protection, FALSE);
+ PMAP_ENTER(map->pmap, addr, m,
+ protection, cache_attr, FALSE);
vm_object_lock(object);
+
PAGE_WAKEUP_DONE(m);
vm_page_lock_queues();
if (!m->active && !m->inactive)
}
}
+boolean_t vm_map_pmap_is_empty(
+ vm_map_t map,
+ vm_map_offset_t start,
+ vm_map_offset_t end);
+boolean_t vm_map_pmap_is_empty(
+ vm_map_t map,
+ vm_map_offset_t start,
+ vm_map_offset_t end)
+{
+ vm_map_offset_t offset;
+ ppnum_t phys_page;
+
+ if (map->pmap == NULL) {
+ return TRUE;
+ }
+ for (offset = start;
+ offset < end;
+ offset += PAGE_SIZE) {
+ phys_page = pmap_find_phys(map->pmap, offset);
+ if (phys_page) {
+ kprintf("vm_map_pmap_is_empty(%p,0x%llx,0x%llx): "
+ "page %d at 0x%llx\n",
+ map, start, end, phys_page, offset);
+ return FALSE;
+ }
+ }
+ return TRUE;
+}
+
/*
* Routine: vm_map_enter
*
*
* Arguments are as defined in the vm_map call.
*/
+int _map_enter_debug = 0;
+static unsigned int vm_map_enter_restore_successes = 0;
+static unsigned int vm_map_enter_restore_failures = 0;
kern_return_t
vm_map_enter(
- register vm_map_t map,
- vm_offset_t *address, /* IN/OUT */
- vm_size_t size,
- vm_offset_t mask,
+ vm_map_t map,
+ vm_map_offset_t *address, /* IN/OUT */
+ vm_map_size_t size,
+ vm_map_offset_t mask,
int flags,
vm_object_t object,
vm_object_offset_t offset,
vm_prot_t max_protection,
vm_inherit_t inheritance)
{
- vm_map_entry_t entry;
- register vm_offset_t start;
- register vm_offset_t end;
+ vm_map_entry_t entry, new_entry;
+ vm_map_offset_t start, tmp_start;
+ vm_map_offset_t end, tmp_end;
kern_return_t result = KERN_SUCCESS;
-
- boolean_t anywhere = VM_FLAGS_ANYWHERE & flags;
+ vm_map_t zap_old_map = VM_MAP_NULL;
+ vm_map_t zap_new_map = VM_MAP_NULL;
+ boolean_t map_locked = FALSE;
+ boolean_t pmap_empty = TRUE;
+ boolean_t new_mapping_established = FALSE;
+ boolean_t anywhere = ((flags & VM_FLAGS_ANYWHERE) != 0);
+ boolean_t purgable = ((flags & VM_FLAGS_PURGABLE) != 0);
+ boolean_t overwrite = ((flags & VM_FLAGS_OVERWRITE) != 0);
char alias;
+ if (size == 0) {
+ *address = 0;
+ return KERN_INVALID_ARGUMENT;
+ }
+
VM_GET_FLAGS_ALIAS(flags, alias);
#define RETURN(value) { result = value; goto BailOut; }
assert(page_aligned(*address));
assert(page_aligned(size));
+
+ /*
+ * Only zero-fill objects are allowed to be purgable.
+ * LP64todo - limit purgable objects to 32-bits for now
+ */
+ if (purgable &&
+ (offset != 0 ||
+ (object != VM_OBJECT_NULL &&
+ (object->size != size ||
+ object->purgable == VM_OBJECT_NONPURGABLE))
+ || size > VM_MAX_ADDRESS)) /* LP64todo: remove when dp capable */
+ return KERN_INVALID_ARGUMENT;
+
+ if (!anywhere && overwrite) {
+ /*
+ * Create a temporary VM map to hold the old mappings in the
+ * affected area while we create the new one.
+ * This avoids releasing the VM map lock in
+ * vm_map_entry_delete() and allows atomicity
+ * when we want to replace some mappings with a new one.
+ * It also allows us to restore the old VM mappings if the
+ * new mapping fails.
+ */
+ zap_old_map = vm_map_create(PMAP_NULL,
+ *address,
+ *address + size,
+ TRUE);
+ }
+
StartAgain: ;
start = *address;
if (anywhere) {
vm_map_lock(map);
+ map_locked = TRUE;
/*
* Calculate the first possible address.
assert_wait((event_t)map,
THREAD_ABORTSAFE);
vm_map_unlock(map);
- thread_block((void (*)(void))0);
+ map_locked = FALSE;
+ thread_block(THREAD_CONTINUE_NULL);
goto StartAgain;
}
}
*/
vm_map_lock(map);
+ map_locked = TRUE;
if ((start & mask) != 0)
RETURN(KERN_NO_SPACE);
RETURN(KERN_INVALID_ADDRESS);
}
+ if (overwrite && zap_old_map != VM_MAP_NULL) {
+ /*
+ * Fixed mapping and "overwrite" flag: attempt to
+ * remove all existing mappings in the specified
+ * address range, saving them in our "zap_old_map".
+ */
+ (void) vm_map_delete(map, start, end,
+ VM_MAP_REMOVE_SAVE_ENTRIES,
+ zap_old_map);
+ }
+
/*
* ... the starting address isn't allocated
*/
/*
* See whether we can avoid creating a new entry (and object) by
* extending one of our neighbors. [So far, we only attempt to
- * extend from below.]
+ * extend from below.] Note that we can never extend/join
+ * purgable objects because they need to remain distinct
+ * entities in order to implement their "volatile object"
+ * semantics.
*/
- if ((object == VM_OBJECT_NULL) &&
+ if (purgable) {
+ if (object == VM_OBJECT_NULL) {
+ object = vm_object_allocate(size);
+ object->copy_strategy = MEMORY_OBJECT_COPY_NONE;
+ object->purgable = VM_OBJECT_PURGABLE_NONVOLATILE;
+ offset = (vm_object_offset_t)0;
+ }
+ } else if ((object == VM_OBJECT_NULL) &&
(entry != vm_map_to_entry(map)) &&
(entry->vme_end == start) &&
(!entry->is_shared) &&
(entry->max_protection == max_protection) &&
(entry->behavior == VM_BEHAVIOR_DEFAULT) &&
(entry->in_transition == 0) &&
+ ((alias == VM_MEMORY_REALLOC) || ((entry->vme_end - entry->vme_start) + size < NO_COALESCE_LIMIT)) &&
(entry->wired_count == 0)) { /* implies user_wired_count == 0 */
if (vm_object_coalesce(entry->object.vm_object,
VM_OBJECT_NULL,
entry->offset,
(vm_object_offset_t) 0,
- (vm_size_t)(entry->vme_end - entry->vme_start),
- (vm_size_t)(end - entry->vme_end))) {
+ (vm_map_size_t)(entry->vme_end - entry->vme_start),
+ (vm_map_size_t)(end - entry->vme_end))) {
/*
* Coalesced the two objects - can extend
/*
* Create a new entry
+ * LP64todo - for now, we can only allocate 4GB internal objects
+ * because the default pager can't page bigger ones. Remove this
+ * when it can.
*/
-
- { /**/
- register vm_map_entry_t new_entry;
-
- new_entry = vm_map_entry_insert(map, entry, start, end, object,
- offset, needs_copy, FALSE, FALSE,
+ tmp_start = start;
+ if (object == VM_OBJECT_NULL && size > (vm_map_size_t)VM_MAX_ADDRESS)
+ tmp_end = tmp_start + (vm_map_size_t)VM_MAX_ADDRESS;
+ else
+ tmp_end = end;
+ do {
+ new_entry = vm_map_entry_insert(map, entry, tmp_start, tmp_end,
+ object, offset, needs_copy, FALSE, FALSE,
cur_protection, max_protection,
VM_BEHAVIOR_DEFAULT, inheritance, 0);
- new_entry->alias = alias;
+ new_entry->alias = alias;
+ entry = new_entry;
+ } while (object == VM_OBJECT_NULL &&
+ tmp_end != end &&
+ (tmp_start = tmp_end) &&
+ (tmp_end = (end - tmp_end > (vm_map_size_t)VM_MAX_ADDRESS) ?
+ tmp_end + (vm_map_size_t)VM_MAX_ADDRESS : end));
+
vm_map_unlock(map);
+ map_locked = FALSE;
+
+ new_mapping_established = TRUE;
/* Wire down the new entry if the user
* requested all new map entries be wired.
*/
if (map->wiring_required) {
+ pmap_empty = FALSE; /* pmap won't be empty */
result = vm_map_wire(map, start, end,
- new_entry->protection, TRUE);
- return(result);
+ new_entry->protection, TRUE);
+ RETURN(result);
}
if ((object != VM_OBJECT_NULL) &&
(!anywhere) &&
(!needs_copy) &&
(size < (128*1024))) {
+ pmap_empty = FALSE; /* pmap won't be empty */
vm_map_pmap_enter(map, start, end,
object, offset, cur_protection);
}
- return(result);
- } /**/
-
BailOut: ;
- vm_map_unlock(map);
- return(result);
+ if (result == KERN_SUCCESS &&
+ pmap_empty &&
+ !(flags & VM_FLAGS_NO_PMAP_CHECK)) {
+ assert(vm_map_pmap_is_empty(map, *address, *address+size));
+ }
+
+ if (result != KERN_SUCCESS) {
+ if (new_mapping_established) {
+ /*
+ * We have to get rid of the new mappings since we
+ * won't make them available to the user.
+ * Try and do that atomically, to minimize the risk
+ * that someone else create new mappings that range.
+ */
+ zap_new_map = vm_map_create(PMAP_NULL,
+ *address,
+ *address + size,
+ TRUE);
+ if (!map_locked) {
+ vm_map_lock(map);
+ map_locked = TRUE;
+ }
+ (void) vm_map_delete(map, *address, *address+size,
+ VM_MAP_REMOVE_SAVE_ENTRIES,
+ zap_new_map);
+ }
+ if (zap_old_map != VM_MAP_NULL &&
+ zap_old_map->hdr.nentries != 0) {
+ vm_map_entry_t entry1, entry2;
+
+ /*
+ * The new mapping failed. Attempt to restore
+ * the old mappings, saved in the "zap_old_map".
+ */
+ if (!map_locked) {
+ vm_map_lock(map);
+ map_locked = TRUE;
+ }
+
+ /* first check if the coast is still clear */
+ start = vm_map_first_entry(zap_old_map)->vme_start;
+ end = vm_map_last_entry(zap_old_map)->vme_end;
+ if (vm_map_lookup_entry(map, start, &entry1) ||
+ vm_map_lookup_entry(map, end, &entry2) ||
+ entry1 != entry2) {
+ /*
+ * Part of that range has already been
+ * re-mapped: we can't restore the old
+ * mappings...
+ */
+ vm_map_enter_restore_failures++;
+ } else {
+ /*
+ * Transfer the saved map entries from
+ * "zap_old_map" to the original "map",
+ * inserting them all after "entry1".
+ */
+ for (entry2 = vm_map_first_entry(zap_old_map);
+ entry2 != vm_map_to_entry(zap_old_map);
+ entry2 = vm_map_first_entry(zap_old_map)) {
+ vm_map_entry_unlink(zap_old_map,
+ entry2);
+ vm_map_entry_link(map, entry1, entry2);
+ entry1 = entry2;
+ }
+ if (map->wiring_required) {
+ /*
+ * XXX TODO: we should rewire the
+ * old pages here...
+ */
+ }
+ vm_map_enter_restore_successes++;
+ }
+ }
+ }
+
+ if (map_locked) {
+ vm_map_unlock(map);
+ }
+
+ /*
+ * Get rid of the "zap_maps" and all the map entries that
+ * they may still contain.
+ */
+ if (zap_old_map != VM_MAP_NULL) {
+ vm_map_destroy(zap_old_map);
+ zap_old_map = VM_MAP_NULL;
+ }
+ if (zap_new_map != VM_MAP_NULL) {
+ vm_map_destroy(zap_new_map);
+ zap_new_map = VM_MAP_NULL;
+ }
+
+ return result;
#undef RETURN
}
+
+#if VM_CPM
+
+#ifdef MACH_ASSERT
+extern vm_offset_t avail_start, avail_end;
+#endif
+
/*
- * vm_map_clip_start: [ internal use only ]
- *
- * Asserts that the given entry begins at or after
- * the specified address; if necessary,
- * it splits the entry into two.
+ * Allocate memory in the specified map, with the caveat that
+ * the memory is physically contiguous. This call may fail
+ * if the system can't find sufficient contiguous memory.
+ * This call may cause or lead to heart-stopping amounts of
+ * paging activity.
+ *
+ * Memory obtained from this call should be freed in the
+ * normal way, viz., via vm_deallocate.
*/
-#ifndef i386
-#define vm_map_clip_start(map, entry, startaddr) \
-MACRO_BEGIN \
- vm_map_t VMCS_map; \
- vm_map_entry_t VMCS_entry; \
- vm_offset_t VMCS_startaddr; \
- VMCS_map = (map); \
- VMCS_entry = (entry); \
- VMCS_startaddr = (startaddr); \
- if (VMCS_startaddr > VMCS_entry->vme_start) { \
- if(entry->use_pmap) { \
- vm_offset_t pmap_base_addr; \
- vm_offset_t pmap_end_addr; \
- \
- pmap_base_addr = 0xF0000000 & entry->vme_start; \
- pmap_end_addr = (pmap_base_addr + 0x10000000) - 1; \
- pmap_unnest(map->pmap, pmap_base_addr, \
- (pmap_end_addr - pmap_base_addr) + 1); \
- entry->use_pmap = FALSE; \
- } \
- _vm_map_clip_start(&VMCS_map->hdr,VMCS_entry,VMCS_startaddr);\
- } \
- UPDATE_FIRST_FREE(VMCS_map, VMCS_map->first_free); \
-MACRO_END
-#else
-#define vm_map_clip_start(map, entry, startaddr) \
-MACRO_BEGIN \
- vm_map_t VMCS_map; \
- vm_map_entry_t VMCS_entry; \
- vm_offset_t VMCS_startaddr; \
- VMCS_map = (map); \
- VMCS_entry = (entry); \
- VMCS_startaddr = (startaddr); \
- if (VMCS_startaddr > VMCS_entry->vme_start) { \
- _vm_map_clip_start(&VMCS_map->hdr,VMCS_entry,VMCS_startaddr);\
- } \
- UPDATE_FIRST_FREE(VMCS_map, VMCS_map->first_free); \
+kern_return_t
+vm_map_enter_cpm(
+ vm_map_t map,
+ vm_map_offset_t *addr,
+ vm_map_size_t size,
+ int flags)
+{
+ vm_object_t cpm_obj;
+ pmap_t pmap;
+ vm_page_t m, pages;
+ kern_return_t kr;
+ vm_map_offset_t va, start, end, offset;
+#if MACH_ASSERT
+ vm_map_offset_t prev_addr;
+#endif /* MACH_ASSERT */
+
+ boolean_t anywhere = ((VM_FLAGS_ANYWHERE & flags) != 0);
+
+ if (!vm_allocate_cpm_enabled)
+ return KERN_FAILURE;
+
+ if (size == 0) {
+ *addr = 0;
+ return KERN_SUCCESS;
+ }
+
+ if (anywhere)
+ *addr = vm_map_min(map);
+ else
+ *addr = vm_map_trunc_page(*addr);
+ size = vm_map_round_page(size);
+
+ /*
+ * LP64todo - cpm_allocate should probably allow
+ * allocations of >4GB, but not with the current
+ * algorithm, so just cast down the size for now.
+ */
+ if (size > VM_MAX_ADDRESS)
+ return KERN_RESOURCE_SHORTAGE;
+ if ((kr = cpm_allocate(CAST_DOWN(vm_size_t, size),
+ &pages, TRUE)) != KERN_SUCCESS)
+ return kr;
+
+ cpm_obj = vm_object_allocate((vm_object_size_t)size);
+ assert(cpm_obj != VM_OBJECT_NULL);
+ assert(cpm_obj->internal);
+ assert(cpm_obj->size == (vm_object_size_t)size);
+ assert(cpm_obj->can_persist == FALSE);
+ assert(cpm_obj->pager_created == FALSE);
+ assert(cpm_obj->pageout == FALSE);
+ assert(cpm_obj->shadow == VM_OBJECT_NULL);
+
+ /*
+ * Insert pages into object.
+ */
+
+ vm_object_lock(cpm_obj);
+ for (offset = 0; offset < size; offset += PAGE_SIZE) {
+ m = pages;
+ pages = NEXT_PAGE(m);
+
+ assert(!m->gobbled);
+ assert(!m->wanted);
+ assert(!m->pageout);
+ assert(!m->tabled);
+ /*
+ * ENCRYPTED SWAP:
+ * "m" is not supposed to be pageable, so it
+ * should not be encrypted. It wouldn't be safe
+ * to enter it in a new VM object while encrypted.
+ */
+ ASSERT_PAGE_DECRYPTED(m);
+ assert(m->busy);
+ assert(m->phys_page>=avail_start && m->phys_page<=avail_end);
+
+ m->busy = FALSE;
+ vm_page_insert(m, cpm_obj, offset);
+ }
+ assert(cpm_obj->resident_page_count == size / PAGE_SIZE);
+ vm_object_unlock(cpm_obj);
+
+ /*
+ * Hang onto a reference on the object in case a
+ * multi-threaded application for some reason decides
+ * to deallocate the portion of the address space into
+ * which we will insert this object.
+ *
+ * Unfortunately, we must insert the object now before
+ * we can talk to the pmap module about which addresses
+ * must be wired down. Hence, the race with a multi-
+ * threaded app.
+ */
+ vm_object_reference(cpm_obj);
+
+ /*
+ * Insert object into map.
+ */
+
+ kr = vm_map_enter(
+ map,
+ addr,
+ size,
+ (vm_map_offset_t)0,
+ flags,
+ cpm_obj,
+ (vm_object_offset_t)0,
+ FALSE,
+ VM_PROT_ALL,
+ VM_PROT_ALL,
+ VM_INHERIT_DEFAULT);
+
+ if (kr != KERN_SUCCESS) {
+ /*
+ * A CPM object doesn't have can_persist set,
+ * so all we have to do is deallocate it to
+ * free up these pages.
+ */
+ assert(cpm_obj->pager_created == FALSE);
+ assert(cpm_obj->can_persist == FALSE);
+ assert(cpm_obj->pageout == FALSE);
+ assert(cpm_obj->shadow == VM_OBJECT_NULL);
+ vm_object_deallocate(cpm_obj); /* kill acquired ref */
+ vm_object_deallocate(cpm_obj); /* kill creation ref */
+ }
+
+ /*
+ * Inform the physical mapping system that the
+ * range of addresses may not fault, so that
+ * page tables and such can be locked down as well.
+ */
+ start = *addr;
+ end = start + size;
+ pmap = vm_map_pmap(map);
+ pmap_pageable(pmap, start, end, FALSE);
+
+ /*
+ * Enter each page into the pmap, to avoid faults.
+ * Note that this loop could be coded more efficiently,
+ * if the need arose, rather than looking up each page
+ * again.
+ */
+ for (offset = 0, va = start; offset < size;
+ va += PAGE_SIZE, offset += PAGE_SIZE) {
+ vm_object_lock(cpm_obj);
+ m = vm_page_lookup(cpm_obj, (vm_object_offset_t)offset);
+ vm_object_unlock(cpm_obj);
+ assert(m != VM_PAGE_NULL);
+ PMAP_ENTER(pmap, va, m, VM_PROT_ALL,
+ ((unsigned int)(m->object->wimg_bits)) & VM_WIMG_MASK,
+ TRUE);
+ }
+
+#if MACH_ASSERT
+ /*
+ * Verify ordering in address space.
+ */
+ for (offset = 0; offset < size; offset += PAGE_SIZE) {
+ vm_object_lock(cpm_obj);
+ m = vm_page_lookup(cpm_obj, (vm_object_offset_t)offset);
+ vm_object_unlock(cpm_obj);
+ if (m == VM_PAGE_NULL)
+ panic("vm_allocate_cpm: obj 0x%x off 0x%x no page",
+ cpm_obj, offset);
+ assert(m->tabled);
+ assert(!m->busy);
+ assert(!m->wanted);
+ assert(!m->fictitious);
+ assert(!m->private);
+ assert(!m->absent);
+ assert(!m->error);
+ assert(!m->cleaning);
+ assert(!m->precious);
+ assert(!m->clustered);
+ if (offset != 0) {
+ if (m->phys_page != prev_addr + 1) {
+ printf("start 0x%x end 0x%x va 0x%x\n",
+ start, end, va);
+ printf("obj 0x%x off 0x%x\n", cpm_obj, offset);
+ printf("m 0x%x prev_address 0x%x\n", m,
+ prev_addr);
+ panic("vm_allocate_cpm: pages not contig!");
+ }
+ }
+ prev_addr = m->phys_page;
+ }
+#endif /* MACH_ASSERT */
+
+ vm_object_deallocate(cpm_obj); /* kill extra ref */
+
+ return kr;
+}
+
+
+#else /* VM_CPM */
+
+/*
+ * Interface is defined in all cases, but unless the kernel
+ * is built explicitly for this option, the interface does
+ * nothing.
+ */
+
+kern_return_t
+vm_map_enter_cpm(
+ __unused vm_map_t map,
+ __unused vm_map_offset_t *addr,
+ __unused vm_map_size_t size,
+ __unused int flags)
+{
+ return KERN_FAILURE;
+}
+#endif /* VM_CPM */
+
+/*
+ * vm_map_clip_start: [ internal use only ]
+ *
+ * Asserts that the given entry begins at or after
+ * the specified address; if necessary,
+ * it splits the entry into two.
+ */
+#ifndef i386
+#define vm_map_clip_start(map, entry, startaddr) \
+MACRO_BEGIN \
+ vm_map_t VMCS_map; \
+ vm_map_entry_t VMCS_entry; \
+ vm_map_offset_t VMCS_startaddr; \
+ VMCS_map = (map); \
+ VMCS_entry = (entry); \
+ VMCS_startaddr = (startaddr); \
+ if (VMCS_startaddr > VMCS_entry->vme_start) { \
+ if(entry->use_pmap) { \
+ vm_map_offset_t pmap_base_addr; \
+ \
+ pmap_base_addr = 0xF0000000 & entry->vme_start; \
+ pmap_unnest(map->pmap, (addr64_t)pmap_base_addr); \
+ entry->use_pmap = FALSE; \
+ } else if(entry->object.vm_object \
+ && !entry->is_sub_map \
+ && entry->object.vm_object->phys_contiguous) { \
+ pmap_remove(map->pmap, \
+ (addr64_t)(entry->vme_start), \
+ (addr64_t)(entry->vme_end)); \
+ } \
+ _vm_map_clip_start(&VMCS_map->hdr,VMCS_entry,VMCS_startaddr);\
+ } \
+ UPDATE_FIRST_FREE(VMCS_map, VMCS_map->first_free); \
+MACRO_END
+#else
+#define vm_map_clip_start(map, entry, startaddr) \
+MACRO_BEGIN \
+ vm_map_t VMCS_map; \
+ vm_map_entry_t VMCS_entry; \
+ vm_map_offset_t VMCS_startaddr; \
+ VMCS_map = (map); \
+ VMCS_entry = (entry); \
+ VMCS_startaddr = (startaddr); \
+ if (VMCS_startaddr > VMCS_entry->vme_start) { \
+ _vm_map_clip_start(&VMCS_map->hdr,VMCS_entry,VMCS_startaddr);\
+ } \
+ UPDATE_FIRST_FREE(VMCS_map, VMCS_map->first_free); \
MACRO_END
#endif
* This routine is called only when it is known that
* the entry must be split.
*/
-void
+static void
_vm_map_clip_start(
register struct vm_map_header *map_header,
register vm_map_entry_t entry,
- register vm_offset_t start)
+ register vm_map_offset_t start)
{
register vm_map_entry_t new_entry;
MACRO_BEGIN \
vm_map_t VMCE_map; \
vm_map_entry_t VMCE_entry; \
- vm_offset_t VMCE_endaddr; \
+ vm_map_offset_t VMCE_endaddr; \
VMCE_map = (map); \
VMCE_entry = (entry); \
VMCE_endaddr = (endaddr); \
if (VMCE_endaddr < VMCE_entry->vme_end) { \
if(entry->use_pmap) { \
- vm_offset_t pmap_base_addr; \
- vm_offset_t pmap_end_addr; \
+ vm_map_offset_t pmap_base_addr; \
\
pmap_base_addr = 0xF0000000 & entry->vme_start; \
- pmap_end_addr = (pmap_base_addr + 0x10000000) - 1; \
- pmap_unnest(map->pmap, pmap_base_addr, \
- (pmap_end_addr - pmap_base_addr) + 1); \
+ pmap_unnest(map->pmap, (addr64_t)pmap_base_addr); \
entry->use_pmap = FALSE; \
+ } else if(entry->object.vm_object \
+ && !entry->is_sub_map \
+ && entry->object.vm_object->phys_contiguous) { \
+ pmap_remove(map->pmap, \
+ (addr64_t)(entry->vme_start), \
+ (addr64_t)(entry->vme_end)); \
} \
_vm_map_clip_end(&VMCE_map->hdr,VMCE_entry,VMCE_endaddr); \
} \
MACRO_BEGIN \
vm_map_t VMCE_map; \
vm_map_entry_t VMCE_entry; \
- vm_offset_t VMCE_endaddr; \
+ vm_map_offset_t VMCE_endaddr; \
VMCE_map = (map); \
VMCE_entry = (entry); \
VMCE_endaddr = (endaddr); \
* This routine is called only when it is known that
* the entry must be split.
*/
-void
+static void
_vm_map_clip_end(
register struct vm_map_header *map_header,
register vm_map_entry_t entry,
- register vm_offset_t end)
+ register vm_map_offset_t end)
{
register vm_map_entry_t new_entry;
*
* The map is locked for reading on entry and is left locked.
*/
-boolean_t
+static boolean_t
vm_map_range_check(
register vm_map_t map,
- register vm_offset_t start,
- register vm_offset_t end,
+ register vm_map_offset_t start,
+ register vm_map_offset_t end,
vm_map_entry_t *entry)
{
vm_map_entry_t cur;
- register vm_offset_t prev;
+ register vm_map_offset_t prev;
/*
* Basic sanity checks first
*/
kern_return_t
vm_map_submap(
- register vm_map_t map,
- register vm_offset_t start,
- register vm_offset_t end,
+ vm_map_t map,
+ vm_map_offset_t start,
+ vm_map_offset_t end,
vm_map_t submap,
- vm_offset_t offset,
+ vm_map_offset_t offset,
+#ifdef i386
+ __unused
+#endif
boolean_t use_pmap)
{
vm_map_entry_t entry;
vm_map_lock(map);
+ submap->mapped = TRUE;
+
VM_MAP_RANGE_CHECK(map, start, end);
if (vm_map_lookup_entry(map, start, &entry)) {
(object->copy == VM_OBJECT_NULL) &&
(object->shadow == VM_OBJECT_NULL) &&
(!object->pager_created)) {
- entry->offset = (vm_object_offset_t)offset;
- entry->object.vm_object = VM_OBJECT_NULL;
- vm_object_deallocate(object);
- entry->is_sub_map = TRUE;
- vm_map_reference(entry->object.sub_map = submap);
+ entry->offset = (vm_object_offset_t)offset;
+ entry->object.vm_object = VM_OBJECT_NULL;
+ vm_object_deallocate(object);
+ entry->is_sub_map = TRUE;
+ entry->object.sub_map = submap;
+ vm_map_reference(submap);
#ifndef i386
- if ((use_pmap) && (offset == 0)) {
- /* nest if platform code will allow */
- result = pmap_nest(map->pmap, (entry->object.sub_map)->pmap,
- start, end - start);
- if(result)
- panic("pmap_nest failed!");
- entry->use_pmap = TRUE;
- }
+ if ((use_pmap) && (offset == 0)) {
+ /* nest if platform code will allow */
+ if(submap->pmap == NULL) {
+ submap->pmap = pmap_create((vm_map_size_t) 0);
+ if(submap->pmap == PMAP_NULL) {
+ vm_map_unlock(map);
+ return(KERN_NO_SPACE);
+ }
+ }
+ result = pmap_nest(map->pmap, (entry->object.sub_map)->pmap,
+ (addr64_t)start,
+ (addr64_t)start,
+ (uint64_t)(end - start));
+ if(result)
+ panic("vm_map_submap: pmap_nest failed, rc = %08X\n", result);
+ entry->use_pmap = TRUE;
+ }
#endif
#ifdef i386
- pmap_remove(map->pmap, start, end);
+ pmap_remove(map->pmap, (addr64_t)start, (addr64_t)end);
#endif
- result = KERN_SUCCESS;
+ result = KERN_SUCCESS;
}
vm_map_unlock(map);
kern_return_t
vm_map_protect(
register vm_map_t map,
- register vm_offset_t start,
- register vm_offset_t end,
+ register vm_map_offset_t start,
+ register vm_map_offset_t end,
register vm_prot_t new_prot,
register boolean_t set_max)
{
register vm_map_entry_t current;
- register vm_offset_t prev;
+ register vm_map_offset_t prev;
vm_map_entry_t entry;
vm_prot_t new_max;
boolean_t clip;
vm_map_lock(map);
+ /* LP64todo - remove this check when vm_map_commpage64()
+ * no longer has to stuff in a map_entry for the commpage
+ * above the map's max_offset.
+ */
+ if (start >= map->max_offset) {
+ vm_map_unlock(map);
+ return(KERN_INVALID_ADDRESS);
+ }
+
/*
* Lookup the entry. If it doesn't start in a valid
* entry, return an error. Remember if we need to
/* the 256M hack for existing hardware limitations */
if (current->protection != old_prot) {
if(current->is_sub_map && current->use_pmap) {
- vm_offset_t pmap_base_addr;
- vm_offset_t pmap_end_addr;
+ vm_map_offset_t pmap_base_addr;
+ vm_map_offset_t pmap_end_addr;
+#ifdef i386
+ __unused
+#endif
vm_map_entry_t local_entry;
pmap_base_addr = 0xF0000000 & current->vme_start;
local_entry->use_pmap = FALSE;
local_entry = local_entry->vme_next;
}
- pmap_unnest(map->pmap, pmap_base_addr,
- (pmap_end_addr - pmap_base_addr) + 1);
+ pmap_unnest(map->pmap, (addr64_t)pmap_base_addr);
#endif
}
if (!(current->protection & VM_PROT_WRITE)) {
current = current->vme_next;
}
+ current = entry;
+ while ((current != vm_map_to_entry(map)) &&
+ (current->vme_start <= end)) {
+ vm_map_simplify_entry(map, current);
+ current = current->vme_next;
+ }
+
vm_map_unlock(map);
return(KERN_SUCCESS);
}
kern_return_t
vm_map_inherit(
register vm_map_t map,
- register vm_offset_t start,
- register vm_offset_t end,
+ register vm_map_offset_t start,
+ register vm_map_offset_t end,
register vm_inherit_t new_inheritance)
{
register vm_map_entry_t entry;
/* given inheritance. */
while ((entry != vm_map_to_entry(map)) && (entry->vme_start < end)) {
if(entry->is_sub_map) {
- if(new_inheritance == VM_INHERIT_COPY)
+ if(new_inheritance == VM_INHERIT_COPY) {
+ vm_map_unlock(map);
return(KERN_INVALID_ARGUMENT);
+ }
}
entry = entry->vme_next;
* The map must not be locked, but a reference must remain to the
* map throughout the call.
*/
-kern_return_t
+static kern_return_t
vm_map_wire_nested(
register vm_map_t map,
- register vm_offset_t start,
- register vm_offset_t end,
+ register vm_map_offset_t start,
+ register vm_map_offset_t end,
register vm_prot_t access_type,
boolean_t user_wire,
- pmap_t map_pmap)
+ pmap_t map_pmap,
+ vm_map_offset_t pmap_addr)
{
register vm_map_entry_t entry;
struct vm_map_entry *first_entry, tmp_entry;
- vm_map_t pmap_map;
- register vm_offset_t s,e;
+ vm_map_t real_map;
+ register vm_map_offset_t s,e;
kern_return_t rc;
boolean_t need_wakeup;
boolean_t main_map = FALSE;
+ wait_interrupt_t interruptible_state;
+ thread_t cur_thread;
unsigned int last_timestamp;
- vm_size_t size;
+ vm_map_size_t size;
vm_map_lock(map);
if(map_pmap == NULL)
VM_MAP_RANGE_CHECK(map, start, end);
assert(page_aligned(start));
assert(page_aligned(end));
+ if (start == end) {
+ /* We wired what the caller asked for, zero pages */
+ vm_map_unlock(map);
+ return KERN_SUCCESS;
+ }
if (vm_map_lookup_entry(map, start, &first_entry)) {
entry = first_entry;
s=start;
need_wakeup = FALSE;
+ cur_thread = current_thread();
while ((entry != vm_map_to_entry(map)) && (entry->vme_start < end)) {
/*
* If another thread is wiring/unwiring this entry then
* block after informing other thread to wake us up.
*/
if (entry->in_transition) {
+ wait_result_t wait_result;
+
/*
* We have not clipped the entry. Make sure that
* the start address is in range so that the lookup
/*
* User wiring is interruptible
*/
- vm_map_entry_wait(map,
+ wait_result = vm_map_entry_wait(map,
(user_wire) ? THREAD_ABORTSAFE :
THREAD_UNINT);
- if (user_wire && current_thread()->wait_result ==
- THREAD_INTERRUPTED) {
+ if (user_wire && wait_result == THREAD_INTERRUPTED) {
/*
* undo the wirings we have done so far
* We do not clear the needs_wakeup flag,
* because we cannot tell if we were the
* only one waiting.
*/
+ vm_map_unlock(map);
vm_map_unwire(map, start, s, user_wire);
return(KERN_FAILURE);
}
- vm_map_lock(map);
/*
* Cannot avoid a lookup here. reset timestamp.
*/
}
if(entry->is_sub_map) {
- vm_offset_t sub_start;
- vm_offset_t sub_end;
- vm_offset_t local_end;
+ vm_map_offset_t sub_start;
+ vm_map_offset_t sub_end;
+ vm_map_offset_t local_start;
+ vm_map_offset_t local_end;
pmap_t pmap;
vm_map_clip_start(map, entry, start);
vm_map_clip_end(map, entry, end);
- sub_start += entry->offset;
+ sub_start = entry->offset;
sub_end = entry->vme_end - entry->vme_start;
sub_end += entry->offset;
if(map_pmap == NULL) {
if(entry->use_pmap) {
pmap = entry->object.sub_map->pmap;
+ /* ppc implementation requires that */
+ /* submaps pmap address ranges line */
+ /* up with parent map */
+#ifdef notdef
+ pmap_addr = sub_start;
+#endif
+ pmap_addr = start;
} else {
pmap = map->pmap;
+ pmap_addr = start;
}
if (entry->wired_count) {
if (entry->wired_count
entry->vme_start, user_wire);
return(KERN_FAILURE);
}
- if (!user_wire ||
- (entry->user_wired_count++ == 0))
+ if(user_wire)
+ entry->user_wired_count++;
+ if((!user_wire) ||
+ (entry->user_wired_count == 0))
entry->wired_count++;
entry = entry->vme_next;
continue;
} else {
vm_object_t object;
- vm_object_offset_t offset_hi;
- vm_object_offset_t offset_lo;
+ vm_map_offset_t offset_hi;
+ vm_map_offset_t offset_lo;
vm_object_offset_t offset;
vm_prot_t prot;
boolean_t wired;
vm_behavior_t behavior;
- vm_offset_t local_start;
vm_map_entry_t local_entry;
vm_map_version_t version;
vm_map_t lookup_map;
vm_map_lock_write_to_read(map);
if(vm_map_lookup_locked(
&lookup_map, local_start,
- VM_PROT_WRITE,
+ access_type,
&version, &object,
&offset, &prot, &wired,
&behavior, &offset_lo,
- &offset_hi, &pmap_map)) {
+ &offset_hi, &real_map)) {
- vm_map_unlock(lookup_map);
+ vm_map_unlock_read(lookup_map);
vm_map_unwire(map, start,
entry->vme_start, user_wire);
return(KERN_FAILURE);
}
- if(pmap_map != lookup_map)
- vm_map_unlock(pmap_map);
- if(lookup_map != map) {
- vm_map_unlock(lookup_map);
- vm_map_lock(map);
- } else {
- vm_map_unlock(map);
- vm_map_lock(map);
- }
- last_timestamp =
- version.main_timestamp;
+ if(real_map != lookup_map)
+ vm_map_unlock(real_map);
+ vm_map_unlock_read(lookup_map);
+ vm_map_lock(map);
vm_object_unlock(object);
- if (vm_map_lookup_entry(map,
+
+ if (!vm_map_lookup_entry(map,
local_start, &local_entry)) {
vm_map_unlock(map);
vm_map_unwire(map, start,
return(KERN_FAILURE);
}
/* did we have a change of type? */
- if (!local_entry->is_sub_map)
+ if (!local_entry->is_sub_map) {
+ last_timestamp = map->timestamp;
continue;
+ }
entry = local_entry;
if (user_wire)
entry->user_wired_count++;
+ if((!user_wire) ||
+ (entry->user_wired_count == 1))
entry->wired_count++;
entry->in_transition = TRUE;
entry->object.sub_map,
sub_start, sub_end,
access_type,
- user_wire, pmap);
+ user_wire, pmap, pmap_addr);
vm_map_lock(map);
- last_timestamp = map->timestamp;
}
} else {
+ local_start = entry->vme_start;
+ if (user_wire)
+ entry->user_wired_count++;
+ if((!user_wire) ||
+ (entry->user_wired_count == 1))
+ entry->wired_count++;
vm_map_unlock(map);
rc = vm_map_wire_nested(entry->object.sub_map,
sub_start, sub_end,
access_type,
- user_wire, pmap);
+ user_wire, map_pmap, pmap_addr);
vm_map_lock(map);
- last_timestamp = map->timestamp;
}
s = entry->vme_start;
e = entry->vme_end;
- if (last_timestamp+1 != map->timestamp) {
+
/*
* Find the entry again. It could have been clipped
* after we unlocked the map.
*/
- if (!vm_map_lookup_entry(map, local_end,
- &first_entry))
- panic("vm_map_wire: re-lookup failed");
-
- entry = first_entry;
- }
+ if (!vm_map_lookup_entry(map, local_start,
+ &first_entry))
+ panic("vm_map_wire: re-lookup failed");
+ entry = first_entry;
last_timestamp = map->timestamp;
while ((entry != vm_map_to_entry(map)) &&
need_wakeup = TRUE;
}
if (rc != KERN_SUCCESS) {/* from vm_*_wire */
- if(main_map) {
if (user_wire)
entry->user_wired_count--;
- entry->wired_count--;
- }
+ if ((!user_wire) ||
+ (entry->user_wired_count == 0))
+ entry->wired_count--;
}
entry = entry->vme_next;
}
* If this entry is already wired then increment
* the appropriate wire reference count.
*/
- if (entry->wired_count && main_map) {
+ if (entry->wired_count) {
/* sanity check: wired_count is a short */
if (entry->wired_count >= MAX_WIRE_COUNT)
panic("vm_map_wire: too many wirings");
*/
vm_map_clip_start(map, entry, start);
vm_map_clip_end(map, entry, end);
- if (!user_wire || (entry->user_wired_count++ == 0))
+ if (user_wire)
+ entry->user_wired_count++;
+ if ((!user_wire) || (entry->user_wired_count == 1))
entry->wired_count++;
entry = entry->vme_next;
assert(entry->wired_count == 0 && entry->user_wired_count == 0);
- if(main_map) {
- if (user_wire)
- entry->user_wired_count++;
+ if (user_wire)
+ entry->user_wired_count++;
+ if ((!user_wire) || (entry->user_wired_count == 1))
entry->wired_count++;
- }
entry->in_transition = TRUE;
* there when the map lock is acquired for the second time.
*/
vm_map_unlock(map);
+
+ if (!user_wire && cur_thread != THREAD_NULL)
+ interruptible_state = thread_interrupt_level(THREAD_UNINT);
+ else
+ interruptible_state = THREAD_UNINT;
+
if(map_pmap)
- rc = vm_fault_wire(map, &tmp_entry, map_pmap);
+ rc = vm_fault_wire(map,
+ &tmp_entry, map_pmap, pmap_addr);
else
- rc = vm_fault_wire(map, &tmp_entry, map->pmap);
+ rc = vm_fault_wire(map,
+ &tmp_entry, map->pmap,
+ tmp_entry.vme_start);
+
+ if (!user_wire && cur_thread != THREAD_NULL)
+ thread_interrupt_level(interruptible_state);
+
vm_map_lock(map);
if (last_timestamp+1 != map->timestamp) {
need_wakeup = TRUE;
}
if (rc != KERN_SUCCESS) { /* from vm_*_wire */
- if(main_map) {
- if (user_wire)
- entry->user_wired_count--;
+ if (user_wire)
+ entry->user_wired_count--;
+ if ((!user_wire) ||
+ (entry->user_wired_count == 0))
entry->wired_count--;
- }
}
entry = entry->vme_next;
}
kern_return_t
vm_map_wire(
register vm_map_t map,
- register vm_offset_t start,
- register vm_offset_t end,
+ register vm_map_offset_t start,
+ register vm_map_offset_t end,
register vm_prot_t access_type,
boolean_t user_wire)
{
mapping_prealloc(end - start);
#endif
kret = vm_map_wire_nested(map, start, end, access_type,
- user_wire, (pmap_t)NULL);
+ user_wire, (pmap_t)NULL, 0);
#ifdef ppc
mapping_relpre();
#endif
* unwired and intransition entries to avoid losing memory by leaving
* it unwired.
*/
-kern_return_t
+static kern_return_t
vm_map_unwire_nested(
register vm_map_t map,
- register vm_offset_t start,
- register vm_offset_t end,
+ register vm_map_offset_t start,
+ register vm_map_offset_t end,
boolean_t user_wire,
- pmap_t map_pmap)
+ pmap_t map_pmap,
+ vm_map_offset_t pmap_addr)
{
register vm_map_entry_t entry;
struct vm_map_entry *first_entry, tmp_entry;
}
if(entry->is_sub_map) {
- vm_offset_t sub_start;
- vm_offset_t sub_end;
- vm_offset_t local_end;
+ vm_map_offset_t sub_start;
+ vm_map_offset_t sub_end;
+ vm_map_offset_t local_end;
pmap_t pmap;
if(map_pmap == NULL) {
if(entry->use_pmap) {
pmap = entry->object.sub_map->pmap;
+ pmap_addr = sub_start;
} else {
pmap = map->pmap;
+ pmap_addr = start;
}
if (entry->wired_count == 0 ||
(user_wire && entry->user_wired_count == 0)) {
*/
vm_map_unlock(map);
vm_map_unwire_nested(entry->object.sub_map,
- sub_start, sub_end, user_wire, pmap);
+ sub_start, sub_end, user_wire, pmap, pmap_addr);
vm_map_lock(map);
if (last_timestamp+1 != map->timestamp) {
continue;
} else {
vm_map_unlock(map);
- vm_map_unwire_nested(entry->object.sub_map,
- sub_start, sub_end, user_wire, pmap);
+ vm_map_unwire_nested(entry->object.sub_map,
+ sub_start, sub_end, user_wire, map_pmap,
+ pmap_addr);
vm_map_lock(map);
if (last_timestamp+1 != map->timestamp) {
}
- if (main_map && (entry->wired_count == 0 ||
- (user_wire && entry->user_wired_count == 0))) {
+ if ((entry->wired_count == 0) ||
+ (user_wire && entry->user_wired_count == 0)) {
if (!user_wire)
panic("vm_map_unwire: entry is unwired");
continue;
}
- if(main_map) {
- if (!user_wire || (--entry->user_wired_count == 0))
+ if (!user_wire || (--entry->user_wired_count == 0))
entry->wired_count--;
- if (entry->wired_count != 0) {
+ if (entry->wired_count != 0) {
entry = entry->vme_next;
continue;
- }
}
entry->in_transition = TRUE;
*/
vm_map_unlock(map);
if(map_pmap) {
- vm_fault_unwire(map, &tmp_entry, FALSE, map_pmap);
+ vm_fault_unwire(map,
+ &tmp_entry, FALSE, map_pmap, pmap_addr);
} else {
- vm_fault_unwire(map, &tmp_entry, FALSE, map->pmap);
+ vm_fault_unwire(map,
+ &tmp_entry, FALSE, map->pmap,
+ tmp_entry.vme_start);
}
vm_map_lock(map);
entry = entry->vme_next;
}
}
+
+ /*
+ * We might have fragmented the address space when we wired this
+ * range of addresses. Attempt to re-coalesce these VM map entries
+ * with their neighbors now that they're no longer wired.
+ * Under some circumstances, address space fragmentation can
+ * prevent VM object shadow chain collapsing, which can cause
+ * swap space leaks.
+ */
+ vm_map_simplify_range(map, start, end);
+
vm_map_unlock(map);
/*
* wake up anybody waiting on entries that we have unwired.
kern_return_t
vm_map_unwire(
register vm_map_t map,
- register vm_offset_t start,
- register vm_offset_t end,
+ register vm_map_offset_t start,
+ register vm_map_offset_t end,
boolean_t user_wire)
{
- return vm_map_unwire_nested(map, start, end, user_wire, (pmap_t)NULL);
+ return vm_map_unwire_nested(map, start, end,
+ user_wire, (pmap_t)NULL, 0);
}
*
* Deallocate the given entry from the target map.
*/
-void
+static void
vm_map_entry_delete(
register vm_map_t map,
register vm_map_entry_t entry)
{
- register vm_offset_t s, e;
+ register vm_map_offset_t s, e;
register vm_object_t object;
register vm_map_t submap;
- extern vm_object_t kernel_object;
s = entry->vme_start;
e = entry->vme_end;
void
vm_map_submap_pmap_clean(
vm_map_t map,
- vm_offset_t start,
- vm_offset_t end,
+ vm_map_offset_t start,
+ vm_map_offset_t end,
vm_map_t sub_map,
- vm_offset_t offset)
+ vm_map_offset_t offset)
{
- vm_offset_t submap_start;
- vm_offset_t submap_end;
- vm_offset_t addr;
- vm_size_t remove_size;
+ vm_map_offset_t submap_start;
+ vm_map_offset_t submap_end;
+ vm_map_size_t remove_size;
vm_map_entry_t entry;
submap_end = offset + (end - start);
entry->object.sub_map,
entry->offset);
} else {
- pmap_remove(map->pmap, start, start + remove_size);
+
+ if((map->mapped) && (map->ref_count)
+ && (entry->object.vm_object != NULL)) {
+ vm_object_pmap_protect(
+ entry->object.vm_object,
+ entry->offset,
+ remove_size,
+ PMAP_NULL,
+ entry->vme_start,
+ VM_PROT_NONE);
+ } else {
+ pmap_remove(map->pmap,
+ (addr64_t)start,
+ (addr64_t)(start + remove_size));
+ }
}
}
entry->object.sub_map,
entry->offset);
} else {
- pmap_remove(map->pmap,
- (start + entry->vme_start) - offset,
- ((start + entry->vme_start) - offset) + remove_size);
+ if((map->mapped) && (map->ref_count)
+ && (entry->object.vm_object != NULL)) {
+ vm_object_pmap_protect(
+ entry->object.vm_object,
+ entry->offset,
+ remove_size,
+ PMAP_NULL,
+ entry->vme_start,
+ VM_PROT_NONE);
+ } else {
+ pmap_remove(map->pmap,
+ (addr64_t)((start + entry->vme_start)
+ - offset),
+ (addr64_t)(((start + entry->vme_start)
+ - offset) + remove_size));
+ }
}
entry = entry->vme_next;
}
*
* This routine is called with map locked and leaves map locked.
*/
-kern_return_t
+static kern_return_t
vm_map_delete(
- register vm_map_t map,
- vm_offset_t start,
- register vm_offset_t end,
- int flags)
+ vm_map_t map,
+ vm_map_offset_t start,
+ vm_map_offset_t end,
+ int flags,
+ vm_map_t zap_map)
{
vm_map_entry_t entry, next;
struct vm_map_entry *first_entry, tmp_entry;
- register vm_offset_t s, e;
+ register vm_map_offset_t s, e;
register vm_object_t object;
boolean_t need_wakeup;
unsigned int last_timestamp = ~0; /* unlikely value */
int interruptible;
- extern vm_map_t kernel_map;
interruptible = (flags & VM_MAP_REMOVE_INTERRUPTIBLE) ?
THREAD_ABORTSAFE : THREAD_UNINT;
vm_map_clip_end(map, entry, end);
if (entry->in_transition) {
+ wait_result_t wait_result;
+
/*
* Another thread is wiring/unwiring this entry.
* Let the other thread know we are waiting.
need_wakeup = FALSE;
}
- vm_map_entry_wait(map, interruptible);
+ wait_result = vm_map_entry_wait(map, interruptible);
if (interruptible &&
- current_thread()->wait_result == THREAD_INTERRUPTED)
+ wait_result == THREAD_INTERRUPTED) {
/*
* We do not clear the needs_wakeup flag,
* since we cannot tell if we were the only one.
*/
+ vm_map_unlock(map);
return KERN_ABORTED;
-
- vm_map_lock(map);
- /*
- * Cannot avoid a lookup here. reset timestamp.
- */
- last_timestamp = map->timestamp;
+ }
/*
* The entry could have been clipped or it
entry = first_entry;
SAVE_HINT(map, entry->vme_prev);
}
+ last_timestamp = map->timestamp;
continue;
} /* end in_transition */
* error.
*/
if (flags & VM_MAP_REMOVE_WAIT_FOR_KWIRE) {
+ wait_result_t wait_result;
s = entry->vme_start;
entry->needs_wakeup = TRUE;
- vm_map_entry_wait(map, interruptible);
+ wait_result = vm_map_entry_wait(map,
+ interruptible);
if (interruptible &&
- current_thread()->wait_result ==
- THREAD_INTERRUPTED)
+ wait_result == THREAD_INTERRUPTED) {
/*
* We do not clear the
* needs_wakeup flag, since we
* cannot tell if we were the
* only one.
*/
+ vm_map_unlock(map);
return KERN_ABORTED;
-
- vm_map_lock(map);
- /*
- * Cannot avoid a lookup here. reset
- * timestamp.
- */
- last_timestamp = map->timestamp;
+ }
/*
* The entry could have been clipped or
entry = first_entry;
SAVE_HINT(map, entry->vme_prev);
}
+ last_timestamp = map->timestamp;
continue;
}
else {
vm_map_unlock(map);
vm_fault_unwire(map, &tmp_entry,
tmp_entry.object.vm_object == kernel_object,
- map->pmap);
+ map->pmap, tmp_entry.vme_start);
vm_map_lock(map);
if (last_timestamp+1 != map->timestamp) {
if(entry->is_sub_map) {
if(entry->use_pmap) {
#ifndef i386
- pmap_unnest(map->pmap, entry->vme_start,
- entry->vme_end - entry->vme_start);
+ pmap_unnest(map->pmap, (addr64_t)entry->vme_start);
#endif
+ if((map->mapped) && (map->ref_count)) {
+ /* clean up parent map/maps */
+ vm_map_submap_pmap_clean(
+ map, entry->vme_start,
+ entry->vme_end,
+ entry->object.sub_map,
+ entry->offset);
+ }
} else {
vm_map_submap_pmap_clean(
map, entry->vme_start, entry->vme_end,
entry->offset);
}
} else {
- pmap_remove(map->pmap,
- entry->vme_start, entry->vme_end);
+ object = entry->object.vm_object;
+ if((map->mapped) && (map->ref_count)) {
+ vm_object_pmap_protect(
+ object, entry->offset,
+ entry->vme_end - entry->vme_start,
+ PMAP_NULL,
+ entry->vme_start,
+ VM_PROT_NONE);
+ } else {
+ pmap_remove(map->pmap,
+ entry->vme_start,
+ entry->vme_end);
+ }
}
}
+ /*
+ * All pmap mappings for this map entry must have been
+ * cleared by now.
+ */
+ assert(vm_map_pmap_is_empty(map,
+ entry->vme_start,
+ entry->vme_end));
+
next = entry->vme_next;
s = next->vme_start;
last_timestamp = map->timestamp;
- vm_map_entry_delete(map, entry);
- /* vm_map_entry_delete unlocks the map */
- vm_map_lock(map);
+
+ if ((flags & VM_MAP_REMOVE_SAVE_ENTRIES) &&
+ zap_map != VM_MAP_NULL) {
+ /*
+ * The caller wants to save the affected VM map entries
+ * into the "zap_map". The caller will take care of
+ * these entries.
+ */
+ /* unlink the entry from "map" ... */
+ vm_map_entry_unlink(map, entry);
+ /* ... and add it to the end of the "zap_map" */
+ vm_map_entry_link(zap_map,
+ vm_map_last_entry(zap_map),
+ entry);
+ } else {
+ vm_map_entry_delete(map, entry);
+ /* vm_map_entry_delete unlocks the map */
+ vm_map_lock(map);
+ }
+
entry = next;
if(entry == vm_map_to_entry(map)) {
kern_return_t
vm_map_remove(
register vm_map_t map,
- register vm_offset_t start,
- register vm_offset_t end,
+ register vm_map_offset_t start,
+ register vm_map_offset_t end,
register boolean_t flags)
{
register kern_return_t result;
vm_map_lock(map);
VM_MAP_RANGE_CHECK(map, start, end);
- result = vm_map_delete(map, start, end, flags);
+ result = vm_map_delete(map, start, end, flags, VM_MAP_NULL);
vm_map_unlock(map);
return(result);
/*
- * vm_map_copy_steal_pages:
+ * Routine: vm_map_copy_discard
*
- * Steal all the pages from a vm_map_copy page_list by copying ones
- * that have not already been stolen.
+ * Description:
+ * Dispose of a map copy object (returned by
+ * vm_map_copyin).
*/
void
-vm_map_copy_steal_pages(
+vm_map_copy_discard(
vm_map_copy_t copy)
{
- register vm_page_t m, new_m;
- register int i;
- vm_object_t object;
-
- assert(copy->type == VM_MAP_COPY_PAGE_LIST);
- for (i = 0; i < copy->cpy_npages; i++) {
+ TR_DECL("vm_map_copy_discard");
- /*
- * If the page is not tabled, then it's already stolen.
- */
- m = copy->cpy_page_list[i];
- if (!m->tabled)
- continue;
+/* tr3("enter: copy 0x%x type %d", copy, copy->type);*/
- /*
- * Page was not stolen, get a new
- * one and do the copy now.
- */
- while ((new_m = vm_page_grab()) == VM_PAGE_NULL) {
- VM_PAGE_WAIT();
- }
+ if (copy == VM_MAP_COPY_NULL)
+ return;
- vm_page_gobble(new_m); /* mark as consumed internally */
- vm_page_copy(m, new_m);
-
- object = m->object;
- vm_object_lock(object);
- vm_page_lock_queues();
- if (!m->active && !m->inactive)
- vm_page_activate(m);
- vm_page_unlock_queues();
- PAGE_WAKEUP_DONE(m);
- vm_object_paging_end(object);
- vm_object_unlock(object);
-
- copy->cpy_page_list[i] = new_m;
- }
- copy->cpy_page_loose = TRUE;
-}
-
-/*
- * vm_map_copy_page_discard:
- *
- * Get rid of the pages in a page_list copy. If the pages are
- * stolen, they are freed. If the pages are not stolen, they
- * are unbusied, and associated state is cleaned up.
- */
-void
-vm_map_copy_page_discard(
- vm_map_copy_t copy)
-{
- assert(copy->type == VM_MAP_COPY_PAGE_LIST);
- while (copy->cpy_npages > 0) {
- vm_page_t m;
-
- if ((m = copy->cpy_page_list[--(copy->cpy_npages)]) !=
- VM_PAGE_NULL) {
-
- /*
- * If it's not in the table, then it's
- * a stolen page that goes back
- * to the free list. Else it belongs
- * to some object, and we hold a
- * paging reference on that object.
- */
- if (!m->tabled) {
- VM_PAGE_FREE(m);
- }
- else {
- vm_object_t object;
-
- object = m->object;
-
- vm_object_lock(object);
- vm_page_lock_queues();
- if (!m->active && !m->inactive)
- vm_page_activate(m);
- vm_page_unlock_queues();
-
- if ((!m->busy)) {
- kern_return_t kr;
- kr = vm_page_unpin(m);
- assert(kr == KERN_SUCCESS);
- } else {
- PAGE_WAKEUP_DONE(m);
- }
- vm_object_paging_end(object);
- vm_object_unlock(object);
- }
- }
- }
-}
-
-/*
- * Routine: vm_map_copy_discard
- *
- * Description:
- * Dispose of a map copy object (returned by
- * vm_map_copyin).
- */
-void
-vm_map_copy_discard(
- vm_map_copy_t copy)
-{
- TR_DECL("vm_map_copy_discard");
-
-/* tr3("enter: copy 0x%x type %d", copy, copy->type);*/
-free_next_copy:
- if (copy == VM_MAP_COPY_NULL)
- return;
-
- switch (copy->type) {
- case VM_MAP_COPY_ENTRY_LIST:
- while (vm_map_copy_first_entry(copy) !=
- vm_map_copy_to_entry(copy)) {
- vm_map_entry_t entry = vm_map_copy_first_entry(copy);
+ switch (copy->type) {
+ case VM_MAP_COPY_ENTRY_LIST:
+ while (vm_map_copy_first_entry(copy) !=
+ vm_map_copy_to_entry(copy)) {
+ vm_map_entry_t entry = vm_map_copy_first_entry(copy);
vm_map_copy_entry_unlink(copy, entry);
vm_object_deallocate(entry->object.vm_object);
case VM_MAP_COPY_OBJECT:
vm_object_deallocate(copy->cpy_object);
break;
- case VM_MAP_COPY_PAGE_LIST:
-
- /*
- * To clean this up, we have to unbusy all the pages
- * and release the paging references in their objects.
- */
- if (copy->cpy_npages > 0)
- vm_map_copy_page_discard(copy);
-
- /*
- * If there's a continuation, abort it. The
- * abort routine releases any storage.
- */
- if (vm_map_copy_has_cont(copy)) {
-
- assert(vm_map_copy_cont_is_valid(copy));
- /*
- * Special case: recognize
- * vm_map_copy_discard_cont and optimize
- * here to avoid tail recursion.
- */
- if (copy->cpy_cont == vm_map_copy_discard_cont) {
- register vm_map_copy_t new_copy;
-
- new_copy = (vm_map_copy_t) copy->cpy_cont_args;
- zfree(vm_map_copy_zone, (vm_offset_t) copy);
- copy = new_copy;
- goto free_next_copy;
- } else {
- vm_map_copy_abort_cont(copy);
- }
- }
-
- break;
-
case VM_MAP_COPY_KERNEL_BUFFER:
/*
* allocated by a single call to kalloc(), i.e. the
* vm_map_copy_t was not allocated out of the zone.
*/
- kfree((vm_offset_t) copy, copy->cpy_kalloc_size);
+ kfree(copy, copy->cpy_kalloc_size);
return;
}
- zfree(vm_map_copy_zone, (vm_offset_t) copy);
+ zfree(vm_map_copy_zone, copy);
}
/*
return new_copy;
}
-/*
- * Routine: vm_map_copy_discard_cont
- *
- * Description:
- * A version of vm_map_copy_discard that can be called
- * as a continuation from a vm_map_copy page list.
- */
-kern_return_t
-vm_map_copy_discard_cont(
- vm_map_copyin_args_t cont_args,
- vm_map_copy_t *copy_result) /* OUT */
-{
- vm_map_copy_discard((vm_map_copy_t) cont_args);
- if (copy_result != (vm_map_copy_t *)0)
- *copy_result = VM_MAP_COPY_NULL;
- return(KERN_SUCCESS);
-}
-
-kern_return_t
+static kern_return_t
vm_map_overwrite_submap_recurse(
vm_map_t dst_map,
- vm_offset_t dst_addr,
- vm_size_t dst_size)
+ vm_map_offset_t dst_addr,
+ vm_map_size_t dst_size)
{
- vm_offset_t dst_end;
+ vm_map_offset_t dst_end;
vm_map_entry_t tmp_entry;
vm_map_entry_t entry;
kern_return_t result;
* splitting entries in strange ways.
*/
- dst_end = round_page(dst_addr + dst_size);
+ dst_end = vm_map_round_page(dst_addr + dst_size);
+ vm_map_lock(dst_map);
start_pass_1:
- vm_map_lock(dst_map);
if (!vm_map_lookup_entry(dst_map, dst_addr, &tmp_entry)) {
vm_map_unlock(dst_map);
return(KERN_INVALID_ADDRESS);
}
- vm_map_clip_start(dst_map, tmp_entry, trunc_page(dst_addr));
+ vm_map_clip_start(dst_map, tmp_entry, vm_map_trunc_page(dst_addr));
for (entry = tmp_entry;;) {
vm_map_entry_t next;
next = entry->vme_next;
while(entry->is_sub_map) {
- vm_offset_t sub_start;
- vm_offset_t sub_end;
- vm_offset_t local_end;
+ vm_map_offset_t sub_start;
+ vm_map_offset_t sub_end;
+ vm_map_offset_t local_end;
if (entry->in_transition) {
/*
* returned.
*/
-kern_return_t
+static kern_return_t
vm_map_copy_overwrite_nested(
- vm_map_t dst_map,
- vm_offset_t dst_addr,
- vm_map_copy_t copy,
- boolean_t interruptible,
- pmap_t pmap)
+ vm_map_t dst_map,
+ vm_map_address_t dst_addr,
+ vm_map_copy_t copy,
+ boolean_t interruptible,
+ pmap_t pmap)
{
- vm_offset_t dst_end;
- vm_map_entry_t tmp_entry;
- vm_map_entry_t entry;
- kern_return_t kr;
- boolean_t aligned = TRUE;
- boolean_t contains_permanent_objects = FALSE;
- boolean_t encountered_sub_map = FALSE;
- vm_offset_t base_addr;
- vm_size_t copy_size;
- vm_size_t total_size;
+ vm_map_offset_t dst_end;
+ vm_map_entry_t tmp_entry;
+ vm_map_entry_t entry;
+ kern_return_t kr;
+ boolean_t aligned = TRUE;
+ boolean_t contains_permanent_objects = FALSE;
+ boolean_t encountered_sub_map = FALSE;
+ vm_map_offset_t base_addr;
+ vm_map_size_t copy_size;
+ vm_map_size_t total_size;
/*
*/
if (copy->type == VM_MAP_COPY_KERNEL_BUFFER) {
- return(vm_map_copyout_kernel_buffer(dst_map, &dst_addr,
- copy, TRUE));
+ return(vm_map_copyout_kernel_buffer(
+ dst_map, &dst_addr,
+ copy, TRUE));
}
/*
!page_aligned (dst_addr))
{
aligned = FALSE;
- dst_end = round_page(dst_addr + copy->size);
+ dst_end = vm_map_round_page(dst_addr + copy->size);
} else {
dst_end = dst_addr + copy->size;
}
-start_pass_1:
vm_map_lock(dst_map);
+
+ /* LP64todo - remove this check when vm_map_commpage64()
+ * no longer has to stuff in a map_entry for the commpage
+ * above the map's max_offset.
+ */
+ if (dst_addr >= dst_map->max_offset) {
+ vm_map_unlock(dst_map);
+ return(KERN_INVALID_ADDRESS);
+ }
+
+start_pass_1:
if (!vm_map_lookup_entry(dst_map, dst_addr, &tmp_entry)) {
vm_map_unlock(dst_map);
return(KERN_INVALID_ADDRESS);
}
- vm_map_clip_start(dst_map, tmp_entry, trunc_page(dst_addr));
+ vm_map_clip_start(dst_map, tmp_entry, vm_map_trunc_page(dst_addr));
for (entry = tmp_entry;;) {
vm_map_entry_t next = entry->vme_next;
while(entry->is_sub_map) {
- vm_offset_t sub_start;
- vm_offset_t sub_end;
- vm_offset_t local_end;
+ vm_map_offset_t sub_start;
+ vm_map_offset_t sub_end;
+ vm_map_offset_t local_end;
if (entry->in_transition) {
/* deconstruct the copy object and do in parts */
/* only in sub_map, interruptable case */
vm_map_entry_t copy_entry;
- vm_map_entry_t previous_prev;
- vm_map_entry_t next_copy;
+ vm_map_entry_t previous_prev = VM_MAP_ENTRY_NULL;
+ vm_map_entry_t next_copy = VM_MAP_ENTRY_NULL;
int nentries;
- int remaining_entries;
- int new_offset;
+ int remaining_entries = 0;
+ int new_offset = 0;
for (entry = tmp_entry; copy_size == 0;) {
vm_map_entry_t next;
entry->needs_wakeup = TRUE;
vm_map_entry_wait(dst_map, THREAD_UNINT);
- vm_map_lock(dst_map);
if(!vm_map_lookup_entry(dst_map, base_addr,
&tmp_entry)) {
vm_map_unlock(dst_map);
continue;
}
if(entry->is_sub_map) {
- vm_offset_t sub_start;
- vm_offset_t sub_end;
- vm_offset_t local_end;
+ vm_map_offset_t sub_start;
+ vm_map_offset_t sub_end;
+ vm_map_offset_t local_end;
if (entry->needs_copy) {
/* if this is a COW submap */
/* adjust the copy object */
if (total_size > copy_size) {
- vm_size_t local_size = 0;
- vm_size_t entry_size;
+ vm_map_size_t local_size = 0;
+ vm_map_size_t entry_size;
nentries = 1;
new_offset = copy->offset;
/* adjust the copy object */
if (total_size > copy_size) {
- vm_size_t local_size = 0;
- vm_size_t entry_size;
+ vm_map_size_t local_size = 0;
+ vm_map_size_t entry_size;
new_offset = copy->offset;
copy_entry = vm_map_copy_first_entry(copy);
break;
}
}
- vm_map_clip_start(dst_map, tmp_entry, trunc_page(base_addr));
+ vm_map_clip_start(dst_map, tmp_entry, vm_map_trunc_page(base_addr));
entry = tmp_entry;
} /* while */
kern_return_t
vm_map_copy_overwrite(
vm_map_t dst_map,
- vm_offset_t dst_addr,
+ vm_map_offset_t dst_addr,
vm_map_copy_t copy,
boolean_t interruptible)
{
/*
- * Routine: vm_map_copy_overwrite_unaligned
+ * Routine: vm_map_copy_overwrite_unaligned [internal use only]
*
* Decription:
* Physically copy unaligned data
* unlocked on error.
*/
-kern_return_t
+static kern_return_t
vm_map_copy_overwrite_unaligned(
vm_map_t dst_map,
vm_map_entry_t entry,
vm_map_copy_t copy,
- vm_offset_t start)
+ vm_map_offset_t start)
{
vm_map_entry_t copy_entry = vm_map_copy_first_entry(copy);
vm_map_version_t version;
vm_object_offset_t dst_offset;
vm_object_offset_t src_offset;
vm_object_offset_t entry_offset;
- vm_offset_t entry_end;
- vm_size_t src_size,
+ vm_map_offset_t entry_end;
+ vm_map_size_t src_size,
dst_size,
copy_size,
amount_left;
vm_map_lock_write_to_read(dst_map);
- src_offset = copy->offset - trunc_page_64(copy->offset);
+ src_offset = copy->offset - vm_object_trunc_page(copy->offset);
amount_left = copy->size;
/*
* unaligned so we never clipped this entry, we need the offset into
}
vm_object_shadow(&entry->object.vm_object,
&entry->offset,
- (vm_size_t)(entry->vme_end
+ (vm_map_size_t)(entry->vme_end
- entry->vme_start));
entry->needs_copy = FALSE;
vm_map_lock_write_to_read(dst_map);
vm_map_lock_read(dst_map);
goto RetryLookup;
}
- dst_object = vm_object_allocate((vm_size_t)
+ dst_object = vm_object_allocate((vm_map_size_t)
entry->vme_end - entry->vme_start);
entry->object.vm_object = dst_object;
entry->offset = 0;
}
}/* while */
- /* NOTREACHED ?? */
- vm_map_unlock_read(dst_map);
-
return KERN_SUCCESS;
}/* vm_map_copy_overwrite_unaligned */
/*
- * Routine: vm_map_copy_overwrite_aligned
+ * Routine: vm_map_copy_overwrite_aligned [internal use only]
*
* Description:
* Does all the vm_trickery possible for whole pages.
* to the above pass and make sure that no wiring is involved.
*/
-kern_return_t
+static kern_return_t
vm_map_copy_overwrite_aligned(
vm_map_t dst_map,
vm_map_entry_t tmp_entry,
vm_map_copy_t copy,
- vm_offset_t start,
+ vm_map_offset_t start,
+#if !BAD_OPTIMIZATION
+ __unused
+#endif /* !BAD_OPTIMIZATION */
pmap_t pmap)
{
vm_object_t object;
vm_map_entry_t copy_entry;
- vm_size_t copy_size;
- vm_size_t size;
+ vm_map_size_t copy_size;
+ vm_map_size_t size;
vm_map_entry_t entry;
while ((copy_entry = vm_map_copy_first_entry(copy))
if (old_object != VM_OBJECT_NULL) {
if(entry->is_sub_map) {
- if(entry->use_pmap) {
+ if(entry->use_pmap) {
#ifndef i386
- pmap_unnest(dst_map->pmap,
- entry->vme_start,
- entry->vme_end - entry->vme_start);
+ pmap_unnest(dst_map->pmap,
+ entry->vme_start);
#endif
- } else {
- vm_map_submap_pmap_clean(
- dst_map, entry->vme_start,
- entry->vme_end,
- entry->object.sub_map,
- entry->offset);
- }
- vm_map_deallocate(
+ if(dst_map->mapped) {
+ /* clean up parent */
+ /* map/maps */
+ vm_map_submap_pmap_clean(
+ dst_map, entry->vme_start,
+ entry->vme_end,
+ entry->object.sub_map,
+ entry->offset);
+ }
+ } else {
+ vm_map_submap_pmap_clean(
+ dst_map, entry->vme_start,
+ entry->vme_end,
+ entry->object.sub_map,
+ entry->offset);
+ }
+ vm_map_deallocate(
entry->object.sub_map);
- } else {
- vm_object_pmap_protect(
- old_object,
- old_offset,
- size,
- pmap,
- tmp_entry->vme_start,
- VM_PROT_NONE);
-
+ } else {
+ if(dst_map->mapped) {
+ vm_object_pmap_protect(
+ entry->object.vm_object,
+ entry->offset,
+ entry->vme_end
+ - entry->vme_start,
+ PMAP_NULL,
+ entry->vme_start,
+ VM_PROT_NONE);
+ } else {
+ pmap_remove(dst_map->pmap,
+ (addr64_t)(entry->vme_start),
+ (addr64_t)(entry->vme_end));
+ }
vm_object_deallocate(old_object);
- }
+ }
}
entry->is_sub_map = FALSE;
* (but avoid uninstantiated objects)
*/
if (object != VM_OBJECT_NULL) {
- vm_offset_t va = entry->vme_start;
+ vm_map_offset_t va = entry->vme_start;
while (va < entry->vme_end) {
register vm_page_t m;
vm_object_lock(object);
vm_object_paging_begin(object);
+ /*
+ * ENCRYPTED SWAP:
+ * If the page is encrypted, skip it:
+ * we can't let the user see the encrypted
+ * contents. The page will get decrypted
+ * on demand when the user generates a
+ * soft-fault when trying to access it.
+ */
if ((m = vm_page_lookup(object,offset)) !=
- VM_PAGE_NULL && !m->busy &&
- !m->fictitious &&
+ VM_PAGE_NULL && !m->busy &&
+ !m->fictitious && !m->encrypted &&
(!m->unusual || (!m->error &&
!m->restart && !m->absent &&
(prot & m->page_lock) == 0))) {
*/
if (entry->needs_copy)
prot &= ~VM_PROT_WRITE;
-
- PMAP_ENTER(pmap, va, m,
- prot, FALSE);
+ /* It is our policy to require */
+ /* explicit sync from anyone */
+ /* writing code and then */
+ /* a pc to execute it. */
+ /* No isync here */
+
+ PMAP_ENTER(pmap, va, m, prot,
+ ((unsigned int)
+ (m->object->wimg_bits))
+ & VM_WIMG_MASK,
+ FALSE);
vm_object_lock(object);
vm_page_lock_queues();
vm_object_reference(dst_object);
- version.main_timestamp = dst_map->timestamp;
+ /* account for unlock bumping up timestamp */
+ version.main_timestamp = dst_map->timestamp + 1;
vm_map_unlock(dst_map);
start += copy_size;
vm_map_lock(dst_map);
- if ((version.main_timestamp + 1) == dst_map->timestamp) {
+ if (version.main_timestamp == dst_map->timestamp) {
/* We can safely use saved tmp_entry value */
vm_map_clip_end(dst_map, tmp_entry, start);
}/* vm_map_copy_overwrite_aligned */
/*
- * Routine: vm_map_copyin_kernel_buffer
+ * Routine: vm_map_copyin_kernel_buffer [internal use only]
*
* Description:
* Copy in data to a kernel buffer from space in the
- * source map. The original space may be otpionally
+ * source map. The original space may be optionally
* deallocated.
*
* If successful, returns a new copy object.
*/
-kern_return_t
+static kern_return_t
vm_map_copyin_kernel_buffer(
vm_map_t src_map,
- vm_offset_t src_addr,
- vm_size_t len,
+ vm_map_offset_t src_addr,
+ vm_map_size_t len,
boolean_t src_destroy,
vm_map_copy_t *copy_result)
{
- boolean_t flags;
+ kern_return_t kr;
vm_map_copy_t copy;
- vm_size_t kalloc_size = sizeof(struct vm_map_copy) + len;
+ vm_map_size_t kalloc_size = sizeof(struct vm_map_copy) + len;
copy = (vm_map_copy_t) kalloc(kalloc_size);
if (copy == VM_MAP_COPY_NULL) {
copy->type = VM_MAP_COPY_KERNEL_BUFFER;
copy->size = len;
copy->offset = 0;
- copy->cpy_kdata = (vm_offset_t) (copy + 1);
+ copy->cpy_kdata = (void *) (copy + 1);
copy->cpy_kalloc_size = kalloc_size;
- if (src_map == kernel_map) {
- bcopy((char *)src_addr, (char *)copy->cpy_kdata, len);
- flags = VM_MAP_REMOVE_KUNWIRE | VM_MAP_REMOVE_WAIT_FOR_KWIRE |
- VM_MAP_REMOVE_INTERRUPTIBLE;
- } else {
- kern_return_t kr;
- kr = copyinmap(src_map, src_addr, copy->cpy_kdata, len);
- if (kr != KERN_SUCCESS) {
- kfree((vm_offset_t)copy, kalloc_size);
- return kr;
- }
- flags = VM_MAP_REMOVE_WAIT_FOR_KWIRE |
- VM_MAP_REMOVE_INTERRUPTIBLE;
+ kr = copyinmap(src_map, src_addr, copy->cpy_kdata, len);
+ if (kr != KERN_SUCCESS) {
+ kfree(copy, kalloc_size);
+ return kr;
}
if (src_destroy) {
- (void) vm_map_remove(src_map, trunc_page(src_addr),
- round_page(src_addr + len),
- flags);
+ (void) vm_map_remove(src_map, vm_map_trunc_page(src_addr),
+ vm_map_round_page(src_addr + len),
+ VM_MAP_REMOVE_INTERRUPTIBLE |
+ VM_MAP_REMOVE_WAIT_FOR_KWIRE |
+ (src_map == kernel_map) ?
+ VM_MAP_REMOVE_KUNWIRE : 0);
}
*copy_result = copy;
return KERN_SUCCESS;
}
/*
- * Routine: vm_map_copyout_kernel_buffer
+ * Routine: vm_map_copyout_kernel_buffer [internal use only]
*
* Description:
* Copy out data from a kernel buffer into space in the
* If successful, consumes the copy object.
* Otherwise, the caller is responsible for it.
*/
-kern_return_t
+static int vm_map_copyout_kernel_buffer_failures = 0;
+static kern_return_t
vm_map_copyout_kernel_buffer(
- vm_map_t map,
- vm_offset_t *addr, /* IN/OUT */
- vm_map_copy_t copy,
- boolean_t overwrite)
+ vm_map_t map,
+ vm_map_address_t *addr, /* IN/OUT */
+ vm_map_copy_t copy,
+ boolean_t overwrite)
{
kern_return_t kr = KERN_SUCCESS;
- thread_act_t thr_act = current_act();
+ thread_t thread = current_thread();
if (!overwrite) {
*addr = 0;
kr = vm_map_enter(map,
addr,
- round_page(copy->size),
- (vm_offset_t) 0,
- TRUE,
+ vm_map_round_page(copy->size),
+ (vm_map_offset_t) 0,
+ VM_FLAGS_ANYWHERE,
VM_OBJECT_NULL,
(vm_object_offset_t) 0,
FALSE,
VM_PROT_ALL,
VM_INHERIT_DEFAULT);
if (kr != KERN_SUCCESS)
- return(kr);
+ return kr;
}
/*
* Copyout the data from the kernel buffer to the target map.
*/
- if (thr_act->map == map) {
+ if (thread->map == map) {
/*
* If the target map is the current map, just do
* the copy.
*/
- if (copyout((char *)copy->cpy_kdata, (char *)*addr,
- copy->size)) {
+ if (copyout(copy->cpy_kdata, *addr, copy->size)) {
kr = KERN_INVALID_ADDRESS;
}
}
vm_map_reference(map);
oldmap = vm_map_switch(map);
- if (copyout((char *)copy->cpy_kdata, (char *)*addr,
- copy->size)) {
+ if (copyout(copy->cpy_kdata, *addr, copy->size)) {
+ vm_map_copyout_kernel_buffer_failures++;
kr = KERN_INVALID_ADDRESS;
}
vm_map_deallocate(map);
}
- kfree((vm_offset_t)copy, copy->cpy_kalloc_size);
+ if (kr != KERN_SUCCESS) {
+ /* the copy failed, clean up */
+ if (!overwrite) {
+ /*
+ * Deallocate the space we allocated in the target map.
+ */
+ (void) vm_map_remove(map,
+ vm_map_trunc_page(*addr),
+ vm_map_round_page(*addr +
+ vm_map_round_page(copy->size)),
+ VM_MAP_NO_FLAGS);
+ *addr = 0;
+ }
+ } else {
+ /* copy was successful, dicard the copy structure */
+ kfree(copy, copy->cpy_kalloc_size);
+ }
- return(kr);
+ return kr;
}
/*
->vme_prev = VMCI_where; \
VMCI_map->hdr.nentries += VMCI_copy->cpy_hdr.nentries; \
UPDATE_FIRST_FREE(VMCI_map, VMCI_map->first_free); \
- zfree(vm_map_copy_zone, (vm_offset_t) VMCI_copy); \
+ zfree(vm_map_copy_zone, VMCI_copy); \
MACRO_END
/*
*/
kern_return_t
vm_map_copyout(
- register vm_map_t dst_map,
- vm_offset_t *dst_addr, /* OUT */
- register vm_map_copy_t copy)
+ vm_map_t dst_map,
+ vm_map_address_t *dst_addr, /* OUT */
+ vm_map_copy_t copy)
{
- vm_size_t size;
- vm_size_t adjustment;
- vm_offset_t start;
+ vm_map_size_t size;
+ vm_map_size_t adjustment;
+ vm_map_offset_t start;
vm_object_offset_t vm_copy_start;
vm_map_entry_t last;
register
kern_return_t kr;
vm_object_offset_t offset;
- offset = trunc_page_64(copy->offset);
- size = round_page(copy->size +
- (vm_size_t)(copy->offset - offset));
+ offset = vm_object_trunc_page(copy->offset);
+ size = vm_map_round_page(copy->size +
+ (vm_map_size_t)(copy->offset - offset));
*dst_addr = 0;
kr = vm_map_enter(dst_map, dst_addr, size,
- (vm_offset_t) 0, TRUE,
+ (vm_map_offset_t) 0, VM_FLAGS_ANYWHERE,
object, offset, FALSE,
VM_PROT_DEFAULT, VM_PROT_ALL,
VM_INHERIT_DEFAULT);
if (kr != KERN_SUCCESS)
return(kr);
/* Account for non-pagealigned copy object */
- *dst_addr += (vm_offset_t)(copy->offset - offset);
- zfree(vm_map_copy_zone, (vm_offset_t) copy);
+ *dst_addr += (vm_map_offset_t)(copy->offset - offset);
+ zfree(vm_map_copy_zone, copy);
return(KERN_SUCCESS);
}
copy, FALSE));
}
- if (copy->type == VM_MAP_COPY_PAGE_LIST)
- return(vm_map_copyout_page_list(dst_map, dst_addr, copy));
-
/*
* Find space for the data
*/
- vm_copy_start = trunc_page_64(copy->offset);
- size = round_page((vm_size_t)copy->offset + copy->size)
+ vm_copy_start = vm_object_trunc_page(copy->offset);
+ size = vm_map_round_page((vm_map_size_t)copy->offset + copy->size)
- vm_copy_start;
StartAgain: ;
while (TRUE) {
vm_map_entry_t next = last->vme_next;
- vm_offset_t end = start + size;
+ vm_map_offset_t end = start + size;
if ((end > dst_map->max_offset) || (end < start)) {
if (dst_map->wait_for_space) {
assert_wait((event_t) dst_map,
THREAD_INTERRUPTIBLE);
vm_map_unlock(dst_map);
- thread_block((void (*)(void))0);
+ thread_block(THREAD_CONTINUE_NULL);
goto StartAgain;
}
}
vm_map_copy_last_entry(copy),
new);
next = entry->vme_next;
- zfree(old_zone, (vm_offset_t) entry);
+ zfree(old_zone, entry);
entry = next;
}
}
* map the pages into the destination map.
*/
if (entry->wired_count != 0) {
- register vm_offset_t va;
+ register vm_map_offset_t va;
vm_object_offset_t offset;
register vm_object_t object;
m->absent)
panic("vm_map_copyout: wiring 0x%x", m);
+ /*
+ * ENCRYPTED SWAP:
+ * The page is assumed to be wired here, so it
+ * shouldn't be encrypted. Otherwise, we
+ * couldn't enter it in the page table, since
+ * we don't want the user to see the encrypted
+ * data.
+ */
+ ASSERT_PAGE_DECRYPTED(m);
+
m->busy = TRUE;
vm_object_unlock(object);
- PMAP_ENTER(dst_map->pmap, va, m,
- entry->protection, TRUE);
+ PMAP_ENTER(dst_map->pmap, va, m, entry->protection,
+ ((unsigned int)
+ (m->object->wimg_bits))
+ & VM_WIMG_MASK,
+ TRUE);
vm_object_lock(object);
PAGE_WAKEUP_DONE(m);
}
else if (size <= vm_map_aggressive_enter_max) {
- register vm_offset_t va;
+ register vm_map_offset_t va;
vm_object_offset_t offset;
register vm_object_t object;
vm_prot_t prot;
vm_object_lock(object);
vm_object_paging_begin(object);
+ /*
+ * ENCRYPTED SWAP:
+ * If the page is encrypted, skip it:
+ * we can't let the user see the
+ * encrypted contents. The page will
+ * get decrypted on demand when the
+ * user generates a soft-fault when
+ * trying to access it.
+ */
if (((m = vm_page_lookup(object,
offset))
!= VM_PAGE_NULL) &&
!m->busy && !m->fictitious &&
+ !m->encrypted &&
!m->absent && !m->error) {
m->busy = TRUE;
vm_object_unlock(object);
prot &= ~VM_PROT_WRITE;
PMAP_ENTER(dst_map->pmap, va,
- m, prot, FALSE);
+ m, prot,
+ ((unsigned int)
+ (m->object->wimg_bits))
+ & VM_WIMG_MASK,
+ FALSE);
vm_object_lock(object);
vm_page_lock_queues();
return(KERN_SUCCESS);
}
-boolean_t vm_map_aggressive_enter; /* not used yet */
-
/*
+ * Routine: vm_map_copyin
*
- * vm_map_copyout_page_list:
+ * Description:
+ * Copy the specified region (src_addr, len) from the
+ * source address space (src_map), possibly removing
+ * the region from the source address space (src_destroy).
*
- * Version of vm_map_copyout() for page list vm map copies.
+ * Returns:
+ * A vm_map_copy_t object (copy_result), suitable for
+ * insertion into another address space (using vm_map_copyout),
+ * copying over another address space region (using
+ * vm_map_copy_overwrite). If the copy is unused, it
+ * should be destroyed (using vm_map_copy_discard).
*
+ * In/out conditions:
+ * The source map should not be locked on entry.
*/
+
+typedef struct submap_map {
+ vm_map_t parent_map;
+ vm_map_offset_t base_start;
+ vm_map_offset_t base_end;
+ struct submap_map *next;
+} submap_map_t;
+
kern_return_t
-vm_map_copyout_page_list(
- register vm_map_t dst_map,
- vm_offset_t *dst_addr, /* OUT */
- register vm_map_copy_t copy)
+vm_map_copyin_common(
+ vm_map_t src_map,
+ vm_map_address_t src_addr,
+ vm_map_size_t len,
+ boolean_t src_destroy,
+ __unused boolean_t src_volatile,
+ vm_map_copy_t *copy_result, /* OUT */
+ boolean_t use_maxprot)
{
- vm_size_t size;
- vm_offset_t start;
- vm_offset_t end;
- vm_object_offset_t offset;
- vm_map_entry_t last;
+ vm_map_entry_t tmp_entry; /* Result of last map lookup --
+ * in multi-level lookup, this
+ * entry contains the actual
+ * vm_object/offset.
+ */
register
- vm_object_t object;
- vm_page_t *page_list, m;
- vm_map_entry_t entry;
- vm_object_offset_t old_last_offset;
- boolean_t cont_invoked, needs_wakeup;
- kern_return_t result = KERN_SUCCESS;
- vm_map_copy_t orig_copy;
- vm_object_offset_t dst_offset;
- boolean_t must_wire;
- boolean_t aggressive_enter;
+ vm_map_entry_t new_entry = VM_MAP_ENTRY_NULL; /* Map entry for copy */
+
+ vm_map_offset_t src_start; /* Start of current entry --
+ * where copy is taking place now
+ */
+ vm_map_offset_t src_end; /* End of entire region to be
+ * copied */
+ vm_map_t base_map = src_map;
+ boolean_t map_share=FALSE;
+ submap_map_t *parent_maps = NULL;
+
+ register
+ vm_map_copy_t copy; /* Resulting copy */
+ vm_map_address_t copy_addr;
/*
- * Check for null copy object.
+ * Check for copies of zero bytes.
*/
- if (copy == VM_MAP_COPY_NULL) {
- *dst_addr = 0;
+ if (len == 0) {
+ *copy_result = VM_MAP_COPY_NULL;
return(KERN_SUCCESS);
}
- assert(copy->type == VM_MAP_COPY_PAGE_LIST);
-
/*
- * Make sure the pages are stolen, because we are
- * going to put them in a new object. Assume that
- * all pages are identical to first in this regard.
+ * Check that the end address doesn't overflow
*/
-
- page_list = ©->cpy_page_list[0];
- if (!copy->cpy_page_loose)
- vm_map_copy_steal_pages(copy);
+ src_end = src_addr + len;
+ if (src_end < src_addr)
+ return KERN_INVALID_ADDRESS;
/*
- * Find space for the data
+ * If the copy is sufficiently small, use a kernel buffer instead
+ * of making a virtual copy. The theory being that the cost of
+ * setting up VM (and taking C-O-W faults) dominates the copy costs
+ * for small regions.
*/
+ if ((len < msg_ool_size_small) && !use_maxprot)
+ return vm_map_copyin_kernel_buffer(src_map, src_addr, len,
+ src_destroy, copy_result);
- size = round_page_64(copy->offset + (vm_object_offset_t)copy->size) -
- trunc_page_64(copy->offset);
-StartAgain:
- vm_map_lock(dst_map);
- must_wire = dst_map->wiring_required;
-
- assert(first_free_is_valid(dst_map));
- last = dst_map->first_free;
- if (last == vm_map_to_entry(dst_map)) {
- start = vm_map_min(dst_map);
- } else {
- start = last->vme_end;
- }
-
- while (TRUE) {
- vm_map_entry_t next = last->vme_next;
- end = start + size;
-
- if ((end > dst_map->max_offset) || (end < start)) {
- if (dst_map->wait_for_space) {
- if (size <= (dst_map->max_offset -
- dst_map->min_offset)) {
- assert_wait((event_t) dst_map,
- THREAD_INTERRUPTIBLE);
- vm_map_unlock(dst_map);
- thread_block((void (*)(void))0);
- goto StartAgain;
- }
- }
- vm_map_unlock(dst_map);
- return(KERN_NO_SPACE);
- }
-
- if ((next == vm_map_to_entry(dst_map)) ||
- (next->vme_start >= end)) {
- break;
- }
+ /*
+ * Compute (page aligned) start and end of region
+ */
+ src_start = vm_map_trunc_page(src_addr);
+ src_end = vm_map_round_page(src_end);
- last = next;
- start = last->vme_end;
- }
+ XPR(XPR_VM_MAP, "vm_map_copyin_common map 0x%x addr 0x%x len 0x%x dest %d\n", (natural_t)src_map, src_addr, len, src_destroy, 0);
/*
- * See whether we can avoid creating a new entry (and object) by
- * extending one of our neighbors. [So far, we only attempt to
- * extend from below.]
+ * Allocate a header element for the list.
*
- * The code path below here is a bit twisted. If any of the
- * extension checks fails, we branch to create_object. If
- * it all works, we fall out the bottom and goto insert_pages.
- */
- if (last == vm_map_to_entry(dst_map) ||
- last->vme_end != start ||
- last->is_shared != FALSE ||
- last->is_sub_map != FALSE ||
- last->inheritance != VM_INHERIT_DEFAULT ||
- last->protection != VM_PROT_DEFAULT ||
- last->max_protection != VM_PROT_ALL ||
- last->behavior != VM_BEHAVIOR_DEFAULT ||
- last->in_transition ||
- (must_wire ? (last->wired_count != 1 ||
- last->user_wired_count != 0) :
- (last->wired_count != 0))) {
- goto create_object;
- }
-
- /*
- * If this entry needs an object, make one.
+ * Use the start and end in the header to
+ * remember the endpoints prior to rounding.
*/
- if (last->object.vm_object == VM_OBJECT_NULL) {
- object = vm_object_allocate(
- (vm_size_t)(last->vme_end - last->vme_start + size));
- last->object.vm_object = object;
- last->offset = 0;
- }
- else {
- vm_object_offset_t prev_offset = last->offset;
- vm_size_t prev_size = start - last->vme_start;
- vm_size_t new_size;
-
- /*
- * This is basically vm_object_coalesce.
- */
- object = last->object.vm_object;
- vm_object_lock(object);
+ copy = (vm_map_copy_t) zalloc(vm_map_copy_zone);
+ vm_map_copy_first_entry(copy) =
+ vm_map_copy_last_entry(copy) = vm_map_copy_to_entry(copy);
+ copy->type = VM_MAP_COPY_ENTRY_LIST;
+ copy->cpy_hdr.nentries = 0;
+ copy->cpy_hdr.entries_pageable = TRUE;
- /*
- * Try to collapse the object first
- */
- vm_object_collapse(object);
-
-
- /*
- * Can't coalesce if pages not mapped to
- * last may be in use anyway:
- * . more than one reference
- * . paged out
- * . shadows another object
- * . has a copy elsewhere
- * . paging references (pages might be in page-list)
- */
-
- if ((object->ref_count > 1) ||
- object->pager_created ||
- (object->shadow != VM_OBJECT_NULL) ||
- (object->copy != VM_OBJECT_NULL) ||
- (object->paging_in_progress != 0)) {
- vm_object_unlock(object);
- goto create_object;
- }
-
- /*
- * Extend the object if necessary. Don't have to call
- * vm_object_page_remove because the pages aren't mapped,
- * and vm_page_replace will free up any old ones it encounters.
- */
- new_size = prev_offset + prev_size + size;
- if (new_size > object->size) {
-#if MACH_PAGEMAP
- /*
- * We cannot extend an object that has existence info,
- * since the existence info might then fail to cover
- * the entire object.
- *
- * This assertion must be true because the object
- * has no pager, and we only create existence info
- * for objects with pagers.
- */
- assert(object->existence_map == VM_EXTERNAL_NULL);
-#endif /* MACH_PAGEMAP */
- object->size = new_size;
- }
- vm_object_unlock(object);
- }
-
- /*
- * Coalesced the two objects - can extend
- * the previous map entry to include the
- * new range.
- */
- dst_map->size += size;
- last->vme_end = end;
- UPDATE_FIRST_FREE(dst_map, dst_map->first_free);
-
- SAVE_HINT(dst_map, last);
-
- goto insert_pages;
-
-create_object:
-
- /*
- * Create object
- */
- object = vm_object_allocate(size);
-
- /*
- * Create entry
- */
- last = vm_map_entry_insert(dst_map, last, start, start + size,
- object, 0, FALSE, FALSE, TRUE,
- VM_PROT_DEFAULT, VM_PROT_ALL,
- VM_BEHAVIOR_DEFAULT,
- VM_INHERIT_DEFAULT, (must_wire ? 1 : 0));
-
- /*
- * Transfer pages into new object.
- * Scan page list in vm_map_copy.
- */
-insert_pages:
- dst_offset = copy->offset & PAGE_MASK_64;
- cont_invoked = FALSE;
- orig_copy = copy;
- last->in_transition = TRUE;
- old_last_offset = last->offset
- + (start - last->vme_start);
-
- aggressive_enter = (size <= vm_map_aggressive_enter_max);
-
- for (offset = 0; offset < size; offset += PAGE_SIZE_64) {
- m = *page_list;
- assert(m && !m->tabled);
-
- /*
- * Must clear busy bit in page before inserting it.
- * Ok to skip wakeup logic because nobody else
- * can possibly know about this page. Also set
- * dirty bit on the assumption that the page is
- * not a page of zeros.
- */
-
- m->busy = FALSE;
- m->dirty = TRUE;
- vm_object_lock(object);
- vm_page_lock_queues();
- vm_page_replace(m, object, old_last_offset + offset);
- if (must_wire) {
- vm_page_wire(m);
- } else if (aggressive_enter) {
- vm_page_activate(m);
- }
- vm_page_unlock_queues();
- vm_object_unlock(object);
-
- if (aggressive_enter || must_wire) {
- PMAP_ENTER(dst_map->pmap,
- last->vme_start + m->offset - last->offset,
- m, last->protection, must_wire);
- }
-
- *page_list++ = VM_PAGE_NULL;
- assert(copy != VM_MAP_COPY_NULL);
- assert(copy->type == VM_MAP_COPY_PAGE_LIST);
- if (--(copy->cpy_npages) == 0 &&
- vm_map_copy_has_cont(copy)) {
- vm_map_copy_t new_copy;
-
- /*
- * Ok to unlock map because entry is
- * marked in_transition.
- */
- cont_invoked = TRUE;
- vm_map_unlock(dst_map);
- vm_map_copy_invoke_cont(copy, &new_copy, &result);
-
- if (result == KERN_SUCCESS) {
-
- /*
- * If we got back a copy with real pages,
- * steal them now. Either all of the
- * pages in the list are tabled or none
- * of them are; mixtures are not possible.
- *
- * Save original copy for consume on
- * success logic at end of routine.
- */
- if (copy != orig_copy)
- vm_map_copy_discard(copy);
-
- if ((copy = new_copy) != VM_MAP_COPY_NULL) {
- page_list = ©->cpy_page_list[0];
- if (!copy->cpy_page_loose)
- vm_map_copy_steal_pages(copy);
- }
- }
- else {
- /*
- * Continuation failed.
- */
- vm_map_lock(dst_map);
- goto error;
- }
-
- vm_map_lock(dst_map);
- }
- }
-
- *dst_addr = start + dst_offset;
-
- /*
- * Clear the in transition bits. This is easy if we
- * didn't have a continuation.
- */
-error:
- needs_wakeup = FALSE;
- if (!cont_invoked) {
- /*
- * We didn't unlock the map, so nobody could
- * be waiting.
- */
- last->in_transition = FALSE;
- assert(!last->needs_wakeup);
- }
- else {
- if (!vm_map_lookup_entry(dst_map, start, &entry))
- panic("vm_map_copyout_page_list: missing entry");
-
- /*
- * Clear transition bit for all constituent entries that
- * were in the original entry. Also check for waiters.
- */
- while ((entry != vm_map_to_entry(dst_map)) &&
- (entry->vme_start < end)) {
- assert(entry->in_transition);
- entry->in_transition = FALSE;
- if (entry->needs_wakeup) {
- entry->needs_wakeup = FALSE;
- needs_wakeup = TRUE;
- }
- entry = entry->vme_next;
- }
- }
-
- if (result != KERN_SUCCESS)
- (void) vm_map_delete(dst_map, start, end, VM_MAP_NO_FLAGS);
-
- vm_map_unlock(dst_map);
-
- if (needs_wakeup)
- vm_map_entry_wakeup(dst_map);
-
- /*
- * Consume on success logic.
- */
- if (copy != VM_MAP_COPY_NULL && copy != orig_copy) {
- zfree(vm_map_copy_zone, (vm_offset_t) copy);
- }
- if (result == KERN_SUCCESS) {
- assert(orig_copy != VM_MAP_COPY_NULL);
- assert(orig_copy->type == VM_MAP_COPY_PAGE_LIST);
- zfree(vm_map_copy_zone, (vm_offset_t) orig_copy);
- }
-
- return(result);
-}
-
-/*
- * Routine: vm_map_copyin
- *
- * Description:
- * Copy the specified region (src_addr, len) from the
- * source address space (src_map), possibly removing
- * the region from the source address space (src_destroy).
- *
- * Returns:
- * A vm_map_copy_t object (copy_result), suitable for
- * insertion into another address space (using vm_map_copyout),
- * copying over another address space region (using
- * vm_map_copy_overwrite). If the copy is unused, it
- * should be destroyed (using vm_map_copy_discard).
- *
- * In/out conditions:
- * The source map should not be locked on entry.
- */
-
-typedef struct submap_map {
- vm_map_t parent_map;
- vm_offset_t base_start;
- vm_offset_t base_end;
- struct submap_map *next;
-} submap_map_t;
-
-kern_return_t
-vm_map_copyin_common(
- vm_map_t src_map,
- vm_offset_t src_addr,
- vm_size_t len,
- boolean_t src_destroy,
- boolean_t src_volatile,
- vm_map_copy_t *copy_result, /* OUT */
- boolean_t use_maxprot)
-{
- extern int msg_ool_size_small;
-
- vm_map_entry_t tmp_entry; /* Result of last map lookup --
- * in multi-level lookup, this
- * entry contains the actual
- * vm_object/offset.
- */
- register
- vm_map_entry_t new_entry = VM_MAP_ENTRY_NULL; /* Map entry for copy */
-
- vm_offset_t src_start; /* Start of current entry --
- * where copy is taking place now
- */
- vm_offset_t src_end; /* End of entire region to be
- * copied */
- vm_offset_t base_start; /* submap fields to save offsets */
- /* in original map */
- vm_offset_t base_end;
- vm_map_t base_map=src_map;
- vm_map_entry_t base_entry;
- boolean_t map_share=FALSE;
- submap_map_t *parent_maps = NULL;
-
- register
- vm_map_copy_t copy; /* Resulting copy */
- vm_offset_t copy_addr;
-
- /*
- * Check for copies of zero bytes.
- */
-
- if (len == 0) {
- *copy_result = VM_MAP_COPY_NULL;
- return(KERN_SUCCESS);
- }
-
- /*
- * If the copy is sufficiently small, use a kernel buffer instead
- * of making a virtual copy. The theory being that the cost of
- * setting up VM (and taking C-O-W faults) dominates the copy costs
- * for small regions.
- */
- if ((len < msg_ool_size_small) && !use_maxprot)
- return vm_map_copyin_kernel_buffer(src_map, src_addr, len,
- src_destroy, copy_result);
-
- /*
- * Compute start and end of region
- */
-
- src_start = trunc_page(src_addr);
- src_end = round_page(src_addr + len);
-
- XPR(XPR_VM_MAP, "vm_map_copyin_common map 0x%x addr 0x%x len 0x%x dest %d\n", (natural_t)src_map, src_addr, len, src_destroy, 0);
-
- /*
- * Check that the end address doesn't overflow
- */
-
- if (src_end <= src_start)
- if ((src_end < src_start) || (src_start != 0))
- return(KERN_INVALID_ADDRESS);
-
- /*
- * Allocate a header element for the list.
- *
- * Use the start and end in the header to
- * remember the endpoints prior to rounding.
- */
-
- copy = (vm_map_copy_t) zalloc(vm_map_copy_zone);
- vm_map_copy_first_entry(copy) =
- vm_map_copy_last_entry(copy) = vm_map_copy_to_entry(copy);
- copy->type = VM_MAP_COPY_ENTRY_LIST;
- copy->cpy_hdr.nentries = 0;
- copy->cpy_hdr.entries_pageable = TRUE;
-
- copy->offset = src_addr;
- copy->size = len;
-
- new_entry = vm_map_copy_entry_create(copy);
+ copy->offset = src_addr;
+ copy->size = len;
+
+ new_entry = vm_map_copy_entry_create(copy);
#define RETURN(x) \
MACRO_BEGIN \
vm_map_unlock(src_map); \
+ if(src_map != base_map) \
+ vm_map_deallocate(src_map); \
if (new_entry != VM_MAP_ENTRY_NULL) \
vm_map_copy_entry_dispose(copy,new_entry); \
vm_map_copy_discard(copy); \
{ \
- submap_map_t *ptr; \
+ submap_map_t *_ptr; \
\
- for(ptr = parent_maps; ptr != NULL; ptr = parent_maps) { \
+ for(_ptr = parent_maps; _ptr != NULL; _ptr = parent_maps) { \
parent_maps=parent_maps->next; \
- kfree((vm_offset_t)ptr, sizeof(submap_map_t)); \
+ if (_ptr->parent_map != base_map) \
+ vm_map_deallocate(_ptr->parent_map); \
+ kfree(_ptr, sizeof(submap_map_t)); \
} \
} \
MACRO_RETURN(x); \
while (TRUE) {
register
vm_map_entry_t src_entry = tmp_entry; /* Top-level entry */
- vm_size_t src_size; /* Size of source
+ vm_map_size_t src_size; /* Size of source
* map entry (in both
* maps)
*/
* copy_strategically.
*/
while(tmp_entry->is_sub_map) {
- vm_size_t submap_len;
+ vm_map_size_t submap_len;
submap_map_t *ptr;
ptr = (submap_map_t *)kalloc(sizeof(submap_map_t));
src_end = src_start + submap_len;
src_map = tmp_entry->object.sub_map;
vm_map_lock(src_map);
+ /* keep an outstanding reference for all maps in */
+ /* the parents tree except the base map */
+ vm_map_reference(src_map);
vm_map_unlock(ptr->parent_map);
if (!vm_map_lookup_entry(
src_map, src_start, &tmp_entry))
vm_map_clip_start(src_map, tmp_entry, src_start);
src_entry = tmp_entry;
}
+ if ((tmp_entry->object.vm_object != VM_OBJECT_NULL) &&
+ (tmp_entry->object.vm_object->phys_contiguous)) {
+ /* This is not, supported for now.In future */
+ /* we will need to detect the phys_contig */
+ /* condition and then upgrade copy_slowly */
+ /* to do physical copy from the device mem */
+ /* based object. We can piggy-back off of */
+ /* the was wired boolean to set-up the */
+ /* proper handling */
+ RETURN(KERN_PROTECTION_FAILURE);
+ }
/*
* Create a new address map entry to hold the result.
* Fill in the fields from the appropriate source entries.
XPR(XPR_VM_MAP, "vm_map_copyin_common src_obj 0x%x ent 0x%x obj 0x%x was_wired %d\n",
src_object, new_entry, new_entry->object.vm_object,
was_wired, 0);
- if (!was_wired &&
- vm_object_copy_quickly(
+ if ((src_object == VM_OBJECT_NULL ||
+ (!was_wired && !map_share && !tmp_entry->is_shared)) &&
+ vm_object_copy_quickly(
&new_entry->object.vm_object,
src_offset,
src_size,
*/
if (src_needs_copy && !tmp_entry->needs_copy) {
- if (tmp_entry->is_shared ||
- tmp_entry->object.vm_object->true_share ||
- map_share) {
- vm_map_unlock(src_map);
- new_entry->object.vm_object =
- vm_object_copy_delayed(
- src_object,
- src_offset,
- src_size);
- /* dec ref gained in copy_quickly */
- vm_object_lock(src_object);
- src_object->ref_count--;
- vm_object_res_deallocate(src_object);
- vm_object_unlock(src_object);
- vm_map_lock(src_map);
- /*
- * it turns out that we have
- * finished our copy. No matter
- * what the state of the map
- * we will lock it again here
- * knowing that if there is
- * additional data to copy
- * it will be checked at
- * the top of the loop
- *
- * Don't do timestamp check
- */
-
- } else {
- vm_object_pmap_protect(
- src_object,
- src_offset,
- src_size,
- (src_entry->is_shared ?
- PMAP_NULL
- : src_map->pmap),
- src_entry->vme_start,
- src_entry->protection &
- ~VM_PROT_WRITE);
-
- tmp_entry->needs_copy = TRUE;
- }
+ vm_object_pmap_protect(
+ src_object,
+ src_offset,
+ src_size,
+ (src_entry->is_shared ?
+ PMAP_NULL
+ : src_map->pmap),
+ src_entry->vme_start,
+ src_entry->protection &
+ ~VM_PROT_WRITE);
+ tmp_entry->needs_copy = TRUE;
}
/*
goto CopySuccessful;
}
- new_entry->needs_copy = FALSE;
-
/*
* Take an object reference, so that we may
* release the map lock(s).
*/
version.main_timestamp = src_map->timestamp;
- vm_map_unlock(src_map);
+ vm_map_unlock(src_map); /* Increments timestamp once! */
/*
* Perform the copy
*/
if (was_wired) {
+ CopySlowly:
vm_object_lock(src_object);
result = vm_object_copy_slowly(
src_object,
&new_entry->object.vm_object);
new_entry->offset = 0;
new_entry->needs_copy = FALSE;
+
+ }
+ else if (src_object->copy_strategy == MEMORY_OBJECT_COPY_SYMMETRIC &&
+ (tmp_entry->is_shared || map_share)) {
+ vm_object_t new_object;
+
+ vm_object_lock(src_object);
+ new_object = vm_object_copy_delayed(
+ src_object,
+ src_offset,
+ src_size);
+ if (new_object == VM_OBJECT_NULL)
+ goto CopySlowly;
+
+ new_entry->object.vm_object = new_object;
+ new_entry->needs_copy = TRUE;
+ result = KERN_SUCCESS;
+
} else {
result = vm_object_copy_strategically(src_object,
src_offset,
&new_entry_needs_copy);
new_entry->needs_copy = new_entry_needs_copy;
-
}
if (result != KERN_SUCCESS &&
* changed while the copy was being made.
*/
- vm_map_lock(src_map); /* Increments timestamp once! */
+ vm_map_lock(src_map);
if ((version.main_timestamp + 1) == src_map->timestamp)
goto VerificationSuccessful;
src_entry = tmp_entry;
vm_map_clip_start(src_map, src_entry, src_start);
- if ((src_entry->protection & VM_PROT_READ == VM_PROT_NONE &&
- !use_maxprot) ||
- src_entry->max_protection & VM_PROT_READ == 0)
+ if ((((src_entry->protection & VM_PROT_READ) == VM_PROT_NONE) &&
+ !use_maxprot) ||
+ ((src_entry->max_protection & VM_PROT_READ) == 0))
goto VerificationFailed;
if (src_entry->vme_end < new_entry->vme_end)
ptr = parent_maps;
assert(ptr != NULL);
parent_maps = parent_maps->next;
- vm_map_lock(ptr->parent_map);
vm_map_unlock(src_map);
+ vm_map_deallocate(src_map);
+ vm_map_lock(ptr->parent_map);
src_map = ptr->parent_map;
src_start = ptr->base_start;
src_end = ptr->base_end;
!vm_map_lookup_entry(
src_map, src_start, &tmp_entry))
RETURN(KERN_INVALID_ADDRESS);
- kfree((vm_offset_t)ptr, sizeof(submap_map_t));
+ kfree(ptr, sizeof(submap_map_t));
if(parent_maps == NULL)
map_share = FALSE;
src_entry = tmp_entry->vme_prev;
*/
if (src_destroy) {
(void) vm_map_delete(src_map,
- trunc_page(src_addr),
+ vm_map_trunc_page(src_addr),
src_end,
(src_map == kernel_map) ?
VM_MAP_REMOVE_KUNWIRE :
- VM_MAP_NO_FLAGS);
+ VM_MAP_NO_FLAGS,
+ VM_MAP_NULL);
}
vm_map_unlock(src_map);
copy = (vm_map_copy_t) zalloc(vm_map_copy_zone);
copy->type = VM_MAP_COPY_OBJECT;
copy->cpy_object = object;
- copy->cpy_index = 0;
copy->offset = offset;
copy->size = size;
return(KERN_SUCCESS);
}
-/*
- * vm_map_copyin_page_list_cont:
- *
- * Continuation routine for vm_map_copyin_page_list.
- *
- * If vm_map_copyin_page_list can't fit the entire vm range
- * into a single page list object, it creates a continuation.
- * When the target of the operation has used the pages in the
- * initial page list, it invokes the continuation, which calls
- * this routine. If an error happens, the continuation is aborted
- * (abort arg to this routine is TRUE). To avoid deadlocks, the
- * pages are discarded from the initial page list before invoking
- * the continuation.
- *
- * NOTE: This is not the same sort of continuation used by
- * the scheduler.
- */
-
-kern_return_t
-vm_map_copyin_page_list_cont(
- vm_map_copyin_args_t cont_args,
- vm_map_copy_t *copy_result) /* OUT */
+static void
+vm_map_fork_share(
+ vm_map_t old_map,
+ vm_map_entry_t old_entry,
+ vm_map_t new_map)
{
- kern_return_t result = KERN_SUCCESS;
- register boolean_t abort, src_destroy, src_destroy_only;
+ vm_object_t object;
+ vm_map_entry_t new_entry;
/*
- * Check for cases that only require memory destruction.
+ * New sharing code. New map entry
+ * references original object. Internal
+ * objects use asynchronous copy algorithm for
+ * future copies. First make sure we have
+ * the right object. If we need a shadow,
+ * or someone else already has one, then
+ * make a new shadow and share it.
*/
- abort = (copy_result == (vm_map_copy_t *) 0);
- src_destroy = (cont_args->destroy_len != (vm_size_t) 0);
- src_destroy_only = (cont_args->src_len == (vm_size_t) 0);
-
- if (abort || src_destroy_only) {
- if (src_destroy)
- result = vm_map_remove(cont_args->map,
- cont_args->destroy_addr,
- cont_args->destroy_addr + cont_args->destroy_len,
- VM_MAP_NO_FLAGS);
- if (!abort)
- *copy_result = VM_MAP_COPY_NULL;
- }
- else {
- result = vm_map_copyin_page_list(cont_args->map,
- cont_args->src_addr, cont_args->src_len,
- cont_args->options, copy_result, TRUE);
-
- if (src_destroy &&
- (cont_args->options & VM_MAP_COPYIN_OPT_STEAL_PAGES) &&
- vm_map_copy_has_cont(*copy_result)) {
- vm_map_copyin_args_t new_args;
- /*
- * Transfer old destroy info.
- */
- new_args = (vm_map_copyin_args_t)
- (*copy_result)->cpy_cont_args;
- new_args->destroy_addr = cont_args->destroy_addr;
- new_args->destroy_len = cont_args->destroy_len;
- }
- }
- vm_map_deallocate(cont_args->map);
- kfree((vm_offset_t)cont_args, sizeof(vm_map_copyin_args_data_t));
-
- return(result);
-}
-
-/*
- * vm_map_copyin_page_list:
- *
- * This is a variant of vm_map_copyin that copies in a list of pages.
- * If steal_pages is TRUE, the pages are only in the returned list.
- * If steal_pages is FALSE, the pages are busy and still in their
- * objects. A continuation may be returned if not all the pages fit:
- * the recipient of this copy_result must be prepared to deal with it.
- */
-
-kern_return_t
-vm_map_copyin_page_list(
- vm_map_t src_map,
- vm_offset_t src_addr,
- vm_size_t len,
- int options,
- vm_map_copy_t *copy_result, /* OUT */
- boolean_t is_cont)
-{
- vm_map_entry_t src_entry;
- vm_page_t m;
- vm_offset_t src_start;
- vm_offset_t src_end;
- vm_size_t src_size;
- register vm_object_t src_object;
- register vm_object_offset_t src_offset;
- vm_object_offset_t src_last_offset;
- register vm_map_copy_t copy; /* Resulting copy */
- kern_return_t result = KERN_SUCCESS;
- boolean_t need_map_lookup;
- vm_map_copyin_args_t cont_args;
- kern_return_t error_code;
- vm_prot_t prot;
- boolean_t wired;
- boolean_t no_zero_fill;
-
- submap_map_t *parent_maps = NULL;
- vm_map_t base_map = src_map;
-
- prot = (options & VM_MAP_COPYIN_OPT_VM_PROT);
- no_zero_fill = (options & VM_MAP_COPYIN_OPT_NO_ZERO_FILL);
-
- /*
- * If steal_pages is FALSE, this leaves busy pages in
- * the object. A continuation must be used if src_destroy
- * is true in this case (!steal_pages && src_destroy).
- *
- * XXX Still have a more general problem of what happens
- * XXX if the same page occurs twice in a list. Deadlock
- * XXX can happen if vm_fault_page was called. A
- * XXX possible solution is to use a continuation if vm_fault_page
- * XXX is called and we cross a map entry boundary.
- */
-
- /*
- * Check for copies of zero bytes.
- */
-
- if (len == 0) {
- *copy_result = VM_MAP_COPY_NULL;
- return(KERN_SUCCESS);
- }
-
- /*
- * Compute start and end of region
- */
+ object = old_entry->object.vm_object;
+ if (old_entry->is_sub_map) {
+ assert(old_entry->wired_count == 0);
+#ifndef i386
+ if(old_entry->use_pmap) {
+ kern_return_t result;
- src_start = trunc_page(src_addr);
- src_end = round_page(src_addr + len);
-
- /*
- * If the region is not page aligned, override the no_zero_fill
- * argument.
- */
-
- if (options & VM_MAP_COPYIN_OPT_NO_ZERO_FILL) {
- if (!page_aligned(src_addr) || !page_aligned(src_addr +len))
- options &= ~VM_MAP_COPYIN_OPT_NO_ZERO_FILL;
- }
-
- /*
- * Check that the end address doesn't overflow
- */
-
- if (src_end <= src_start && (src_end < src_start || src_start != 0)) {
- return KERN_INVALID_ADDRESS;
- }
-
- /*
- * Allocate a header element for the page list.
- *
- * Record original offset and size, as caller may not
- * be page-aligned.
- */
-
- copy = (vm_map_copy_t) zalloc(vm_map_copy_zone);
- copy->type = VM_MAP_COPY_PAGE_LIST;
- copy->cpy_npages = 0;
- copy->cpy_page_loose = FALSE;
- copy->offset = src_addr;
- copy->size = len;
- copy->cpy_cont = VM_MAP_COPY_CONT_NULL;
- copy->cpy_cont_args = VM_MAP_COPYIN_ARGS_NULL;
-
- /*
- * Find the beginning of the region.
- */
-
-do_map_lookup:
-
- vm_map_lock(src_map);
-
- if (!vm_map_lookup_entry(src_map, src_start, &src_entry)) {
- result = KERN_INVALID_ADDRESS;
- goto error;
- }
- need_map_lookup = FALSE;
-
- /*
- * Go through entries until we get to the end.
- */
-
- while (TRUE) {
- if ((src_entry->protection & prot) != prot) {
- result = KERN_PROTECTION_FAILURE;
- goto error;
- }
-
- /* translate down through submaps to find the target entry */
- while(src_entry->is_sub_map) {
- vm_size_t submap_len;
- submap_map_t *ptr;
-
- ptr = (submap_map_t *)kalloc(sizeof(submap_map_t));
- ptr->next = parent_maps;
- parent_maps = ptr;
- ptr->parent_map = src_map;
- ptr->base_start = src_start;
- ptr->base_end = src_end;
- submap_len = src_entry->vme_end - src_entry->vme_start;
- if(submap_len > (src_end-src_start))
- submap_len = src_end-src_start;
- ptr->base_start += submap_len;
-
- src_start -= src_entry->vme_start;
- src_start += src_entry->offset;
- src_end = src_start + submap_len;
- src_map = src_entry->object.sub_map;
- vm_map_lock(src_map);
- vm_map_unlock(ptr->parent_map);
- if (!vm_map_lookup_entry(
- src_map, src_start, &src_entry)) {
- result = KERN_INVALID_ADDRESS;
- goto error;
- }
- vm_map_clip_start(src_map, src_entry, src_start);
- }
-
- wired = (src_entry->wired_count != 0);
-
- if (src_end > src_entry->vme_end)
- src_size = src_entry->vme_end - src_start;
- else
- src_size = src_end - src_start;
-
- src_object = src_entry->object.vm_object;
-
- /*
- * If src_object is NULL, allocate it now;
- * we're going to fault on it shortly.
- */
- if (src_object == VM_OBJECT_NULL) {
- src_object = vm_object_allocate((vm_size_t)
- src_entry->vme_end -
- src_entry->vme_start);
- src_entry->object.vm_object = src_object;
- }
- else if (src_entry->needs_copy && (prot & VM_PROT_WRITE)) {
- vm_object_shadow(
- &src_entry->object.vm_object,
- &src_entry->offset,
- (vm_size_t) (src_entry->vme_end -
- src_entry->vme_start));
-
- src_entry->needs_copy = FALSE;
-
- /* reset src_object */
- src_object = src_entry->object.vm_object;
- }
-
- /*
- * calculate src_offset now, since vm_object_shadow
- * may have changed src_entry->offset.
- */
- src_offset = src_entry->offset + (src_start - src_entry->vme_start);
-
- /*
- * Iterate over pages. Fault in ones that aren't present.
- */
- src_last_offset = src_offset + src_size;
- for (; (src_offset < src_last_offset);
- src_offset += PAGE_SIZE_64, src_start += PAGE_SIZE) {
-
- if (copy->cpy_npages == VM_MAP_COPY_PAGE_LIST_MAX) {
- vm_offset_t src_delta;
-make_continuation:
- /*
- * At this point we have the max number of
- * pages busy for this thread that we're
- * willing to allow. Stop here and record
- * arguments for the remainder. Note:
- * this means that this routine isn't atomic,
- * but that's the breaks. Note that only
- * the first vm_map_copy_t that comes back
- * from this routine has the right offset
- * and size; those from continuations are
- * page rounded, and short by the amount
- * already done.
- *
- * Reset src_end so the src_destroy
- * code at the bottom doesn't do
- * something stupid.
- */
-
- src_delta = src_end - src_start;
- while (src_map != base_map) {
- submap_map_t *ptr;
-
- if(!need_map_lookup) {
- vm_map_unlock(src_map);
- }
- ptr = parent_maps;
- assert(ptr != NULL);
- parent_maps = parent_maps->next;
- src_map = ptr->parent_map;
- src_start = ptr->base_start - src_delta;
- src_delta = ptr->base_end - src_start;
- kfree((vm_offset_t)ptr, sizeof(submap_map_t));
-
- need_map_lookup = TRUE;
- }
- src_end = src_start;
-
-
- cont_args = (vm_map_copyin_args_t)
- kalloc(sizeof(vm_map_copyin_args_data_t));
- cont_args->map = src_map;
- vm_map_reference(src_map);
- cont_args->src_addr = src_start;
- cont_args->src_len = len - (src_start - src_addr);
- if (options & VM_MAP_COPYIN_OPT_SRC_DESTROY) {
- cont_args->destroy_addr = cont_args->src_addr;
- cont_args->destroy_len = cont_args->src_len;
- } else {
- cont_args->destroy_addr = (vm_offset_t) 0;
- cont_args->destroy_len = (vm_offset_t) 0;
- }
- cont_args->options = options;
-
- copy->cpy_cont_args = cont_args;
- copy->cpy_cont = vm_map_copyin_page_list_cont;
-
- break;
- }
-
- /*
- * Try to find the page of data. Have to
- * fault it in if there's no page, or something
- * going on with the page, or the object has
- * a copy object.
- */
- vm_object_lock(src_object);
- vm_object_paging_begin(src_object);
- if (((m = vm_page_lookup(src_object, src_offset)) !=
- VM_PAGE_NULL) &&
- !m->busy && !m->fictitious && !m->unusual &&
- ((prot & VM_PROT_WRITE) == 0 ||
- (m->object->copy == VM_OBJECT_NULL))) {
-
- if (!m->absent &&
- !(options & VM_MAP_COPYIN_OPT_STEAL_PAGES)) {
-
- /*
- * The page is present and will not be
- * replaced, prep it. Thus allowing
- * mutiple access on this page
- */
- kern_return_t kr;
-
- kr = vm_page_prep(m);
- assert(kr == KERN_SUCCESS);
- kr = vm_page_pin(m);
- assert(kr == KERN_SUCCESS);
- } else {
- /*
- * This is the page. Mark it busy
- * and keep the paging reference on
- * the object whilst we do our thing.
- */
-
- m->busy = TRUE;
- }
- } else {
- vm_prot_t result_prot;
- vm_page_t top_page;
- kern_return_t kr;
- boolean_t data_supply;
-
- /*
- * Have to fault the page in; must
- * unlock the map to do so. While
- * the map is unlocked, anything
- * can happen, we must lookup the
- * map entry before continuing.
- */
- vm_map_unlock(src_map);
- need_map_lookup = TRUE;
- data_supply = src_object->silent_overwrite &&
- (prot & VM_PROT_WRITE) &&
- src_start >= src_addr &&
- src_start + PAGE_SIZE <=
- src_addr + len;
-
-retry:
- result_prot = prot;
-
- XPR(XPR_VM_FAULT,
- "vm_map_copyin_page_list -> vm_fault_page\n",
- 0,0,0,0,0);
- kr = vm_fault_page(src_object, src_offset,
- prot, FALSE, THREAD_UNINT,
- src_entry->offset,
- src_entry->offset +
- (src_entry->vme_end -
- src_entry->vme_start),
- VM_BEHAVIOR_SEQUENTIAL,
- &result_prot, &m, &top_page,
- (int *)0,
- &error_code,
- options & VM_MAP_COPYIN_OPT_NO_ZERO_FILL,
- data_supply);
- /*
- * Cope with what happened.
- */
- switch (kr) {
- case VM_FAULT_SUCCESS:
-
- /*
- * If we lost write access,
- * try again.
- */
- if ((prot & VM_PROT_WRITE) &&
- !(result_prot & VM_PROT_WRITE)) {
- vm_object_lock(src_object);
- vm_object_paging_begin(src_object);
- goto retry;
- }
- break;
- case VM_FAULT_MEMORY_SHORTAGE:
- VM_PAGE_WAIT();
- /* fall thru */
- case VM_FAULT_INTERRUPTED: /* ??? */
- case VM_FAULT_RETRY:
- vm_object_lock(src_object);
- vm_object_paging_begin(src_object);
- goto retry;
- case VM_FAULT_FICTITIOUS_SHORTAGE:
- vm_page_more_fictitious();
- vm_object_lock(src_object);
- vm_object_paging_begin(src_object);
- goto retry;
- case VM_FAULT_MEMORY_ERROR:
- /*
- * Something broke. If this
- * is a continuation, return
- * a partial result if possible,
- * else fail the whole thing.
- * In the continuation case, the
- * next continuation call will
- * get this error if it persists.
- */
- vm_map_lock(src_map);
- if (is_cont &&
- copy->cpy_npages != 0)
- goto make_continuation;
-
- result = error_code ? error_code : KERN_MEMORY_ERROR;
- goto error;
- }
-
- if (top_page != VM_PAGE_NULL) {
- vm_object_lock(src_object);
- VM_PAGE_FREE(top_page);
- vm_object_paging_end(src_object);
- vm_object_unlock(src_object);
- }
-
- }
-
- /*
- * The page is busy, its object is locked, and
- * we have a paging reference on it. Either
- * the map is locked, or need_map_lookup is
- * TRUE.
- */
-
- /*
- * Put the page in the page list.
- */
- copy->cpy_page_list[copy->cpy_npages++] = m;
- vm_object_unlock(m->object);
-
- /*
- * Pmap enter support. Only used for
- * device I/O for colocated server.
- *
- * WARNING: This code assumes that this
- * option is only used for well behaved
- * memory. If the mapping has changed,
- * the following code will make mistakes.
- *
- * XXXO probably ought to do pmap_extract first,
- * XXXO to avoid needless pmap_enter, but this
- * XXXO can't detect protection mismatch??
- */
-
- if (options & VM_MAP_COPYIN_OPT_PMAP_ENTER) {
- /*
- * XXX Only used on kernel map.
- * XXX Must not remove VM_PROT_WRITE on
- * XXX an I/O only requiring VM_PROT_READ
- * XXX as another I/O may be active on same page
- * XXX assume that if mapping exists, it must
- * XXX have the equivalent of at least VM_PROT_READ,
- * XXX but don't assume it has VM_PROT_WRITE as the
- * XXX pmap might not all the rights of the object
- */
- assert(vm_map_pmap(src_map) == kernel_pmap);
-
- if ((prot & VM_PROT_WRITE) ||
- (pmap_extract(vm_map_pmap(src_map),
- src_start) != m->phys_addr))
-
- PMAP_ENTER(vm_map_pmap(src_map), src_start,
- m, prot, wired);
- }
- if(need_map_lookup) {
- need_map_lookup = FALSE;
- vm_map_lock(src_map);
- if (!vm_map_lookup_entry(src_map, src_start, &src_entry)) {
- result = KERN_INVALID_ADDRESS;
- goto error;
- }
- }
- }
-
- /*
- * Verify that there are no gaps in the region
- */
- src_start = src_entry->vme_end;
- if (src_start < src_end) {
- src_entry = src_entry->vme_next;
- if (need_map_lookup) {
- need_map_lookup = FALSE;
- vm_map_lock(src_map);
- if(!vm_map_lookup_entry(src_map,
- src_start, &src_entry)) {
- result = KERN_INVALID_ADDRESS;
- goto error;
- }
- } else if (src_entry->vme_start != src_start) {
- result = KERN_INVALID_ADDRESS;
- goto error;
- }
- }
-
- /*
- * DETERMINE whether the entire region
- * has been copied.
- */
-
- while ((src_start >= src_end) && (src_end != 0)) {
- if (src_map != base_map) {
- submap_map_t *ptr;
-
- ptr = parent_maps;
- assert(ptr != NULL);
- parent_maps = parent_maps->next;
- src_start = ptr->base_start;
- src_end = ptr->base_end;
- if(need_map_lookup) {
- need_map_lookup = FALSE;
- }
- else {
- vm_map_unlock(src_map);
- }
- src_map = ptr->parent_map;
- vm_map_lock(src_map);
- if((src_start < src_end) &&
- (!vm_map_lookup_entry(ptr->parent_map,
- src_start, &src_entry))) {
- result = KERN_INVALID_ADDRESS;
- kfree((vm_offset_t)ptr, sizeof(submap_map_t));
- goto error;
- }
- kfree((vm_offset_t)ptr, sizeof(submap_map_t));
- } else
- break;
- }
- if ((src_start >= src_end) && (src_end != 0)) {
- if (need_map_lookup)
- vm_map_lock(src_map);
- break;
- }
-
- }
-
- /*
- * If steal_pages is true, make sure all
- * pages in the copy are not in any object
- * We try to remove them from the original
- * object, but we may have to copy them.
- *
- * At this point every page in the list is busy
- * and holds a paging reference to its object.
- * When we're done stealing, every page is busy,
- * and in no object (m->tabled == FALSE).
- */
- src_start = trunc_page(src_addr);
- if (options & VM_MAP_COPYIN_OPT_STEAL_PAGES) {
- register int i;
- vm_offset_t page_vaddr;
- vm_offset_t unwire_end;
- vm_offset_t map_entry_end;
- boolean_t share_map = FALSE;
-
- unwire_end = src_start;
- map_entry_end = src_start;
- for (i = 0; i < copy->cpy_npages; i++) {
-
-
- /*
- * Remove the page from its object if it
- * can be stolen. It can be stolen if:
- *
- * (1) The source is being destroyed,
- * the object is internal (hence
- * temporary), and not shared.
- * (2) The page is not precious.
- *
- * The not shared check consists of two
- * parts: (a) there are no objects that
- * shadow this object. (b) it is not the
- * object in any shared map entries (i.e.,
- * use_shared_copy is not set).
- *
- * The first check (a) means that we can't
- * steal pages from objects that are not
- * at the top of their shadow chains. This
- * should not be a frequent occurrence.
- *
- * Stealing wired pages requires telling the
- * pmap module to let go of them.
- *
- * NOTE: stealing clean pages from objects
- * whose mappings survive requires a call to
- * the pmap module. Maybe later.
- */
- m = copy->cpy_page_list[i];
- src_object = m->object;
- vm_object_lock(src_object);
-
- page_vaddr = src_start + (i * PAGE_SIZE);
- if(page_vaddr > map_entry_end) {
- if (!vm_map_lookup_entry(src_map, page_vaddr, &src_entry))
- share_map = TRUE;
- else if (src_entry->is_sub_map) {
- map_entry_end = src_entry->vme_end;
- share_map = TRUE;
- } else {
- map_entry_end = src_entry->vme_end;
- share_map = FALSE;
- }
- }
-
-
- if ((options & VM_MAP_COPYIN_OPT_SRC_DESTROY) &&
- src_object->internal &&
- !src_object->true_share &&
- (!src_object->shadowed) &&
- (src_object->copy_strategy ==
- MEMORY_OBJECT_COPY_SYMMETRIC) &&
- !m->precious &&
- !share_map) {
-
- if (m->wire_count > 0) {
-
- assert(m->wire_count == 1);
- /*
- * In order to steal a wired
- * page, we have to unwire it
- * first. We do this inline
- * here because we have the page.
- *
- * Step 1: Unwire the map entry.
- * Also tell the pmap module
- * that this piece of the
- * pmap is pageable.
- */
- vm_object_unlock(src_object);
- if (page_vaddr >= unwire_end) {
- if (!vm_map_lookup_entry(src_map,
- page_vaddr, &src_entry))
- panic("vm_map_copyin_page_list: missing wired map entry");
-
- vm_map_clip_start(src_map, src_entry,
- page_vaddr);
- vm_map_clip_end(src_map, src_entry,
- src_start + src_size);
-
-/* revisit why this assert fails CDY
- assert(src_entry->wired_count > 0);
-*/
- src_entry->wired_count = 0;
- src_entry->user_wired_count = 0;
- unwire_end = src_entry->vme_end;
- pmap_pageable(vm_map_pmap(src_map),
- page_vaddr, unwire_end, TRUE);
- }
-
- /*
- * Step 2: Unwire the page.
- * pmap_remove handles this for us.
- */
- vm_object_lock(src_object);
- }
-
- /*
- * Don't need to remove the mapping;
- * vm_map_delete will handle it.
- *
- * Steal the page. Setting the wire count
- * to zero is vm_page_unwire without
- * activating the page.
- */
- vm_page_lock_queues();
- vm_page_remove(m);
- if (m->wire_count > 0) {
- m->wire_count = 0;
- vm_page_wire_count--;
- } else {
- VM_PAGE_QUEUES_REMOVE(m);
- }
- vm_page_unlock_queues();
- } else {
- /*
- * Have to copy this page. Have to
- * unlock the map while copying,
- * hence no further page stealing.
- * Hence just copy all the pages.
- * Unlock the map while copying;
- * This means no further page stealing.
- */
- vm_object_unlock(src_object);
- vm_map_unlock(src_map);
- vm_map_copy_steal_pages(copy);
- vm_map_lock(src_map);
- break;
- }
-
- vm_object_paging_end(src_object);
- vm_object_unlock(src_object);
- }
-
- copy->cpy_page_loose = TRUE;
-
- /*
- * If the source should be destroyed, do it now, since the
- * copy was successful.
- */
-
- if (options & VM_MAP_COPYIN_OPT_SRC_DESTROY) {
- (void) vm_map_delete(src_map, src_start,
- src_end, VM_MAP_NO_FLAGS);
- }
- } else {
- /*
- * Not stealing pages leaves busy or prepped pages in the map.
- * This will cause source destruction to hang. Use
- * a continuation to prevent this.
- */
- if ((options & VM_MAP_COPYIN_OPT_SRC_DESTROY) &&
- !vm_map_copy_has_cont(copy)) {
- cont_args = (vm_map_copyin_args_t)
- kalloc(sizeof(vm_map_copyin_args_data_t));
- vm_map_reference(src_map);
- cont_args->map = src_map;
- cont_args->src_addr = (vm_offset_t) 0;
- cont_args->src_len = (vm_size_t) 0;
- cont_args->destroy_addr = src_start;
- cont_args->destroy_len = src_end - src_start;
- cont_args->options = options;
-
- copy->cpy_cont_args = cont_args;
- copy->cpy_cont = vm_map_copyin_page_list_cont;
- }
- }
-
- vm_map_unlock(src_map);
-
- *copy_result = copy;
- return(result);
-
-error:
- {
- submap_map_t *ptr;
-
- vm_map_unlock(src_map);
- vm_map_copy_discard(copy);
-
- for(ptr = parent_maps; ptr != NULL; ptr = parent_maps) {
- parent_maps=parent_maps->next;
- kfree((vm_offset_t)ptr, sizeof(submap_map_t));
- }
- return(result);
- }
-}
-
-void
-vm_map_fork_share(
- vm_map_t old_map,
- vm_map_entry_t old_entry,
- vm_map_t new_map)
-{
- vm_object_t object;
- vm_map_entry_t new_entry;
- kern_return_t result;
-
- /*
- * New sharing code. New map entry
- * references original object. Internal
- * objects use asynchronous copy algorithm for
- * future copies. First make sure we have
- * the right object. If we need a shadow,
- * or someone else already has one, then
- * make a new shadow and share it.
- */
-
- object = old_entry->object.vm_object;
- if (old_entry->is_sub_map) {
- assert(old_entry->wired_count == 0);
-#ifndef i386
- if(old_entry->use_pmap) {
result = pmap_nest(new_map->pmap,
(old_entry->object.sub_map)->pmap,
- old_entry->vme_start,
- old_entry->vme_end - old_entry->vme_start);
+ (addr64_t)old_entry->vme_start,
+ (addr64_t)old_entry->vme_start,
+ (uint64_t)(old_entry->vme_end - old_entry->vme_start));
if(result)
panic("vm_map_fork_share: pmap_nest failed!");
}
#endif
} else if (object == VM_OBJECT_NULL) {
- object = vm_object_allocate((vm_size_t)(old_entry->vme_end -
+ object = vm_object_allocate((vm_map_size_t)(old_entry->vme_end -
old_entry->vme_start));
old_entry->offset = 0;
old_entry->object.vm_object = object;
(!object->true_share && /* case 3 */
!old_entry->is_shared &&
(object->size >
- (vm_size_t)(old_entry->vme_end -
+ (vm_map_size_t)(old_entry->vme_end -
old_entry->vme_start)))) {
/*
assert(!(object->shadowed && old_entry->is_shared));
vm_object_shadow(&old_entry->object.vm_object,
&old_entry->offset,
- (vm_size_t) (old_entry->vme_end -
+ (vm_map_size_t) (old_entry->vme_end -
old_entry->vme_start));
/*
* to remove write permission.
*/
-/* CDY FIX this! page_protect! */
if (!old_entry->needs_copy &&
(old_entry->protection & VM_PROT_WRITE)) {
- if(old_entry->is_sub_map && old_entry->use_pmap) {
- pmap_protect(old_entry->object.sub_map->pmap,
- old_entry->vme_start,
- old_entry->vme_end,
- old_entry->protection & ~VM_PROT_WRITE);
+ if(old_map->mapped) {
+ vm_object_pmap_protect(
+ old_entry->object.vm_object,
+ old_entry->offset,
+ (old_entry->vme_end -
+ old_entry->vme_start),
+ PMAP_NULL,
+ old_entry->vme_start,
+ old_entry->protection & ~VM_PROT_WRITE);
} else {
- pmap_protect(vm_map_pmap(old_map),
+ pmap_protect(old_map->pmap,
old_entry->vme_start,
old_entry->vme_end,
old_entry->protection & ~VM_PROT_WRITE);
}
}
-boolean_t
+static boolean_t
vm_map_fork_copy(
vm_map_t old_map,
vm_map_entry_t *old_entry_p,
vm_map_t new_map)
{
vm_map_entry_t old_entry = *old_entry_p;
- vm_size_t entry_size = old_entry->vme_end - old_entry->vme_start;
- vm_offset_t start = old_entry->vme_start;
+ vm_map_size_t entry_size = old_entry->vme_end - old_entry->vme_start;
+ vm_map_offset_t start = old_entry->vme_start;
vm_map_copy_t copy;
vm_map_entry_t last = vm_map_last_entry(new_map);
*/
vm_map_lock(old_map);
if (!vm_map_lookup_entry(old_map, start, &last) ||
- last->max_protection & VM_PROT_READ ==
- VM_PROT_NONE) {
+ (last->max_protection & VM_PROT_READ) == VM_PROT_NONE) {
last = last->vme_next;
}
*old_entry_p = last;
vm_map_fork(
vm_map_t old_map)
{
- pmap_t new_pmap = pmap_create((vm_size_t) 0);
+ pmap_t new_pmap = pmap_create((vm_map_size_t) 0);
vm_map_t new_map;
vm_map_entry_t old_entry;
- vm_size_t new_size = 0, entry_size;
+ vm_map_size_t new_size = 0, entry_size;
vm_map_entry_t new_entry;
boolean_t src_needs_copy;
boolean_t new_entry_needs_copy;
if(old_entry->is_sub_map)
break;
- if (old_entry->wired_count != 0) {
+ if ((old_entry->wired_count != 0) ||
+ ((old_entry->object.vm_object != NULL) &&
+ (old_entry->object.vm_object->true_share))) {
goto slow_vm_map_fork_copy;
}
(old_entry->vme_end -
old_entry->vme_start),
((old_entry->is_shared
- || old_entry->is_sub_map)
+ || old_map->mapped)
? PMAP_NULL :
old_map->pmap),
old_entry->vme_start,
kern_return_t
vm_map_lookup_locked(
vm_map_t *var_map, /* IN/OUT */
- register vm_offset_t vaddr,
- register vm_prot_t fault_type,
+ vm_map_offset_t vaddr,
+ vm_prot_t fault_type,
vm_map_version_t *out_version, /* OUT */
vm_object_t *object, /* OUT */
vm_object_offset_t *offset, /* OUT */
vm_prot_t *out_prot, /* OUT */
boolean_t *wired, /* OUT */
int *behavior, /* OUT */
- vm_object_offset_t *lo_offset, /* OUT */
- vm_object_offset_t *hi_offset, /* OUT */
- vm_map_t *pmap_map)
+ vm_map_offset_t *lo_offset, /* OUT */
+ vm_map_offset_t *hi_offset, /* OUT */
+ vm_map_t *real_map)
{
vm_map_entry_t entry;
register vm_map_t map = *var_map;
vm_map_t old_map = *var_map;
vm_map_t cow_sub_map_parent = VM_MAP_NULL;
- vm_offset_t cow_parent_vaddr;
- vm_offset_t old_start;
- vm_offset_t old_end;
+ vm_map_offset_t cow_parent_vaddr = 0;
+ vm_map_offset_t old_start = 0;
+ vm_map_offset_t old_end = 0;
register vm_prot_t prot;
- *pmap_map = map;
+ *real_map = map;
RetryLookup: ;
/*
if (!vm_map_lookup_entry(map, vaddr, &tmp_entry)) {
if((cow_sub_map_parent) && (cow_sub_map_parent != map))
vm_map_unlock(cow_sub_map_parent);
- if((*pmap_map != map)
- && (*pmap_map != cow_sub_map_parent))
- vm_map_unlock(*pmap_map);
+ if((*real_map != map)
+ && (*real_map != cow_sub_map_parent))
+ vm_map_unlock(*real_map);
return KERN_INVALID_ADDRESS;
}
submap_recurse:
if (entry->is_sub_map) {
- vm_offset_t local_vaddr;
- vm_offset_t end_delta;
- vm_offset_t start_delta;
- vm_offset_t object_start_delta;
+ vm_map_offset_t local_vaddr;
+ vm_map_offset_t end_delta;
+ vm_map_offset_t start_delta;
vm_map_entry_t submap_entry;
boolean_t mapped_needs_copy=FALSE;
local_vaddr = vaddr;
if ((!entry->needs_copy) && (entry->use_pmap)) {
- /* if pmap_map equals map we unlock below */
- if ((*pmap_map != map) &&
- (*pmap_map != cow_sub_map_parent))
- vm_map_unlock(*pmap_map);
- *pmap_map = entry->object.sub_map;
+ /* if real_map equals map we unlock below */
+ if ((*real_map != map) &&
+ (*real_map != cow_sub_map_parent))
+ vm_map_unlock(*real_map);
+ *real_map = entry->object.sub_map;
}
if(entry->needs_copy) {
if (!mapped_needs_copy) {
if (vm_map_lock_read_to_write(map)) {
vm_map_lock_read(map);
- if(*pmap_map == entry->object.sub_map)
- *pmap_map = map;
+ if(*real_map == entry->object.sub_map)
+ *real_map = map;
goto RetryLookup;
}
vm_map_lock_read(entry->object.sub_map);
} else {
vm_map_lock_read(entry->object.sub_map);
if((cow_sub_map_parent != map) &&
- (*pmap_map != map))
+ (*real_map != map))
vm_map_unlock(map);
}
} else {
/* follow the maps down to the object */
/* here we unlock knowing we are not */
/* revisiting the map. */
- if((*pmap_map != map) && (map != cow_sub_map_parent))
+ if((*real_map != map) && (map != cow_sub_map_parent))
vm_map_unlock_read(map);
}
if((cow_sub_map_parent) && (cow_sub_map_parent != map)){
vm_map_unlock(cow_sub_map_parent);
}
- if((*pmap_map != map)
- && (*pmap_map != cow_sub_map_parent)) {
- vm_map_unlock(*pmap_map);
+ if((*real_map != map)
+ && (*real_map != cow_sub_map_parent)) {
+ vm_map_unlock(*real_map);
}
- *pmap_map = map;
+ *real_map = map;
return KERN_INVALID_ADDRESS;
}
/* find the attenuated shadow of the underlying object */
if(((fault_type & VM_PROT_WRITE) && cow_sub_map_parent)) {
vm_object_t copy_object;
- vm_offset_t local_start;
- vm_offset_t local_end;
+ vm_map_offset_t local_start;
+ vm_map_offset_t local_end;
+ boolean_t copied_slowly = FALSE;
if (vm_map_lock_read_to_write(map)) {
vm_map_lock_read(map);
old_end += end_delta;
goto RetrySubMap;
}
+
+
if (submap_entry->object.vm_object == VM_OBJECT_NULL) {
submap_entry->object.vm_object =
vm_object_allocate(
- (vm_size_t)
+ (vm_map_size_t)
(submap_entry->vme_end
- submap_entry->vme_start));
- submap_entry->offset = 0;
+ submap_entry->offset = 0;
}
local_start = local_vaddr -
(cow_parent_vaddr - old_start);
/* an entry in our space to the underlying */
/* object in the submap, bypassing the */
/* submap. */
+
+
+ if(submap_entry->wired_count != 0) {
+ vm_object_lock(
+ submap_entry->object.vm_object);
+ vm_object_copy_slowly(
+ submap_entry->object.vm_object,
+ submap_entry->offset,
+ submap_entry->vme_end -
+ submap_entry->vme_start,
+ FALSE,
+ ©_object);
+ copied_slowly = TRUE;
+ } else {
- /* set up shadow object */
- copy_object = submap_entry->object.vm_object;
- submap_entry->object.vm_object->shadowed = TRUE;
- submap_entry->needs_copy = TRUE;
- vm_object_pmap_protect(submap_entry->object.vm_object,
+ /* set up shadow object */
+ copy_object = submap_entry->object.vm_object;
+ vm_object_reference(copy_object);
+ submap_entry->object.vm_object->shadowed = TRUE;
+ submap_entry->needs_copy = TRUE;
+ vm_object_pmap_protect(
+ submap_entry->object.vm_object,
submap_entry->offset,
submap_entry->vme_end -
submap_entry->vme_start,
- submap_entry->is_shared ?
+ (submap_entry->is_shared
+ || map->mapped) ?
PMAP_NULL : map->pmap,
submap_entry->vme_start,
submap_entry->protection &
~VM_PROT_WRITE);
+ }
/* This works diffently than the */
/* shared map entry */
vm_map_deallocate(entry->object.sub_map);
entry->is_sub_map = FALSE;
- vm_object_reference(copy_object);
entry->object.vm_object = copy_object;
- entry->offset = submap_entry->offset;
entry->protection |= VM_PROT_WRITE;
entry->max_protection |= VM_PROT_WRITE;
- entry->needs_copy = TRUE;
+ if(copied_slowly) {
+ entry->offset = 0;
+ entry->needs_copy = FALSE;
+ entry->is_shared = FALSE;
+ } else {
+ entry->offset = submap_entry->offset;
+ entry->needs_copy = TRUE;
+ if(entry->inheritance == VM_INHERIT_SHARE)
+ entry->inheritance = VM_INHERIT_COPY;
+ if (map != old_map)
+ entry->is_shared = TRUE;
+ }
if(entry->inheritance == VM_INHERIT_SHARE)
- entry->inheritance = VM_INHERIT_COPY;
- if (map != old_map)
- entry->is_shared = TRUE;
+ entry->inheritance = VM_INHERIT_COPY;
vm_map_lock_write_to_read(map);
} else {
if((cow_sub_map_parent)
- && (cow_sub_map_parent != *pmap_map)
+ && (cow_sub_map_parent != *real_map)
&& (cow_sub_map_parent != map)) {
vm_map_unlock(cow_sub_map_parent);
}
prot = entry->protection;
if ((fault_type & (prot)) != fault_type) {
- if (*pmap_map != map) {
- vm_map_unlock(*pmap_map);
+ if (*real_map != map) {
+ vm_map_unlock(*real_map);
}
- *pmap_map = map;
+ *real_map = map;
return KERN_PROTECTION_FAILURE;
}
* it for all possible accesses.
*/
- if (*wired = (entry->wired_count != 0))
+ *wired = (entry->wired_count != 0);
+ if (*wired)
prot = fault_type = entry->protection;
/*
* demote the permissions allowed.
*/
- if (fault_type & VM_PROT_WRITE || *wired) {
+ if ((fault_type & VM_PROT_WRITE) || *wired) {
/*
* Make a new object, and place it in the
* object chain. Note that no new references
}
vm_object_shadow(&entry->object.vm_object,
&entry->offset,
- (vm_size_t) (entry->vme_end -
+ (vm_map_size_t) (entry->vme_end -
entry->vme_start));
entry->object.vm_object->shadowed = TRUE;
}
entry->object.vm_object = vm_object_allocate(
- (vm_size_t)(entry->vme_end - entry->vme_start));
+ (vm_map_size_t)(entry->vme_end - entry->vme_start));
entry->offset = 0;
vm_map_lock_write_to_read(map);
}
/*
- * vm_region:
- *
- * User call to obtain information about a region in
- * a task's address map. Currently, only one flavor is
- * supported.
- *
- * XXX The reserved and behavior fields cannot be filled
- * in until the vm merge from the IK is completed, and
- * vm_reserve is implemented.
+ * TEMPORARYTEMPORARYTEMPORARYTEMPORARYTEMPORARYTEMPORARY
+ * Goes away after regular vm_region_recurse function migrates to
+ * 64 bits
+ * vm_region_recurse: A form of vm_region which follows the
+ * submaps in a target map
*
- * XXX Dependency: syscall_vm_region() also supports only one flavor.
*/
kern_return_t
-vm_region(
+vm_map_region_recurse_64(
vm_map_t map,
- vm_offset_t *address, /* IN/OUT */
- vm_size_t *size, /* OUT */
- vm_region_flavor_t flavor, /* IN */
- vm_region_info_t info, /* OUT */
- mach_msg_type_number_t *count, /* IN/OUT */
- ipc_port_t *object_name) /* OUT */
+ vm_map_offset_t *address, /* IN/OUT */
+ vm_map_size_t *size, /* OUT */
+ natural_t *nesting_depth, /* IN/OUT */
+ vm_region_submap_info_64_t submap_info, /* IN/OUT */
+ mach_msg_type_number_t *count) /* IN/OUT */
{
- vm_map_entry_t tmp_entry;
- register
- vm_map_entry_t entry;
- register
- vm_offset_t start;
- vm_region_basic_info_t basic;
- vm_region_extended_info_t extended;
- vm_region_top_info_t top;
+ vm_region_extended_info_data_t extended;
+ vm_map_entry_t tmp_entry;
+ vm_map_offset_t user_address;
+ unsigned int user_max_depth;
- if (map == VM_MAP_NULL)
- return(KERN_INVALID_ARGUMENT);
+ /*
+ * "curr_entry" is the VM map entry preceding or including the
+ * address we're looking for.
+ * "curr_map" is the map or sub-map containing "curr_entry".
+ * "curr_offset" is the cumulated offset of "curr_map" in the
+ * target task's address space.
+ * "curr_depth" is the depth of "curr_map" in the chain of
+ * sub-maps.
+ * "curr_max_offset" is the maximum offset we should take into
+ * account in the current map. It may be smaller than the current
+ * map's "max_offset" because we might not have mapped it all in
+ * the upper level map.
+ */
+ vm_map_entry_t curr_entry;
+ vm_map_offset_t curr_offset;
+ vm_map_t curr_map;
+ unsigned int curr_depth;
+ vm_map_offset_t curr_max_offset;
- switch (flavor) {
-
- case VM_REGION_BASIC_INFO:
- {
- if (*count < VM_REGION_BASIC_INFO_COUNT)
- return(KERN_INVALID_ARGUMENT);
+ /*
+ * "next_" is the same as "curr_" but for the VM region immediately
+ * after the address we're looking for. We need to keep track of this
+ * too because we want to return info about that region if the
+ * address we're looking for is not mapped.
+ */
+ vm_map_entry_t next_entry;
+ vm_map_offset_t next_offset;
+ vm_map_t next_map;
+ unsigned int next_depth;
+ vm_map_offset_t next_max_offset;
+
+ if (map == VM_MAP_NULL) {
+ /* no address space to work on */
+ return KERN_INVALID_ARGUMENT;
+ }
- basic = (vm_region_basic_info_t) info;
- *count = VM_REGION_BASIC_INFO_COUNT;
+ if (*count < VM_REGION_SUBMAP_INFO_COUNT_64) {
+ /* "info" structure is not big enough and would overflow */
+ return KERN_INVALID_ARGUMENT;
+ }
- vm_map_lock_read(map);
+ *count = VM_REGION_SUBMAP_INFO_COUNT_64;
- start = *address;
+ user_address = *address;
+ user_max_depth = *nesting_depth;
+
+ curr_entry = NULL;
+ curr_map = map;
+ curr_offset = 0;
+ curr_depth = 0;
+ curr_max_offset = curr_map->max_offset;
+
+ next_entry = NULL;
+ next_map = NULL;
+ next_offset = 0;
+ next_depth = 0;
+ next_max_offset = curr_max_offset;
+
+ if (not_in_kdp) {
+ vm_map_lock_read(curr_map);
+ }
+
+ for (;;) {
+ if (vm_map_lookup_entry(curr_map,
+ user_address - curr_offset,
+ &tmp_entry)) {
+ /* tmp_entry contains the address we're looking for */
+ curr_entry = tmp_entry;
+ } else {
+ /*
+ * The address is not mapped. "tmp_entry" is the
+ * map entry preceding the address. We want the next
+ * one, if it exists.
+ */
+ curr_entry = tmp_entry->vme_next;
+ if (curr_entry == vm_map_to_entry(curr_map) ||
+ curr_entry->vme_start >= curr_max_offset) {
+ /* no next entry at this level: stop looking */
+ if (not_in_kdp) {
+ vm_map_unlock_read(curr_map);
+ }
+ curr_entry = NULL;
+ curr_map = NULL;
+ curr_offset = 0;
+ curr_depth = 0;
+ curr_max_offset = 0;
+ break;
+ }
+ }
+
+ /*
+ * Is the next entry at this level closer to the address (or
+ * deeper in the submap chain) than the one we had
+ * so far ?
+ */
+ tmp_entry = curr_entry->vme_next;
+ if (tmp_entry == vm_map_to_entry(curr_map)) {
+ /* no next entry at this level */
+ } else if (tmp_entry->vme_start >= curr_max_offset) {
+ /*
+ * tmp_entry is beyond the scope of what we mapped of
+ * this submap in the upper level: ignore it.
+ */
+ } else if ((next_entry == NULL) ||
+ (tmp_entry->vme_start + curr_offset <=
+ next_entry->vme_start + next_offset)) {
+ /*
+ * We didn't have a "next_entry" or this one is
+ * closer to the address we're looking for:
+ * use this "tmp_entry" as the new "next_entry".
+ */
+ if (next_entry != NULL) {
+ /* unlock the last "next_map" */
+ if (next_map != curr_map && not_in_kdp) {
+ vm_map_unlock_read(next_map);
+ }
+ }
+ next_entry = tmp_entry;
+ next_map = curr_map;
+ next_offset = curr_offset;
+ next_depth = curr_depth;
+ next_max_offset = curr_max_offset;
+ }
+
+ if (!curr_entry->is_sub_map ||
+ curr_depth >= user_max_depth) {
+ /*
+ * We hit a leaf map or we reached the maximum depth
+ * we could, so stop looking. Keep the current map
+ * locked.
+ */
+ break;
+ }
+
+ /*
+ * Get down to the next submap level.
+ */
+
+ /*
+ * Lock the next level and unlock the current level,
+ * unless we need to keep it locked to access the "next_entry"
+ * later.
+ */
+ if (not_in_kdp) {
+ vm_map_lock_read(curr_entry->object.sub_map);
+ }
+ if (curr_map == next_map) {
+ /* keep "next_map" locked in case we need it */
+ } else {
+ /* release this map */
+ vm_map_unlock_read(curr_map);
+ }
+
+ /*
+ * Adjust the offset. "curr_entry" maps the submap
+ * at relative address "curr_entry->vme_start" in the
+ * curr_map but skips the first "curr_entry->offset"
+ * bytes of the submap.
+ * "curr_offset" always represents the offset of a virtual
+ * address in the curr_map relative to the absolute address
+ * space (i.e. the top-level VM map).
+ */
+ curr_offset +=
+ (curr_entry->vme_start - curr_entry->offset);
+ /* switch to the submap */
+ curr_map = curr_entry->object.sub_map;
+ curr_depth++;
+ /*
+ * "curr_max_offset" allows us to keep track of the
+ * portion of the submap that is actually mapped at this level:
+ * the rest of that submap is irrelevant to us, since it's not
+ * mapped here.
+ * The relevant portion of the map starts at
+ * "curr_entry->offset" up to the size of "curr_entry".
+ */
+ curr_max_offset =
+ curr_entry->vme_end - curr_entry->vme_start +
+ curr_entry->offset;
+ curr_entry = NULL;
+ }
+
+ if (curr_entry == NULL) {
+ /* no VM region contains the address... */
+ if (next_entry == NULL) {
+ /* ... and no VM region follows it either */
+ return KERN_INVALID_ADDRESS;
+ }
+ /* ... gather info about the next VM region */
+ curr_entry = next_entry;
+ curr_map = next_map; /* still locked ... */
+ curr_offset = next_offset;
+ curr_depth = next_depth;
+ curr_max_offset = next_max_offset;
+ } else {
+ /* we won't need "next_entry" after all */
+ if (next_entry != NULL) {
+ /* release "next_map" */
+ if (next_map != curr_map && not_in_kdp) {
+ vm_map_unlock_read(next_map);
+ }
+ }
+ }
+ next_entry = NULL;
+ next_map = NULL;
+ next_offset = 0;
+ next_depth = 0;
+ next_max_offset = 0;
+
+ *nesting_depth = curr_depth;
+ *size = curr_entry->vme_end - curr_entry->vme_start;
+ *address = curr_entry->vme_start + curr_offset;
+
+ submap_info->user_tag = curr_entry->alias;
+ submap_info->offset = curr_entry->offset;
+ submap_info->protection = curr_entry->protection;
+ submap_info->inheritance = curr_entry->inheritance;
+ submap_info->max_protection = curr_entry->max_protection;
+ submap_info->behavior = curr_entry->behavior;
+ submap_info->user_wired_count = curr_entry->user_wired_count;
+ submap_info->is_submap = curr_entry->is_sub_map;
+ submap_info->object_id = (uint32_t) curr_entry->object.vm_object;
+
+ extended.pages_resident = 0;
+ extended.pages_swapped_out = 0;
+ extended.pages_shared_now_private = 0;
+ extended.pages_dirtied = 0;
+ extended.external_pager = 0;
+ extended.shadow_depth = 0;
+
+ if (not_in_kdp) {
+ if (!curr_entry->is_sub_map) {
+ vm_map_region_walk(curr_map,
+ curr_entry->vme_start,
+ curr_entry,
+ curr_entry->offset,
+ (curr_entry->vme_end -
+ curr_entry->vme_start),
+ &extended);
+ submap_info->share_mode = extended.share_mode;
+ if (extended.external_pager &&
+ extended.ref_count == 2 &&
+ extended.share_mode == SM_SHARED) {
+ submap_info->share_mode = SM_PRIVATE;
+ }
+ submap_info->ref_count = extended.ref_count;
+ } else {
+ if (curr_entry->use_pmap) {
+ submap_info->share_mode = SM_TRUESHARED;
+ } else {
+ submap_info->share_mode = SM_PRIVATE;
+ }
+ submap_info->ref_count =
+ curr_entry->object.sub_map->ref_count;
+ }
+ }
+
+ submap_info->pages_resident = extended.pages_resident;
+ submap_info->pages_swapped_out = extended.pages_swapped_out;
+ submap_info->pages_shared_now_private =
+ extended.pages_shared_now_private;
+ submap_info->pages_dirtied = extended.pages_dirtied;
+ submap_info->external_pager = extended.external_pager;
+ submap_info->shadow_depth = extended.shadow_depth;
+
+ if (not_in_kdp) {
+ vm_map_unlock_read(curr_map);
+ }
+
+ return KERN_SUCCESS;
+}
+
+/*
+ * vm_region:
+ *
+ * User call to obtain information about a region in
+ * a task's address map. Currently, only one flavor is
+ * supported.
+ *
+ * XXX The reserved and behavior fields cannot be filled
+ * in until the vm merge from the IK is completed, and
+ * vm_reserve is implemented.
+ */
+
+kern_return_t
+vm_map_region(
+ vm_map_t map,
+ vm_map_offset_t *address, /* IN/OUT */
+ vm_map_size_t *size, /* OUT */
+ vm_region_flavor_t flavor, /* IN */
+ vm_region_info_t info, /* OUT */
+ mach_msg_type_number_t *count, /* IN/OUT */
+ mach_port_t *object_name) /* OUT */
+{
+ vm_map_entry_t tmp_entry;
+ vm_map_entry_t entry;
+ vm_map_offset_t start;
+
+ if (map == VM_MAP_NULL)
+ return(KERN_INVALID_ARGUMENT);
+
+ switch (flavor) {
+
+ case VM_REGION_BASIC_INFO:
+ /* legacy for old 32-bit objects info */
+ {
+ vm_region_basic_info_t basic;
+
+ if (*count < VM_REGION_BASIC_INFO_COUNT)
+ return(KERN_INVALID_ARGUMENT);
+
+ basic = (vm_region_basic_info_t) info;
+ *count = VM_REGION_BASIC_INFO_COUNT;
+
+ vm_map_lock_read(map);
+
+ start = *address;
+ if (!vm_map_lookup_entry(map, start, &tmp_entry)) {
+ if ((entry = tmp_entry->vme_next) == vm_map_to_entry(map)) {
+ vm_map_unlock_read(map);
+ return(KERN_INVALID_ADDRESS);
+ }
+ } else {
+ entry = tmp_entry;
+ }
+
+ start = entry->vme_start;
+
+ basic->offset = (uint32_t)entry->offset;
+ basic->protection = entry->protection;
+ basic->inheritance = entry->inheritance;
+ basic->max_protection = entry->max_protection;
+ basic->behavior = entry->behavior;
+ basic->user_wired_count = entry->user_wired_count;
+ basic->reserved = entry->is_sub_map;
+ *address = start;
+ *size = (entry->vme_end - start);
+
+ if (object_name) *object_name = IP_NULL;
+ if (entry->is_sub_map) {
+ basic->shared = FALSE;
+ } else {
+ basic->shared = entry->is_shared;
+ }
+
+ vm_map_unlock_read(map);
+ return(KERN_SUCCESS);
+ }
+
+ case VM_REGION_BASIC_INFO_64:
+ {
+ vm_region_basic_info_64_t basic;
+
+ if (*count < VM_REGION_BASIC_INFO_COUNT_64)
+ return(KERN_INVALID_ARGUMENT);
+
+ basic = (vm_region_basic_info_64_t) info;
+ *count = VM_REGION_BASIC_INFO_COUNT_64;
+
+ vm_map_lock_read(map);
+
+ start = *address;
if (!vm_map_lookup_entry(map, start, &tmp_entry)) {
if ((entry = tmp_entry->vme_next) == vm_map_to_entry(map)) {
vm_map_unlock_read(map);
}
case VM_REGION_EXTENDED_INFO:
{
+ vm_region_extended_info_t extended;
if (*count < VM_REGION_EXTENDED_INFO_COUNT)
return(KERN_INVALID_ARGUMENT);
extended->pages_resident = 0;
extended->pages_swapped_out = 0;
extended->pages_shared_now_private = 0;
- extended->pages_referenced = 0;
+ extended->pages_dirtied = 0;
extended->external_pager = 0;
extended->shadow_depth = 0;
- vm_region_walk(entry, extended, entry->offset, entry->vme_end - start, map, start);
+ vm_map_region_walk(map, start, entry, entry->offset, entry->vme_end - start, extended);
if (extended->external_pager && extended->ref_count == 2 && extended->share_mode == SM_SHARED)
extended->share_mode = SM_PRIVATE;
}
case VM_REGION_TOP_INFO:
{
+ vm_region_top_info_t top;
if (*count < VM_REGION_TOP_INFO_COUNT)
return(KERN_INVALID_ARGUMENT);
top->private_pages_resident = 0;
top->shared_pages_resident = 0;
- vm_region_top_walk(entry, top);
+ vm_map_region_top_walk(entry, top);
if (object_name)
*object_name = IP_NULL;
}
}
-/*
- * vm_region_recurse: A form of vm_region which follows the
- * submaps in a target map
- *
- */
-
-kern_return_t
-vm_region_recurse(
- vm_map_t map,
- vm_offset_t *address, /* IN/OUT */
- vm_size_t *size, /* OUT */
- natural_t *nesting_depth, /* IN/OUT */
- vm_region_recurse_info_t info, /* IN/OUT */
- mach_msg_type_number_t *count) /* IN/OUT */
+static void
+vm_map_region_top_walk(
+ vm_map_entry_t entry,
+ vm_region_top_info_t top)
{
- vm_map_entry_t tmp_entry;
- register
- vm_map_entry_t entry;
- register
- vm_offset_t start;
-
- unsigned int recurse_count;
- vm_map_t submap;
- vm_map_t base_map;
- vm_map_entry_t base_entry;
- vm_offset_t base_next;
- vm_offset_t base_addr;
- vm_offset_t baddr_start_delta;
- vm_region_submap_info_t submap_info;
- vm_region_extended_info_data_t extended;
-
- if (map == VM_MAP_NULL)
- return(KERN_INVALID_ARGUMENT);
-
- submap_info = (vm_region_submap_info_t) info;
- *count = VM_REGION_SUBMAP_INFO_COUNT;
-
- if (*count < VM_REGION_SUBMAP_INFO_COUNT)
- return(KERN_INVALID_ARGUMENT);
-
- start = *address;
- base_map = map;
- recurse_count = *nesting_depth;
+ register struct vm_object *obj, *tmp_obj;
+ register int ref_count;
-LOOKUP_NEXT_BASE_ENTRY:
- vm_map_lock_read(map);
- if (!vm_map_lookup_entry(map, start, &tmp_entry)) {
- if ((entry = tmp_entry->vme_next) == vm_map_to_entry(map)) {
- vm_map_unlock_read(map);
- return(KERN_INVALID_ADDRESS);
- }
- } else {
- entry = tmp_entry;
+ if (entry->object.vm_object == 0 || entry->is_sub_map) {
+ top->share_mode = SM_EMPTY;
+ top->ref_count = 0;
+ top->obj_id = 0;
+ return;
}
- *size = entry->vme_end - entry->vme_start;
- start = entry->vme_start;
- base_addr = start;
- baddr_start_delta = *address - start;
- base_next = entry->vme_end;
- base_entry = entry;
-
- while(entry->is_sub_map && recurse_count) {
- recurse_count--;
- vm_map_lock_read(entry->object.sub_map);
+ {
+ obj = entry->object.vm_object;
+ vm_object_lock(obj);
- if(entry == base_entry) {
- start = entry->offset;
- start += *address - entry->vme_start;
- }
+ if ((ref_count = obj->ref_count) > 1 && obj->paging_in_progress)
+ ref_count--;
- submap = entry->object.sub_map;
- vm_map_unlock_read(map);
- map = submap;
+ if (obj->shadow) {
+ if (ref_count == 1)
+ top->private_pages_resident = obj->resident_page_count;
+ else
+ top->shared_pages_resident = obj->resident_page_count;
+ top->ref_count = ref_count;
+ top->share_mode = SM_COW;
+
+ while ((tmp_obj = obj->shadow)) {
+ vm_object_lock(tmp_obj);
+ vm_object_unlock(obj);
+ obj = tmp_obj;
- if (!vm_map_lookup_entry(map, start, &tmp_entry)) {
- if ((entry = tmp_entry->vme_next)
- == vm_map_to_entry(map)) {
- vm_map_unlock_read(map);
- map = base_map;
- start = base_next;
- recurse_count = 0;
- *nesting_depth = 0;
- goto LOOKUP_NEXT_BASE_ENTRY;
- }
- } else {
- entry = tmp_entry;
+ if ((ref_count = obj->ref_count) > 1 && obj->paging_in_progress)
+ ref_count--;
+ top->shared_pages_resident += obj->resident_page_count;
+ top->ref_count += ref_count - 1;
}
- if(start <= entry->vme_start) {
- vm_offset_t old_start = start;
- if(baddr_start_delta) {
- base_addr += (baddr_start_delta);
- *size -= baddr_start_delta;
- baddr_start_delta = 0;
- }
- if(base_next <=
- (base_addr += (entry->vme_start - start))) {
- vm_map_unlock_read(map);
- map = base_map;
- start = base_next;
- recurse_count = 0;
- *nesting_depth = 0;
- goto LOOKUP_NEXT_BASE_ENTRY;
- }
- *size -= entry->vme_start - start;
- if (*size > (entry->vme_end - entry->vme_start)) {
- *size = entry->vme_end - entry->vme_start;
- }
- start = 0;
+ } else {
+ if (entry->needs_copy) {
+ top->share_mode = SM_COW;
+ top->shared_pages_resident = obj->resident_page_count;
} else {
- if(baddr_start_delta) {
- if((start - entry->vme_start)
- < baddr_start_delta) {
- base_addr += start - entry->vme_start;
- *size -= start - entry->vme_start;
- } else {
- base_addr += baddr_start_delta;
- *size += baddr_start_delta;
- }
- baddr_start_delta = 0;
- }
- base_addr += entry->vme_start;
- if(base_addr >= base_next) {
- vm_map_unlock_read(map);
- map = base_map;
- start = base_next;
- recurse_count = 0;
- *nesting_depth = 0;
- goto LOOKUP_NEXT_BASE_ENTRY;
- }
- if (*size > (entry->vme_end - start))
- *size = entry->vme_end - start;
-
- start = entry->vme_start - start;
+ if (ref_count == 1 ||
+ (ref_count == 2 && !(obj->pager_trusted) && !(obj->internal))) {
+ top->share_mode = SM_PRIVATE;
+ top->private_pages_resident = obj->resident_page_count;
+ } else {
+ top->share_mode = SM_SHARED;
+ top->shared_pages_resident = obj->resident_page_count;
+ }
}
+ top->ref_count = ref_count;
+ }
+ top->obj_id = (int)obj;
- start += entry->offset;
-
- }
- *nesting_depth -= recurse_count;
- if(entry != base_entry) {
- start = entry->vme_start + (start - entry->offset);
+ vm_object_unlock(obj);
}
+}
+static void
+vm_map_region_walk(
+ vm_map_t map,
+ vm_map_offset_t va,
+ vm_map_entry_t entry,
+ vm_object_offset_t offset,
+ vm_object_size_t range,
+ vm_region_extended_info_t extended)
+{
+ register struct vm_object *obj, *tmp_obj;
+ register vm_map_offset_t last_offset;
+ register int i;
+ register int ref_count;
+ struct vm_object *shadow_object;
+ int shadow_depth;
- submap_info->user_tag = entry->alias;
- submap_info->offset = entry->offset;
- submap_info->protection = entry->protection;
- submap_info->inheritance = entry->inheritance;
- submap_info->max_protection = entry->max_protection;
- submap_info->behavior = entry->behavior;
- submap_info->user_wired_count = entry->user_wired_count;
- submap_info->is_submap = entry->is_sub_map;
- submap_info->object_id = (vm_offset_t)entry->object.vm_object;
- *address = base_addr;
+ if ((entry->object.vm_object == 0) ||
+ (entry->is_sub_map) ||
+ (entry->object.vm_object->phys_contiguous)) {
+ extended->share_mode = SM_EMPTY;
+ extended->ref_count = 0;
+ return;
+ }
+ {
+ obj = entry->object.vm_object;
+ vm_object_lock(obj);
- extended.pages_resident = 0;
- extended.pages_swapped_out = 0;
- extended.pages_shared_now_private = 0;
- extended.pages_referenced = 0;
- extended.external_pager = 0;
- extended.shadow_depth = 0;
+ if ((ref_count = obj->ref_count) > 1 && obj->paging_in_progress)
+ ref_count--;
- if(!entry->is_sub_map) {
- vm_region_walk(entry, &extended, entry->offset,
- entry->vme_end - start, map, start);
- submap_info->share_mode = extended.share_mode;
- if (extended.external_pager && extended.ref_count == 2
- && extended.share_mode == SM_SHARED)
- submap_info->share_mode = SM_PRIVATE;
- submap_info->ref_count = extended.ref_count;
- } else {
- if(entry->use_pmap)
- submap_info->share_mode = SM_TRUESHARED;
- else
- submap_info->share_mode = SM_PRIVATE;
- submap_info->ref_count = entry->object.sub_map->ref_count;
- }
+ for (last_offset = offset + range; offset < last_offset; offset += PAGE_SIZE_64, va += PAGE_SIZE)
+ vm_map_region_look_for_page(map, va, obj, offset, ref_count, 0, extended);
+
+ shadow_object = obj->shadow;
+ shadow_depth = 0;
+ if (shadow_object != VM_OBJECT_NULL) {
+ vm_object_lock(shadow_object);
+ for (;
+ shadow_object != VM_OBJECT_NULL;
+ shadow_depth++) {
+ vm_object_t next_shadow;
+
+ next_shadow = shadow_object->shadow;
+ if (next_shadow) {
+ vm_object_lock(next_shadow);
+ }
+ vm_object_unlock(shadow_object);
+ shadow_object = next_shadow;
+ }
+ }
+ extended->shadow_depth = shadow_depth;
- submap_info->pages_resident = extended.pages_resident;
- submap_info->pages_swapped_out = extended.pages_swapped_out;
- submap_info->pages_shared_now_private =
- extended.pages_shared_now_private;
- submap_info->pages_referenced = extended.pages_referenced;
- submap_info->external_pager = extended.external_pager;
- submap_info->shadow_depth = extended.shadow_depth;
+ if (extended->shadow_depth || entry->needs_copy)
+ extended->share_mode = SM_COW;
+ else {
+ if (ref_count == 1)
+ extended->share_mode = SM_PRIVATE;
+ else {
+ if (obj->true_share)
+ extended->share_mode = SM_TRUESHARED;
+ else
+ extended->share_mode = SM_SHARED;
+ }
+ }
+ extended->ref_count = ref_count - extended->shadow_depth;
+
+ for (i = 0; i < extended->shadow_depth; i++) {
+ if ((tmp_obj = obj->shadow) == 0)
+ break;
+ vm_object_lock(tmp_obj);
+ vm_object_unlock(obj);
- vm_map_unlock_read(map);
- return(KERN_SUCCESS);
+ if ((ref_count = tmp_obj->ref_count) > 1 && tmp_obj->paging_in_progress)
+ ref_count--;
+
+ extended->ref_count += ref_count;
+ obj = tmp_obj;
+ }
+ vm_object_unlock(obj);
+
+ if (extended->share_mode == SM_SHARED) {
+ register vm_map_entry_t cur;
+ register vm_map_entry_t last;
+ int my_refs;
+
+ obj = entry->object.vm_object;
+ last = vm_map_to_entry(map);
+ my_refs = 0;
+
+ if ((ref_count = obj->ref_count) > 1 && obj->paging_in_progress)
+ ref_count--;
+ for (cur = vm_map_first_entry(map); cur != last; cur = cur->vme_next)
+ my_refs += vm_map_region_count_obj_refs(cur, obj);
+
+ if (my_refs == ref_count)
+ extended->share_mode = SM_PRIVATE_ALIASED;
+ else if (my_refs > 1)
+ extended->share_mode = SM_SHARED_ALIASED;
+ }
+ }
}
-/*
- * TEMPORARYTEMPORARYTEMPORARYTEMPORARYTEMPORARYTEMPORARY
- * Goes away after regular vm_region_recurse function migrates to
- * 64 bits
- * vm_region_recurse: A form of vm_region which follows the
- * submaps in a target map
- *
- */
-kern_return_t
-vm_region_recurse_64(
- vm_map_t map,
- vm_offset_t *address, /* IN/OUT */
- vm_size_t *size, /* OUT */
- natural_t *nesting_depth, /* IN/OUT */
- vm_region_recurse_info_t info, /* IN/OUT */
- mach_msg_type_number_t *count) /* IN/OUT */
-{
- vm_map_entry_t tmp_entry;
- register
- vm_map_entry_t entry;
- register
- vm_offset_t start;
-
- unsigned int recurse_count;
- vm_map_t submap;
- vm_map_t base_map;
- vm_map_entry_t base_entry;
- vm_offset_t base_next;
- vm_offset_t base_addr;
- vm_offset_t baddr_start_delta;
- vm_region_submap_info_64_t submap_info;
- vm_region_extended_info_data_t extended;
+/* object is locked on entry and locked on return */
- if (map == VM_MAP_NULL)
- return(KERN_INVALID_ARGUMENT);
- submap_info = (vm_region_submap_info_64_t) info;
- *count = VM_REGION_SUBMAP_INFO_COUNT;
+static void
+vm_map_region_look_for_page(
+ __unused vm_map_t map,
+ __unused vm_map_offset_t va,
+ vm_object_t object,
+ vm_object_offset_t offset,
+ int max_refcnt,
+ int depth,
+ vm_region_extended_info_t extended)
+{
+ register vm_page_t p;
+ register vm_object_t shadow;
+ register int ref_count;
+ vm_object_t caller_object;
+
+ shadow = object->shadow;
+ caller_object = object;
+
+
+ while (TRUE) {
- if (*count < VM_REGION_SUBMAP_INFO_COUNT)
- return(KERN_INVALID_ARGUMENT);
+ if ( !(object->pager_trusted) && !(object->internal))
+ extended->external_pager = 1;
- start = *address;
- base_map = map;
- recurse_count = *nesting_depth;
+ if ((p = vm_page_lookup(object, offset)) != VM_PAGE_NULL) {
+ if (shadow && (max_refcnt == 1))
+ extended->pages_shared_now_private++;
-LOOKUP_NEXT_BASE_ENTRY:
- vm_map_lock_read(map);
- if (!vm_map_lookup_entry(map, start, &tmp_entry)) {
- if ((entry = tmp_entry->vme_next) == vm_map_to_entry(map)) {
- vm_map_unlock_read(map);
- return(KERN_INVALID_ADDRESS);
- }
- } else {
- entry = tmp_entry;
- }
- *size = entry->vme_end - entry->vme_start;
- start = entry->vme_start;
- base_addr = start;
- baddr_start_delta = *address - start;
- base_next = entry->vme_end;
- base_entry = entry;
+ if (!p->fictitious &&
+ (p->dirty || pmap_is_modified(p->phys_page)))
+ extended->pages_dirtied++;
- while(entry->is_sub_map && recurse_count) {
- recurse_count--;
- vm_map_lock_read(entry->object.sub_map);
+ extended->pages_resident++;
+ if(object != caller_object)
+ vm_object_unlock(object);
- if(entry == base_entry) {
- start = entry->offset;
- start += *address - entry->vme_start;
+ return;
}
+ if (object->existence_map) {
+ if (vm_external_state_get(object->existence_map, offset) == VM_EXTERNAL_STATE_EXISTS) {
- submap = entry->object.sub_map;
- vm_map_unlock_read(map);
- map = submap;
+ extended->pages_swapped_out++;
- if (!vm_map_lookup_entry(map, start, &tmp_entry)) {
- if ((entry = tmp_entry->vme_next)
- == vm_map_to_entry(map)) {
- vm_map_unlock_read(map);
- map = base_map;
- start = base_next;
- recurse_count = 0;
- *nesting_depth = 0;
- goto LOOKUP_NEXT_BASE_ENTRY;
- }
- } else {
- entry = tmp_entry;
+ if(object != caller_object)
+ vm_object_unlock(object);
+ return;
+ }
}
- if(start <= entry->vme_start) {
- vm_offset_t old_start = start;
- if(baddr_start_delta) {
- base_addr += (baddr_start_delta);
- *size -= baddr_start_delta;
- baddr_start_delta = 0;
- }
- if(base_next <=
- (base_addr += (entry->vme_start - start))) {
- vm_map_unlock_read(map);
- map = base_map;
- start = base_next;
- recurse_count = 0;
- *nesting_depth = 0;
- goto LOOKUP_NEXT_BASE_ENTRY;
- }
- *size -= entry->vme_start - start;
- if (*size > (entry->vme_end - entry->vme_start)) {
- *size = entry->vme_end - entry->vme_start;
- }
- start = 0;
- } else {
- if(baddr_start_delta) {
- if((start - entry->vme_start)
- < baddr_start_delta) {
- base_addr += start - entry->vme_start;
- *size -= start - entry->vme_start;
- } else {
- base_addr += baddr_start_delta;
- *size += baddr_start_delta;
- }
- baddr_start_delta = 0;
- }
- base_addr += entry->vme_start;
- if(base_addr >= base_next) {
- vm_map_unlock_read(map);
- map = base_map;
- start = base_next;
- recurse_count = 0;
- *nesting_depth = 0;
- goto LOOKUP_NEXT_BASE_ENTRY;
- }
- if (*size > (entry->vme_end - start))
- *size = entry->vme_end - start;
+ if (shadow) {
+ vm_object_lock(shadow);
- start = entry->vme_start - start;
- }
+ if ((ref_count = shadow->ref_count) > 1 && shadow->paging_in_progress)
+ ref_count--;
- start += entry->offset;
+ if (++depth > extended->shadow_depth)
+ extended->shadow_depth = depth;
- }
- *nesting_depth -= recurse_count;
- if(entry != base_entry) {
- start = entry->vme_start + (start - entry->offset);
- }
+ if (ref_count > max_refcnt)
+ max_refcnt = ref_count;
+
+ if(object != caller_object)
+ vm_object_unlock(object);
+ offset = offset + object->shadow_offset;
+ object = shadow;
+ shadow = object->shadow;
+ continue;
+ }
+ if(object != caller_object)
+ vm_object_unlock(object);
+ break;
+ }
+}
- submap_info->user_tag = entry->alias;
- submap_info->offset = entry->offset;
- submap_info->protection = entry->protection;
- submap_info->inheritance = entry->inheritance;
- submap_info->max_protection = entry->max_protection;
- submap_info->behavior = entry->behavior;
- submap_info->user_wired_count = entry->user_wired_count;
- submap_info->is_submap = entry->is_sub_map;
- submap_info->object_id = (vm_offset_t)entry->object.vm_object;
- *address = base_addr;
+static int
+vm_map_region_count_obj_refs(
+ vm_map_entry_t entry,
+ vm_object_t object)
+{
+ register int ref_count;
+ register vm_object_t chk_obj;
+ register vm_object_t tmp_obj;
+ if (entry->object.vm_object == 0)
+ return(0);
- extended.pages_resident = 0;
- extended.pages_swapped_out = 0;
- extended.pages_shared_now_private = 0;
- extended.pages_referenced = 0;
- extended.external_pager = 0;
- extended.shadow_depth = 0;
+ if (entry->is_sub_map)
+ return(0);
+ else {
+ ref_count = 0;
- if(!entry->is_sub_map) {
- vm_region_walk(entry, &extended, entry->offset,
- entry->vme_end - start, map, start);
- submap_info->share_mode = extended.share_mode;
- if (extended.external_pager && extended.ref_count == 2
- && extended.share_mode == SM_SHARED)
- submap_info->share_mode = SM_PRIVATE;
- submap_info->ref_count = extended.ref_count;
- } else {
- if(entry->use_pmap)
- submap_info->share_mode = SM_TRUESHARED;
- else
- submap_info->share_mode = SM_PRIVATE;
- submap_info->ref_count = entry->object.sub_map->ref_count;
- }
+ chk_obj = entry->object.vm_object;
+ vm_object_lock(chk_obj);
- submap_info->pages_resident = extended.pages_resident;
- submap_info->pages_swapped_out = extended.pages_swapped_out;
- submap_info->pages_shared_now_private =
- extended.pages_shared_now_private;
- submap_info->pages_referenced = extended.pages_referenced;
- submap_info->external_pager = extended.external_pager;
- submap_info->shadow_depth = extended.shadow_depth;
+ while (chk_obj) {
+ if (chk_obj == object)
+ ref_count++;
+ tmp_obj = chk_obj->shadow;
+ if (tmp_obj)
+ vm_object_lock(tmp_obj);
+ vm_object_unlock(chk_obj);
- vm_map_unlock_read(map);
- return(KERN_SUCCESS);
+ chk_obj = tmp_obj;
+ }
+ }
+ return(ref_count);
}
/*
- * TEMPORARYTEMPORARYTEMPORARYTEMPORARYTEMPORARYTEMPORARY
- * Goes away after regular vm_region function migrates to
- * 64 bits
+ * Routine: vm_map_simplify
+ *
+ * Description:
+ * Attempt to simplify the map representation in
+ * the vicinity of the given starting address.
+ * Note:
+ * This routine is intended primarily to keep the
+ * kernel maps more compact -- they generally don't
+ * benefit from the "expand a map entry" technology
+ * at allocation time because the adjacent entry
+ * is often wired down.
*/
+void
+vm_map_simplify_entry(
+ vm_map_t map,
+ vm_map_entry_t this_entry)
+{
+ vm_map_entry_t prev_entry;
+ counter(c_vm_map_simplify_entry_called++);
+
+ prev_entry = this_entry->vme_prev;
+
+ if ((this_entry != vm_map_to_entry(map)) &&
+ (prev_entry != vm_map_to_entry(map)) &&
+
+ (prev_entry->vme_end == this_entry->vme_start) &&
+
+ (prev_entry->is_sub_map == FALSE) &&
+ (this_entry->is_sub_map == FALSE) &&
+
+ (prev_entry->object.vm_object == this_entry->object.vm_object) &&
+ ((prev_entry->offset + (prev_entry->vme_end -
+ prev_entry->vme_start))
+ == this_entry->offset) &&
+
+ (prev_entry->inheritance == this_entry->inheritance) &&
+ (prev_entry->protection == this_entry->protection) &&
+ (prev_entry->max_protection == this_entry->max_protection) &&
+ (prev_entry->behavior == this_entry->behavior) &&
+ (prev_entry->alias == this_entry->alias) &&
+ (prev_entry->wired_count == this_entry->wired_count) &&
+ (prev_entry->user_wired_count == this_entry->user_wired_count) &&
+
+ (prev_entry->needs_copy == this_entry->needs_copy) &&
+
+ (prev_entry->use_pmap == FALSE) &&
+ (this_entry->use_pmap == FALSE) &&
+ (prev_entry->in_transition == FALSE) &&
+ (this_entry->in_transition == FALSE) &&
+ (prev_entry->needs_wakeup == FALSE) &&
+ (this_entry->needs_wakeup == FALSE) &&
+ (prev_entry->is_shared == FALSE) &&
+ (this_entry->is_shared == FALSE)
+ ) {
+ _vm_map_entry_unlink(&map->hdr, prev_entry);
+ this_entry->vme_start = prev_entry->vme_start;
+ this_entry->offset = prev_entry->offset;
+ vm_object_deallocate(prev_entry->object.vm_object);
+ vm_map_entry_dispose(map, prev_entry);
+ SAVE_HINT(map, this_entry);
+ counter(c_vm_map_simplified++);
+ }
+}
-kern_return_t
-vm_region_64(
- vm_map_t map,
- vm_offset_t *address, /* IN/OUT */
- vm_size_t *size, /* OUT */
- vm_region_flavor_t flavor, /* IN */
- vm_region_info_t info, /* OUT */
- mach_msg_type_number_t *count, /* IN/OUT */
- ipc_port_t *object_name) /* OUT */
+void
+vm_map_simplify(
+ vm_map_t map,
+ vm_map_offset_t start)
{
- vm_map_entry_t tmp_entry;
- register
- vm_map_entry_t entry;
- register
- vm_offset_t start;
- vm_region_basic_info_64_t basic;
- vm_region_extended_info_t extended;
- vm_region_top_info_t top;
+ vm_map_entry_t this_entry;
- if (map == VM_MAP_NULL)
- return(KERN_INVALID_ARGUMENT);
+ vm_map_lock(map);
+ if (vm_map_lookup_entry(map, start, &this_entry)) {
+ vm_map_simplify_entry(map, this_entry);
+ vm_map_simplify_entry(map, this_entry->vme_next);
+ }
+ counter(c_vm_map_simplify_called++);
+ vm_map_unlock(map);
+}
- switch (flavor) {
-
- case VM_REGION_BASIC_INFO:
- {
- if (*count < VM_REGION_BASIC_INFO_COUNT)
- return(KERN_INVALID_ARGUMENT);
+static void
+vm_map_simplify_range(
+ vm_map_t map,
+ vm_map_offset_t start,
+ vm_map_offset_t end)
+{
+ vm_map_entry_t entry;
- basic = (vm_region_basic_info_64_t) info;
- *count = VM_REGION_BASIC_INFO_COUNT;
+ /*
+ * The map should be locked (for "write") by the caller.
+ */
- vm_map_lock_read(map);
+ if (start >= end) {
+ /* invalid address range */
+ return;
+ }
- start = *address;
- if (!vm_map_lookup_entry(map, start, &tmp_entry)) {
- if ((entry = tmp_entry->vme_next) == vm_map_to_entry(map)) {
- vm_map_unlock_read(map);
- return(KERN_INVALID_ADDRESS);
+ if (!vm_map_lookup_entry(map, start, &entry)) {
+ /* "start" is not mapped and "entry" ends before "start" */
+ if (entry == vm_map_to_entry(map)) {
+ /* start with first entry in the map */
+ entry = vm_map_first_entry(map);
+ } else {
+ /* start with next entry */
+ entry = entry->vme_next;
}
- } else {
- entry = tmp_entry;
- }
+ }
+
+ while (entry != vm_map_to_entry(map) &&
+ entry->vme_start <= end) {
+ /* try and coalesce "entry" with its previous entry */
+ vm_map_simplify_entry(map, entry);
+ entry = entry->vme_next;
+ }
+}
- start = entry->vme_start;
- basic->offset = entry->offset;
- basic->protection = entry->protection;
- basic->inheritance = entry->inheritance;
- basic->max_protection = entry->max_protection;
- basic->behavior = entry->behavior;
- basic->user_wired_count = entry->user_wired_count;
- basic->reserved = entry->is_sub_map;
- *address = start;
- *size = (entry->vme_end - start);
+/*
+ * Routine: vm_map_machine_attribute
+ * Purpose:
+ * Provide machine-specific attributes to mappings,
+ * such as cachability etc. for machines that provide
+ * them. NUMA architectures and machines with big/strange
+ * caches will use this.
+ * Note:
+ * Responsibilities for locking and checking are handled here,
+ * everything else in the pmap module. If any non-volatile
+ * information must be kept, the pmap module should handle
+ * it itself. [This assumes that attributes do not
+ * need to be inherited, which seems ok to me]
+ */
+kern_return_t
+vm_map_machine_attribute(
+ vm_map_t map,
+ vm_map_offset_t start,
+ vm_map_offset_t end,
+ vm_machine_attribute_t attribute,
+ vm_machine_attribute_val_t* value) /* IN/OUT */
+{
+ kern_return_t ret;
+ vm_map_size_t sync_size;
+ vm_map_entry_t entry;
+
+ if (start < vm_map_min(map) || end > vm_map_max(map))
+ return KERN_INVALID_ADDRESS;
- if (object_name) *object_name = IP_NULL;
- if (entry->is_sub_map) {
- basic->shared = FALSE;
- } else {
- basic->shared = entry->is_shared;
- }
+ /* Figure how much memory we need to flush (in page increments) */
+ sync_size = end - start;
- vm_map_unlock_read(map);
- return(KERN_SUCCESS);
+ vm_map_lock(map);
+
+ if (attribute != MATTR_CACHE) {
+ /* If we don't have to find physical addresses, we */
+ /* don't have to do an explicit traversal here. */
+ ret = pmap_attribute(map->pmap, start, end-start,
+ attribute, value);
+ vm_map_unlock(map);
+ return ret;
}
- case VM_REGION_EXTENDED_INFO:
- {
- if (*count < VM_REGION_EXTENDED_INFO_COUNT)
- return(KERN_INVALID_ARGUMENT);
+ ret = KERN_SUCCESS; /* Assume it all worked */
- extended = (vm_region_extended_info_t) info;
- *count = VM_REGION_EXTENDED_INFO_COUNT;
+ while(sync_size) {
+ if (vm_map_lookup_entry(map, start, &entry)) {
+ vm_map_size_t sub_size;
+ if((entry->vme_end - start) > sync_size) {
+ sub_size = sync_size;
+ sync_size = 0;
+ } else {
+ sub_size = entry->vme_end - start;
+ sync_size -= sub_size;
+ }
+ if(entry->is_sub_map) {
+ vm_map_offset_t sub_start;
+ vm_map_offset_t sub_end;
+
+ sub_start = (start - entry->vme_start)
+ + entry->offset;
+ sub_end = sub_start + sub_size;
+ vm_map_machine_attribute(
+ entry->object.sub_map,
+ sub_start,
+ sub_end,
+ attribute, value);
+ } else {
+ if(entry->object.vm_object) {
+ vm_page_t m;
+ vm_object_t object;
+ vm_object_t base_object;
+ vm_object_t last_object;
+ vm_object_offset_t offset;
+ vm_object_offset_t base_offset;
+ vm_map_size_t range;
+ range = sub_size;
+ offset = (start - entry->vme_start)
+ + entry->offset;
+ base_offset = offset;
+ object = entry->object.vm_object;
+ base_object = object;
+ last_object = NULL;
- vm_map_lock_read(map);
+ vm_object_lock(object);
- start = *address;
- if (!vm_map_lookup_entry(map, start, &tmp_entry)) {
- if ((entry = tmp_entry->vme_next) == vm_map_to_entry(map)) {
- vm_map_unlock_read(map);
- return(KERN_INVALID_ADDRESS);
+ while (range) {
+ m = vm_page_lookup(
+ object, offset);
+
+ if (m && !m->fictitious) {
+ ret =
+ pmap_attribute_cache_sync(
+ m->phys_page,
+ PAGE_SIZE,
+ attribute, value);
+
+ } else if (object->shadow) {
+ offset = offset + object->shadow_offset;
+ last_object = object;
+ object = object->shadow;
+ vm_object_lock(last_object->shadow);
+ vm_object_unlock(last_object);
+ continue;
+ }
+ range -= PAGE_SIZE;
+
+ if (base_object != object) {
+ vm_object_unlock(object);
+ vm_object_lock(base_object);
+ object = base_object;
+ }
+ /* Bump to the next page */
+ base_offset += PAGE_SIZE;
+ offset = base_offset;
+ }
+ vm_object_unlock(object);
+ }
+ }
+ start += sub_size;
+ } else {
+ vm_map_unlock(map);
+ return KERN_FAILURE;
}
- } else {
- entry = tmp_entry;
- }
- start = entry->vme_start;
+
+ }
- extended->protection = entry->protection;
- extended->user_tag = entry->alias;
- extended->pages_resident = 0;
- extended->pages_swapped_out = 0;
- extended->pages_shared_now_private = 0;
- extended->pages_referenced = 0;
- extended->external_pager = 0;
- extended->shadow_depth = 0;
+ vm_map_unlock(map);
- vm_region_walk(entry, extended, entry->offset, entry->vme_end - start, map, start);
+ return ret;
+}
- if (extended->external_pager && extended->ref_count == 2 && extended->share_mode == SM_SHARED)
- extended->share_mode = SM_PRIVATE;
+/*
+ * vm_map_behavior_set:
+ *
+ * Sets the paging reference behavior of the specified address
+ * range in the target map. Paging reference behavior affects
+ * how pagein operations resulting from faults on the map will be
+ * clustered.
+ */
+kern_return_t
+vm_map_behavior_set(
+ vm_map_t map,
+ vm_map_offset_t start,
+ vm_map_offset_t end,
+ vm_behavior_t new_behavior)
+{
+ register vm_map_entry_t entry;
+ vm_map_entry_t temp_entry;
- if (object_name)
- *object_name = IP_NULL;
- *address = start;
- *size = (entry->vme_end - start);
+ XPR(XPR_VM_MAP,
+ "vm_map_behavior_set, 0x%X start 0x%X end 0x%X behavior %d",
+ (integer_t)map, start, end, new_behavior, 0);
- vm_map_unlock_read(map);
- return(KERN_SUCCESS);
+ switch (new_behavior) {
+ case VM_BEHAVIOR_DEFAULT:
+ case VM_BEHAVIOR_RANDOM:
+ case VM_BEHAVIOR_SEQUENTIAL:
+ case VM_BEHAVIOR_RSEQNTL:
+ break;
+ case VM_BEHAVIOR_WILLNEED:
+ case VM_BEHAVIOR_DONTNEED:
+ new_behavior = VM_BEHAVIOR_DEFAULT;
+ break;
+ default:
+ return(KERN_INVALID_ARGUMENT);
}
- case VM_REGION_TOP_INFO:
- {
- if (*count < VM_REGION_TOP_INFO_COUNT)
- return(KERN_INVALID_ARGUMENT);
+ vm_map_lock(map);
- top = (vm_region_top_info_t) info;
- *count = VM_REGION_TOP_INFO_COUNT;
+ /*
+ * The entire address range must be valid for the map.
+ * Note that vm_map_range_check() does a
+ * vm_map_lookup_entry() internally and returns the
+ * entry containing the start of the address range if
+ * the entire range is valid.
+ */
+ if (vm_map_range_check(map, start, end, &temp_entry)) {
+ entry = temp_entry;
+ vm_map_clip_start(map, entry, start);
+ }
+ else {
+ vm_map_unlock(map);
+ return(KERN_INVALID_ADDRESS);
+ }
- vm_map_lock_read(map);
+ while ((entry != vm_map_to_entry(map)) && (entry->vme_start < end)) {
+ vm_map_clip_end(map, entry, end);
- start = *address;
- if (!vm_map_lookup_entry(map, start, &tmp_entry)) {
- if ((entry = tmp_entry->vme_next) == vm_map_to_entry(map)) {
- vm_map_unlock_read(map);
- return(KERN_INVALID_ADDRESS);
- }
- } else {
- entry = tmp_entry;
+ entry->behavior = new_behavior;
- }
- start = entry->vme_start;
+ entry = entry->vme_next;
+ }
- top->private_pages_resident = 0;
- top->shared_pages_resident = 0;
+ vm_map_unlock(map);
+ return(KERN_SUCCESS);
+}
- vm_region_top_walk(entry, top);
- if (object_name)
- *object_name = IP_NULL;
- *address = start;
- *size = (entry->vme_end - start);
+#include <mach_kdb.h>
+#if MACH_KDB
+#include <ddb/db_output.h>
+#include <vm/vm_print.h>
- vm_map_unlock_read(map);
- return(KERN_SUCCESS);
- }
- default:
- return(KERN_INVALID_ARGUMENT);
- }
+#define printf db_printf
+
+/*
+ * Forward declarations for internal functions.
+ */
+extern void vm_map_links_print(
+ struct vm_map_links *links);
+
+extern void vm_map_header_print(
+ struct vm_map_header *header);
+
+extern void vm_map_entry_print(
+ vm_map_entry_t entry);
+
+extern void vm_follow_entry(
+ vm_map_entry_t entry);
+
+extern void vm_follow_map(
+ vm_map_t map);
+
+/*
+ * vm_map_links_print: [ debug ]
+ */
+void
+vm_map_links_print(
+ struct vm_map_links *links)
+{
+ iprintf("prev = %08X next = %08X start = %016llX end = %016llX\n",
+ links->prev,
+ links->next,
+ (unsigned long long)links->start,
+ (unsigned long long)links->end);
}
+/*
+ * vm_map_header_print: [ debug ]
+ */
void
-vm_region_top_walk(
- vm_map_entry_t entry,
- vm_region_top_info_t top)
+vm_map_header_print(
+ struct vm_map_header *header)
{
- register struct vm_object *obj, *tmp_obj;
+ vm_map_links_print(&header->links);
+ iprintf("nentries = %08X, %sentries_pageable\n",
+ header->nentries,
+ (header->entries_pageable ? "" : "!"));
+}
- if (entry->object.vm_object == 0) {
- top->share_mode = SM_EMPTY;
- top->ref_count = 0;
- top->obj_id = 0;
- return;
- }
- if (entry->is_sub_map)
- vm_region_top_walk((vm_map_entry_t)entry->object.sub_map, top);
- else {
- obj = entry->object.vm_object;
+/*
+ * vm_follow_entry: [ debug ]
+ */
+void
+vm_follow_entry(
+ vm_map_entry_t entry)
+{
+ int shadows;
- vm_object_lock(obj);
+ iprintf("map entry %08X\n", entry);
- if (obj->shadow) {
- if (obj->ref_count == 1)
- top->private_pages_resident = obj->resident_page_count;
- else
- top->shared_pages_resident = obj->resident_page_count;
- top->ref_count = obj->ref_count;
- top->share_mode = SM_COW;
-
- while (tmp_obj = obj->shadow) {
- vm_object_lock(tmp_obj);
- vm_object_unlock(obj);
- obj = tmp_obj;
+ db_indent += 2;
- top->shared_pages_resident += obj->resident_page_count;
- top->ref_count += obj->ref_count - 1;
- }
- } else {
- if (entry->needs_copy) {
- top->share_mode = SM_COW;
- top->shared_pages_resident = obj->resident_page_count;
- } else {
- if (obj->ref_count == 1 ||
- (obj->ref_count == 2 && !(obj->pager_trusted) && !(obj->internal))) {
- top->share_mode = SM_PRIVATE;
- top->private_pages_resident = obj->resident_page_count;
- } else {
- top->share_mode = SM_SHARED;
- top->shared_pages_resident = obj->resident_page_count;
- }
- }
- top->ref_count = obj->ref_count;
- }
- top->obj_id = (int)obj;
+ shadows = vm_follow_object(entry->object.vm_object);
+ iprintf("Total objects : %d\n",shadows);
- vm_object_unlock(obj);
- }
+ db_indent -= 2;
}
+/*
+ * vm_map_entry_print: [ debug ]
+ */
void
-vm_region_walk(
- vm_map_entry_t entry,
- vm_region_extended_info_t extended,
- vm_object_offset_t offset,
- vm_offset_t range,
- vm_map_t map,
- vm_offset_t va)
+vm_map_entry_print(
+ register vm_map_entry_t entry)
{
- register struct vm_object *obj, *tmp_obj;
- register vm_offset_t last_offset;
- register int i;
- void vm_region_look_for_page();
+ static const char *inheritance_name[4] =
+ { "share", "copy", "none", "?"};
+ static const char *behavior_name[4] =
+ { "dflt", "rand", "seqtl", "rseqntl" };
+
+ iprintf("map entry %08X - prev = %08X next = %08X\n", entry, entry->vme_prev, entry->vme_next);
- if (entry->object.vm_object == 0) {
- extended->share_mode = SM_EMPTY;
- extended->ref_count = 0;
- return;
- }
- if (entry->is_sub_map)
- vm_region_walk((vm_map_entry_t)entry->object.sub_map, extended, offset + entry->offset,
- range, map, va);
- else {
- obj = entry->object.vm_object;
+ db_indent += 2;
- vm_object_lock(obj);
+ vm_map_links_print(&entry->links);
- for (last_offset = offset + range; offset < last_offset; offset += PAGE_SIZE_64, va += PAGE_SIZE)
- vm_region_look_for_page(obj, extended, offset, obj->ref_count, 0, map, va);
+ iprintf("start = %016llX end = %016llX - prot=%x/%x/%s\n",
+ (unsigned long long)entry->vme_start,
+ (unsigned long long)entry->vme_end,
+ entry->protection,
+ entry->max_protection,
+ inheritance_name[(entry->inheritance & 0x3)]);
- if (extended->shadow_depth || entry->needs_copy)
- extended->share_mode = SM_COW;
- else {
- if (obj->ref_count == 1)
- extended->share_mode = SM_PRIVATE;
- else {
- if (obj->true_share)
- extended->share_mode = SM_TRUESHARED;
- else
- extended->share_mode = SM_SHARED;
- }
- }
- extended->ref_count = obj->ref_count - extended->shadow_depth;
-
- for (i = 0; i < extended->shadow_depth; i++) {
- if ((tmp_obj = obj->shadow) == 0)
- break;
- vm_object_lock(tmp_obj);
- vm_object_unlock(obj);
- extended->ref_count += tmp_obj->ref_count;
- obj = tmp_obj;
- }
- vm_object_unlock(obj);
-
- if (extended->share_mode == SM_SHARED) {
- register vm_map_entry_t cur;
- register vm_map_entry_t last;
- int my_refs;
-
- obj = entry->object.vm_object;
- last = vm_map_to_entry(map);
- my_refs = 0;
-
- for (cur = vm_map_first_entry(map); cur != last; cur = cur->vme_next)
- my_refs += vm_region_count_obj_refs(cur, obj);
+ iprintf("behavior = %s, wired_count = %d, user_wired_count = %d\n",
+ behavior_name[(entry->behavior & 0x3)],
+ entry->wired_count,
+ entry->user_wired_count);
+ iprintf("%sin_transition, %sneeds_wakeup\n",
+ (entry->in_transition ? "" : "!"),
+ (entry->needs_wakeup ? "" : "!"));
- if (my_refs == obj->ref_count)
- extended->share_mode = SM_PRIVATE_ALIASED;
- else if (my_refs > 1)
- extended->share_mode = SM_SHARED_ALIASED;
- }
+ if (entry->is_sub_map) {
+ iprintf("submap = %08X - offset = %016llX\n",
+ entry->object.sub_map,
+ (unsigned long long)entry->offset);
+ } else {
+ iprintf("object = %08X offset = %016llX - ",
+ entry->object.vm_object,
+ (unsigned long long)entry->offset);
+ printf("%sis_shared, %sneeds_copy\n",
+ (entry->is_shared ? "" : "!"),
+ (entry->needs_copy ? "" : "!"));
}
-}
-
+ db_indent -= 2;
+}
+/*
+ * vm_follow_map: [ debug ]
+ */
void
-vm_region_look_for_page(
- vm_object_t object,
- vm_region_extended_info_t extended,
- vm_object_offset_t offset,
- int max_refcnt,
- int depth,
- vm_map_t map,
- vm_offset_t va)
+vm_follow_map(
+ vm_map_t map)
{
- register vm_page_t p;
- register vm_object_t shadow;
-
- shadow = object->shadow;
-
- if ( !(object->pager_trusted) && !(object->internal))
- extended->external_pager = 1;
+ register vm_map_entry_t entry;
- if ((p = vm_page_lookup(object, offset)) != VM_PAGE_NULL) {
- if (shadow && (max_refcnt == 1))
- extended->pages_shared_now_private++;
+ iprintf("task map %08X\n", map);
- if (pmap_extract(vm_map_pmap(map), va))
- extended->pages_referenced++;
- extended->pages_resident++;
+ db_indent += 2;
- return;
+ for (entry = vm_map_first_entry(map);
+ entry && entry != vm_map_to_entry(map);
+ entry = entry->vme_next) {
+ vm_follow_entry(entry);
}
- if (object->existence_map) {
- if (vm_external_state_get(object->existence_map, offset) == VM_EXTERNAL_STATE_EXISTS) {
- if (shadow && (max_refcnt == 1))
- extended->pages_shared_now_private++;
- extended->pages_swapped_out++;
- return;
- }
- }
- if (shadow) {
- vm_object_lock(shadow);
+ db_indent -= 2;
+}
- if (++depth > extended->shadow_depth)
- extended->shadow_depth = depth;
+/*
+ * vm_map_print: [ debug ]
+ */
+void
+vm_map_print(
+ db_addr_t inmap)
+{
+ register vm_map_entry_t entry;
+ vm_map_t map;
+#if TASK_SWAPPER
+ char *swstate;
+#endif /* TASK_SWAPPER */
- if (shadow->ref_count > max_refcnt)
- max_refcnt = shadow->ref_count;
-
- vm_region_look_for_page(shadow, extended, offset + object->shadow_offset,
- max_refcnt, depth, map, va);
- vm_object_unlock(shadow);
+ map = (vm_map_t)(long)
+ inmap; /* Make sure we have the right type */
- return;
- }
-}
+ iprintf("task map %08X\n", map);
+ db_indent += 2;
-vm_region_count_obj_refs(
- vm_map_entry_t entry,
- vm_object_t object)
-{
- register int ref_count;
- register vm_object_t chk_obj;
- register vm_object_t tmp_obj;
+ vm_map_header_print(&map->hdr);
- if (entry->object.vm_object == 0)
- return(0);
+ iprintf("pmap = %08X size = %08X ref = %d hint = %08X first_free = %08X\n",
+ map->pmap,
+ map->size,
+ map->ref_count,
+ map->hint,
+ map->first_free);
- if (entry->is_sub_map)
- ref_count = vm_region_count_obj_refs((vm_map_entry_t)entry->object.sub_map, object);
- else {
- ref_count = 0;
+ iprintf("%swait_for_space, %swiring_required, timestamp = %d\n",
+ (map->wait_for_space ? "" : "!"),
+ (map->wiring_required ? "" : "!"),
+ map->timestamp);
- chk_obj = entry->object.vm_object;
- vm_object_lock(chk_obj);
+#if TASK_SWAPPER
+ switch (map->sw_state) {
+ case MAP_SW_IN:
+ swstate = "SW_IN";
+ break;
+ case MAP_SW_OUT:
+ swstate = "SW_OUT";
+ break;
+ default:
+ swstate = "????";
+ break;
+ }
+ iprintf("res = %d, sw_state = %s\n", map->res_count, swstate);
+#endif /* TASK_SWAPPER */
- while (chk_obj) {
- if (chk_obj == object)
- ref_count++;
- if (tmp_obj = chk_obj->shadow)
- vm_object_lock(tmp_obj);
- vm_object_unlock(chk_obj);
-
- chk_obj = tmp_obj;
- }
+ for (entry = vm_map_first_entry(map);
+ entry && entry != vm_map_to_entry(map);
+ entry = entry->vme_next) {
+ vm_map_entry_print(entry);
}
- return(ref_count);
-}
+ db_indent -= 2;
+}
/*
- * Routine: vm_map_simplify
- *
- * Description:
- * Attempt to simplify the map representation in
- * the vicinity of the given starting address.
- * Note:
- * This routine is intended primarily to keep the
- * kernel maps more compact -- they generally don't
- * benefit from the "expand a map entry" technology
- * at allocation time because the adjacent entry
- * is often wired down.
+ * Routine: vm_map_copy_print
+ * Purpose:
+ * Pretty-print a copy object for ddb.
*/
+
void
-vm_map_simplify(
- vm_map_t map,
- vm_offset_t start)
+vm_map_copy_print(
+ db_addr_t incopy)
{
- vm_map_entry_t this_entry;
- vm_map_entry_t prev_entry;
- vm_map_entry_t next_entry;
+ vm_map_copy_t copy;
+ vm_map_entry_t entry;
- vm_map_lock(map);
- if (
- (vm_map_lookup_entry(map, start, &this_entry)) &&
- ((prev_entry = this_entry->vme_prev) != vm_map_to_entry(map)) &&
-
- (prev_entry->vme_end == this_entry->vme_start) &&
-
- (prev_entry->is_shared == FALSE) &&
- (prev_entry->is_sub_map == FALSE) &&
-
- (this_entry->is_shared == FALSE) &&
- (this_entry->is_sub_map == FALSE) &&
-
- (prev_entry->inheritance == this_entry->inheritance) &&
- (prev_entry->protection == this_entry->protection) &&
- (prev_entry->max_protection == this_entry->max_protection) &&
- (prev_entry->behavior == this_entry->behavior) &&
- (prev_entry->wired_count == this_entry->wired_count) &&
- (prev_entry->user_wired_count == this_entry->user_wired_count)&&
- (prev_entry->in_transition == FALSE) &&
- (this_entry->in_transition == FALSE) &&
-
- (prev_entry->needs_copy == this_entry->needs_copy) &&
-
- (prev_entry->object.vm_object == this_entry->object.vm_object)&&
- ((prev_entry->offset +
- (prev_entry->vme_end - prev_entry->vme_start))
- == this_entry->offset)
- ) {
- SAVE_HINT(map, prev_entry);
- vm_map_entry_unlink(map, this_entry);
- prev_entry->vme_end = this_entry->vme_end;
- UPDATE_FIRST_FREE(map, map->first_free);
- vm_object_deallocate(this_entry->object.vm_object);
- vm_map_entry_dispose(map, this_entry);
- counter(c_vm_map_simplified_lower++);
- }
- if (
- (vm_map_lookup_entry(map, start, &this_entry)) &&
- ((next_entry = this_entry->vme_next) != vm_map_to_entry(map)) &&
-
- (next_entry->vme_start == this_entry->vme_end) &&
-
- (next_entry->is_shared == FALSE) &&
- (next_entry->is_sub_map == FALSE) &&
-
- (next_entry->is_shared == FALSE) &&
- (next_entry->is_sub_map == FALSE) &&
-
- (next_entry->inheritance == this_entry->inheritance) &&
- (next_entry->protection == this_entry->protection) &&
- (next_entry->max_protection == this_entry->max_protection) &&
- (next_entry->behavior == this_entry->behavior) &&
- (next_entry->wired_count == this_entry->wired_count) &&
- (next_entry->user_wired_count == this_entry->user_wired_count)&&
- (this_entry->in_transition == FALSE) &&
- (next_entry->in_transition == FALSE) &&
-
- (next_entry->needs_copy == this_entry->needs_copy) &&
-
- (next_entry->object.vm_object == this_entry->object.vm_object)&&
- ((this_entry->offset +
- (this_entry->vme_end - this_entry->vme_start))
- == next_entry->offset)
- ) {
- vm_map_entry_unlink(map, next_entry);
- this_entry->vme_end = next_entry->vme_end;
- UPDATE_FIRST_FREE(map, map->first_free);
- vm_object_deallocate(next_entry->object.vm_object);
- vm_map_entry_dispose(map, next_entry);
- counter(c_vm_map_simplified_upper++);
- }
- counter(c_vm_map_simplify_called++);
- vm_map_unlock(map);
-}
+ copy = (vm_map_copy_t)(long)
+ incopy; /* Make sure we have the right type */
+ printf("copy object 0x%x\n", copy);
-/*
- * Routine: vm_map_machine_attribute
- * Purpose:
- * Provide machine-specific attributes to mappings,
- * such as cachability etc. for machines that provide
- * them. NUMA architectures and machines with big/strange
- * caches will use this.
- * Note:
- * Responsibilities for locking and checking are handled here,
- * everything else in the pmap module. If any non-volatile
- * information must be kept, the pmap module should handle
- * it itself. [This assumes that attributes do not
- * need to be inherited, which seems ok to me]
- */
-kern_return_t
-vm_map_machine_attribute(
- vm_map_t map,
- vm_offset_t address,
- vm_size_t size,
- vm_machine_attribute_t attribute,
- vm_machine_attribute_val_t* value) /* IN/OUT */
-{
- kern_return_t ret;
+ db_indent += 2;
- if (address < vm_map_min(map) ||
- (address + size) > vm_map_max(map))
- return KERN_INVALID_ADDRESS;
+ iprintf("type=%d", copy->type);
+ switch (copy->type) {
+ case VM_MAP_COPY_ENTRY_LIST:
+ printf("[entry_list]");
+ break;
+
+ case VM_MAP_COPY_OBJECT:
+ printf("[object]");
+ break;
+
+ case VM_MAP_COPY_KERNEL_BUFFER:
+ printf("[kernel_buffer]");
+ break;
- vm_map_lock(map);
+ default:
+ printf("[bad type]");
+ break;
+ }
+ printf(", offset=0x%llx", (unsigned long long)copy->offset);
+ printf(", size=0x%x\n", copy->size);
- ret = pmap_attribute(map->pmap, address, size, attribute, value);
+ switch (copy->type) {
+ case VM_MAP_COPY_ENTRY_LIST:
+ vm_map_header_print(©->cpy_hdr);
+ for (entry = vm_map_copy_first_entry(copy);
+ entry && entry != vm_map_copy_to_entry(copy);
+ entry = entry->vme_next) {
+ vm_map_entry_print(entry);
+ }
+ break;
- vm_map_unlock(map);
+ case VM_MAP_COPY_OBJECT:
+ iprintf("object=0x%x\n", copy->cpy_object);
+ break;
- return ret;
+ case VM_MAP_COPY_KERNEL_BUFFER:
+ iprintf("kernel buffer=0x%x", copy->cpy_kdata);
+ printf(", kalloc_size=0x%x\n", copy->cpy_kalloc_size);
+ break;
+
+ }
+
+ db_indent -=2;
}
/*
- * vm_map_behavior_set:
+ * db_vm_map_total_size(map) [ debug ]
*
- * Sets the paging reference behavior of the specified address
- * range in the target map. Paging reference behavior affects
- * how pagein operations resulting from faults on the map will be
- * clustered.
+ * return the total virtual size (in bytes) of the map
*/
-kern_return_t
-vm_map_behavior_set(
- vm_map_t map,
- vm_offset_t start,
- vm_offset_t end,
- vm_behavior_t new_behavior)
+vm_map_size_t
+db_vm_map_total_size(
+ db_addr_t inmap)
{
- register vm_map_entry_t entry;
- vm_map_entry_t temp_entry;
+ vm_map_entry_t entry;
+ vm_map_size_t total;
+ vm_map_t map;
- XPR(XPR_VM_MAP,
- "vm_map_behavior_set, 0x%X start 0x%X end 0x%X behavior %d",
- (integer_t)map, start, end, new_behavior, 0);
+ map = (vm_map_t)(long)
+ inmap; /* Make sure we have the right type */
- switch (new_behavior) {
- case VM_BEHAVIOR_DEFAULT:
- case VM_BEHAVIOR_RANDOM:
- case VM_BEHAVIOR_SEQUENTIAL:
- case VM_BEHAVIOR_RSEQNTL:
- break;
- default:
- return(KERN_INVALID_ARGUMENT);
+ total = 0;
+ for (entry = vm_map_first_entry(map);
+ entry != vm_map_to_entry(map);
+ entry = entry->vme_next) {
+ total += entry->vme_end - entry->vme_start;
}
- vm_map_lock(map);
-
- /*
- * The entire address range must be valid for the map.
- * Note that vm_map_range_check() does a
- * vm_map_lookup_entry() internally and returns the
- * entry containing the start of the address range if
- * the entire range is valid.
- */
- if (vm_map_range_check(map, start, end, &temp_entry)) {
- entry = temp_entry;
- vm_map_clip_start(map, entry, start);
- }
- else {
- vm_map_unlock(map);
- return(KERN_INVALID_ADDRESS);
- }
-
- while ((entry != vm_map_to_entry(map)) && (entry->vme_start < end)) {
- vm_map_clip_end(map, entry, end);
-
- entry->behavior = new_behavior;
-
- entry = entry->vme_next;
- }
-
- vm_map_unlock(map);
- return(KERN_SUCCESS);
-}
+ return total;
+}
+#endif /* MACH_KDB */
-int
-vm_map_copy_cont_is_valid(
- vm_map_copy_t copy)
+/*
+ * Routine: vm_map_entry_insert
+ *
+ * Descritpion: This routine inserts a new vm_entry in a locked map.
+ */
+vm_map_entry_t
+vm_map_entry_insert(
+ vm_map_t map,
+ vm_map_entry_t insp_entry,
+ vm_map_offset_t start,
+ vm_map_offset_t end,
+ vm_object_t object,
+ vm_object_offset_t offset,
+ boolean_t needs_copy,
+ boolean_t is_shared,
+ boolean_t in_transition,
+ vm_prot_t cur_protection,
+ vm_prot_t max_protection,
+ vm_behavior_t behavior,
+ vm_inherit_t inheritance,
+ unsigned wired_count)
{
- vm_map_copy_cont_t cont;
-
- assert(copy->type == VM_MAP_COPY_PAGE_LIST);
- cont = copy->cpy_cont;
- if (
- cont != vm_map_copy_discard_cont &&
- cont != vm_map_copyin_page_list_cont ) {
- printf("vm_map_copy_cont_is_valid: bogus cont 0x%x\n", cont);
- assert((integer_t) cont == 0xdeadbeef);
- }
- return 1;
-}
-
-#include <mach_kdb.h>
-#if MACH_KDB
-#include <ddb/db_output.h>
-#include <vm/vm_print.h>
+ vm_map_entry_t new_entry;
-#define printf db_printf
+ assert(insp_entry != (vm_map_entry_t)0);
-/*
- * Forward declarations for internal functions.
- */
-extern void vm_map_links_print(
- struct vm_map_links *links);
+ new_entry = vm_map_entry_create(map);
-extern void vm_map_header_print(
- struct vm_map_header *header);
+ new_entry->vme_start = start;
+ new_entry->vme_end = end;
+ assert(page_aligned(new_entry->vme_start));
+ assert(page_aligned(new_entry->vme_end));
-extern void vm_map_entry_print(
- vm_map_entry_t entry);
+ new_entry->object.vm_object = object;
+ new_entry->offset = offset;
+ new_entry->is_shared = is_shared;
+ new_entry->is_sub_map = FALSE;
+ new_entry->needs_copy = needs_copy;
+ new_entry->in_transition = in_transition;
+ new_entry->needs_wakeup = FALSE;
+ new_entry->inheritance = inheritance;
+ new_entry->protection = cur_protection;
+ new_entry->max_protection = max_protection;
+ new_entry->behavior = behavior;
+ new_entry->wired_count = wired_count;
+ new_entry->user_wired_count = 0;
+ new_entry->use_pmap = FALSE;
-extern void vm_follow_entry(
- vm_map_entry_t entry);
+ /*
+ * Insert the new entry into the list.
+ */
-extern void vm_follow_map(
- vm_map_t map);
+ vm_map_entry_link(map, insp_entry, new_entry);
+ map->size += end - start;
-/*
- * vm_map_links_print: [ debug ]
- */
-void
-vm_map_links_print(
- struct vm_map_links *links)
-{
- iprintf("prev=0x%x, next=0x%x, start=0x%x, end=0x%x\n",
- links->prev,
- links->next,
- links->start,
- links->end);
-}
+ /*
+ * Update the free space hint and the lookup hint.
+ */
-/*
- * vm_map_header_print: [ debug ]
- */
-void
-vm_map_header_print(
- struct vm_map_header *header)
-{
- vm_map_links_print(&header->links);
- iprintf("nentries=0x%x, %sentries_pageable\n",
- header->nentries,
- (header->entries_pageable ? "" : "!"));
+ SAVE_HINT(map, new_entry);
+ return new_entry;
}
/*
- * vm_follow_entry: [ debug ]
+ * Routine: vm_map_remap_extract
+ *
+ * Descritpion: This routine returns a vm_entry list from a map.
*/
-void
-vm_follow_entry(
- vm_map_entry_t entry)
+static kern_return_t
+vm_map_remap_extract(
+ vm_map_t map,
+ vm_map_offset_t addr,
+ vm_map_size_t size,
+ boolean_t copy,
+ struct vm_map_header *map_header,
+ vm_prot_t *cur_protection,
+ vm_prot_t *max_protection,
+ /* What, no behavior? */
+ vm_inherit_t inheritance,
+ boolean_t pageable)
{
- extern int db_indent;
- int shadows;
+ kern_return_t result;
+ vm_map_size_t mapped_size;
+ vm_map_size_t tmp_size;
+ vm_map_entry_t src_entry; /* result of last map lookup */
+ vm_map_entry_t new_entry;
+ vm_object_offset_t offset;
+ vm_map_offset_t map_address;
+ vm_map_offset_t src_start; /* start of entry to map */
+ vm_map_offset_t src_end; /* end of region to be mapped */
+ vm_object_t object;
+ vm_map_version_t version;
+ boolean_t src_needs_copy;
+ boolean_t new_entry_needs_copy;
- iprintf("map entry 0x%x:\n", entry);
+ assert(map != VM_MAP_NULL);
+ assert(size != 0 && size == vm_map_round_page(size));
+ assert(inheritance == VM_INHERIT_NONE ||
+ inheritance == VM_INHERIT_COPY ||
+ inheritance == VM_INHERIT_SHARE);
- db_indent += 2;
+ /*
+ * Compute start and end of region.
+ */
+ src_start = vm_map_trunc_page(addr);
+ src_end = vm_map_round_page(src_start + size);
- shadows = vm_follow_object(entry->object.vm_object);
- iprintf("Total objects : %d\n",shadows);
+ /*
+ * Initialize map_header.
+ */
+ map_header->links.next = (struct vm_map_entry *)&map_header->links;
+ map_header->links.prev = (struct vm_map_entry *)&map_header->links;
+ map_header->nentries = 0;
+ map_header->entries_pageable = pageable;
- db_indent -= 2;
-}
+ *cur_protection = VM_PROT_ALL;
+ *max_protection = VM_PROT_ALL;
-/*
- * vm_map_entry_print: [ debug ]
- */
-void
-vm_map_entry_print(
- register vm_map_entry_t entry)
-{
- extern int db_indent;
- static char *inheritance_name[4] = { "share", "copy", "none", "?"};
- static char *behavior_name[4] = { "dflt", "rand", "seqtl", "rseqntl" };
-
- iprintf("map entry 0x%x:\n", entry);
+ map_address = 0;
+ mapped_size = 0;
+ result = KERN_SUCCESS;
- db_indent += 2;
+ /*
+ * The specified source virtual space might correspond to
+ * multiple map entries, need to loop on them.
+ */
+ vm_map_lock(map);
+ while (mapped_size != size) {
+ vm_map_size_t entry_size;
- vm_map_links_print(&entry->links);
+ /*
+ * Find the beginning of the region.
+ */
+ if (! vm_map_lookup_entry(map, src_start, &src_entry)) {
+ result = KERN_INVALID_ADDRESS;
+ break;
+ }
- iprintf("start=0x%x, end=0x%x, prot=%x/%x/%s\n",
- entry->vme_start,
- entry->vme_end,
- entry->protection,
- entry->max_protection,
- inheritance_name[(entry->inheritance & 0x3)]);
+ if (src_start < src_entry->vme_start ||
+ (mapped_size && src_start != src_entry->vme_start)) {
+ result = KERN_INVALID_ADDRESS;
+ break;
+ }
- iprintf("behavior=%s, wired_count=%d, user_wired_count=%d\n",
- behavior_name[(entry->behavior & 0x3)],
- entry->wired_count,
- entry->user_wired_count);
- iprintf("%sin_transition, %sneeds_wakeup\n",
- (entry->in_transition ? "" : "!"),
- (entry->needs_wakeup ? "" : "!"));
+ if(src_entry->is_sub_map) {
+ result = KERN_INVALID_ADDRESS;
+ break;
+ }
- if (entry->is_sub_map) {
- iprintf("submap=0x%x, offset=0x%x\n",
- entry->object.sub_map,
- entry->offset);
- } else {
- iprintf("object=0x%x, offset=0x%x, ",
- entry->object.vm_object,
- entry->offset);
- printf("%sis_shared, %sneeds_copy\n",
- (entry->is_shared ? "" : "!"),
- (entry->needs_copy ? "" : "!"));
- }
+ tmp_size = size - mapped_size;
+ if (src_end > src_entry->vme_end)
+ tmp_size -= (src_end - src_entry->vme_end);
- db_indent -= 2;
-}
+ entry_size = (vm_map_size_t)(src_entry->vme_end -
+ src_entry->vme_start);
-/*
- * vm_follow_map: [ debug ]
- */
-void
-vm_follow_map(
- vm_map_t map)
-{
- register vm_map_entry_t entry;
- extern int db_indent;
+ if(src_entry->is_sub_map) {
+ vm_map_reference(src_entry->object.sub_map);
+ object = VM_OBJECT_NULL;
+ } else {
+ object = src_entry->object.vm_object;
- iprintf("task map 0x%x:\n", map);
+ if (object == VM_OBJECT_NULL) {
+ object = vm_object_allocate(entry_size);
+ src_entry->offset = 0;
+ src_entry->object.vm_object = object;
+ } else if (object->copy_strategy !=
+ MEMORY_OBJECT_COPY_SYMMETRIC) {
+ /*
+ * We are already using an asymmetric
+ * copy, and therefore we already have
+ * the right object.
+ */
+ assert(!src_entry->needs_copy);
+ } else if (src_entry->needs_copy || object->shadowed ||
+ (object->internal && !object->true_share &&
+ !src_entry->is_shared &&
+ object->size > entry_size)) {
- db_indent += 2;
+ vm_object_shadow(&src_entry->object.vm_object,
+ &src_entry->offset,
+ entry_size);
- for (entry = vm_map_first_entry(map);
- entry && entry != vm_map_to_entry(map);
- entry = entry->vme_next) {
- vm_follow_entry(entry);
- }
+ if (!src_entry->needs_copy &&
+ (src_entry->protection & VM_PROT_WRITE)) {
+ if(map->mapped) {
+ vm_object_pmap_protect(
+ src_entry->object.vm_object,
+ src_entry->offset,
+ entry_size,
+ PMAP_NULL,
+ src_entry->vme_start,
+ src_entry->protection &
+ ~VM_PROT_WRITE);
+ } else {
+ pmap_protect(vm_map_pmap(map),
+ src_entry->vme_start,
+ src_entry->vme_end,
+ src_entry->protection &
+ ~VM_PROT_WRITE);
+ }
+ }
- db_indent -= 2;
-}
+ object = src_entry->object.vm_object;
+ src_entry->needs_copy = FALSE;
+ }
-/*
- * vm_map_print: [ debug ]
- */
-void
-vm_map_print(
- register vm_map_t map)
-{
- register vm_map_entry_t entry;
- extern int db_indent;
- char *swstate;
- iprintf("task map 0x%x:\n", map);
+ vm_object_lock(object);
+ object->ref_count++; /* object ref. for new entry */
+ VM_OBJ_RES_INCR(object);
+ if (object->copy_strategy ==
+ MEMORY_OBJECT_COPY_SYMMETRIC) {
+ object->copy_strategy =
+ MEMORY_OBJECT_COPY_DELAY;
+ }
+ vm_object_unlock(object);
+ }
- db_indent += 2;
+ offset = src_entry->offset + (src_start - src_entry->vme_start);
- vm_map_header_print(&map->hdr);
+ new_entry = _vm_map_entry_create(map_header);
+ vm_map_entry_copy(new_entry, src_entry);
+ new_entry->use_pmap = FALSE; /* clr address space specifics */
- iprintf("pmap=0x%x, size=%d, ref=%d, hint=0x%x, first_free=0x%x\n",
- map->pmap,
- map->size,
- map->ref_count,
- map->hint,
- map->first_free);
+ new_entry->vme_start = map_address;
+ new_entry->vme_end = map_address + tmp_size;
+ new_entry->inheritance = inheritance;
+ new_entry->offset = offset;
- iprintf("%swait_for_space, %swiring_required, timestamp=%d\n",
- (map->wait_for_space ? "" : "!"),
- (map->wiring_required ? "" : "!"),
- map->timestamp);
+ /*
+ * The new region has to be copied now if required.
+ */
+ RestartCopy:
+ if (!copy) {
+ src_entry->is_shared = TRUE;
+ new_entry->is_shared = TRUE;
+ if (!(new_entry->is_sub_map))
+ new_entry->needs_copy = FALSE;
-#if TASK_SWAPPER
- switch (map->sw_state) {
- case MAP_SW_IN:
- swstate = "SW_IN";
- break;
- case MAP_SW_OUT:
- swstate = "SW_OUT";
- break;
- default:
- swstate = "????";
- break;
- }
- iprintf("res=%d, sw_state=%s\n", map->res_count, swstate);
-#endif /* TASK_SWAPPER */
+ } else if (src_entry->is_sub_map) {
+ /* make this a COW sub_map if not already */
+ new_entry->needs_copy = TRUE;
+ object = VM_OBJECT_NULL;
+ } else if (src_entry->wired_count == 0 &&
+ vm_object_copy_quickly(&new_entry->object.vm_object,
+ new_entry->offset,
+ (new_entry->vme_end -
+ new_entry->vme_start),
+ &src_needs_copy,
+ &new_entry_needs_copy)) {
- for (entry = vm_map_first_entry(map);
- entry && entry != vm_map_to_entry(map);
- entry = entry->vme_next) {
- vm_map_entry_print(entry);
- }
+ new_entry->needs_copy = new_entry_needs_copy;
+ new_entry->is_shared = FALSE;
- db_indent -= 2;
-}
+ /*
+ * Handle copy_on_write semantics.
+ */
+ if (src_needs_copy && !src_entry->needs_copy) {
+ vm_object_pmap_protect(object,
+ offset,
+ entry_size,
+ ((src_entry->is_shared
+ || map->mapped) ?
+ PMAP_NULL : map->pmap),
+ src_entry->vme_start,
+ src_entry->protection &
+ ~VM_PROT_WRITE);
-/*
- * Routine: vm_map_copy_print
- * Purpose:
- * Pretty-print a copy object for ddb.
- */
+ src_entry->needs_copy = TRUE;
+ }
+ /*
+ * Throw away the old object reference of the new entry.
+ */
+ vm_object_deallocate(object);
-void
-vm_map_copy_print(
- vm_map_copy_t copy)
-{
- extern int db_indent;
- int i, npages;
- vm_map_entry_t entry;
+ } else {
+ new_entry->is_shared = FALSE;
- printf("copy object 0x%x\n", copy);
+ /*
+ * The map can be safely unlocked since we
+ * already hold a reference on the object.
+ *
+ * Record the timestamp of the map for later
+ * verification, and unlock the map.
+ */
+ version.main_timestamp = map->timestamp;
+ vm_map_unlock(map); /* Increments timestamp once! */
- db_indent += 2;
+ /*
+ * Perform the copy.
+ */
+ if (src_entry->wired_count > 0) {
+ vm_object_lock(object);
+ result = vm_object_copy_slowly(
+ object,
+ offset,
+ entry_size,
+ THREAD_UNINT,
+ &new_entry->object.vm_object);
- iprintf("type=%d", copy->type);
- switch (copy->type) {
- case VM_MAP_COPY_ENTRY_LIST:
- printf("[entry_list]");
- break;
-
- case VM_MAP_COPY_OBJECT:
- printf("[object]");
- break;
-
- case VM_MAP_COPY_PAGE_LIST:
- printf("[page_list]");
- break;
-
- case VM_MAP_COPY_KERNEL_BUFFER:
- printf("[kernel_buffer]");
- break;
+ new_entry->offset = 0;
+ new_entry->needs_copy = FALSE;
+ } else {
+ result = vm_object_copy_strategically(
+ object,
+ offset,
+ entry_size,
+ &new_entry->object.vm_object,
+ &new_entry->offset,
+ &new_entry_needs_copy);
- default:
- printf("[bad type]");
- break;
- }
- printf(", offset=0x%x", copy->offset);
- printf(", size=0x%x\n", copy->size);
+ new_entry->needs_copy = new_entry_needs_copy;
+ }
- switch (copy->type) {
- case VM_MAP_COPY_ENTRY_LIST:
- vm_map_header_print(©->cpy_hdr);
- for (entry = vm_map_copy_first_entry(copy);
- entry && entry != vm_map_copy_to_entry(copy);
- entry = entry->vme_next) {
- vm_map_entry_print(entry);
- }
- break;
+ /*
+ * Throw away the old object reference of the new entry.
+ */
+ vm_object_deallocate(object);
- case VM_MAP_COPY_OBJECT:
- iprintf("object=0x%x\n", copy->cpy_object);
- break;
+ if (result != KERN_SUCCESS &&
+ result != KERN_MEMORY_RESTART_COPY) {
+ _vm_map_entry_dispose(map_header, new_entry);
+ break;
+ }
- case VM_MAP_COPY_KERNEL_BUFFER:
- iprintf("kernel buffer=0x%x", copy->cpy_kdata);
- printf(", kalloc_size=0x%x\n", copy->cpy_kalloc_size);
- break;
+ /*
+ * Verify that the map has not substantially
+ * changed while the copy was being made.
+ */
- case VM_MAP_COPY_PAGE_LIST:
- iprintf("npages=%d", copy->cpy_npages);
- printf(", cont=%x", copy->cpy_cont);
- printf(", cont_args=%x\n", copy->cpy_cont_args);
- if (copy->cpy_npages < 0) {
- npages = 0;
- } else if (copy->cpy_npages > VM_MAP_COPY_PAGE_LIST_MAX) {
- npages = VM_MAP_COPY_PAGE_LIST_MAX;
- } else {
- npages = copy->cpy_npages;
- }
- iprintf("copy->cpy_page_list[0..%d] = {", npages);
- for (i = 0; i < npages - 1; i++) {
- printf("0x%x, ", copy->cpy_page_list[i]);
+ vm_map_lock(map);
+ if (version.main_timestamp + 1 != map->timestamp) {
+ /*
+ * Simple version comparison failed.
+ *
+ * Retry the lookup and verify that the
+ * same object/offset are still present.
+ */
+ vm_object_deallocate(new_entry->
+ object.vm_object);
+ _vm_map_entry_dispose(map_header, new_entry);
+ if (result == KERN_MEMORY_RESTART_COPY)
+ result = KERN_SUCCESS;
+ continue;
+ }
+
+ if (result == KERN_MEMORY_RESTART_COPY) {
+ vm_object_reference(object);
+ goto RestartCopy;
+ }
}
- if (npages > 0) {
- printf("0x%x", copy->cpy_page_list[npages - 1]);
+
+ _vm_map_entry_link(map_header,
+ map_header->links.prev, new_entry);
+
+ *cur_protection &= src_entry->protection;
+ *max_protection &= src_entry->max_protection;
+
+ map_address += tmp_size;
+ mapped_size += tmp_size;
+ src_start += tmp_size;
+
+ } /* end while */
+
+ vm_map_unlock(map);
+ if (result != KERN_SUCCESS) {
+ /*
+ * Free all allocated elements.
+ */
+ for (src_entry = map_header->links.next;
+ src_entry != (struct vm_map_entry *)&map_header->links;
+ src_entry = new_entry) {
+ new_entry = src_entry->vme_next;
+ _vm_map_entry_unlink(map_header, src_entry);
+ vm_object_deallocate(src_entry->object.vm_object);
+ _vm_map_entry_dispose(map_header, src_entry);
}
- printf("}\n");
- break;
}
-
- db_indent -=2;
+ return result;
}
/*
- * db_vm_map_total_size(map) [ debug ]
+ * Routine: vm_remap
*
- * return the total virtual size (in bytes) of the map
+ * Map portion of a task's address space.
+ * Mapped region must not overlap more than
+ * one vm memory object. Protections and
+ * inheritance attributes remain the same
+ * as in the original task and are out parameters.
+ * Source and Target task can be identical
+ * Other attributes are identical as for vm_map()
*/
-vm_size_t
-db_vm_map_total_size(
- vm_map_t map)
+kern_return_t
+vm_map_remap(
+ vm_map_t target_map,
+ vm_map_address_t *address,
+ vm_map_size_t size,
+ vm_map_offset_t mask,
+ boolean_t anywhere,
+ vm_map_t src_map,
+ vm_map_offset_t memory_address,
+ boolean_t copy,
+ vm_prot_t *cur_protection,
+ vm_prot_t *max_protection,
+ vm_inherit_t inheritance)
{
- vm_map_entry_t entry;
- vm_size_t total;
-
- total = 0;
- for (entry = vm_map_first_entry(map);
- entry != vm_map_to_entry(map);
- entry = entry->vme_next) {
- total += entry->vme_end - entry->vme_start;
- }
-
- return total;
-}
-
-#endif /* MACH_KDB */
+ kern_return_t result;
+ vm_map_entry_t entry;
+ vm_map_entry_t insp_entry;
+ vm_map_entry_t new_entry;
+ struct vm_map_header map_header;
-/*
- * Routine: vm_map_entry_insert
- *
- * Descritpion: This routine inserts a new vm_entry in a locked map.
- */
-vm_map_entry_t
-vm_map_entry_insert(
- vm_map_t map,
- vm_map_entry_t insp_entry,
- vm_offset_t start,
- vm_offset_t end,
- vm_object_t object,
- vm_object_offset_t offset,
- boolean_t needs_copy,
- boolean_t is_shared,
- boolean_t in_transition,
- vm_prot_t cur_protection,
- vm_prot_t max_protection,
- vm_behavior_t behavior,
- vm_inherit_t inheritance,
- unsigned wired_count)
-{
- vm_map_entry_t new_entry;
+ if (target_map == VM_MAP_NULL)
+ return KERN_INVALID_ARGUMENT;
- assert(insp_entry != (vm_map_entry_t)0);
+ switch (inheritance) {
+ case VM_INHERIT_NONE:
+ case VM_INHERIT_COPY:
+ case VM_INHERIT_SHARE:
+ if (size != 0 && src_map != VM_MAP_NULL)
+ break;
+ /*FALL THRU*/
+ default:
+ return KERN_INVALID_ARGUMENT;
+ }
- new_entry = vm_map_entry_create(map);
+ size = vm_map_round_page(size);
- new_entry->vme_start = start;
- new_entry->vme_end = end;
- assert(page_aligned(new_entry->vme_start));
- assert(page_aligned(new_entry->vme_end));
+ result = vm_map_remap_extract(src_map, memory_address,
+ size, copy, &map_header,
+ cur_protection,
+ max_protection,
+ inheritance,
+ target_map->hdr.
+ entries_pageable);
- new_entry->object.vm_object = object;
- new_entry->offset = offset;
- new_entry->is_shared = is_shared;
- new_entry->is_sub_map = FALSE;
- new_entry->needs_copy = needs_copy;
- new_entry->in_transition = in_transition;
- new_entry->needs_wakeup = FALSE;
- new_entry->inheritance = inheritance;
- new_entry->protection = cur_protection;
- new_entry->max_protection = max_protection;
- new_entry->behavior = behavior;
- new_entry->wired_count = wired_count;
- new_entry->user_wired_count = 0;
- new_entry->use_pmap = FALSE;
+ if (result != KERN_SUCCESS) {
+ return result;
+ }
/*
- * Insert the new entry into the list.
+ * Allocate/check a range of free virtual address
+ * space for the target
*/
+ *address = vm_map_trunc_page(*address);
+ vm_map_lock(target_map);
+ result = vm_map_remap_range_allocate(target_map, address, size,
+ mask, anywhere, &insp_entry);
- vm_map_entry_link(map, insp_entry, new_entry);
- map->size += end - start;
+ for (entry = map_header.links.next;
+ entry != (struct vm_map_entry *)&map_header.links;
+ entry = new_entry) {
+ new_entry = entry->vme_next;
+ _vm_map_entry_unlink(&map_header, entry);
+ if (result == KERN_SUCCESS) {
+ entry->vme_start += *address;
+ entry->vme_end += *address;
+ vm_map_entry_link(target_map, insp_entry, entry);
+ insp_entry = entry;
+ } else {
+ if (!entry->is_sub_map) {
+ vm_object_deallocate(entry->object.vm_object);
+ } else {
+ vm_map_deallocate(entry->object.sub_map);
+ }
+ _vm_map_entry_dispose(&map_header, entry);
+ }
+ }
- /*
- * Update the free space hint and the lookup hint.
- */
+ if (result == KERN_SUCCESS) {
+ target_map->size += size;
+ SAVE_HINT(target_map, insp_entry);
+ }
+ vm_map_unlock(target_map);
- SAVE_HINT(map, new_entry);
- return new_entry;
+ if (result == KERN_SUCCESS && target_map->wiring_required)
+ result = vm_map_wire(target_map, *address,
+ *address + size, *cur_protection, TRUE);
+ return result;
}
/*
- * Routine: vm_remap_extract
+ * Routine: vm_map_remap_range_allocate
*
- * Descritpion: This routine returns a vm_entry list from a map.
+ * Description:
+ * Allocate a range in the specified virtual address map.
+ * returns the address and the map entry just before the allocated
+ * range
+ *
+ * Map must be locked.
*/
-kern_return_t
-vm_remap_extract(
+
+static kern_return_t
+vm_map_remap_range_allocate(
vm_map_t map,
- vm_offset_t addr,
- vm_size_t size,
- boolean_t copy,
- struct vm_map_header *map_header,
- vm_prot_t *cur_protection,
- vm_prot_t *max_protection,
- /* What, no behavior? */
- vm_inherit_t inheritance,
- boolean_t pageable)
+ vm_map_address_t *address, /* IN/OUT */
+ vm_map_size_t size,
+ vm_map_offset_t mask,
+ boolean_t anywhere,
+ vm_map_entry_t *map_entry) /* OUT */
{
- kern_return_t result;
- vm_size_t mapped_size;
- vm_size_t tmp_size;
- vm_map_entry_t src_entry; /* result of last map lookup */
- vm_map_entry_t new_entry;
- vm_object_offset_t offset;
- vm_offset_t map_address;
- vm_offset_t src_start; /* start of entry to map */
- vm_offset_t src_end; /* end of region to be mapped */
- vm_object_t object;
- vm_map_version_t version;
- boolean_t src_needs_copy;
- boolean_t new_entry_needs_copy;
+ register vm_map_entry_t entry;
+ register vm_map_offset_t start;
+ register vm_map_offset_t end;
- assert(map != VM_MAP_NULL);
- assert(size != 0 && size == round_page(size));
- assert(inheritance == VM_INHERIT_NONE ||
- inheritance == VM_INHERIT_COPY ||
- inheritance == VM_INHERIT_SHARE);
+ StartAgain: ;
+
+ start = *address;
+ if (anywhere)
+ {
/*
- * Compute start and end of region.
+ * Calculate the first possible address.
*/
- src_start = trunc_page(addr);
- src_end = round_page(src_start + size);
+ if (start < map->min_offset)
+ start = map->min_offset;
+ if (start > map->max_offset)
+ return(KERN_NO_SPACE);
+
/*
- * Initialize map_header.
+ * Look for the first possible address;
+ * if there's already something at this
+ * address, we have to start after it.
*/
- map_header->links.next = (struct vm_map_entry *)&map_header->links;
- map_header->links.prev = (struct vm_map_entry *)&map_header->links;
- map_header->nentries = 0;
- map_header->entries_pageable = pageable;
-
- *cur_protection = VM_PROT_ALL;
- *max_protection = VM_PROT_ALL;
-
- map_address = 0;
- mapped_size = 0;
- result = KERN_SUCCESS;
- /*
- * The specified source virtual space might correspond to
- * multiple map entries, need to loop on them.
+ assert(first_free_is_valid(map));
+ if (start == map->min_offset) {
+ if ((entry = map->first_free) != vm_map_to_entry(map))
+ start = entry->vme_end;
+ } else {
+ vm_map_entry_t tmp_entry;
+ if (vm_map_lookup_entry(map, start, &tmp_entry))
+ start = tmp_entry->vme_end;
+ entry = tmp_entry;
+ }
+
+ /*
+ * In any case, the "entry" always precedes
+ * the proposed new region throughout the
+ * loop:
*/
- vm_map_lock(map);
- while (mapped_size != size) {
- vm_size_t entry_size;
-
- /*
- * Find the beginning of the region.
- */
- if (! vm_map_lookup_entry(map, src_start, &src_entry)) {
- result = KERN_INVALID_ADDRESS;
- break;
- }
-
- if (src_start < src_entry->vme_start ||
- (mapped_size && src_start != src_entry->vme_start)) {
- result = KERN_INVALID_ADDRESS;
- break;
- }
-
- if(src_entry->is_sub_map) {
- result = KERN_INVALID_ADDRESS;
- break;
- }
-
- tmp_size = size - mapped_size;
- if (src_end > src_entry->vme_end)
- tmp_size -= (src_end - src_entry->vme_end);
- entry_size = (vm_size_t)(src_entry->vme_end -
- src_entry->vme_start);
-
- if(src_entry->is_sub_map) {
- vm_map_reference(src_entry->object.sub_map);
- } else {
- object = src_entry->object.vm_object;
+ while (TRUE) {
+ register vm_map_entry_t next;
- if (object == VM_OBJECT_NULL) {
- object = vm_object_allocate(entry_size);
- src_entry->offset = 0;
- src_entry->object.vm_object = object;
- } else if (object->copy_strategy !=
- MEMORY_OBJECT_COPY_SYMMETRIC) {
- /*
- * We are already using an asymmetric
- * copy, and therefore we already have
- * the right object.
- */
- assert(!src_entry->needs_copy);
- } else if (src_entry->needs_copy || object->shadowed ||
- (object->internal && !object->true_share &&
- !src_entry->is_shared &&
- object->size > entry_size)) {
+ /*
+ * Find the end of the proposed new region.
+ * Be sure we didn't go beyond the end, or
+ * wrap around the address.
+ */
- vm_object_shadow(&src_entry->object.vm_object,
- &src_entry->offset,
- entry_size);
+ end = ((start + mask) & ~mask);
+ if (end < start)
+ return(KERN_NO_SPACE);
+ start = end;
+ end += size;
- if (!src_entry->needs_copy &&
- (src_entry->protection & VM_PROT_WRITE)) {
- pmap_protect(vm_map_pmap(map),
- src_entry->vme_start,
- src_entry->vme_end,
- src_entry->protection &
- ~VM_PROT_WRITE);
- }
+ if ((end > map->max_offset) || (end < start)) {
+ if (map->wait_for_space) {
+ if (size <= (map->max_offset -
+ map->min_offset)) {
+ assert_wait((event_t) map, THREAD_INTERRUPTIBLE);
+ vm_map_unlock(map);
+ thread_block(THREAD_CONTINUE_NULL);
+ vm_map_lock(map);
+ goto StartAgain;
+ }
+ }
+
+ return(KERN_NO_SPACE);
+ }
- object = src_entry->object.vm_object;
- src_entry->needs_copy = FALSE;
- }
+ /*
+ * If there are no more entries, we must win.
+ */
+ next = entry->vme_next;
+ if (next == vm_map_to_entry(map))
+ break;
- vm_object_lock(object);
- object->ref_count++; /* object ref. for new entry */
- VM_OBJ_RES_INCR(object);
- if (object->copy_strategy ==
- MEMORY_OBJECT_COPY_SYMMETRIC) {
- object->copy_strategy =
- MEMORY_OBJECT_COPY_DELAY;
- }
- vm_object_unlock(object);
- }
+ /*
+ * If there is another entry, it must be
+ * after the end of the potential new region.
+ */
- offset = src_entry->offset + (src_start - src_entry->vme_start);
+ if (next->vme_start >= end)
+ break;
- new_entry = _vm_map_entry_create(map_header);
- vm_map_entry_copy(new_entry, src_entry);
- new_entry->use_pmap = FALSE; /* clr address space specifics */
+ /*
+ * Didn't fit -- move to the next entry.
+ */
- new_entry->vme_start = map_address;
- new_entry->vme_end = map_address + tmp_size;
- new_entry->inheritance = inheritance;
- new_entry->offset = offset;
+ entry = next;
+ start = entry->vme_end;
+ }
+ *address = start;
+ } else {
+ vm_map_entry_t temp_entry;
+
+ /*
+ * Verify that:
+ * the address doesn't itself violate
+ * the mask requirement.
+ */
- /*
- * The new region has to be copied now if required.
- */
- RestartCopy:
- if (!copy) {
- src_entry->is_shared = TRUE;
- new_entry->is_shared = TRUE;
- if (!(new_entry->is_sub_map))
- new_entry->needs_copy = FALSE;
+ if ((start & mask) != 0)
+ return(KERN_NO_SPACE);
- } else if (src_entry->is_sub_map) {
- /* make this a COW sub_map if not already */
- new_entry->needs_copy = TRUE;
- } else if (src_entry->wired_count == 0 &&
- vm_object_copy_quickly(&new_entry->object.vm_object,
- new_entry->offset,
- (new_entry->vme_end -
- new_entry->vme_start),
- &src_needs_copy,
- &new_entry_needs_copy)) {
- new_entry->needs_copy = new_entry_needs_copy;
- new_entry->is_shared = FALSE;
+ /*
+ * ... the address is within bounds
+ */
- /*
- * Handle copy_on_write semantics.
- */
- if (src_needs_copy && !src_entry->needs_copy) {
- vm_object_pmap_protect(object,
- offset,
- entry_size,
- (src_entry->is_shared ?
- PMAP_NULL : map->pmap),
- src_entry->vme_start,
- src_entry->protection &
- ~VM_PROT_WRITE);
+ end = start + size;
- src_entry->needs_copy = TRUE;
- }
- /*
- * Throw away the old object reference of the new entry.
- */
- vm_object_deallocate(object);
+ if ((start < map->min_offset) ||
+ (end > map->max_offset) ||
+ (start >= end)) {
+ return(KERN_INVALID_ADDRESS);
+ }
- } else {
- new_entry->is_shared = FALSE;
+ /*
+ * ... the starting address isn't allocated
+ */
- /*
- * The map can be safely unlocked since we
- * already hold a reference on the object.
- *
- * Record the timestamp of the map for later
- * verification, and unlock the map.
- */
- version.main_timestamp = map->timestamp;
- vm_map_unlock(map);
+ if (vm_map_lookup_entry(map, start, &temp_entry))
+ return(KERN_NO_SPACE);
- /*
- * Perform the copy.
- */
- if (src_entry->wired_count > 0) {
- vm_object_lock(object);
- result = vm_object_copy_slowly(
- object,
- offset,
- entry_size,
- THREAD_UNINT,
- &new_entry->object.vm_object);
+ entry = temp_entry;
- new_entry->offset = 0;
- new_entry->needs_copy = FALSE;
- } else {
- result = vm_object_copy_strategically(
- object,
- offset,
- entry_size,
- &new_entry->object.vm_object,
- &new_entry->offset,
- &new_entry_needs_copy);
+ /*
+ * ... the next region doesn't overlap the
+ * end point.
+ */
- new_entry->needs_copy = new_entry_needs_copy;
- }
+ if ((entry->vme_next != vm_map_to_entry(map)) &&
+ (entry->vme_next->vme_start < end))
+ return(KERN_NO_SPACE);
+ }
+ *map_entry = entry;
+ return(KERN_SUCCESS);
+}
- /*
- * Throw away the old object reference of the new entry.
- */
- vm_object_deallocate(object);
+/*
+ * vm_map_switch:
+ *
+ * Set the address map for the current thread to the specified map
+ */
- if (result != KERN_SUCCESS &&
- result != KERN_MEMORY_RESTART_COPY) {
- _vm_map_entry_dispose(map_header, new_entry);
- break;
- }
+vm_map_t
+vm_map_switch(
+ vm_map_t map)
+{
+ int mycpu;
+ thread_t thread = current_thread();
+ vm_map_t oldmap = thread->map;
- /*
- * Verify that the map has not substantially
- * changed while the copy was being made.
- */
+ mp_disable_preemption();
+ mycpu = cpu_number();
- vm_map_lock(map); /* Increments timestamp once! */
- if (version.main_timestamp + 1 != map->timestamp) {
- /*
- * Simple version comparison failed.
- *
- * Retry the lookup and verify that the
- * same object/offset are still present.
- */
- vm_object_deallocate(new_entry->
- object.vm_object);
- _vm_map_entry_dispose(map_header, new_entry);
- if (result == KERN_MEMORY_RESTART_COPY)
- result = KERN_SUCCESS;
- continue;
- }
+ /*
+ * Deactivate the current map and activate the requested map
+ */
+ PMAP_SWITCH_USER(thread, map, mycpu);
- if (result == KERN_MEMORY_RESTART_COPY) {
- vm_object_reference(object);
- goto RestartCopy;
- }
- }
+ mp_enable_preemption();
+ return(oldmap);
+}
- _vm_map_entry_link(map_header,
- map_header->links.prev, new_entry);
- *cur_protection &= src_entry->protection;
- *max_protection &= src_entry->max_protection;
+/*
+ * Routine: vm_map_write_user
+ *
+ * Description:
+ * Copy out data from a kernel space into space in the
+ * destination map. The space must already exist in the
+ * destination map.
+ * NOTE: This routine should only be called by threads
+ * which can block on a page fault. i.e. kernel mode user
+ * threads.
+ *
+ */
+kern_return_t
+vm_map_write_user(
+ vm_map_t map,
+ void *src_p,
+ vm_map_address_t dst_addr,
+ vm_size_t size)
+{
+ kern_return_t kr = KERN_SUCCESS;
- map_address += tmp_size;
- mapped_size += tmp_size;
- src_start += tmp_size;
+ if(current_map() == map) {
+ if (copyout(src_p, dst_addr, size)) {
+ kr = KERN_INVALID_ADDRESS;
+ }
+ } else {
+ vm_map_t oldmap;
- } /* end while */
+ /* take on the identity of the target map while doing */
+ /* the transfer */
- vm_map_unlock(map);
- if (result != KERN_SUCCESS) {
- /*
- * Free all allocated elements.
- */
- for (src_entry = map_header->links.next;
- src_entry != (struct vm_map_entry *)&map_header->links;
- src_entry = new_entry) {
- new_entry = src_entry->vme_next;
- _vm_map_entry_unlink(map_header, src_entry);
- vm_object_deallocate(src_entry->object.vm_object);
- _vm_map_entry_dispose(map_header, src_entry);
+ vm_map_reference(map);
+ oldmap = vm_map_switch(map);
+ if (copyout(src_p, dst_addr, size)) {
+ kr = KERN_INVALID_ADDRESS;
}
+ vm_map_switch(oldmap);
+ vm_map_deallocate(map);
}
- return result;
+ return kr;
}
/*
- * Routine: vm_remap
+ * Routine: vm_map_read_user
+ *
+ * Description:
+ * Copy in data from a user space source map into the
+ * kernel map. The space must already exist in the
+ * kernel map.
+ * NOTE: This routine should only be called by threads
+ * which can block on a page fault. i.e. kernel mode user
+ * threads.
*
- * Map portion of a task's address space.
- * Mapped region must not overlap more than
- * one vm memory object. Protections and
- * inheritance attributes remain the same
- * as in the original task and are out parameters.
- * Source and Target task can be identical
- * Other attributes are identical as for vm_map()
*/
kern_return_t
-vm_remap(
- vm_map_t target_map,
- vm_offset_t *address,
- vm_size_t size,
- vm_offset_t mask,
- boolean_t anywhere,
- vm_map_t src_map,
- vm_offset_t memory_address,
- boolean_t copy,
- vm_prot_t *cur_protection,
- vm_prot_t *max_protection,
- vm_inherit_t inheritance)
+vm_map_read_user(
+ vm_map_t map,
+ vm_map_address_t src_addr,
+ void *dst_p,
+ vm_size_t size)
{
- kern_return_t result;
- vm_map_entry_t entry;
- vm_map_entry_t insp_entry;
- vm_map_entry_t new_entry;
- struct vm_map_header map_header;
+ kern_return_t kr = KERN_SUCCESS;
- if (target_map == VM_MAP_NULL)
- return KERN_INVALID_ARGUMENT;
+ if(current_map() == map) {
+ if (copyin(src_addr, dst_p, size)) {
+ kr = KERN_INVALID_ADDRESS;
+ }
+ } else {
+ vm_map_t oldmap;
- switch (inheritance) {
- case VM_INHERIT_NONE:
- case VM_INHERIT_COPY:
- case VM_INHERIT_SHARE:
- if (size != 0 && src_map != VM_MAP_NULL)
- break;
- /*FALL THRU*/
- default:
- return KERN_INVALID_ARGUMENT;
- }
+ /* take on the identity of the target map while doing */
+ /* the transfer */
- size = round_page(size);
+ vm_map_reference(map);
+ oldmap = vm_map_switch(map);
+ if (copyin(src_addr, dst_p, size)) {
+ kr = KERN_INVALID_ADDRESS;
+ }
+ vm_map_switch(oldmap);
+ vm_map_deallocate(map);
+ }
+ return kr;
+}
- result = vm_remap_extract(src_map, memory_address,
- size, copy, &map_header,
- cur_protection,
- max_protection,
- inheritance,
- target_map->hdr.
- entries_pageable);
- vm_map_deallocate(src_map);
- if (result != KERN_SUCCESS) {
- return result;
- }
+/*
+ * vm_map_check_protection:
+ *
+ * Assert that the target map allows the specified
+ * privilege on the entire address region given.
+ * The entire region must be allocated.
+ */
+boolean_t vm_map_check_protection(map, start, end, protection)
+ register vm_map_t map;
+ register vm_map_offset_t start;
+ register vm_map_offset_t end;
+ register vm_prot_t protection;
+{
+ register vm_map_entry_t entry;
+ vm_map_entry_t tmp_entry;
- /*
- * Allocate/check a range of free virtual address
- * space for the target
- */
- *address = trunc_page(*address);
- vm_map_lock(target_map);
- result = vm_remap_range_allocate(target_map, address, size,
- mask, anywhere, &insp_entry);
+ vm_map_lock(map);
- for (entry = map_header.links.next;
- entry != (struct vm_map_entry *)&map_header.links;
- entry = new_entry) {
- new_entry = entry->vme_next;
- _vm_map_entry_unlink(&map_header, entry);
- if (result == KERN_SUCCESS) {
- entry->vme_start += *address;
- entry->vme_end += *address;
- vm_map_entry_link(target_map, insp_entry, entry);
- insp_entry = entry;
- } else {
- if (!entry->is_sub_map) {
- vm_object_deallocate(entry->object.vm_object);
- } else {
- vm_map_deallocate(entry->object.sub_map);
- }
- _vm_map_entry_dispose(&map_header, entry);
- }
+ if (start < vm_map_min(map) || end > vm_map_max(map) || start > end)
+ {
+ vm_map_unlock(map);
+ return (FALSE);
}
- if (result == KERN_SUCCESS) {
- target_map->size += size;
- SAVE_HINT(target_map, insp_entry);
+ if (!vm_map_lookup_entry(map, start, &tmp_entry)) {
+ vm_map_unlock(map);
+ return(FALSE);
}
- vm_map_unlock(target_map);
- if (result == KERN_SUCCESS && target_map->wiring_required)
- result = vm_map_wire(target_map, *address,
- *address + size, *cur_protection, TRUE);
- return result;
-}
+ entry = tmp_entry;
-/*
- * Routine: vm_remap_range_allocate
- *
- * Description:
- * Allocate a range in the specified virtual address map.
- * returns the address and the map entry just before the allocated
- * range
- *
- * Map must be locked.
- */
+ while (start < end) {
+ if (entry == vm_map_to_entry(map)) {
+ vm_map_unlock(map);
+ return(FALSE);
+ }
-kern_return_t
-vm_remap_range_allocate(
- vm_map_t map,
- vm_offset_t *address, /* IN/OUT */
- vm_size_t size,
- vm_offset_t mask,
- boolean_t anywhere,
- vm_map_entry_t *map_entry) /* OUT */
-{
- register vm_map_entry_t entry;
- register vm_offset_t start;
- register vm_offset_t end;
- kern_return_t result = KERN_SUCCESS;
+ /*
+ * No holes allowed!
+ */
- StartAgain: ;
+ if (start < entry->vme_start) {
+ vm_map_unlock(map);
+ return(FALSE);
+ }
- start = *address;
+ /*
+ * Check protection associated with entry.
+ */
- if (anywhere)
- {
- /*
- * Calculate the first possible address.
- */
+ if ((entry->protection & protection) != protection) {
+ vm_map_unlock(map);
+ return(FALSE);
+ }
- if (start < map->min_offset)
- start = map->min_offset;
- if (start > map->max_offset)
- return(KERN_NO_SPACE);
-
- /*
- * Look for the first possible address;
- * if there's already something at this
- * address, we have to start after it.
- */
+ /* go to next entry */
- assert(first_free_is_valid(map));
- if (start == map->min_offset) {
- if ((entry = map->first_free) != vm_map_to_entry(map))
start = entry->vme_end;
- } else {
- vm_map_entry_t tmp_entry;
- if (vm_map_lookup_entry(map, start, &tmp_entry))
- start = tmp_entry->vme_end;
- entry = tmp_entry;
+ entry = entry->vme_next;
}
-
+ vm_map_unlock(map);
+ return(TRUE);
+}
+
+kern_return_t
+vm_map_purgable_control(
+ vm_map_t map,
+ vm_map_offset_t address,
+ vm_purgable_t control,
+ int *state)
+{
+ vm_map_entry_t entry;
+ vm_object_t object;
+ kern_return_t kr;
+
/*
- * In any case, the "entry" always precedes
- * the proposed new region throughout the
- * loop:
+ * Vet all the input parameters and current type and state of the
+ * underlaying object. Return with an error if anything is amiss.
*/
+ if (map == VM_MAP_NULL)
+ return(KERN_INVALID_ARGUMENT);
- while (TRUE) {
- register vm_map_entry_t next;
+ if (control != VM_PURGABLE_SET_STATE &&
+ control != VM_PURGABLE_GET_STATE)
+ return(KERN_INVALID_ARGUMENT);
- /*
- * Find the end of the proposed new region.
- * Be sure we didn't go beyond the end, or
- * wrap around the address.
- */
+ if (control == VM_PURGABLE_SET_STATE &&
+ (*state < VM_PURGABLE_STATE_MIN ||
+ *state > VM_PURGABLE_STATE_MAX))
+ return(KERN_INVALID_ARGUMENT);
- end = ((start + mask) & ~mask);
- if (end < start)
- return(KERN_NO_SPACE);
- start = end;
- end += size;
+ vm_map_lock(map);
- if ((end > map->max_offset) || (end < start)) {
- if (map->wait_for_space) {
- if (size <= (map->max_offset -
- map->min_offset)) {
- assert_wait((event_t) map, THREAD_INTERRUPTIBLE);
- vm_map_unlock(map);
- thread_block((void (*)(void))0);
- vm_map_lock(map);
- goto StartAgain;
- }
- }
-
- return(KERN_NO_SPACE);
- }
+ if (!vm_map_lookup_entry(map, address, &entry) || entry->is_sub_map) {
- /*
- * If there are no more entries, we must win.
- */
+ /*
+ * Must pass a valid non-submap address.
+ */
+ vm_map_unlock(map);
+ return(KERN_INVALID_ADDRESS);
+ }
- next = entry->vme_next;
- if (next == vm_map_to_entry(map))
- break;
+ if ((entry->protection & VM_PROT_WRITE) == 0) {
+ /*
+ * Can't apply purgable controls to something you can't write.
+ */
+ vm_map_unlock(map);
+ return(KERN_PROTECTION_FAILURE);
+ }
- /*
- * If there is another entry, it must be
- * after the end of the potential new region.
- */
+ object = entry->object.vm_object;
+ if (object == VM_OBJECT_NULL) {
+ /*
+ * Object must already be present or it can't be purgable.
+ */
+ vm_map_unlock(map);
+ return KERN_INVALID_ARGUMENT;
+ }
+
+ vm_object_lock(object);
- if (next->vme_start >= end)
- break;
+ if (entry->offset != 0 ||
+ entry->vme_end - entry->vme_start != object->size) {
+ /*
+ * Can only apply purgable controls to the whole (existing)
+ * object at once.
+ */
+ vm_map_unlock(map);
+ vm_object_unlock(object);
+ return KERN_INVALID_ARGUMENT;
+ }
+
+ vm_map_unlock(map);
- /*
- * Didn't fit -- move to the next entry.
- */
+ kr = vm_object_purgable_control(object, control, state);
- entry = next;
- start = entry->vme_end;
- }
- *address = start;
- } else {
- vm_map_entry_t temp_entry;
-
- /*
- * Verify that:
- * the address doesn't itself violate
- * the mask requirement.
- */
+ vm_object_unlock(object);
- if ((start & mask) != 0)
- return(KERN_NO_SPACE);
+ return kr;
+}
+kern_return_t
+vm_map_page_info(
+ vm_map_t target_map,
+ vm_map_offset_t offset,
+ int *disposition,
+ int *ref_count)
+{
+ vm_map_entry_t map_entry;
+ vm_object_t object;
+ vm_page_t m;
- /*
- * ... the address is within bounds
- */
+restart_page_query:
+ *disposition = 0;
+ *ref_count = 0;
+ vm_map_lock(target_map);
+ if(!vm_map_lookup_entry(target_map, offset, &map_entry)) {
+ vm_map_unlock(target_map);
+ return KERN_FAILURE;
+ }
+ offset -= map_entry->vme_start; /* adjust to offset within entry */
+ offset += map_entry->offset; /* adjust to target object offset */
+ if(map_entry->object.vm_object != VM_OBJECT_NULL) {
+ if(!map_entry->is_sub_map) {
+ object = map_entry->object.vm_object;
+ } else {
+ vm_map_unlock(target_map);
+ target_map = map_entry->object.sub_map;
+ goto restart_page_query;
+ }
+ } else {
+ vm_map_unlock(target_map);
+ return KERN_FAILURE;
+ }
+ vm_object_lock(object);
+ vm_map_unlock(target_map);
+ while(TRUE) {
+ m = vm_page_lookup(object, offset);
+ if (m != VM_PAGE_NULL) {
+ *disposition |= VM_PAGE_QUERY_PAGE_PRESENT;
+ break;
+ } else {
+ if(object->shadow) {
+ offset += object->shadow_offset;
+ vm_object_unlock(object);
+ object = object->shadow;
+ vm_object_lock(object);
+ continue;
+ }
+ vm_object_unlock(object);
+ return KERN_FAILURE;
+ }
+ }
- end = start + size;
+ /* The ref_count is not strictly accurate, it measures the number */
+ /* of entities holding a ref on the object, they may not be mapping */
+ /* the object or may not be mapping the section holding the */
+ /* target page but its still a ball park number and though an over- */
+ /* count, it picks up the copy-on-write cases */
- if ((start < map->min_offset) ||
- (end > map->max_offset) ||
- (start >= end)) {
- return(KERN_INVALID_ADDRESS);
+ /* We could also get a picture of page sharing from pmap_attributes */
+ /* but this would under count as only faulted-in mappings would */
+ /* show up. */
+
+ *ref_count = object->ref_count;
+
+ if (m->fictitious) {
+ *disposition |= VM_PAGE_QUERY_PAGE_FICTITIOUS;
+ vm_object_unlock(object);
+ return KERN_SUCCESS;
}
- /*
- * ... the starting address isn't allocated
- */
+ if (m->dirty)
+ *disposition |= VM_PAGE_QUERY_PAGE_DIRTY;
+ else if(pmap_is_modified(m->phys_page))
+ *disposition |= VM_PAGE_QUERY_PAGE_DIRTY;
- if (vm_map_lookup_entry(map, start, &temp_entry))
- return(KERN_NO_SPACE);
+ if (m->reference)
+ *disposition |= VM_PAGE_QUERY_PAGE_REF;
+ else if(pmap_is_referenced(m->phys_page))
+ *disposition |= VM_PAGE_QUERY_PAGE_REF;
- entry = temp_entry;
+ vm_object_unlock(object);
+ return KERN_SUCCESS;
+
+}
- /*
- * ... the next region doesn't overlap the
- * end point.
- */
- if ((entry->vme_next != vm_map_to_entry(map)) &&
- (entry->vme_next->vme_start < end))
- return(KERN_NO_SPACE);
- }
- *map_entry = entry;
- return(KERN_SUCCESS);
+/* For a given range, check all map entries. If the entry coresponds to */
+/* the old vm_region/map provided on the call, replace it with the */
+/* corresponding range in the new vm_region/map */
+kern_return_t vm_map_region_replace(
+ vm_map_t target_map,
+ ipc_port_t old_region,
+ ipc_port_t new_region,
+ vm_map_offset_t start,
+ vm_map_offset_t end)
+{
+ vm_named_entry_t old_object;
+ vm_named_entry_t new_object;
+ vm_map_t old_submap;
+ vm_map_t new_submap;
+ vm_map_offset_t addr;
+ vm_map_entry_t entry;
+ int nested_pmap = 0;
+
+
+ vm_map_lock(target_map);
+ old_object = (vm_named_entry_t)old_region->ip_kobject;
+ new_object = (vm_named_entry_t)new_region->ip_kobject;
+ if((!old_object->is_sub_map) || (!new_object->is_sub_map)) {
+ vm_map_unlock(target_map);
+ return KERN_INVALID_ARGUMENT;
+ }
+ old_submap = (vm_map_t)old_object->backing.map;
+ new_submap = (vm_map_t)new_object->backing.map;
+ vm_map_lock(old_submap);
+ if((old_submap->min_offset != new_submap->min_offset) ||
+ (old_submap->max_offset != new_submap->max_offset)) {
+ vm_map_unlock(old_submap);
+ vm_map_unlock(target_map);
+ return KERN_INVALID_ARGUMENT;
+ }
+ if(!vm_map_lookup_entry(target_map, start, &entry)) {
+ /* if the src is not contained, the entry preceeds */
+ /* our range */
+ addr = entry->vme_start;
+ if(entry == vm_map_to_entry(target_map)) {
+ vm_map_unlock(old_submap);
+ vm_map_unlock(target_map);
+ return KERN_SUCCESS;
+ }
+ }
+ if ((entry->use_pmap) &&
+ (new_submap->pmap == NULL)) {
+ new_submap->pmap = pmap_create((vm_map_size_t) 0);
+ if(new_submap->pmap == PMAP_NULL) {
+ vm_map_unlock(old_submap);
+ vm_map_unlock(target_map);
+ return(KERN_NO_SPACE);
+ }
+ }
+ addr = entry->vme_start;
+ vm_map_reference(old_submap);
+ while((entry != vm_map_to_entry(target_map)) &&
+ (entry->vme_start < end)) {
+ if((entry->is_sub_map) &&
+ (entry->object.sub_map == old_submap)) {
+ if(entry->use_pmap) {
+ if((start & 0x0fffffff) ||
+ ((end - start) != 0x10000000)) {
+ vm_map_unlock(old_submap);
+ vm_map_deallocate(old_submap);
+ vm_map_unlock(target_map);
+ return KERN_INVALID_ARGUMENT;
+ }
+ nested_pmap = 1;
+ }
+ entry->object.sub_map = new_submap;
+ vm_map_reference(new_submap);
+ vm_map_deallocate(old_submap);
+ }
+ entry = entry->vme_next;
+ addr = entry->vme_start;
+ }
+ if(nested_pmap) {
+#ifndef i386
+ pmap_unnest(target_map->pmap, (addr64_t)start);
+ if(target_map->mapped) {
+ vm_map_submap_pmap_clean(target_map,
+ start, end, old_submap, 0);
+ }
+ pmap_nest(target_map->pmap, new_submap->pmap,
+ (addr64_t)start, (addr64_t)start,
+ (uint64_t)(end - start));
+#endif /* i386 */
+ } else {
+ vm_map_submap_pmap_clean(target_map,
+ start, end, old_submap, 0);
+ }
+ vm_map_unlock(old_submap);
+ vm_map_deallocate(old_submap);
+ vm_map_unlock(target_map);
+ return KERN_SUCCESS;
}
/*
- * vm_map_switch:
- *
- * Set the address map for the current thr_act to the specified map
+ * vm_map_msync
+ *
+ * Synchronises the memory range specified with its backing store
+ * image by either flushing or cleaning the contents to the appropriate
+ * memory manager engaging in a memory object synchronize dialog with
+ * the manager. The client doesn't return until the manager issues
+ * m_o_s_completed message. MIG Magically converts user task parameter
+ * to the task's address map.
+ *
+ * interpretation of sync_flags
+ * VM_SYNC_INVALIDATE - discard pages, only return precious
+ * pages to manager.
+ *
+ * VM_SYNC_INVALIDATE & (VM_SYNC_SYNCHRONOUS | VM_SYNC_ASYNCHRONOUS)
+ * - discard pages, write dirty or precious
+ * pages back to memory manager.
+ *
+ * VM_SYNC_SYNCHRONOUS | VM_SYNC_ASYNCHRONOUS
+ * - write dirty or precious pages back to
+ * the memory manager.
+ *
+ * VM_SYNC_CONTIGUOUS - does everything normally, but if there
+ * is a hole in the region, and we would
+ * have returned KERN_SUCCESS, return
+ * KERN_INVALID_ADDRESS instead.
+ *
+ * NOTE
+ * The memory object attributes have not yet been implemented, this
+ * function will have to deal with the invalidate attribute
+ *
+ * RETURNS
+ * KERN_INVALID_TASK Bad task parameter
+ * KERN_INVALID_ARGUMENT both sync and async were specified.
+ * KERN_SUCCESS The usual.
+ * KERN_INVALID_ADDRESS There was a hole in the region.
*/
-vm_map_t
-vm_map_switch(
- vm_map_t map)
+kern_return_t
+vm_map_msync(
+ vm_map_t map,
+ vm_map_address_t address,
+ vm_map_size_t size,
+ vm_sync_t sync_flags)
{
- int mycpu;
- thread_act_t thr_act = current_act();
- vm_map_t oldmap = thr_act->map;
-
- mp_disable_preemption();
- mycpu = cpu_number();
+ msync_req_t msr;
+ msync_req_t new_msr;
+ queue_chain_t req_q; /* queue of requests for this msync */
+ vm_map_entry_t entry;
+ vm_map_size_t amount_left;
+ vm_object_offset_t offset;
+ boolean_t do_sync_req;
+ boolean_t modifiable;
+ boolean_t had_hole = FALSE;
+
+ if ((sync_flags & VM_SYNC_ASYNCHRONOUS) &&
+ (sync_flags & VM_SYNC_SYNCHRONOUS))
+ return(KERN_INVALID_ARGUMENT);
/*
- * Deactivate the current map and activate the requested map
+ * align address and size on page boundaries
*/
- PMAP_SWITCH_USER(thr_act, map, mycpu);
+ size = vm_map_round_page(address + size) - vm_map_trunc_page(address);
+ address = vm_map_trunc_page(address);
- mp_enable_preemption();
- return(oldmap);
-}
+ if (map == VM_MAP_NULL)
+ return(KERN_INVALID_TASK);
+ if (size == 0)
+ return(KERN_SUCCESS);
-/*
- * Routine: vm_map_write_user
- *
- * Description:
- * Copy out data from a kernel space into space in the
- * destination map. The space must already exist in the
- * destination map.
- * NOTE: This routine should only be called by threads
- * which can block on a page fault. i.e. kernel mode user
- * threads.
- *
- */
-kern_return_t
-vm_map_write_user(
- vm_map_t map,
- vm_offset_t src_addr,
- vm_offset_t dst_addr,
- vm_size_t size)
-{
- thread_act_t thr_act = current_act();
- kern_return_t kr = KERN_SUCCESS;
+ queue_init(&req_q);
+ amount_left = size;
+
+ while (amount_left > 0) {
+ vm_object_size_t flush_size;
+ vm_object_t object;
+
+ vm_map_lock(map);
+ if (!vm_map_lookup_entry(map,
+ vm_map_trunc_page(address), &entry)) {
+
+ vm_size_t skip;
+
+ /*
+ * hole in the address map.
+ */
+ had_hole = TRUE;
+
+ /*
+ * Check for empty map.
+ */
+ if (entry == vm_map_to_entry(map) &&
+ entry->vme_next == entry) {
+ vm_map_unlock(map);
+ break;
+ }
+ /*
+ * Check that we don't wrap and that
+ * we have at least one real map entry.
+ */
+ if ((map->hdr.nentries == 0) ||
+ (entry->vme_next->vme_start < address)) {
+ vm_map_unlock(map);
+ break;
+ }
+ /*
+ * Move up to the next entry if needed
+ */
+ skip = (entry->vme_next->vme_start - address);
+ if (skip >= amount_left)
+ amount_left = 0;
+ else
+ amount_left -= skip;
+ address = entry->vme_next->vme_start;
+ vm_map_unlock(map);
+ continue;
+ }
- if(thr_act->map == map) {
- if (copyout((char *)src_addr, (char *)dst_addr, size)) {
- kr = KERN_INVALID_ADDRESS;
+ offset = address - entry->vme_start;
+
+ /*
+ * do we have more to flush than is contained in this
+ * entry ?
+ */
+ if (amount_left + entry->vme_start + offset > entry->vme_end) {
+ flush_size = entry->vme_end -
+ (entry->vme_start + offset);
+ } else {
+ flush_size = amount_left;
}
- } else {
- vm_map_t oldmap;
+ amount_left -= flush_size;
+ address += flush_size;
- /* take on the identity of the target map while doing */
- /* the transfer */
+ if (entry->is_sub_map == TRUE) {
+ vm_map_t local_map;
+ vm_map_offset_t local_offset;
- vm_map_reference(map);
- oldmap = vm_map_switch(map);
- if (copyout((char *)src_addr, (char *)dst_addr, size)) {
- kr = KERN_INVALID_ADDRESS;
+ local_map = entry->object.sub_map;
+ local_offset = entry->offset;
+ vm_map_unlock(map);
+ if (vm_map_msync(
+ local_map,
+ local_offset,
+ flush_size,
+ sync_flags) == KERN_INVALID_ADDRESS) {
+ had_hole = TRUE;
+ }
+ continue;
}
- vm_map_switch(oldmap);
- vm_map_deallocate(map);
- }
- return kr;
-}
+ object = entry->object.vm_object;
-/*
- * Routine: vm_map_read_user
- *
- * Description:
- * Copy in data from a user space source map into the
- * kernel map. The space must already exist in the
- * kernel map.
- * NOTE: This routine should only be called by threads
- * which can block on a page fault. i.e. kernel mode user
- * threads.
- *
- */
-kern_return_t
-vm_map_read_user(
- vm_map_t map,
- vm_offset_t src_addr,
- vm_offset_t dst_addr,
- vm_size_t size)
-{
- thread_act_t thr_act = current_act();
- kern_return_t kr = KERN_SUCCESS;
+ /*
+ * We can't sync this object if the object has not been
+ * created yet
+ */
+ if (object == VM_OBJECT_NULL) {
+ vm_map_unlock(map);
+ continue;
+ }
+ offset += entry->offset;
+ modifiable = (entry->protection & VM_PROT_WRITE)
+ != VM_PROT_NONE;
- if(thr_act->map == map) {
- if (copyin((char *)src_addr, (char *)dst_addr, size)) {
- kr = KERN_INVALID_ADDRESS;
+ vm_object_lock(object);
+
+ if (sync_flags & (VM_SYNC_KILLPAGES | VM_SYNC_DEACTIVATE)) {
+ boolean_t kill_pages = 0;
+
+ if (sync_flags & VM_SYNC_KILLPAGES) {
+ if (object->ref_count == 1 && !entry->needs_copy && !object->shadow)
+ kill_pages = 1;
+ else
+ kill_pages = -1;
+ }
+ if (kill_pages != -1)
+ vm_object_deactivate_pages(object, offset,
+ (vm_object_size_t)flush_size, kill_pages);
+ vm_object_unlock(object);
+ vm_map_unlock(map);
+ continue;
}
- } else {
- vm_map_t oldmap;
+ /*
+ * We can't sync this object if there isn't a pager.
+ * Don't bother to sync internal objects, since there can't
+ * be any "permanent" storage for these objects anyway.
+ */
+ if ((object->pager == MEMORY_OBJECT_NULL) ||
+ (object->internal) || (object->private)) {
+ vm_object_unlock(object);
+ vm_map_unlock(map);
+ continue;
+ }
+ /*
+ * keep reference on the object until syncing is done
+ */
+ assert(object->ref_count > 0);
+ object->ref_count++;
+ vm_object_res_reference(object);
+ vm_object_unlock(object);
- /* take on the identity of the target map while doing */
- /* the transfer */
+ vm_map_unlock(map);
- vm_map_reference(map);
- oldmap = vm_map_switch(map);
- if (copyin((char *)src_addr, (char *)dst_addr, size)) {
- kr = KERN_INVALID_ADDRESS;
+ do_sync_req = vm_object_sync(object,
+ offset,
+ flush_size,
+ sync_flags & VM_SYNC_INVALIDATE,
+ (modifiable &&
+ (sync_flags & VM_SYNC_SYNCHRONOUS ||
+ sync_flags & VM_SYNC_ASYNCHRONOUS)),
+ sync_flags & VM_SYNC_SYNCHRONOUS);
+ /*
+ * only send a m_o_s if we returned pages or if the entry
+ * is writable (ie dirty pages may have already been sent back)
+ */
+ if (!do_sync_req && !modifiable) {
+ vm_object_deallocate(object);
+ continue;
}
- vm_map_switch(oldmap);
- vm_map_deallocate(map);
- }
- return kr;
-}
+ msync_req_alloc(new_msr);
+
+ vm_object_lock(object);
+ offset += object->paging_offset;
+
+ new_msr->offset = offset;
+ new_msr->length = flush_size;
+ new_msr->object = object;
+ new_msr->flag = VM_MSYNC_SYNCHRONIZING;
+re_iterate:
+ queue_iterate(&object->msr_q, msr, msync_req_t, msr_q) {
+ /*
+ * need to check for overlapping entry, if found, wait
+ * on overlapping msr to be done, then reiterate
+ */
+ msr_lock(msr);
+ if (msr->flag == VM_MSYNC_SYNCHRONIZING &&
+ ((offset >= msr->offset &&
+ offset < (msr->offset + msr->length)) ||
+ (msr->offset >= offset &&
+ msr->offset < (offset + flush_size))))
+ {
+ assert_wait((event_t) msr,THREAD_INTERRUPTIBLE);
+ msr_unlock(msr);
+ vm_object_unlock(object);
+ thread_block(THREAD_CONTINUE_NULL);
+ vm_object_lock(object);
+ goto re_iterate;
+ }
+ msr_unlock(msr);
+ }/* queue_iterate */
+
+ queue_enter(&object->msr_q, new_msr, msync_req_t, msr_q);
+ vm_object_unlock(object);
+
+ queue_enter(&req_q, new_msr, msync_req_t, req_q);
+
+ (void) memory_object_synchronize(
+ object->pager,
+ offset,
+ flush_size,
+ sync_flags & ~VM_SYNC_CONTIGUOUS);
+ }/* while */
+
+ /*
+ * wait for memory_object_sychronize_completed messages from pager(s)
+ */
+
+ while (!queue_empty(&req_q)) {
+ msr = (msync_req_t)queue_first(&req_q);
+ msr_lock(msr);
+ while(msr->flag != VM_MSYNC_DONE) {
+ assert_wait((event_t) msr, THREAD_INTERRUPTIBLE);
+ msr_unlock(msr);
+ thread_block(THREAD_CONTINUE_NULL);
+ msr_lock(msr);
+ }/* while */
+ queue_remove(&req_q, msr, msync_req_t, req_q);
+ msr_unlock(msr);
+ vm_object_deallocate(msr->object);
+ msync_req_free(msr);
+ }/* queue_iterate */
+
+ /* for proper msync() behaviour */
+ if (had_hole == TRUE && (sync_flags & VM_SYNC_CONTIGUOUS))
+ return(KERN_INVALID_ADDRESS);
+
+ return(KERN_SUCCESS);
+}/* vm_msync */
/* Takes existing source and destination sub-maps and clones the contents of */
/* the source map */
-
kern_return_t
vm_region_clone(
ipc_port_t src_region,
vm_named_entry_t dst_object;
vm_map_t src_map;
vm_map_t dst_map;
- vm_offset_t addr;
- vm_offset_t max_off;
+ vm_map_offset_t addr;
+ vm_map_offset_t max_off;
vm_map_entry_t entry;
vm_map_entry_t new_entry;
vm_map_entry_t insert_point;
return KERN_SUCCESS;
}
+/*
+ * Routine: convert_port_entry_to_map
+ * Purpose:
+ * Convert from a port specifying an entry or a task
+ * to a map. Doesn't consume the port ref; produces a map ref,
+ * which may be null. Unlike convert_port_to_map, the
+ * port may be task or a named entry backed.
+ * Conditions:
+ * Nothing locked.
+ */
+
+
+vm_map_t
+convert_port_entry_to_map(
+ ipc_port_t port)
+{
+ vm_map_t map;
+ vm_named_entry_t named_entry;
+
+ if(IP_VALID(port) && (ip_kotype(port) == IKOT_NAMED_ENTRY)) {
+ while(TRUE) {
+ ip_lock(port);
+ if(ip_active(port) && (ip_kotype(port)
+ == IKOT_NAMED_ENTRY)) {
+ named_entry =
+ (vm_named_entry_t)port->ip_kobject;
+ if (!(mutex_try(&(named_entry)->Lock))) {
+ ip_unlock(port);
+ mutex_pause();
+ continue;
+ }
+ named_entry->ref_count++;
+ mutex_unlock(&(named_entry)->Lock);
+ ip_unlock(port);
+ if ((named_entry->is_sub_map) &&
+ (named_entry->protection
+ & VM_PROT_WRITE)) {
+ map = named_entry->backing.map;
+ } else {
+ mach_destroy_memory_entry(port);
+ return VM_MAP_NULL;
+ }
+ vm_map_reference_swap(map);
+ mach_destroy_memory_entry(port);
+ break;
+ }
+ else
+ return VM_MAP_NULL;
+ }
+ }
+ else
+ map = convert_port_to_map(port);
+
+ return map;
+}
+
+/*
+ * Routine: convert_port_entry_to_object
+ * Purpose:
+ * Convert from a port specifying a named entry to an
+ * object. Doesn't consume the port ref; produces a map ref,
+ * which may be null.
+ * Conditions:
+ * Nothing locked.
+ */
+
+
+vm_object_t
+convert_port_entry_to_object(
+ ipc_port_t port)
+{
+ vm_object_t object;
+ vm_named_entry_t named_entry;
+
+ if(IP_VALID(port) && (ip_kotype(port) == IKOT_NAMED_ENTRY)) {
+ while(TRUE) {
+ ip_lock(port);
+ if(ip_active(port) && (ip_kotype(port)
+ == IKOT_NAMED_ENTRY)) {
+ named_entry =
+ (vm_named_entry_t)port->ip_kobject;
+ if (!(mutex_try(&(named_entry)->Lock))) {
+ ip_unlock(port);
+ mutex_pause();
+ continue;
+ }
+ named_entry->ref_count++;
+ mutex_unlock(&(named_entry)->Lock);
+ ip_unlock(port);
+ if ((!named_entry->is_sub_map) &&
+ (!named_entry->is_pager) &&
+ (named_entry->protection
+ & VM_PROT_WRITE)) {
+ object = named_entry->backing.object;
+ } else {
+ mach_destroy_memory_entry(port);
+ return (vm_object_t)NULL;
+ }
+ vm_object_reference(named_entry->backing.object);
+ mach_destroy_memory_entry(port);
+ break;
+ }
+ else
+ return (vm_object_t)NULL;
+ }
+ } else {
+ return (vm_object_t)NULL;
+ }
+
+ return object;
+}
+
/*
* Export routines to other components for the things we access locally through
* macros.
}
/*
- * vm_map_check_protection:
+ * vm_map_reference:
*
- * Assert that the target map allows the specified
- * privilege on the entire address region given.
- * The entire region must be allocated.
+ * Most code internal to the osfmk will go through a
+ * macro defining this. This is always here for the
+ * use of other kernel components.
*/
-boolean_t vm_map_check_protection(map, start, end, protection)
- register vm_map_t map;
- register vm_offset_t start;
- register vm_offset_t end;
- register vm_prot_t protection;
+#undef vm_map_reference
+void
+vm_map_reference(
+ register vm_map_t map)
{
- register vm_map_entry_t entry;
- vm_map_entry_t tmp_entry;
+ if (map == VM_MAP_NULL)
+ return;
- vm_map_lock(map);
+ mutex_lock(&map->s_lock);
+#if TASK_SWAPPER
+ assert(map->res_count > 0);
+ assert(map->ref_count >= map->res_count);
+ map->res_count++;
+#endif
+ map->ref_count++;
+ mutex_unlock(&map->s_lock);
+}
- if (start < vm_map_min(map) || end > vm_map_max(map) || start > end)
- {
- vm_map_unlock(map);
- return (FALSE);
- }
+/*
+ * vm_map_deallocate:
+ *
+ * Removes a reference from the specified map,
+ * destroying it if no references remain.
+ * The map should not be locked.
+ */
+void
+vm_map_deallocate(
+ register vm_map_t map)
+{
+ unsigned int ref;
- if (!vm_map_lookup_entry(map, start, &tmp_entry)) {
- vm_map_unlock(map);
- return(FALSE);
+ if (map == VM_MAP_NULL)
+ return;
+
+ mutex_lock(&map->s_lock);
+ ref = --map->ref_count;
+ if (ref > 0) {
+ vm_map_res_deallocate(map);
+ mutex_unlock(&map->s_lock);
+ return;
}
+ assert(map->ref_count == 0);
+ mutex_unlock(&map->s_lock);
- entry = tmp_entry;
+#if TASK_SWAPPER
+ /*
+ * The map residence count isn't decremented here because
+ * the vm_map_delete below will traverse the entire map,
+ * deleting entries, and the residence counts on objects
+ * and sharing maps will go away then.
+ */
+#endif
- while (start < end) {
- if (entry == vm_map_to_entry(map)) {
- vm_map_unlock(map);
- return(FALSE);
- }
+ vm_map_destroy(map);
+}
- /*
- * No holes allowed!
- */
+#ifdef __PPC__
- if (start < entry->vme_start) {
+/* LP64todo - this whole mechanism is temporary. It should be redone when
+ * the pmap layer can handle 64-bit address spaces. Until then, we trump
+ * up a map entry for the 64-bit commpage above the map's max_offset.
+ */
+extern vm_map_t com_region_map64; /* the submap for 64-bit commpage */
+SInt32 commpage64s_in_use = 0;
+
+void
+vm_map_commpage64(
+ vm_map_t map )
+{
+ vm_map_entry_t entry;
+ vm_object_t object;
+
+ vm_map_lock(map);
+
+ /* The commpage is necessarily the last entry in the map.
+ * See if one is already there (not sure if this can happen???)
+ */
+ entry = vm_map_last_entry(map);
+ if (entry != vm_map_to_entry(map)) {
+ if (entry->vme_end >= (vm_map_offset_t)_COMM_PAGE_BASE_ADDRESS) {
vm_map_unlock(map);
- return(FALSE);
+ return;
}
+ }
+
+ entry = vm_map_first_entry(com_region_map64); /* the 64-bit commpage */
+ object = entry->object.vm_object;
+ vm_object_reference(object);
+
+ /* We bypass vm_map_enter() because we are adding the entry past the
+ * map's max_offset.
+ */
+ entry = vm_map_entry_insert(
+ map,
+ vm_map_last_entry(map), /* insert after last entry */
+ _COMM_PAGE_BASE_ADDRESS,
+ _COMM_PAGE_BASE_ADDRESS+_COMM_PAGE_AREA_USED,
+ object,
+ 0, /* offset */
+ FALSE, /* needs_copy */
+ FALSE, /* is_shared */
+ FALSE, /* in_transition */
+ VM_PROT_READ,
+ VM_PROT_READ,
+ VM_BEHAVIOR_DEFAULT,
+ VM_INHERIT_NONE,
+ 1 ); /* wired_count */
+
+ vm_map_unlock(map);
+
+ OSIncrementAtomic(&commpage64s_in_use);
+}
- /*
- * Check protection associated with entry.
- */
-
- if ((entry->protection & protection) != protection) {
- vm_map_unlock(map);
- return(FALSE);
- }
- /* go to next entry */
+/* LP64todo - remove this! */
- start = entry->vme_end;
- entry = entry->vme_next;
+void
+vm_map_remove_commpage64(
+ vm_map_t map )
+{
+ vm_map_entry_t entry;
+ int deleted = 0;
+
+ while( 1 ) {
+ vm_map_lock(map);
+
+ entry = vm_map_last_entry(map);
+ if ((entry == vm_map_to_entry(map)) ||
+ (entry->vme_start < (vm_map_offset_t)_COMM_PAGE_BASE_ADDRESS))
+ break;
+
+ /* clearing the wired count isn't strictly correct */
+ entry->wired_count = 0;
+ vm_map_entry_delete(map,entry);
+ deleted++;
}
+
vm_map_unlock(map);
- return(TRUE);
+
+ if (deleted != 0)
+ OSDecrementAtomic(&commpage64s_in_use);
}
+
+#endif /* __PPC__ */
projects / apple / xnu.git / blobdiff