#include <mach/vm_param.h>
#include <mach/vm_statistics.h>
#include <mach/mach_syscalls.h>
+#include <mach/sdt.h>
#include <mach/host_priv_server.h>
#include <mach/mach_vm_server.h>
#include <vm/memory_object.h>
#include <vm/vm_pageout.h>
#include <vm/vm_protos.h>
+#include <vm/vm_purgeable_internal.h>
+#include <vm/vm_init.h>
vm_size_t upl_offset_to_pagelist = 0;
#include <vm/cpm.h>
#endif /* VM_CPM */
+lck_grp_t dynamic_pager_control_port_lock_group;
+decl_lck_mtx_data(, dynamic_pager_control_port_lock);
ipc_port_t dynamic_pager_control_port=NULL;
/*
*/
map_addr = vm_map_min(map);
if (map_addr == 0)
- map_addr += PAGE_SIZE;
+ map_addr += VM_MAP_PAGE_SIZE(map);
} else
- map_addr = vm_map_trunc_page(*addr);
- map_size = vm_map_round_page(size);
+ map_addr = vm_map_trunc_page(*addr,
+ VM_MAP_PAGE_MASK(map));
+ map_size = vm_map_round_page(size,
+ VM_MAP_PAGE_MASK(map));
if (map_size == 0) {
return(KERN_INVALID_ARGUMENT);
}
*/
map_addr = vm_map_min(map);
if (map_addr == 0)
- map_addr += PAGE_SIZE;
+ map_addr += VM_MAP_PAGE_SIZE(map);
} else
- map_addr = vm_map_trunc_page(*addr);
- map_size = vm_map_round_page(size);
+ map_addr = vm_map_trunc_page(*addr,
+ VM_MAP_PAGE_MASK(map));
+ map_size = vm_map_round_page(size,
+ VM_MAP_PAGE_MASK(map));
if (map_size == 0) {
return(KERN_INVALID_ARGUMENT);
}
if (size == (mach_vm_offset_t) 0)
return(KERN_SUCCESS);
- return(vm_map_remove(map, vm_map_trunc_page(start),
- vm_map_round_page(start+size), VM_MAP_NO_FLAGS));
+ return(vm_map_remove(map,
+ vm_map_trunc_page(start,
+ VM_MAP_PAGE_MASK(map)),
+ vm_map_round_page(start+size,
+ VM_MAP_PAGE_MASK(map)),
+ VM_MAP_NO_FLAGS));
}
/*
*/
kern_return_t
vm_deallocate(
- register vm_map_t map,
+ vm_map_t map,
vm_offset_t start,
vm_size_t size)
{
if (size == (vm_offset_t) 0)
return(KERN_SUCCESS);
- return(vm_map_remove(map, vm_map_trunc_page(start),
- vm_map_round_page(start+size), VM_MAP_NO_FLAGS));
+ return(vm_map_remove(map,
+ vm_map_trunc_page(start,
+ VM_MAP_PAGE_MASK(map)),
+ vm_map_round_page(start+size,
+ VM_MAP_PAGE_MASK(map)),
+ VM_MAP_NO_FLAGS));
}
/*
return KERN_SUCCESS;
return(vm_map_inherit(map,
- vm_map_trunc_page(start),
- vm_map_round_page(start+size),
+ vm_map_trunc_page(start,
+ VM_MAP_PAGE_MASK(map)),
+ vm_map_round_page(start+size,
+ VM_MAP_PAGE_MASK(map)),
new_inheritance));
}
*/
kern_return_t
vm_inherit(
- register vm_map_t map,
+ vm_map_t map,
vm_offset_t start,
vm_size_t size,
vm_inherit_t new_inheritance)
return KERN_SUCCESS;
return(vm_map_inherit(map,
- vm_map_trunc_page(start),
- vm_map_round_page(start+size),
+ vm_map_trunc_page(start,
+ VM_MAP_PAGE_MASK(map)),
+ vm_map_round_page(start+size,
+ VM_MAP_PAGE_MASK(map)),
new_inheritance));
}
return KERN_SUCCESS;
return(vm_map_protect(map,
- vm_map_trunc_page(start),
- vm_map_round_page(start+size),
+ vm_map_trunc_page(start,
+ VM_MAP_PAGE_MASK(map)),
+ vm_map_round_page(start+size,
+ VM_MAP_PAGE_MASK(map)),
new_protection,
set_maximum));
}
return KERN_SUCCESS;
return(vm_map_protect(map,
- vm_map_trunc_page(start),
- vm_map_round_page(start+size),
+ vm_map_trunc_page(start,
+ VM_MAP_PAGE_MASK(map)),
+ vm_map_round_page(start+size,
+ VM_MAP_PAGE_MASK(map)),
new_protection,
set_maximum));
}
if (size == 0)
return KERN_SUCCESS;
- return vm_map_machine_attribute(map,
- vm_map_trunc_page(addr),
- vm_map_round_page(addr+size),
- attribute,
- value);
+ return vm_map_machine_attribute(
+ map,
+ vm_map_trunc_page(addr,
+ VM_MAP_PAGE_MASK(map)),
+ vm_map_round_page(addr+size,
+ VM_MAP_PAGE_MASK(map)),
+ attribute,
+ value);
}
/*
if (size == 0)
return KERN_SUCCESS;
- return vm_map_machine_attribute(map,
- vm_map_trunc_page(addr),
- vm_map_round_page(addr+size),
- attribute,
- value);
+ return vm_map_machine_attribute(
+ map,
+ vm_map_trunc_page(addr,
+ VM_MAP_PAGE_MASK(map)),
+ vm_map_round_page(addr+size,
+ VM_MAP_PAGE_MASK(map)),
+ attribute,
+ value);
}
/*
vm_prot_t max_protection,
vm_inherit_t inheritance)
{
+ kern_return_t kr;
+ vm_map_offset_t vmmaddr;
+
+ vmmaddr = (vm_map_offset_t) *address;
+
/* filter out any kernel-only flags */
if (flags & ~VM_FLAGS_USER_MAP)
return KERN_INVALID_ARGUMENT;
- return vm_map_enter_mem_object(target_map,
- address,
+ kr = vm_map_enter_mem_object(target_map,
+ &vmmaddr,
initial_size,
mask,
flags,
cur_protection,
max_protection,
inheritance);
+
+ *address = vmmaddr;
+ return kr;
}
return KERN_INVALID_ARGUMENT;
if (access != VM_PROT_NONE) {
- rc = vm_map_wire(map, vm_map_trunc_page(start),
- vm_map_round_page(start+size), access, TRUE);
+ rc = vm_map_wire(map,
+ vm_map_trunc_page(start,
+ VM_MAP_PAGE_MASK(map)),
+ vm_map_round_page(start+size,
+ VM_MAP_PAGE_MASK(map)),
+ access | VM_PROT_MEMORY_TAG_MAKE(VM_KERN_MEMORY_MLOCK),
+ TRUE);
} else {
- rc = vm_map_unwire(map, vm_map_trunc_page(start),
- vm_map_round_page(start+size), TRUE);
+ rc = vm_map_unwire(map,
+ vm_map_trunc_page(start,
+ VM_MAP_PAGE_MASK(map)),
+ vm_map_round_page(start+size,
+ VM_MAP_PAGE_MASK(map)),
+ TRUE);
}
return rc;
}
kern_return_t
vm_wire(
host_priv_t host_priv,
- register vm_map_t map,
+ vm_map_t map,
vm_offset_t start,
vm_size_t size,
vm_prot_t access)
if (size == 0) {
rc = KERN_SUCCESS;
} else if (access != VM_PROT_NONE) {
- rc = vm_map_wire(map, vm_map_trunc_page(start),
- vm_map_round_page(start+size), access, TRUE);
+ rc = vm_map_wire(map,
+ vm_map_trunc_page(start,
+ VM_MAP_PAGE_MASK(map)),
+ vm_map_round_page(start+size,
+ VM_MAP_PAGE_MASK(map)),
+ access | VM_PROT_MEMORY_TAG_MAKE(VM_KERN_MEMORY_OSFMK),
+ TRUE);
} else {
- rc = vm_map_unwire(map, vm_map_trunc_page(start),
- vm_map_round_page(start+size), TRUE);
+ rc = vm_map_unwire(map,
+ vm_map_trunc_page(start,
+ VM_MAP_PAGE_MASK(map)),
+ vm_map_round_page(start+size,
+ VM_MAP_PAGE_MASK(map)),
+ TRUE);
}
return rc;
}
{
vm_map_t map = current_map();
+ assert(!map->is_nested_map);
if(toggle == VM_TOGGLE_GETVALUE && old_value != NULL){
*old_value = map->disable_vmentry_reuse;
} else if(toggle == VM_TOGGLE_SET){
+ vm_map_entry_t map_to_entry;
+
vm_map_lock(map);
+ vm_map_disable_hole_optimization(map);
map->disable_vmentry_reuse = TRUE;
- if (map->first_free == vm_map_to_entry(map)) {
+ __IGNORE_WCASTALIGN(map_to_entry = vm_map_to_entry(map));
+ if (map->first_free == map_to_entry) {
map->highest_entry_end = vm_map_min(map);
} else {
map->highest_entry_end = map->first_free->vme_end;
mach_vm_behavior_set(
vm_map_t map,
mach_vm_offset_t start,
- mach_vm_size_t size,
+ mach_vm_size_t size,
vm_behavior_t new_behavior)
{
+ vm_map_offset_t align_mask;
+
if ((map == VM_MAP_NULL) || (start + size < start))
return(KERN_INVALID_ARGUMENT);
if (size == 0)
return KERN_SUCCESS;
- return(vm_map_behavior_set(map, vm_map_trunc_page(start),
- vm_map_round_page(start+size), new_behavior));
+ switch (new_behavior) {
+ case VM_BEHAVIOR_REUSABLE:
+ case VM_BEHAVIOR_REUSE:
+ case VM_BEHAVIOR_CAN_REUSE:
+ /*
+ * Align to the hardware page size, to allow
+ * malloc() to maximize the amount of re-usability,
+ * even on systems with larger software page size.
+ */
+ align_mask = PAGE_MASK;
+ break;
+ default:
+ align_mask = VM_MAP_PAGE_MASK(map);
+ break;
+ }
+
+ return vm_map_behavior_set(map,
+ vm_map_trunc_page(start, align_mask),
+ vm_map_round_page(start+size, align_mask),
+ new_behavior);
}
/*
vm_size_t size,
vm_behavior_t new_behavior)
{
- if ((map == VM_MAP_NULL) || (start + size < start))
- return(KERN_INVALID_ARGUMENT);
-
- if (size == 0)
- return KERN_SUCCESS;
+ if (start + size < start)
+ return KERN_INVALID_ARGUMENT;
- return(vm_map_behavior_set(map, vm_map_trunc_page(start),
- vm_map_round_page(start+size), new_behavior));
+ return mach_vm_behavior_set(map,
+ (mach_vm_offset_t) start,
+ (mach_vm_size_t) size,
+ new_behavior);
}
/*
return KERN_INVALID_ARGUMENT;
return vm_map_purgable_control(map,
- vm_map_trunc_page(address),
+ vm_map_trunc_page(address, PAGE_MASK),
control,
state);
}
return KERN_INVALID_ARGUMENT;
return vm_map_purgable_control(map,
- vm_map_trunc_page(address),
+ vm_map_trunc_page(address, PAGE_MASK),
control,
state);
}
if (VM_MAP_NULL == map)
return KERN_INVALID_ARGUMENT;
- return vm_map_page_query_internal(map,
- vm_map_trunc_page(offset),
- disposition, ref_count);
+ return vm_map_page_query_internal(
+ map,
+ vm_map_trunc_page(offset, PAGE_MASK),
+ disposition, ref_count);
}
kern_return_t
if (VM_MAP_NULL == map)
return KERN_INVALID_ARGUMENT;
- return vm_map_page_query_internal(map,
- vm_map_trunc_page(offset),
- disposition, ref_count);
+ return vm_map_page_query_internal(
+ map,
+ vm_map_trunc_page(offset, PAGE_MASK),
+ disposition, ref_count);
}
kern_return_t
upl_t *upl,
upl_page_info_array_t page_list,
unsigned int *count,
- int *flags,
+ upl_control_flags_t *flags,
int force_data_sync)
{
- int map_flags;
- kern_return_t kr;
+ upl_control_flags_t map_flags;
+ kern_return_t kr;
if (VM_MAP_NULL == map)
return KERN_INVALID_ARGUMENT;
return kr;
}
+
/*
* mach_make_memory_entry_64
*
/* needed for call to vm_map_lookup_locked */
boolean_t wired;
+ boolean_t iskernel;
vm_object_offset_t obj_off;
vm_prot_t prot;
struct vm_object_fault_info fault_info;
vm_map_entry_t next_entry;
vm_map_t local_map;
vm_map_t original_map = target_map;
- vm_map_size_t total_size;
- vm_map_size_t map_size;
- vm_map_offset_t map_offset;
+ vm_map_size_t total_size, map_size;
+ vm_map_offset_t map_start, map_end;
vm_map_offset_t local_offset;
vm_object_size_t mappable_size;
+ /*
+ * Stash the offset in the page for use by vm_map_enter_mem_object()
+ * in the VM_FLAGS_RETURN_DATA_ADDR/MAP_MEM_USE_DATA_ADDR case.
+ */
+ vm_object_offset_t offset_in_page;
+
unsigned int access;
vm_prot_t protections;
vm_prot_t original_protections, mask_protections;
unsigned int wimg_mode;
boolean_t force_shadow = FALSE;
+ boolean_t use_data_addr;
+ boolean_t use_4K_compat;
if (((permission & 0x00FF0000) &
~(MAP_MEM_ONLY |
MAP_MEM_NAMED_CREATE |
+ MAP_MEM_GRAB_SECLUDED | /* XXX FBDP TODO: restrict usage? */
MAP_MEM_PURGABLE |
- MAP_MEM_NAMED_REUSE))) {
+ MAP_MEM_NAMED_REUSE |
+ MAP_MEM_USE_DATA_ADDR |
+ MAP_MEM_VM_COPY |
+ MAP_MEM_4K_DATA_ADDR |
+ MAP_MEM_VM_SHARE))) {
/*
* Unknown flag: reject for forward compatibility.
*/
parent_entry = NULL;
}
+ if (parent_entry && parent_entry->is_copy) {
+ return KERN_INVALID_ARGUMENT;
+ }
+
original_protections = permission & VM_PROT_ALL;
protections = original_protections;
mask_protections = permission & VM_PROT_IS_MASK;
access = GET_MAP_MEM(permission);
+ use_data_addr = ((permission & MAP_MEM_USE_DATA_ADDR) != 0);
+ use_4K_compat = ((permission & MAP_MEM_4K_DATA_ADDR) != 0);
user_handle = IP_NULL;
user_entry = NULL;
- map_offset = vm_map_trunc_page(offset);
- map_size = vm_map_round_page(*size);
+ map_start = vm_map_trunc_page(offset, PAGE_MASK);
if (permission & MAP_MEM_ONLY) {
boolean_t parent_is_object;
- if (parent_entry == NULL) {
+ map_end = vm_map_round_page(offset + *size, PAGE_MASK);
+ map_size = map_end - map_start;
+
+ if (use_data_addr || use_4K_compat || parent_entry == NULL) {
return KERN_INVALID_ARGUMENT;
}
- parent_is_object = !(parent_entry->is_sub_map || parent_entry->is_pager);
+ parent_is_object = !(parent_entry->is_sub_map ||
+ parent_entry->is_pager);
object = parent_entry->backing.object;
if(parent_is_object && object != VM_OBJECT_NULL)
wimg_mode = object->wimg_bits;
} else if (access == MAP_MEM_COPYBACK) {
SET_MAP_MEM(access, parent_entry->protection);
wimg_mode = VM_WIMG_USE_DEFAULT;
+ } else if (access == MAP_MEM_INNERWBACK) {
+ SET_MAP_MEM(access, parent_entry->protection);
+ wimg_mode = VM_WIMG_INNERWBACK;
} else if (access == MAP_MEM_WTHRU) {
SET_MAP_MEM(access, parent_entry->protection);
wimg_mode = VM_WIMG_WTHRU;
if (object_handle)
*object_handle = IP_NULL;
return KERN_SUCCESS;
- }
+ } else if (permission & MAP_MEM_NAMED_CREATE) {
+ map_end = vm_map_round_page(offset + *size, PAGE_MASK);
+ map_size = map_end - map_start;
+
+ if (use_data_addr || use_4K_compat) {
+ return KERN_INVALID_ARGUMENT;
+ }
- if(permission & MAP_MEM_NAMED_CREATE) {
kr = mach_memory_entry_allocate(&user_entry, &user_handle);
if (kr != KERN_SUCCESS) {
return KERN_FAILURE;
goto make_mem_done;
}
object->purgable = VM_PURGABLE_NONVOLATILE;
+ assert(object->vo_purgeable_owner == NULL);
+ assert(object->resident_page_count == 0);
+ assert(object->wired_page_count == 0);
+ vm_object_lock(object);
+ vm_purgeable_nonvolatile_enqueue(object,
+ current_task());
+ vm_object_unlock(object);
+ }
+
+#if CONFIG_SECLUDED_MEMORY
+ if (secluded_for_iokit && /* global boot-arg */
+ ((permission & MAP_MEM_GRAB_SECLUDED)
+#if 11
+ /* XXX FBDP for my testing only */
+ || (secluded_for_fbdp && map_size == 97550336)
+#endif
+ )) {
+#if 11
+ if (!(permission & MAP_MEM_GRAB_SECLUDED) &&
+ secluded_for_fbdp) {
+ printf("FBDP: object %p size %lld can grab secluded\n", object, (uint64_t) map_size);
+ }
+#endif
+ object->can_grab_secluded = TRUE;
+ assert(!object->eligible_for_secluded);
}
+#endif /* CONFIG_SECLUDED_MEMORY */
/*
* The VM object is brand new and nobody else knows about it,
wimg_mode = VM_WIMG_IO;
} else if (access == MAP_MEM_COPYBACK) {
wimg_mode = VM_WIMG_USE_DEFAULT;
+ } else if (access == MAP_MEM_INNERWBACK) {
+ wimg_mode = VM_WIMG_INNERWBACK;
} else if (access == MAP_MEM_WTHRU) {
wimg_mode = VM_WIMG_WTHRU;
} else if (access == MAP_MEM_WCOMB) {
* shadow objects either...
*/
object->copy_strategy = MEMORY_OBJECT_COPY_NONE;
+ object->true_share = TRUE;
user_entry->backing.object = object;
user_entry->internal = TRUE;
user_entry->is_sub_map = FALSE;
user_entry->is_pager = FALSE;
user_entry->offset = 0;
+ user_entry->data_offset = 0;
user_entry->protection = protections;
SET_MAP_MEM(access, user_entry->protection);
user_entry->size = map_size;
/* user_object pager and internal fields are not used */
/* when the object field is filled in. */
- *size = CAST_DOWN(vm_size_t, map_size);
+ *size = CAST_DOWN(vm_size_t, (user_entry->size -
+ user_entry->data_offset));
+ *object_handle = user_handle;
+ return KERN_SUCCESS;
+ }
+
+ if (permission & MAP_MEM_VM_COPY) {
+ vm_map_copy_t copy;
+
+ if (target_map == VM_MAP_NULL) {
+ return KERN_INVALID_TASK;
+ }
+
+ map_end = vm_map_round_page(offset + *size, PAGE_MASK);
+ map_size = map_end - map_start;
+ if (use_data_addr || use_4K_compat) {
+ offset_in_page = offset - map_start;
+ if (use_4K_compat)
+ offset_in_page &= ~((signed)(0xFFF));
+ } else {
+ offset_in_page = 0;
+ }
+
+ kr = vm_map_copyin_internal(target_map,
+ map_start,
+ map_size,
+ VM_MAP_COPYIN_ENTRY_LIST,
+ ©);
+ if (kr != KERN_SUCCESS) {
+ return kr;
+ }
+
+ kr = mach_memory_entry_allocate(&user_entry, &user_handle);
+ if (kr != KERN_SUCCESS) {
+ vm_map_copy_discard(copy);
+ return KERN_FAILURE;
+ }
+
+ user_entry->backing.copy = copy;
+ user_entry->internal = FALSE;
+ user_entry->is_sub_map = FALSE;
+ user_entry->is_pager = FALSE;
+ user_entry->is_copy = TRUE;
+ user_entry->offset = 0;
+ user_entry->protection = protections;
+ user_entry->size = map_size;
+ user_entry->data_offset = offset_in_page;
+
+ *size = CAST_DOWN(vm_size_t, (user_entry->size -
+ user_entry->data_offset));
+ *object_handle = user_handle;
+ return KERN_SUCCESS;
+ }
+
+ if (permission & MAP_MEM_VM_SHARE) {
+ vm_map_copy_t copy;
+ vm_prot_t cur_prot, max_prot;
+
+ if (target_map == VM_MAP_NULL) {
+ return KERN_INVALID_TASK;
+ }
+
+ map_end = vm_map_round_page(offset + *size, PAGE_MASK);
+ map_size = map_end - map_start;
+ if (use_data_addr || use_4K_compat) {
+ offset_in_page = offset - map_start;
+ if (use_4K_compat)
+ offset_in_page &= ~((signed)(0xFFF));
+ } else {
+ offset_in_page = 0;
+ }
+
+ cur_prot = VM_PROT_ALL;
+ kr = vm_map_copy_extract(target_map,
+ map_start,
+ map_size,
+ ©,
+ &cur_prot,
+ &max_prot);
+ if (kr != KERN_SUCCESS) {
+ return kr;
+ }
+
+ if (mask_protections) {
+ /*
+ * We just want as much of "original_protections"
+ * as we can get out of the actual "cur_prot".
+ */
+ protections &= cur_prot;
+ if (protections == VM_PROT_NONE) {
+ /* no access at all: fail */
+ vm_map_copy_discard(copy);
+ return KERN_PROTECTION_FAILURE;
+ }
+ } else {
+ /*
+ * We want exactly "original_protections"
+ * out of "cur_prot".
+ */
+ if ((cur_prot & protections) != protections) {
+ vm_map_copy_discard(copy);
+ return KERN_PROTECTION_FAILURE;
+ }
+ }
+
+ kr = mach_memory_entry_allocate(&user_entry, &user_handle);
+ if (kr != KERN_SUCCESS) {
+ vm_map_copy_discard(copy);
+ return KERN_FAILURE;
+ }
+
+ user_entry->backing.copy = copy;
+ user_entry->internal = FALSE;
+ user_entry->is_sub_map = FALSE;
+ user_entry->is_pager = FALSE;
+ user_entry->is_copy = TRUE;
+ user_entry->offset = 0;
+ user_entry->protection = protections;
+ user_entry->size = map_size;
+ user_entry->data_offset = offset_in_page;
+
+ *size = CAST_DOWN(vm_size_t, (user_entry->size -
+ user_entry->data_offset));
*object_handle = user_handle;
return KERN_SUCCESS;
}
if (parent_entry == NULL ||
(permission & MAP_MEM_NAMED_REUSE)) {
+ map_end = vm_map_round_page(offset + *size, PAGE_MASK);
+ map_size = map_end - map_start;
+ if (use_data_addr || use_4K_compat) {
+ offset_in_page = offset - map_start;
+ if (use_4K_compat)
+ offset_in_page &= ~((signed)(0xFFF));
+ } else {
+ offset_in_page = 0;
+ }
+
/* Create a named object based on address range within the task map */
/* Go find the object at given address */
/* note we check the permission of the range against */
/* that requested by the caller */
- kr = vm_map_lookup_locked(&target_map, map_offset,
+ kr = vm_map_lookup_locked(&target_map, map_start,
protections | mask_protections,
OBJECT_LOCK_EXCLUSIVE, &version,
&object, &obj_off, &prot, &wired,
*/
protections &= prot;
}
+
if (((prot & protections) != protections)
- || (object == kernel_object)) {
+ || (object == kernel_object)) {
kr = KERN_INVALID_RIGHT;
vm_object_unlock(object);
vm_map_unlock_read(target_map);
vm_object_unlock(object);
local_map = original_map;
- local_offset = map_offset;
+ local_offset = map_start;
if(target_map != local_map) {
vm_map_unlock_read(target_map);
if(real_map != target_map)
object = VM_OBJECT_NULL;
goto make_mem_done;
}
+ iskernel = (local_map->pmap == kernel_pmap);
if(!(map_entry->is_sub_map)) {
- if(map_entry->object.vm_object != object) {
+ if (VME_OBJECT(map_entry) != object) {
kr = KERN_INVALID_ARGUMENT;
vm_map_unlock_read(target_map);
if(real_map != target_map)
} else {
vm_map_t tmap;
tmap = local_map;
- local_map = map_entry->object.sub_map;
+ local_map = VME_SUBMAP(map_entry);
vm_map_lock_read(local_map);
vm_map_unlock_read(tmap);
target_map = local_map;
real_map = local_map;
local_offset = local_offset - map_entry->vme_start;
- local_offset += map_entry->offset;
+ local_offset += VME_OFFSET(map_entry);
}
}
/* lets see if the next map entry is still */
/* pointing at this object and is contiguous */
while(map_size > mappable_size) {
- if((next_entry->object.vm_object == object) &&
- (next_entry->vme_start ==
- next_entry->vme_prev->vme_end) &&
- (next_entry->offset ==
- next_entry->vme_prev->offset +
- (next_entry->vme_prev->vme_end -
- next_entry->vme_prev->vme_start))) {
+ if ((VME_OBJECT(next_entry) == object) &&
+ (next_entry->vme_start ==
+ next_entry->vme_prev->vme_end) &&
+ (VME_OFFSET(next_entry) ==
+ (VME_OFFSET(next_entry->vme_prev) +
+ (next_entry->vme_prev->vme_end -
+ next_entry->vme_prev->vme_start)))) {
if (mask_protections) {
/*
* The caller asked us to use
*/
protections &= next_entry->max_protection;
}
+ if ((next_entry->wired_count) &&
+ (map_entry->wired_count == 0)) {
+ break;
+ }
if(((next_entry->max_protection)
& protections) != protections) {
break;
}
}
-#if !CONFIG_EMBEDDED
- if (vm_map_entry_should_cow_for_true_share(map_entry) &&
+ /* vm_map_entry_should_cow_for_true_share() checks for malloc tags,
+ * never true in kernel */
+ if (!iskernel && vm_map_entry_should_cow_for_true_share(map_entry) &&
object->vo_size > map_size &&
map_size != 0) {
/*
goto redo_lookup;
}
- vm_map_clip_start(target_map, map_entry, vm_map_trunc_page(offset));
- vm_map_clip_end(target_map, map_entry, vm_map_round_page(offset) + map_size);
+ vm_map_clip_start(target_map,
+ map_entry,
+ vm_map_trunc_page(map_start,
+ VM_MAP_PAGE_MASK(target_map)));
+ vm_map_clip_end(target_map,
+ map_entry,
+ (vm_map_round_page(map_end,
+ VM_MAP_PAGE_MASK(target_map))));
force_shadow = TRUE;
- map_size = map_entry->vme_end - map_entry->vme_start;
- total_size = map_size;
+ if ((map_entry->vme_end - offset) < map_size) {
+ map_size = map_entry->vme_end - map_start;
+ }
+ total_size = map_entry->vme_end - map_entry->vme_start;
vm_map_lock_write_to_read(target_map);
vm_object_lock(object);
}
-#endif /* !CONFIG_EMBEDDED */
- if(object->internal) {
+ if (object->internal) {
/* vm_map_lookup_locked will create a shadow if */
/* needs_copy is set but does not check for the */
/* other two conditions shown. It is important to */
if (force_shadow ||
((map_entry->needs_copy ||
object->shadowed ||
- (object->vo_size > total_size)) &&
- !object->true_share)) {
+ (object->vo_size > total_size &&
+ (VME_OFFSET(map_entry) != 0 ||
+ object->vo_size >
+ vm_map_round_page(total_size,
+ VM_MAP_PAGE_MASK(target_map)))))
+ && !object->true_share)) {
/*
* We have to unlock the VM object before
* trying to upgrade the VM map lock, to
target_map = original_map;
goto redo_lookup;
}
+#if 00
vm_object_lock(object);
+#endif
/*
* JMM - We need to avoid coming here when the object
*/
/* create a shadow object */
- vm_object_shadow(&map_entry->object.vm_object,
- &map_entry->offset, total_size);
- shadow_object = map_entry->object.vm_object;
+ VME_OBJECT_SHADOW(map_entry, total_size);
+ shadow_object = VME_OBJECT(map_entry);
+#if 00
vm_object_unlock(object);
+#endif
prot = map_entry->protection & ~VM_PROT_WRITE;
- if (override_nx(target_map, map_entry->alias) && prot)
+ if (override_nx(target_map,
+ VME_ALIAS(map_entry))
+ && prot)
prot |= VM_PROT_EXECUTE;
vm_object_pmap_protect(
- object, map_entry->offset,
+ object, VME_OFFSET(map_entry),
total_size,
((map_entry->is_shared
- || target_map->mapped)
+ || target_map->mapped_in_other_pmaps)
? PMAP_NULL :
target_map->pmap),
map_entry->vme_start,
vm_object_lock(shadow_object);
while (total_size) {
- if(next_entry->object.vm_object == object) {
+ assert((next_entry->wired_count == 0) ||
+ (map_entry->wired_count));
+
+ if (VME_OBJECT(next_entry) == object) {
vm_object_reference_locked(shadow_object);
- next_entry->object.vm_object
- = shadow_object;
+ VME_OBJECT_SET(next_entry,
+ shadow_object);
vm_object_deallocate(object);
- next_entry->offset
- = next_entry->vme_prev->offset +
- (next_entry->vme_prev->vme_end
- - next_entry->vme_prev->vme_start);
+ VME_OFFSET_SET(
+ next_entry,
+ (VME_OFFSET(next_entry->vme_prev) +
+ (next_entry->vme_prev->vme_end
+ - next_entry->vme_prev->vme_start)));
next_entry->needs_copy = FALSE;
} else {
panic("mach_make_memory_entry_64:"
vm_object_deallocate(object); /* extra ref */
object = shadow_object;
- obj_off = (local_offset - map_entry->vme_start)
- + map_entry->offset;
+ obj_off = ((local_offset - map_entry->vme_start)
+ + VME_OFFSET(map_entry));
vm_map_lock_write_to_read(target_map);
}
wimg_mode = VM_WIMG_IO;
} else if (access == MAP_MEM_COPYBACK) {
wimg_mode = VM_WIMG_USE_DEFAULT;
+ } else if (access == MAP_MEM_INNERWBACK) {
+ wimg_mode = VM_WIMG_INNERWBACK;
} else if (access == MAP_MEM_WTHRU) {
wimg_mode = VM_WIMG_WTHRU;
} else if (access == MAP_MEM_WCOMB) {
}
}
+#if VM_OBJECT_TRACKING_OP_TRUESHARE
+ if (!object->true_share &&
+ vm_object_tracking_inited) {
+ void *bt[VM_OBJECT_TRACKING_BTDEPTH];
+ int num = 0;
+
+ num = OSBacktrace(bt,
+ VM_OBJECT_TRACKING_BTDEPTH);
+ btlog_add_entry(vm_object_tracking_btlog,
+ object,
+ VM_OBJECT_TRACKING_OP_TRUESHARE,
+ bt,
+ num);
+ }
+#endif /* VM_OBJECT_TRACKING_OP_TRUESHARE */
+
+ vm_object_lock_assert_exclusive(object);
object->true_share = TRUE;
if (object->copy_strategy == MEMORY_OBJECT_COPY_SYMMETRIC)
object->copy_strategy = MEMORY_OBJECT_COPY_DELAY;
parent_entry->is_pager == FALSE &&
parent_entry->offset == obj_off &&
parent_entry->protection == protections &&
- parent_entry->size == map_size) {
+ parent_entry->size == map_size &&
+ ((!(use_data_addr || use_4K_compat) &&
+ (parent_entry->data_offset == 0)) ||
+ ((use_data_addr || use_4K_compat) &&
+ (parent_entry->data_offset == offset_in_page)))) {
/*
* We have a match: re-use "parent_entry".
*/
/* parent_entry->ref_count++; XXX ? */
/* Get an extra send-right on handle */
ipc_port_copy_send(parent_handle);
+
+ *size = CAST_DOWN(vm_size_t,
+ (parent_entry->size -
+ parent_entry->data_offset));
*object_handle = parent_handle;
return KERN_SUCCESS;
} else {
user_entry->is_sub_map = FALSE;
user_entry->is_pager = FALSE;
user_entry->offset = obj_off;
+ user_entry->data_offset = offset_in_page;
user_entry->protection = protections;
SET_MAP_MEM(GET_MAP_MEM(permission), user_entry->protection);
user_entry->size = map_size;
/* user_object pager and internal fields are not used */
/* when the object field is filled in. */
- *size = CAST_DOWN(vm_size_t, map_size);
+ *size = CAST_DOWN(vm_size_t, (user_entry->size -
+ user_entry->data_offset));
*object_handle = user_handle;
return KERN_SUCCESS;
} else {
/* The new object will be base on an existing named object */
-
if (parent_entry == NULL) {
kr = KERN_INVALID_ARGUMENT;
goto make_mem_done;
}
- if((offset + map_size) > parent_entry->size) {
- kr = KERN_INVALID_ARGUMENT;
- goto make_mem_done;
+
+ if (use_data_addr || use_4K_compat) {
+ /*
+ * submaps and pagers should only be accessible from within
+ * the kernel, which shouldn't use the data address flag, so can fail here.
+ */
+ if (parent_entry->is_pager || parent_entry->is_sub_map) {
+ panic("Shouldn't be using data address with a parent entry that is a submap or pager.");
+ }
+ /*
+ * Account for offset to data in parent entry and
+ * compute our own offset to data.
+ */
+ if((offset + *size + parent_entry->data_offset) > parent_entry->size) {
+ kr = KERN_INVALID_ARGUMENT;
+ goto make_mem_done;
+ }
+
+ map_start = vm_map_trunc_page(offset + parent_entry->data_offset, PAGE_MASK);
+ offset_in_page = (offset + parent_entry->data_offset) - map_start;
+ if (use_4K_compat)
+ offset_in_page &= ~((signed)(0xFFF));
+ map_end = vm_map_round_page(offset + parent_entry->data_offset + *size, PAGE_MASK);
+ map_size = map_end - map_start;
+ } else {
+ map_end = vm_map_round_page(offset + *size, PAGE_MASK);
+ map_size = map_end - map_start;
+ offset_in_page = 0;
+
+ if((offset + map_size) > parent_entry->size) {
+ kr = KERN_INVALID_ARGUMENT;
+ goto make_mem_done;
+ }
}
if (mask_protections) {
}
user_entry->size = map_size;
- user_entry->offset = parent_entry->offset + map_offset;
+ user_entry->offset = parent_entry->offset + map_start;
+ user_entry->data_offset = offset_in_page;
user_entry->is_sub_map = parent_entry->is_sub_map;
user_entry->is_pager = parent_entry->is_pager;
+ user_entry->is_copy = parent_entry->is_copy;
user_entry->internal = parent_entry->internal;
user_entry->protection = protections;
assert(object != VM_OBJECT_NULL);
user_entry->backing.object = object;
/* we now point to this object, hold on */
- vm_object_reference(object);
vm_object_lock(object);
+ vm_object_reference_locked(object);
+#if VM_OBJECT_TRACKING_OP_TRUESHARE
+ if (!object->true_share &&
+ vm_object_tracking_inited) {
+ void *bt[VM_OBJECT_TRACKING_BTDEPTH];
+ int num = 0;
+
+ num = OSBacktrace(bt,
+ VM_OBJECT_TRACKING_BTDEPTH);
+ btlog_add_entry(vm_object_tracking_btlog,
+ object,
+ VM_OBJECT_TRACKING_OP_TRUESHARE,
+ bt,
+ num);
+ }
+#endif /* VM_OBJECT_TRACKING_OP_TRUESHARE */
+
object->true_share = TRUE;
if (object->copy_strategy == MEMORY_OBJECT_COPY_SYMMETRIC)
object->copy_strategy = MEMORY_OBJECT_COPY_DELAY;
vm_object_unlock(object);
}
- *size = CAST_DOWN(vm_size_t, map_size);
+ *size = CAST_DOWN(vm_size_t, (user_entry->size -
+ user_entry->data_offset));
*object_handle = user_handle;
return KERN_SUCCESS;
}
user_entry->backing.pager = NULL;
user_entry->is_sub_map = FALSE;
user_entry->is_pager = FALSE;
+ user_entry->is_copy = FALSE;
user_entry->internal = FALSE;
user_entry->size = 0;
user_entry->offset = 0;
+ user_entry->data_offset = 0;
user_entry->protection = VM_PROT_NONE;
user_entry->ref_count = 1;
named_entry_lock(mem_entry);
- if (mem_entry->is_sub_map || mem_entry->is_pager) {
+ if (mem_entry->is_sub_map ||
+ mem_entry->is_pager ||
+ mem_entry->is_copy) {
named_entry_unlock(mem_entry);
return KERN_INVALID_ARGUMENT;
}
return kr;
}
+kern_return_t
+mach_memory_entry_get_page_counts(
+ ipc_port_t entry_port,
+ unsigned int *resident_page_count,
+ unsigned int *dirty_page_count)
+{
+ kern_return_t kr;
+ vm_named_entry_t mem_entry;
+ vm_object_t object;
+ vm_object_offset_t offset;
+ vm_object_size_t size;
+
+ if (entry_port == IP_NULL ||
+ ip_kotype(entry_port) != IKOT_NAMED_ENTRY) {
+ return KERN_INVALID_ARGUMENT;
+ }
+
+ mem_entry = (vm_named_entry_t) entry_port->ip_kobject;
+
+ named_entry_lock(mem_entry);
+
+ if (mem_entry->is_sub_map ||
+ mem_entry->is_pager ||
+ mem_entry->is_copy) {
+ named_entry_unlock(mem_entry);
+ return KERN_INVALID_ARGUMENT;
+ }
+
+ object = mem_entry->backing.object;
+ if (object == VM_OBJECT_NULL) {
+ named_entry_unlock(mem_entry);
+ return KERN_INVALID_ARGUMENT;
+ }
+
+ vm_object_lock(object);
+
+ offset = mem_entry->offset;
+ size = mem_entry->size;
+
+ named_entry_unlock(mem_entry);
+
+ kr = vm_object_get_page_counts(object, offset, size, resident_page_count, dirty_page_count);
+
+ vm_object_unlock(object);
+
+ return kr;
+}
+
/*
* mach_memory_entry_port_release:
*
assert(ip_kotype(port) == IKOT_NAMED_ENTRY);
#endif /* MACH_ASSERT */
named_entry = (vm_named_entry_t)port->ip_kobject;
- lck_mtx_lock(&(named_entry)->Lock);
+
+ named_entry_lock(named_entry);
named_entry->ref_count -= 1;
+
if(named_entry->ref_count == 0) {
if (named_entry->is_sub_map) {
vm_map_deallocate(named_entry->backing.map);
- } else if (!named_entry->is_pager) {
- /* release the memory object we've been pointing to */
+ } else if (named_entry->is_pager) {
+ /* JMM - need to drop reference on pager in that case */
+ } else if (named_entry->is_copy) {
+ vm_map_copy_discard(named_entry->backing.copy);
+ } else {
+ /* release the VM object we've been pointing to */
vm_object_deallocate(named_entry->backing.object);
- } /* else JMM - need to drop reference on pager in that case */
+ }
- lck_mtx_unlock(&(named_entry)->Lock);
+ named_entry_unlock(named_entry);
+ named_entry_lock_destroy(named_entry);
kfree((void *) port->ip_kobject,
sizeof (struct vm_named_entry));
} else
- lck_mtx_unlock(&(named_entry)->Lock);
+ named_entry_unlock(named_entry);
}
/* Allow manipulation of individual page state. This is actually part of */
named_entry_lock(mem_entry);
- if (mem_entry->is_sub_map || mem_entry->is_pager) {
+ if (mem_entry->is_sub_map ||
+ mem_entry->is_pager ||
+ mem_entry->is_copy) {
named_entry_unlock(mem_entry);
return KERN_INVALID_ARGUMENT;
}
named_entry_lock(mem_entry);
- if (mem_entry->is_sub_map || mem_entry->is_pager) {
+ if (mem_entry->is_sub_map ||
+ mem_entry->is_pager ||
+ mem_entry->is_copy) {
named_entry_unlock(mem_entry);
return KERN_INVALID_ARGUMENT;
}
return kr;
}
+static void dp_control_port_init(void)
+{
+ lck_grp_init(&dynamic_pager_control_port_lock_group,"dp_control_port", LCK_GRP_ATTR_NULL);
+ lck_mtx_init(&dynamic_pager_control_port_lock, &dynamic_pager_control_port_lock_group, LCK_ATTR_NULL);
+}
kern_return_t
set_dp_control_port(
host_priv_t host_priv,
ipc_port_t control_port)
{
- if (host_priv == HOST_PRIV_NULL)
- return (KERN_INVALID_HOST);
+ ipc_port_t old_port;
- if (IP_VALID(dynamic_pager_control_port))
- ipc_port_release_send(dynamic_pager_control_port);
+ if (host_priv == HOST_PRIV_NULL)
+ return (KERN_INVALID_HOST);
+ lck_mtx_lock(&dynamic_pager_control_port_lock);
+ old_port = dynamic_pager_control_port;
dynamic_pager_control_port = control_port;
+ lck_mtx_unlock(&dynamic_pager_control_port_lock);
+
+ if (IP_VALID(old_port))
+ ipc_port_release_send(old_port);
+
return KERN_SUCCESS;
}
host_priv_t host_priv,
ipc_port_t *control_port)
{
- if (host_priv == HOST_PRIV_NULL)
+ if (host_priv == HOST_PRIV_NULL)
return (KERN_INVALID_HOST);
+ lck_mtx_lock(&dynamic_pager_control_port_lock);
*control_port = ipc_port_copy_send(dynamic_pager_control_port);
+ lck_mtx_unlock(&dynamic_pager_control_port_lock);
+
return KERN_SUCCESS;
}
/* Create a named object based on a submap of specified size */
new_map = vm_map_create(PMAP_NULL, VM_MAP_MIN_ADDRESS,
- vm_map_round_page(size), TRUE);
+ vm_map_round_page(size,
+ VM_MAP_PAGE_MASK(target_map)),
+ TRUE);
+ vm_map_set_page_shift(new_map, VM_MAP_PAGE_SHIFT(target_map));
user_entry->backing.map = new_map;
user_entry->internal = TRUE;
vm_map_entry_t entry;
ppnum_t phys_page = 0;
- map_offset = vm_map_trunc_page(addr);
+ map_offset = vm_map_trunc_page(addr, PAGE_MASK);
vm_map_lock(map);
while (vm_map_lookup_entry(map, map_offset, &entry)) {
- if (entry->object.vm_object == VM_OBJECT_NULL) {
+ if (VME_OBJECT(entry) == VM_OBJECT_NULL) {
vm_map_unlock(map);
return (ppnum_t) 0;
}
if (entry->is_sub_map) {
vm_map_t old_map;
- vm_map_lock(entry->object.sub_map);
+ vm_map_lock(VME_SUBMAP(entry));
old_map = map;
- map = entry->object.sub_map;
- map_offset = entry->offset + (map_offset - entry->vme_start);
+ map = VME_SUBMAP(entry);
+ map_offset = (VME_OFFSET(entry) +
+ (map_offset - entry->vme_start));
vm_map_unlock(old_map);
continue;
}
- if (entry->object.vm_object->phys_contiguous) {
+ if (VME_OBJECT(entry)->phys_contiguous) {
/* These are not standard pageable memory mappings */
/* If they are not present in the object they will */
/* have to be picked up from the pager through the */
/* fault mechanism. */
- if(entry->object.vm_object->vo_shadow_offset == 0) {
+ if (VME_OBJECT(entry)->vo_shadow_offset == 0) {
/* need to call vm_fault */
vm_map_unlock(map);
vm_fault(map, map_offset, VM_PROT_NONE,
vm_map_lock(map);
continue;
}
- offset = entry->offset + (map_offset - entry->vme_start);
+ offset = (VME_OFFSET(entry) +
+ (map_offset - entry->vme_start));
phys_page = (ppnum_t)
- ((entry->object.vm_object->vo_shadow_offset
- + offset) >> 12);
+ ((VME_OBJECT(entry)->vo_shadow_offset
+ + offset) >> PAGE_SHIFT);
break;
}
- offset = entry->offset + (map_offset - entry->vme_start);
- object = entry->object.vm_object;
+ offset = (VME_OFFSET(entry) + (map_offset - entry->vme_start));
+ object = VME_OBJECT(entry);
vm_object_lock(object);
while (TRUE) {
vm_page_t dst_page = vm_page_lookup(object,offset);
break;
}
} else {
- phys_page = (ppnum_t)(dst_page->phys_page);
+ phys_page = (ppnum_t)(VM_PAGE_GET_PHYS_PAGE(dst_page));
vm_object_unlock(object);
break;
}
return phys_page;
}
+void
+vm_user_init(void)
+{
+ dp_control_port_init();
+}
-
+#if 0
kern_return_t kernel_object_iopl_request( /* forward */
vm_named_entry_t named_entry,
memory_object_offset_t offset,
/* offset from beginning of named entry offset in object */
offset = offset + named_entry->offset;
- if(named_entry->is_sub_map)
- return (KERN_INVALID_ARGUMENT);
+ if (named_entry->is_sub_map ||
+ named_entry->is_copy)
+ return KERN_INVALID_ARGUMENT;
named_entry_lock(named_entry);
}
if (!object->private) {
- if (*upl_size > (MAX_UPL_TRANSFER*PAGE_SIZE))
- *upl_size = (MAX_UPL_TRANSFER*PAGE_SIZE);
+ if (*upl_size > MAX_UPL_TRANSFER_BYTES)
+ *upl_size = MAX_UPL_TRANSFER_BYTES;
if (object->phys_contiguous) {
*flags = UPL_PHYS_CONTIG;
} else {
upl_ptr,
user_page_list,
page_list_count,
- caller_flags);
+ (upl_control_flags_t)(unsigned int)caller_flags);
vm_object_deallocate(object);
return ret;
}
+#endif
projects / apple / xnu.git / blobdiff