+ * The third case is when the object
+ * to be shared has parts sticking
+ * outside of the entry we're working
+ * with, and thus may in the future
+ * be subject to a symmetrical copy.
+ * (This is a preemptive version of
+ * case 2.)
+ */
+ VME_OBJECT_SHADOW(old_entry,
+ (vm_map_size_t) (old_entry->vme_end -
+ old_entry->vme_start));
+
+ /*
+ * If we're making a shadow for other than
+ * copy on write reasons, then we have
+ * to remove write permission.
+ */
+
+ if (!old_entry->needs_copy &&
+ (old_entry->protection & VM_PROT_WRITE)) {
+ vm_prot_t prot;
+
+ assert(!pmap_has_prot_policy(old_entry->protection));
+
+ prot = old_entry->protection & ~VM_PROT_WRITE;
+
+ assert(!pmap_has_prot_policy(prot));
+
+ if (override_nx(old_map, VME_ALIAS(old_entry)) && prot) {
+ prot |= VM_PROT_EXECUTE;
+ }
+
+
+ if (old_map->mapped_in_other_pmaps) {
+ vm_object_pmap_protect(
+ VME_OBJECT(old_entry),
+ VME_OFFSET(old_entry),
+ (old_entry->vme_end -
+ old_entry->vme_start),
+ PMAP_NULL,
+ old_entry->vme_start,
+ prot);
+ } else {
+ pmap_protect(old_map->pmap,
+ old_entry->vme_start,
+ old_entry->vme_end,
+ prot);
+ }
+ }
+
+ old_entry->needs_copy = FALSE;
+ object = VME_OBJECT(old_entry);
+ }
+
+
+ /*
+ * If object was using a symmetric copy strategy,
+ * change its copy strategy to the default
+ * asymmetric copy strategy, which is copy_delay
+ * in the non-norma case and copy_call in the
+ * norma case. Bump the reference count for the
+ * new entry.
+ */
+
+ if (old_entry->is_sub_map) {
+ vm_map_lock(VME_SUBMAP(old_entry));
+ vm_map_reference(VME_SUBMAP(old_entry));
+ vm_map_unlock(VME_SUBMAP(old_entry));
+ } else {
+ vm_object_lock(object);
+ vm_object_reference_locked(object);
+ if (object->copy_strategy == MEMORY_OBJECT_COPY_SYMMETRIC) {
+ object->copy_strategy = MEMORY_OBJECT_COPY_DELAY;
+ }
+ vm_object_unlock(object);
+ }
+
+ /*
+ * Clone the entry, using object ref from above.
+ * Mark both entries as shared.
+ */
+
+ new_entry = vm_map_entry_create(new_map, FALSE); /* Never the kernel
+ * map or descendants */
+ vm_map_entry_copy(new_entry, old_entry);
+ old_entry->is_shared = TRUE;
+ new_entry->is_shared = TRUE;
+
+ /*
+ * We're dealing with a shared mapping, so the resulting mapping
+ * should inherit some of the original mapping's accounting settings.
+ * "iokit_acct" should have been cleared in vm_map_entry_copy().
+ * "use_pmap" should stay the same as before (if it hasn't been reset
+ * to TRUE when we cleared "iokit_acct").
+ */
+ assert(!new_entry->iokit_acct);
+
+ /*
+ * If old entry's inheritence is VM_INHERIT_NONE,
+ * the new entry is for corpse fork, remove the
+ * write permission from the new entry.
+ */
+ if (old_entry->inheritance == VM_INHERIT_NONE) {
+ new_entry->protection &= ~VM_PROT_WRITE;
+ new_entry->max_protection &= ~VM_PROT_WRITE;
+ }
+
+ /*
+ * Insert the entry into the new map -- we
+ * know we're inserting at the end of the new
+ * map.
+ */
+
+ vm_map_store_entry_link(new_map, vm_map_last_entry(new_map), new_entry,
+ VM_MAP_KERNEL_FLAGS_NONE);
+
+ /*
+ * Update the physical map
+ */
+
+ if (old_entry->is_sub_map) {
+ /* Bill Angell pmap support goes here */
+ } else {
+ pmap_copy(new_map->pmap, old_map->pmap, new_entry->vme_start,
+ old_entry->vme_end - old_entry->vme_start,
+ old_entry->vme_start);
+ }
+}
+
+static boolean_t
+vm_map_fork_copy(
+ vm_map_t old_map,
+ vm_map_entry_t *old_entry_p,
+ vm_map_t new_map,
+ int vm_map_copyin_flags)
+{
+ vm_map_entry_t old_entry = *old_entry_p;
+ vm_map_size_t entry_size = old_entry->vme_end - old_entry->vme_start;
+ vm_map_offset_t start = old_entry->vme_start;
+ vm_map_copy_t copy;
+ vm_map_entry_t last = vm_map_last_entry(new_map);
+
+ vm_map_unlock(old_map);
+ /*
+ * Use maxprot version of copyin because we
+ * care about whether this memory can ever
+ * be accessed, not just whether it's accessible
+ * right now.
+ */
+ vm_map_copyin_flags |= VM_MAP_COPYIN_USE_MAXPROT;
+ if (vm_map_copyin_internal(old_map, start, entry_size,
+ vm_map_copyin_flags, ©)
+ != KERN_SUCCESS) {
+ /*
+ * The map might have changed while it
+ * was unlocked, check it again. Skip
+ * any blank space or permanently
+ * unreadable region.
+ */
+ vm_map_lock(old_map);
+ if (!vm_map_lookup_entry(old_map, start, &last) ||
+ (last->max_protection & VM_PROT_READ) == VM_PROT_NONE) {
+ last = last->vme_next;
+ }
+ *old_entry_p = last;
+
+ /*
+ * XXX For some error returns, want to
+ * XXX skip to the next element. Note
+ * that INVALID_ADDRESS and
+ * PROTECTION_FAILURE are handled above.
+ */
+
+ return FALSE;
+ }
+
+ /*
+ * Insert the copy into the new map
+ */
+
+ vm_map_copy_insert(new_map, last, copy);
+
+ /*
+ * Pick up the traversal at the end of
+ * the copied region.
+ */
+
+ vm_map_lock(old_map);
+ start += entry_size;
+ if (!vm_map_lookup_entry(old_map, start, &last)) {
+ last = last->vme_next;
+ } else {
+ if (last->vme_start == start) {
+ /*
+ * No need to clip here and we don't
+ * want to cause any unnecessary
+ * unnesting...
+ */
+ } else {
+ vm_map_clip_start(old_map, last, start);
+ }
+ }
+ *old_entry_p = last;
+
+ return TRUE;
+}
+
+/*
+ * vm_map_fork:
+ *
+ * Create and return a new map based on the old
+ * map, according to the inheritance values on the
+ * regions in that map and the options.
+ *
+ * The source map must not be locked.
+ */
+vm_map_t
+vm_map_fork(
+ ledger_t ledger,
+ vm_map_t old_map,
+ int options)
+{
+ pmap_t new_pmap;
+ vm_map_t new_map;
+ vm_map_entry_t old_entry;
+ vm_map_size_t new_size = 0, entry_size;
+ vm_map_entry_t new_entry;
+ boolean_t src_needs_copy;
+ boolean_t new_entry_needs_copy;
+ boolean_t pmap_is64bit;
+ int vm_map_copyin_flags;
+ vm_inherit_t old_entry_inheritance;
+ int map_create_options;
+ kern_return_t footprint_collect_kr;
+
+ if (options & ~(VM_MAP_FORK_SHARE_IF_INHERIT_NONE |
+ VM_MAP_FORK_PRESERVE_PURGEABLE |
+ VM_MAP_FORK_CORPSE_FOOTPRINT)) {
+ /* unsupported option */
+ return VM_MAP_NULL;
+ }
+
+ pmap_is64bit =
+#if defined(__i386__) || defined(__x86_64__)
+ old_map->pmap->pm_task_map != TASK_MAP_32BIT;
+#elif defined(__arm64__)
+ old_map->pmap->max == MACH_VM_MAX_ADDRESS;
+#elif defined(__arm__)
+ FALSE;
+#else
+#error Unknown architecture.
+#endif
+
+ unsigned int pmap_flags = 0;
+ pmap_flags |= pmap_is64bit ? PMAP_CREATE_64BIT : 0;
+#if defined(HAS_APPLE_PAC)
+ pmap_flags |= old_map->pmap->disable_jop ? PMAP_CREATE_DISABLE_JOP : 0;
+#endif
+ new_pmap = pmap_create_options(ledger, (vm_map_size_t) 0, pmap_flags);
+
+ vm_map_reference_swap(old_map);
+ vm_map_lock(old_map);
+
+ map_create_options = 0;
+ if (old_map->hdr.entries_pageable) {
+ map_create_options |= VM_MAP_CREATE_PAGEABLE;
+ }
+ if (options & VM_MAP_FORK_CORPSE_FOOTPRINT) {
+ map_create_options |= VM_MAP_CREATE_CORPSE_FOOTPRINT;
+ footprint_collect_kr = KERN_SUCCESS;
+ }
+ new_map = vm_map_create_options(new_pmap,
+ old_map->min_offset,
+ old_map->max_offset,
+ map_create_options);
+ vm_map_lock(new_map);
+ vm_commit_pagezero_status(new_map);
+ /* inherit the parent map's page size */
+ vm_map_set_page_shift(new_map, VM_MAP_PAGE_SHIFT(old_map));
+ for (
+ old_entry = vm_map_first_entry(old_map);
+ old_entry != vm_map_to_entry(old_map);
+ ) {
+ entry_size = old_entry->vme_end - old_entry->vme_start;
+
+ old_entry_inheritance = old_entry->inheritance;
+ /*
+ * If caller used the VM_MAP_FORK_SHARE_IF_INHERIT_NONE option
+ * share VM_INHERIT_NONE entries that are not backed by a
+ * device pager.
+ */
+ if (old_entry_inheritance == VM_INHERIT_NONE &&
+ (options & VM_MAP_FORK_SHARE_IF_INHERIT_NONE) &&
+ !(!old_entry->is_sub_map &&
+ VME_OBJECT(old_entry) != NULL &&
+ VME_OBJECT(old_entry)->pager != NULL &&
+ is_device_pager_ops(
+ VME_OBJECT(old_entry)->pager->mo_pager_ops))) {
+ old_entry_inheritance = VM_INHERIT_SHARE;
+ }
+
+ if (old_entry_inheritance != VM_INHERIT_NONE &&
+ (options & VM_MAP_FORK_CORPSE_FOOTPRINT) &&
+ footprint_collect_kr == KERN_SUCCESS) {
+ /*
+ * The corpse won't have old_map->pmap to query
+ * footprint information, so collect that data now
+ * and store it in new_map->vmmap_corpse_footprint
+ * for later autopsy.
+ */
+ footprint_collect_kr =
+ vm_map_corpse_footprint_collect(old_map,
+ old_entry,
+ new_map);
+ }
+
+ switch (old_entry_inheritance) {
+ case VM_INHERIT_NONE:
+ break;
+
+ case VM_INHERIT_SHARE:
+ vm_map_fork_share(old_map, old_entry, new_map);
+ new_size += entry_size;
+ break;
+
+ case VM_INHERIT_COPY:
+
+ /*
+ * Inline the copy_quickly case;
+ * upon failure, fall back on call
+ * to vm_map_fork_copy.
+ */
+
+ if (old_entry->is_sub_map) {
+ break;
+ }
+ if ((old_entry->wired_count != 0) ||
+ ((VME_OBJECT(old_entry) != NULL) &&
+ (VME_OBJECT(old_entry)->true_share))) {
+ goto slow_vm_map_fork_copy;
+ }
+
+ new_entry = vm_map_entry_create(new_map, FALSE); /* never the kernel map or descendants */
+ vm_map_entry_copy(new_entry, old_entry);
+ if (new_entry->is_sub_map) {
+ /* clear address space specifics */
+ new_entry->use_pmap = FALSE;
+ } else {
+ /*
+ * We're dealing with a copy-on-write operation,
+ * so the resulting mapping should not inherit
+ * the original mapping's accounting settings.
+ * "iokit_acct" should have been cleared in
+ * vm_map_entry_copy().
+ * "use_pmap" should be reset to its default
+ * (TRUE) so that the new mapping gets
+ * accounted for in the task's memory footprint.
+ */
+ assert(!new_entry->iokit_acct);
+ new_entry->use_pmap = TRUE;
+ }
+
+ if (!vm_object_copy_quickly(
+ VME_OBJECT_PTR(new_entry),
+ VME_OFFSET(old_entry),
+ (old_entry->vme_end -
+ old_entry->vme_start),
+ &src_needs_copy,
+ &new_entry_needs_copy)) {
+ vm_map_entry_dispose(new_map, new_entry);
+ goto slow_vm_map_fork_copy;
+ }
+
+ /*
+ * Handle copy-on-write obligations
+ */
+
+ if (src_needs_copy && !old_entry->needs_copy) {
+ vm_prot_t prot;
+
+ assert(!pmap_has_prot_policy(old_entry->protection));
+
+ prot = old_entry->protection & ~VM_PROT_WRITE;
+
+ if (override_nx(old_map, VME_ALIAS(old_entry))
+ && prot) {
+ prot |= VM_PROT_EXECUTE;
+ }
+
+ assert(!pmap_has_prot_policy(prot));
+
+ vm_object_pmap_protect(
+ VME_OBJECT(old_entry),
+ VME_OFFSET(old_entry),
+ (old_entry->vme_end -
+ old_entry->vme_start),
+ ((old_entry->is_shared
+ || old_map->mapped_in_other_pmaps)
+ ? PMAP_NULL :
+ old_map->pmap),
+ old_entry->vme_start,
+ prot);
+
+ assert(old_entry->wired_count == 0);
+ old_entry->needs_copy = TRUE;
+ }
+ new_entry->needs_copy = new_entry_needs_copy;
+
+ /*
+ * Insert the entry at the end
+ * of the map.
+ */
+
+ vm_map_store_entry_link(new_map,
+ vm_map_last_entry(new_map),
+ new_entry,
+ VM_MAP_KERNEL_FLAGS_NONE);
+ new_size += entry_size;
+ break;
+
+slow_vm_map_fork_copy:
+ vm_map_copyin_flags = 0;
+ if (options & VM_MAP_FORK_PRESERVE_PURGEABLE) {
+ vm_map_copyin_flags |=
+ VM_MAP_COPYIN_PRESERVE_PURGEABLE;
+ }
+ if (vm_map_fork_copy(old_map,
+ &old_entry,
+ new_map,
+ vm_map_copyin_flags)) {
+ new_size += entry_size;
+ }
+ continue;
+ }
+ old_entry = old_entry->vme_next;
+ }
+
+#if defined(__arm64__)
+ pmap_insert_sharedpage(new_map->pmap);
+#endif
+
+ new_map->size = new_size;
+
+ if (options & VM_MAP_FORK_CORPSE_FOOTPRINT) {
+ vm_map_corpse_footprint_collect_done(new_map);
+ }
+
+ vm_map_unlock(new_map);
+ vm_map_unlock(old_map);
+ vm_map_deallocate(old_map);
+
+ return new_map;
+}
+
+/*
+ * vm_map_exec:
+ *
+ * Setup the "new_map" with the proper execution environment according
+ * to the type of executable (platform, 64bit, chroot environment).
+ * Map the comm page and shared region, etc...
+ */
+kern_return_t
+vm_map_exec(
+ vm_map_t new_map,
+ task_t task,
+ boolean_t is64bit,
+ void *fsroot,
+ cpu_type_t cpu,
+ cpu_subtype_t cpu_subtype)
+{
+ SHARED_REGION_TRACE_DEBUG(
+ ("shared_region: task %p: vm_map_exec(%p,%p,%p,0x%x,0x%x): ->\n",
+ (void *)VM_KERNEL_ADDRPERM(current_task()),
+ (void *)VM_KERNEL_ADDRPERM(new_map),
+ (void *)VM_KERNEL_ADDRPERM(task),
+ (void *)VM_KERNEL_ADDRPERM(fsroot),
+ cpu,
+ cpu_subtype));
+ (void) vm_commpage_enter(new_map, task, is64bit);
+ (void) vm_shared_region_enter(new_map, task, is64bit, fsroot, cpu, cpu_subtype);
+ SHARED_REGION_TRACE_DEBUG(
+ ("shared_region: task %p: vm_map_exec(%p,%p,%p,0x%x,0x%x): <-\n",
+ (void *)VM_KERNEL_ADDRPERM(current_task()),
+ (void *)VM_KERNEL_ADDRPERM(new_map),
+ (void *)VM_KERNEL_ADDRPERM(task),
+ (void *)VM_KERNEL_ADDRPERM(fsroot),
+ cpu,
+ cpu_subtype));
+ return KERN_SUCCESS;
+}
+
+/*
+ * vm_map_lookup_locked:
+ *
+ * Finds the VM object, offset, and
+ * protection for a given virtual address in the
+ * specified map, assuming a page fault of the
+ * type specified.
+ *
+ * Returns the (object, offset, protection) for
+ * this address, whether it is wired down, and whether
+ * this map has the only reference to the data in question.
+ * In order to later verify this lookup, a "version"
+ * is returned.
+ *
+ * The map MUST be locked by the caller and WILL be
+ * locked on exit. In order to guarantee the
+ * existence of the returned object, it is returned
+ * locked.
+ *
+ * If a lookup is requested with "write protection"
+ * specified, the map may be changed to perform virtual
+ * copying operations, although the data referenced will
+ * remain the same.
+ */
+kern_return_t
+vm_map_lookup_locked(
+ vm_map_t *var_map, /* IN/OUT */
+ vm_map_offset_t vaddr,
+ vm_prot_t fault_type,
+ int object_lock_type,
+ vm_map_version_t *out_version, /* OUT */
+ vm_object_t *object, /* OUT */
+ vm_object_offset_t *offset, /* OUT */
+ vm_prot_t *out_prot, /* OUT */
+ boolean_t *wired, /* OUT */
+ vm_object_fault_info_t fault_info, /* OUT */
+ vm_map_t *real_map)
+{
+ vm_map_entry_t entry;
+ vm_map_t map = *var_map;
+ vm_map_t old_map = *var_map;
+ vm_map_t cow_sub_map_parent = VM_MAP_NULL;
+ vm_map_offset_t cow_parent_vaddr = 0;
+ vm_map_offset_t old_start = 0;
+ vm_map_offset_t old_end = 0;
+ vm_prot_t prot;
+ boolean_t mask_protections;
+ boolean_t force_copy;
+ vm_prot_t original_fault_type;
+
+ /*
+ * VM_PROT_MASK means that the caller wants us to use "fault_type"
+ * as a mask against the mapping's actual protections, not as an
+ * absolute value.
+ */
+ mask_protections = (fault_type & VM_PROT_IS_MASK) ? TRUE : FALSE;
+ force_copy = (fault_type & VM_PROT_COPY) ? TRUE : FALSE;
+ fault_type &= VM_PROT_ALL;
+ original_fault_type = fault_type;
+
+ *real_map = map;
+
+RetryLookup:
+ fault_type = original_fault_type;
+
+ /*
+ * If the map has an interesting hint, try it before calling
+ * full blown lookup routine.
+ */
+ entry = map->hint;
+
+ if ((entry == vm_map_to_entry(map)) ||
+ (vaddr < entry->vme_start) || (vaddr >= entry->vme_end)) {
+ vm_map_entry_t tmp_entry;
+
+ /*
+ * Entry was either not a valid hint, or the vaddr
+ * was not contained in the entry, so do a full lookup.
+ */
+ if (!vm_map_lookup_entry(map, vaddr, &tmp_entry)) {
+ if ((cow_sub_map_parent) && (cow_sub_map_parent != map)) {
+ vm_map_unlock(cow_sub_map_parent);
+ }
+ if ((*real_map != map)
+ && (*real_map != cow_sub_map_parent)) {
+ vm_map_unlock(*real_map);
+ }
+ return KERN_INVALID_ADDRESS;
+ }
+
+ entry = tmp_entry;
+ }
+ if (map == old_map) {
+ old_start = entry->vme_start;
+ old_end = entry->vme_end;
+ }
+
+ /*
+ * Handle submaps. Drop lock on upper map, submap is
+ * returned locked.
+ */
+
+submap_recurse:
+ if (entry->is_sub_map) {
+ vm_map_offset_t local_vaddr;
+ vm_map_offset_t end_delta;
+ vm_map_offset_t start_delta;
+ vm_map_entry_t submap_entry;
+ vm_prot_t subentry_protection;
+ vm_prot_t subentry_max_protection;
+ boolean_t subentry_no_copy_on_read;
+ boolean_t mapped_needs_copy = FALSE;
+
+ local_vaddr = vaddr;
+
+ if ((entry->use_pmap &&
+ !((fault_type & VM_PROT_WRITE) ||
+ force_copy))) {
+ /* if real_map equals map we unlock below */
+ if ((*real_map != map) &&
+ (*real_map != cow_sub_map_parent)) {
+ vm_map_unlock(*real_map);
+ }
+ *real_map = VME_SUBMAP(entry);
+ }
+
+ if (entry->needs_copy &&
+ ((fault_type & VM_PROT_WRITE) ||
+ force_copy)) {
+ if (!mapped_needs_copy) {
+ if (vm_map_lock_read_to_write(map)) {
+ vm_map_lock_read(map);
+ *real_map = map;
+ goto RetryLookup;
+ }
+ vm_map_lock_read(VME_SUBMAP(entry));
+ *var_map = VME_SUBMAP(entry);
+ cow_sub_map_parent = map;
+ /* reset base to map before cow object */
+ /* this is the map which will accept */
+ /* the new cow object */
+ old_start = entry->vme_start;
+ old_end = entry->vme_end;
+ cow_parent_vaddr = vaddr;
+ mapped_needs_copy = TRUE;
+ } else {
+ vm_map_lock_read(VME_SUBMAP(entry));
+ *var_map = VME_SUBMAP(entry);
+ if ((cow_sub_map_parent != map) &&
+ (*real_map != map)) {
+ vm_map_unlock(map);
+ }
+ }
+ } else {
+ vm_map_lock_read(VME_SUBMAP(entry));
+ *var_map = VME_SUBMAP(entry);
+ /* leave map locked if it is a target */
+ /* cow sub_map above otherwise, just */
+ /* follow the maps down to the object */
+ /* here we unlock knowing we are not */
+ /* revisiting the map. */
+ if ((*real_map != map) && (map != cow_sub_map_parent)) {
+ vm_map_unlock_read(map);
+ }
+ }
+
+ map = *var_map;
+
+ /* calculate the offset in the submap for vaddr */
+ local_vaddr = (local_vaddr - entry->vme_start) + VME_OFFSET(entry);
+
+RetrySubMap:
+ if (!vm_map_lookup_entry(map, local_vaddr, &submap_entry)) {
+ if ((cow_sub_map_parent) && (cow_sub_map_parent != map)) {
+ vm_map_unlock(cow_sub_map_parent);
+ }
+ if ((*real_map != map)
+ && (*real_map != cow_sub_map_parent)) {
+ vm_map_unlock(*real_map);
+ }
+ *real_map = map;
+ return KERN_INVALID_ADDRESS;
+ }
+
+ /* find the attenuated shadow of the underlying object */
+ /* on our target map */
+
+ /* in english the submap object may extend beyond the */
+ /* region mapped by the entry or, may only fill a portion */
+ /* of it. For our purposes, we only care if the object */
+ /* doesn't fill. In this case the area which will */
+ /* ultimately be clipped in the top map will only need */
+ /* to be as big as the portion of the underlying entry */
+ /* which is mapped */
+ start_delta = submap_entry->vme_start > VME_OFFSET(entry) ?
+ submap_entry->vme_start - VME_OFFSET(entry) : 0;
+
+ end_delta =
+ (VME_OFFSET(entry) + start_delta + (old_end - old_start)) <=
+ submap_entry->vme_end ?
+ 0 : (VME_OFFSET(entry) +
+ (old_end - old_start))
+ - submap_entry->vme_end;
+
+ old_start += start_delta;
+ old_end -= end_delta;
+
+ if (submap_entry->is_sub_map) {
+ entry = submap_entry;
+ vaddr = local_vaddr;
+ goto submap_recurse;
+ }
+
+ if (((fault_type & VM_PROT_WRITE) ||
+ force_copy)
+ && cow_sub_map_parent) {
+ vm_object_t sub_object, copy_object;
+ vm_object_offset_t copy_offset;
+ vm_map_offset_t local_start;
+ vm_map_offset_t local_end;
+ boolean_t copied_slowly = FALSE;
+
+ if (vm_map_lock_read_to_write(map)) {
+ vm_map_lock_read(map);
+ old_start -= start_delta;
+ old_end += end_delta;
+ goto RetrySubMap;
+ }
+
+
+ sub_object = VME_OBJECT(submap_entry);
+ if (sub_object == VM_OBJECT_NULL) {
+ sub_object =
+ vm_object_allocate(
+ (vm_map_size_t)
+ (submap_entry->vme_end -
+ submap_entry->vme_start));
+ VME_OBJECT_SET(submap_entry, sub_object);
+ VME_OFFSET_SET(submap_entry, 0);
+ assert(!submap_entry->is_sub_map);
+ assert(submap_entry->use_pmap);
+ }
+ local_start = local_vaddr -
+ (cow_parent_vaddr - old_start);
+ local_end = local_vaddr +
+ (old_end - cow_parent_vaddr);
+ vm_map_clip_start(map, submap_entry, local_start);
+ vm_map_clip_end(map, submap_entry, local_end);
+ if (submap_entry->is_sub_map) {
+ /* unnesting was done when clipping */
+ assert(!submap_entry->use_pmap);
+ }
+
+ /* This is the COW case, lets connect */
+ /* an entry in our space to the underlying */
+ /* object in the submap, bypassing the */
+ /* submap. */
+
+
+ if (submap_entry->wired_count != 0 ||
+ (sub_object->copy_strategy ==
+ MEMORY_OBJECT_COPY_NONE)) {
+ vm_object_lock(sub_object);
+ vm_object_copy_slowly(sub_object,
+ VME_OFFSET(submap_entry),
+ (submap_entry->vme_end -
+ submap_entry->vme_start),
+ FALSE,
+ ©_object);
+ copied_slowly = TRUE;
+ } else {
+ /* set up shadow object */
+ copy_object = sub_object;
+ vm_object_lock(sub_object);
+ vm_object_reference_locked(sub_object);
+ sub_object->shadowed = TRUE;
+ vm_object_unlock(sub_object);
+
+ assert(submap_entry->wired_count == 0);
+ submap_entry->needs_copy = TRUE;
+
+ prot = submap_entry->protection;
+ assert(!pmap_has_prot_policy(prot));
+ prot = prot & ~VM_PROT_WRITE;
+ assert(!pmap_has_prot_policy(prot));
+
+ if (override_nx(old_map,
+ VME_ALIAS(submap_entry))
+ && prot) {
+ prot |= VM_PROT_EXECUTE;
+ }
+
+ vm_object_pmap_protect(
+ sub_object,
+ VME_OFFSET(submap_entry),
+ submap_entry->vme_end -
+ submap_entry->vme_start,
+ (submap_entry->is_shared
+ || map->mapped_in_other_pmaps) ?
+ PMAP_NULL : map->pmap,
+ submap_entry->vme_start,
+ prot);
+ }
+
+ /*
+ * Adjust the fault offset to the submap entry.
+ */
+ copy_offset = (local_vaddr -
+ submap_entry->vme_start +
+ VME_OFFSET(submap_entry));
+
+ /* This works diffently than the */
+ /* normal submap case. We go back */
+ /* to the parent of the cow map and*/
+ /* clip out the target portion of */
+ /* the sub_map, substituting the */
+ /* new copy object, */
+
+ subentry_protection = submap_entry->protection;
+ subentry_max_protection = submap_entry->max_protection;
+ subentry_no_copy_on_read = submap_entry->vme_no_copy_on_read;
+ vm_map_unlock(map);
+ submap_entry = NULL; /* not valid after map unlock */
+
+ local_start = old_start;
+ local_end = old_end;
+ map = cow_sub_map_parent;
+ *var_map = cow_sub_map_parent;
+ vaddr = cow_parent_vaddr;
+ cow_sub_map_parent = NULL;
+
+ if (!vm_map_lookup_entry(map,
+ vaddr, &entry)) {
+ vm_object_deallocate(
+ copy_object);
+ vm_map_lock_write_to_read(map);
+ return KERN_INVALID_ADDRESS;
+ }
+
+ /* clip out the portion of space */
+ /* mapped by the sub map which */
+ /* corresponds to the underlying */
+ /* object */
+
+ /*
+ * Clip (and unnest) the smallest nested chunk
+ * possible around the faulting address...
+ */
+ local_start = vaddr & ~(pmap_nesting_size_min - 1);
+ local_end = local_start + pmap_nesting_size_min;
+ /*
+ * ... but don't go beyond the "old_start" to "old_end"
+ * range, to avoid spanning over another VM region
+ * with a possibly different VM object and/or offset.
+ */
+ if (local_start < old_start) {
+ local_start = old_start;
+ }
+ if (local_end > old_end) {
+ local_end = old_end;
+ }
+ /*
+ * Adjust copy_offset to the start of the range.
+ */
+ copy_offset -= (vaddr - local_start);
+
+ vm_map_clip_start(map, entry, local_start);
+ vm_map_clip_end(map, entry, local_end);
+ if (entry->is_sub_map) {
+ /* unnesting was done when clipping */
+ assert(!entry->use_pmap);
+ }
+
+ /* substitute copy object for */
+ /* shared map entry */
+ vm_map_deallocate(VME_SUBMAP(entry));
+ assert(!entry->iokit_acct);
+ entry->is_sub_map = FALSE;
+ entry->use_pmap = TRUE;
+ VME_OBJECT_SET(entry, copy_object);
+
+ /* propagate the submap entry's protections */
+ if (entry->protection != VM_PROT_READ) {
+ /*
+ * Someone has already altered the top entry's
+ * protections via vm_protect(VM_PROT_COPY).
+ * Respect these new values and ignore the
+ * submap entry's protections.
+ */
+ } else {
+ /*
+ * Regular copy-on-write: propagate the submap
+ * entry's protections to the top map entry.
+ */
+ entry->protection |= subentry_protection;
+ }
+ entry->max_protection |= subentry_max_protection;
+ /* propagate no_copy_on_read */
+ entry->vme_no_copy_on_read = subentry_no_copy_on_read;
+
+ if ((entry->protection & VM_PROT_WRITE) &&
+ (entry->protection & VM_PROT_EXECUTE) &&
+#if !CONFIG_EMBEDDED
+ map != kernel_map &&
+ cs_process_enforcement(NULL) &&
+#endif /* !CONFIG_EMBEDDED */
+ !(entry->used_for_jit)) {
+ DTRACE_VM3(cs_wx,
+ uint64_t, (uint64_t)entry->vme_start,
+ uint64_t, (uint64_t)entry->vme_end,
+ vm_prot_t, entry->protection);
+ printf("CODE SIGNING: %d[%s] %s can't have both write and exec at the same time\n",
+ proc_selfpid(),
+ (current_task()->bsd_info
+ ? proc_name_address(current_task()->bsd_info)
+ : "?"),
+ __FUNCTION__);
+ entry->protection &= ~VM_PROT_EXECUTE;
+ }
+
+ if (copied_slowly) {
+ VME_OFFSET_SET(entry, local_start - old_start);
+ entry->needs_copy = FALSE;
+ entry->is_shared = FALSE;
+ } else {
+ VME_OFFSET_SET(entry, copy_offset);
+ assert(entry->wired_count == 0);
+ entry->needs_copy = TRUE;
+ if (entry->inheritance == VM_INHERIT_SHARE) {
+ entry->inheritance = VM_INHERIT_COPY;
+ }
+ if (map != old_map) {
+ entry->is_shared = TRUE;
+ }
+ }
+ if (entry->inheritance == VM_INHERIT_SHARE) {
+ entry->inheritance = VM_INHERIT_COPY;
+ }
+
+ vm_map_lock_write_to_read(map);
+ } else {
+ if ((cow_sub_map_parent)
+ && (cow_sub_map_parent != *real_map)
+ && (cow_sub_map_parent != map)) {
+ vm_map_unlock(cow_sub_map_parent);
+ }
+ entry = submap_entry;
+ vaddr = local_vaddr;
+ }
+ }
+
+ /*
+ * Check whether this task is allowed to have
+ * this page.
+ */
+
+ prot = entry->protection;
+
+ if (override_nx(old_map, VME_ALIAS(entry)) && prot) {
+ /*
+ * HACK -- if not a stack, then allow execution
+ */
+ prot |= VM_PROT_EXECUTE;
+ }
+
+ if (mask_protections) {
+ fault_type &= prot;
+ if (fault_type == VM_PROT_NONE) {
+ goto protection_failure;
+ }
+ }
+ if (((fault_type & prot) != fault_type)
+#if __arm64__
+ /* prefetch abort in execute-only page */
+ && !(prot == VM_PROT_EXECUTE && fault_type == (VM_PROT_READ | VM_PROT_EXECUTE))
+#endif
+ ) {
+protection_failure:
+ if (*real_map != map) {
+ vm_map_unlock(*real_map);
+ }
+ *real_map = map;
+
+ if ((fault_type & VM_PROT_EXECUTE) && prot) {
+ log_stack_execution_failure((addr64_t)vaddr, prot);
+ }
+
+ DTRACE_VM2(prot_fault, int, 1, (uint64_t *), NULL);
+ return KERN_PROTECTION_FAILURE;
+ }
+
+ /*
+ * If this page is not pageable, we have to get
+ * it for all possible accesses.
+ */
+
+ *wired = (entry->wired_count != 0);
+ if (*wired) {
+ fault_type = prot;
+ }
+
+ /*
+ * If the entry was copy-on-write, we either ...
+ */
+
+ if (entry->needs_copy) {
+ /*
+ * If we want to write the page, we may as well
+ * handle that now since we've got the map locked.
+ *
+ * If we don't need to write the page, we just
+ * demote the permissions allowed.
+ */
+
+ if ((fault_type & VM_PROT_WRITE) || *wired || force_copy) {
+ /*
+ * Make a new object, and place it in the
+ * object chain. Note that no new references
+ * have appeared -- one just moved from the
+ * map to the new object.
+ */
+
+ if (vm_map_lock_read_to_write(map)) {
+ vm_map_lock_read(map);
+ goto RetryLookup;
+ }
+
+ if (VME_OBJECT(entry)->shadowed == FALSE) {
+ vm_object_lock(VME_OBJECT(entry));
+ VME_OBJECT(entry)->shadowed = TRUE;
+ vm_object_unlock(VME_OBJECT(entry));
+ }
+ VME_OBJECT_SHADOW(entry,
+ (vm_map_size_t) (entry->vme_end -
+ entry->vme_start));
+ entry->needs_copy = FALSE;
+
+ vm_map_lock_write_to_read(map);
+ }
+ if ((fault_type & VM_PROT_WRITE) == 0 && *wired == 0) {
+ /*
+ * We're attempting to read a copy-on-write
+ * page -- don't allow writes.
+ */
+
+ prot &= (~VM_PROT_WRITE);
+ }
+ }
+
+ /*
+ * Create an object if necessary.
+ */
+ if (VME_OBJECT(entry) == VM_OBJECT_NULL) {
+ if (vm_map_lock_read_to_write(map)) {
+ vm_map_lock_read(map);
+ goto RetryLookup;
+ }
+
+ VME_OBJECT_SET(entry,
+ vm_object_allocate(
+ (vm_map_size_t)(entry->vme_end -
+ entry->vme_start)));
+ VME_OFFSET_SET(entry, 0);
+ assert(entry->use_pmap);
+ vm_map_lock_write_to_read(map);
+ }
+
+ /*
+ * Return the object/offset from this entry. If the entry
+ * was copy-on-write or empty, it has been fixed up. Also
+ * return the protection.
+ */
+
+ *offset = (vaddr - entry->vme_start) + VME_OFFSET(entry);
+ *object = VME_OBJECT(entry);
+ *out_prot = prot;
+ KDBG_FILTERED(MACHDBG_CODE(DBG_MACH_WORKINGSET, VM_MAP_LOOKUP_OBJECT), VM_KERNEL_UNSLIDE_OR_PERM(*object), 0, 0, 0, 0);
+
+ if (fault_info) {
+ fault_info->interruptible = THREAD_UNINT; /* for now... */
+ /* ... the caller will change "interruptible" if needed */
+ fault_info->cluster_size = 0;
+ fault_info->user_tag = VME_ALIAS(entry);
+ fault_info->pmap_options = 0;
+ if (entry->iokit_acct ||
+ (!entry->is_sub_map && !entry->use_pmap)) {
+ fault_info->pmap_options |= PMAP_OPTIONS_ALT_ACCT;
+ }
+ fault_info->behavior = entry->behavior;
+ fault_info->lo_offset = VME_OFFSET(entry);
+ fault_info->hi_offset =
+ (entry->vme_end - entry->vme_start) + VME_OFFSET(entry);
+ fault_info->no_cache = entry->no_cache;
+ fault_info->stealth = FALSE;
+ fault_info->io_sync = FALSE;
+ if (entry->used_for_jit ||
+ entry->vme_resilient_codesign) {
+ fault_info->cs_bypass = TRUE;
+ } else {
+ fault_info->cs_bypass = FALSE;
+ }
+ fault_info->pmap_cs_associated = FALSE;
+#if CONFIG_PMAP_CS
+ if (entry->pmap_cs_associated) {
+ /*
+ * The pmap layer will validate this page
+ * before allowing it to be executed from.
+ */
+ fault_info->pmap_cs_associated = TRUE;
+ }
+#endif /* CONFIG_PMAP_CS */
+ fault_info->mark_zf_absent = FALSE;
+ fault_info->batch_pmap_op = FALSE;
+ fault_info->resilient_media = entry->vme_resilient_media;
+ fault_info->no_copy_on_read = entry->vme_no_copy_on_read;
+ }
+
+ /*
+ * Lock the object to prevent it from disappearing
+ */
+ if (object_lock_type == OBJECT_LOCK_EXCLUSIVE) {
+ vm_object_lock(*object);
+ } else {
+ vm_object_lock_shared(*object);
+ }
+
+ /*
+ * Save the version number
+ */
+
+ out_version->main_timestamp = map->timestamp;
+
+ return KERN_SUCCESS;
+}
+
+
+/*
+ * vm_map_verify:
+ *
+ * Verifies that the map in question has not changed
+ * since the given version. The map has to be locked
+ * ("shared" mode is fine) before calling this function
+ * and it will be returned locked too.
+ */
+boolean_t
+vm_map_verify(
+ vm_map_t map,
+ vm_map_version_t *version) /* REF */
+{
+ boolean_t result;
+
+ vm_map_lock_assert_held(map);
+ result = (map->timestamp == version->main_timestamp);
+
+ return result;
+}
+
+/*
+ * TEMPORARYTEMPORARYTEMPORARYTEMPORARYTEMPORARYTEMPORARY
+ * Goes away after regular vm_region_recurse function migrates to
+ * 64 bits
+ * vm_region_recurse: A form of vm_region which follows the
+ * submaps in a target map
+ *
+ */
+
+kern_return_t
+vm_map_region_recurse_64(
+ vm_map_t map,
+ vm_map_offset_t *address, /* IN/OUT */
+ vm_map_size_t *size, /* OUT */
+ natural_t *nesting_depth, /* IN/OUT */
+ vm_region_submap_info_64_t submap_info, /* IN/OUT */
+ mach_msg_type_number_t *count) /* IN/OUT */
+{
+ mach_msg_type_number_t original_count;
+ vm_region_extended_info_data_t extended;
+ vm_map_entry_t tmp_entry;
+ vm_map_offset_t user_address;
+ unsigned int user_max_depth;
+
+ /*
+ * "curr_entry" is the VM map entry preceding or including the
+ * address we're looking for.
+ * "curr_map" is the map or sub-map containing "curr_entry".
+ * "curr_address" is the equivalent of the top map's "user_address"
+ * in the current map.
+ * "curr_offset" is the cumulated offset of "curr_map" in the
+ * target task's address space.
+ * "curr_depth" is the depth of "curr_map" in the chain of
+ * sub-maps.
+ *
+ * "curr_max_below" and "curr_max_above" limit the range (around
+ * "curr_address") we should take into account in the current (sub)map.
+ * They limit the range to what's visible through the map entries
+ * we've traversed from the top map to the current map.
+ *
+ */
+ vm_map_entry_t curr_entry;
+ vm_map_address_t curr_address;
+ vm_map_offset_t curr_offset;
+ vm_map_t curr_map;
+ unsigned int curr_depth;
+ vm_map_offset_t curr_max_below, curr_max_above;
+ vm_map_offset_t curr_skip;
+
+ /*
+ * "next_" is the same as "curr_" but for the VM region immediately
+ * after the address we're looking for. We need to keep track of this
+ * too because we want to return info about that region if the
+ * address we're looking for is not mapped.
+ */
+ vm_map_entry_t next_entry;
+ vm_map_offset_t next_offset;
+ vm_map_offset_t next_address;
+ vm_map_t next_map;
+ unsigned int next_depth;
+ vm_map_offset_t next_max_below, next_max_above;
+ vm_map_offset_t next_skip;
+
+ boolean_t look_for_pages;
+ vm_region_submap_short_info_64_t short_info;
+ boolean_t do_region_footprint;
+
+ if (map == VM_MAP_NULL) {
+ /* no address space to work on */
+ return KERN_INVALID_ARGUMENT;
+ }
+
+
+ if (*count < VM_REGION_SUBMAP_SHORT_INFO_COUNT_64) {
+ /*
+ * "info" structure is not big enough and
+ * would overflow
+ */
+ return KERN_INVALID_ARGUMENT;
+ }
+
+ do_region_footprint = task_self_region_footprint();
+ original_count = *count;
+
+ if (original_count < VM_REGION_SUBMAP_INFO_V0_COUNT_64) {
+ *count = VM_REGION_SUBMAP_SHORT_INFO_COUNT_64;
+ look_for_pages = FALSE;
+ short_info = (vm_region_submap_short_info_64_t) submap_info;
+ submap_info = NULL;
+ } else {
+ look_for_pages = TRUE;
+ *count = VM_REGION_SUBMAP_INFO_V0_COUNT_64;
+ short_info = NULL;
+
+ if (original_count >= VM_REGION_SUBMAP_INFO_V1_COUNT_64) {
+ *count = VM_REGION_SUBMAP_INFO_V1_COUNT_64;
+ }
+ if (original_count >= VM_REGION_SUBMAP_INFO_V2_COUNT_64) {
+ *count = VM_REGION_SUBMAP_INFO_V2_COUNT_64;
+ }
+ }
+
+ user_address = *address;
+ user_max_depth = *nesting_depth;
+
+ if (not_in_kdp) {
+ vm_map_lock_read(map);
+ }
+
+recurse_again:
+ curr_entry = NULL;
+ curr_map = map;
+ curr_address = user_address;
+ curr_offset = 0;
+ curr_skip = 0;
+ curr_depth = 0;
+ curr_max_above = ((vm_map_offset_t) -1) - curr_address;
+ curr_max_below = curr_address;
+
+ next_entry = NULL;
+ next_map = NULL;
+ next_address = 0;
+ next_offset = 0;
+ next_skip = 0;
+ next_depth = 0;
+ next_max_above = (vm_map_offset_t) -1;
+ next_max_below = (vm_map_offset_t) -1;
+
+ for (;;) {
+ if (vm_map_lookup_entry(curr_map,
+ curr_address,
+ &tmp_entry)) {
+ /* tmp_entry contains the address we're looking for */
+ curr_entry = tmp_entry;
+ } else {
+ vm_map_offset_t skip;
+ /*
+ * The address is not mapped. "tmp_entry" is the
+ * map entry preceding the address. We want the next
+ * one, if it exists.
+ */
+ curr_entry = tmp_entry->vme_next;
+
+ if (curr_entry == vm_map_to_entry(curr_map) ||
+ (curr_entry->vme_start >=
+ curr_address + curr_max_above)) {
+ /* no next entry at this level: stop looking */
+ if (not_in_kdp) {
+ vm_map_unlock_read(curr_map);
+ }
+ curr_entry = NULL;
+ curr_map = NULL;
+ curr_skip = 0;
+ curr_offset = 0;
+ curr_depth = 0;
+ curr_max_above = 0;
+ curr_max_below = 0;
+ break;
+ }
+
+ /* adjust current address and offset */
+ skip = curr_entry->vme_start - curr_address;
+ curr_address = curr_entry->vme_start;
+ curr_skip += skip;
+ curr_offset += skip;
+ curr_max_above -= skip;
+ curr_max_below = 0;
+ }
+
+ /*
+ * Is the next entry at this level closer to the address (or
+ * deeper in the submap chain) than the one we had
+ * so far ?
+ */
+ tmp_entry = curr_entry->vme_next;
+ if (tmp_entry == vm_map_to_entry(curr_map)) {
+ /* no next entry at this level */
+ } else if (tmp_entry->vme_start >=
+ curr_address + curr_max_above) {
+ /*
+ * tmp_entry is beyond the scope of what we mapped of
+ * this submap in the upper level: ignore it.
+ */
+ } else if ((next_entry == NULL) ||
+ (tmp_entry->vme_start + curr_offset <=
+ next_entry->vme_start + next_offset)) {
+ /*
+ * We didn't have a "next_entry" or this one is
+ * closer to the address we're looking for:
+ * use this "tmp_entry" as the new "next_entry".
+ */
+ if (next_entry != NULL) {
+ /* unlock the last "next_map" */
+ if (next_map != curr_map && not_in_kdp) {
+ vm_map_unlock_read(next_map);
+ }
+ }
+ next_entry = tmp_entry;
+ next_map = curr_map;
+ next_depth = curr_depth;
+ next_address = next_entry->vme_start;
+ next_skip = curr_skip;
+ next_skip += (next_address - curr_address);
+ next_offset = curr_offset;
+ next_offset += (next_address - curr_address);
+ next_max_above = MIN(next_max_above, curr_max_above);
+ next_max_above = MIN(next_max_above,
+ next_entry->vme_end - next_address);
+ next_max_below = MIN(next_max_below, curr_max_below);
+ next_max_below = MIN(next_max_below,
+ next_address - next_entry->vme_start);
+ }
+
+ /*
+ * "curr_max_{above,below}" allow us to keep track of the
+ * portion of the submap that is actually mapped at this level:
+ * the rest of that submap is irrelevant to us, since it's not
+ * mapped here.
+ * The relevant portion of the map starts at
+ * "VME_OFFSET(curr_entry)" up to the size of "curr_entry".
+ */
+ curr_max_above = MIN(curr_max_above,
+ curr_entry->vme_end - curr_address);
+ curr_max_below = MIN(curr_max_below,
+ curr_address - curr_entry->vme_start);
+
+ if (!curr_entry->is_sub_map ||
+ curr_depth >= user_max_depth) {
+ /*
+ * We hit a leaf map or we reached the maximum depth
+ * we could, so stop looking. Keep the current map
+ * locked.
+ */
+ break;
+ }
+
+ /*
+ * Get down to the next submap level.
+ */
+
+ /*
+ * Lock the next level and unlock the current level,
+ * unless we need to keep it locked to access the "next_entry"
+ * later.
+ */
+ if (not_in_kdp) {
+ vm_map_lock_read(VME_SUBMAP(curr_entry));
+ }
+ if (curr_map == next_map) {
+ /* keep "next_map" locked in case we need it */
+ } else {
+ /* release this map */
+ if (not_in_kdp) {
+ vm_map_unlock_read(curr_map);
+ }
+ }
+
+ /*
+ * Adjust the offset. "curr_entry" maps the submap
+ * at relative address "curr_entry->vme_start" in the
+ * curr_map but skips the first "VME_OFFSET(curr_entry)"
+ * bytes of the submap.
+ * "curr_offset" always represents the offset of a virtual
+ * address in the curr_map relative to the absolute address
+ * space (i.e. the top-level VM map).
+ */
+ curr_offset +=
+ (VME_OFFSET(curr_entry) - curr_entry->vme_start);
+ curr_address = user_address + curr_offset;
+ /* switch to the submap */
+ curr_map = VME_SUBMAP(curr_entry);
+ curr_depth++;
+ curr_entry = NULL;
+ }
+
+// LP64todo: all the current tools are 32bit, obviously never worked for 64b
+// so probably should be a real 32b ID vs. ptr.
+// Current users just check for equality
+
+ if (curr_entry == NULL) {
+ /* no VM region contains the address... */
+
+ if (do_region_footprint && /* we want footprint numbers */
+ next_entry == NULL && /* & there are no more regions */
+ /* & we haven't already provided our fake region: */
+ user_address <= vm_map_last_entry(map)->vme_end) {
+ ledger_amount_t ledger_resident, ledger_compressed;
+
+ /*
+ * Add a fake memory region to account for
+ * purgeable and/or ledger-tagged memory that
+ * counts towards this task's memory footprint,
+ * i.e. the resident/compressed pages of non-volatile
+ * objects owned by that task.
+ */
+ task_ledgers_footprint(map->pmap->ledger,
+ &ledger_resident,
+ &ledger_compressed);
+ if (ledger_resident + ledger_compressed == 0) {
+ /* no purgeable memory usage to report */
+ return KERN_INVALID_ADDRESS;
+ }
+ /* fake region to show nonvolatile footprint */
+ if (look_for_pages) {
+ submap_info->protection = VM_PROT_DEFAULT;
+ submap_info->max_protection = VM_PROT_DEFAULT;
+ submap_info->inheritance = VM_INHERIT_DEFAULT;
+ submap_info->offset = 0;
+ submap_info->user_tag = -1;
+ submap_info->pages_resident = (unsigned int) (ledger_resident / PAGE_SIZE);
+ submap_info->pages_shared_now_private = 0;
+ submap_info->pages_swapped_out = (unsigned int) (ledger_compressed / PAGE_SIZE);
+ submap_info->pages_dirtied = submap_info->pages_resident;
+ submap_info->ref_count = 1;
+ submap_info->shadow_depth = 0;
+ submap_info->external_pager = 0;
+ submap_info->share_mode = SM_PRIVATE;
+ submap_info->is_submap = 0;
+ submap_info->behavior = VM_BEHAVIOR_DEFAULT;
+ submap_info->object_id = INFO_MAKE_FAKE_OBJECT_ID(map, task_ledgers.purgeable_nonvolatile);
+ submap_info->user_wired_count = 0;
+ submap_info->pages_reusable = 0;
+ } else {
+ short_info->user_tag = -1;
+ short_info->offset = 0;
+ short_info->protection = VM_PROT_DEFAULT;
+ short_info->inheritance = VM_INHERIT_DEFAULT;
+ short_info->max_protection = VM_PROT_DEFAULT;
+ short_info->behavior = VM_BEHAVIOR_DEFAULT;
+ short_info->user_wired_count = 0;
+ short_info->is_submap = 0;
+ short_info->object_id = INFO_MAKE_FAKE_OBJECT_ID(map, task_ledgers.purgeable_nonvolatile);
+ short_info->external_pager = 0;
+ short_info->shadow_depth = 0;
+ short_info->share_mode = SM_PRIVATE;
+ short_info->ref_count = 1;
+ }
+ *nesting_depth = 0;
+ *size = (vm_map_size_t) (ledger_resident + ledger_compressed);
+// *address = user_address;
+ *address = vm_map_last_entry(map)->vme_end;
+ return KERN_SUCCESS;
+ }
+
+ if (next_entry == NULL) {
+ /* ... and no VM region follows it either */
+ return KERN_INVALID_ADDRESS;
+ }
+ /* ... gather info about the next VM region */
+ curr_entry = next_entry;
+ curr_map = next_map; /* still locked ... */
+ curr_address = next_address;
+ curr_skip = next_skip;
+ curr_offset = next_offset;
+ curr_depth = next_depth;
+ curr_max_above = next_max_above;
+ curr_max_below = next_max_below;
+ } else {
+ /* we won't need "next_entry" after all */
+ if (next_entry != NULL) {
+ /* release "next_map" */
+ if (next_map != curr_map && not_in_kdp) {
+ vm_map_unlock_read(next_map);
+ }
+ }
+ }
+ next_entry = NULL;
+ next_map = NULL;
+ next_offset = 0;
+ next_skip = 0;
+ next_depth = 0;
+ next_max_below = -1;
+ next_max_above = -1;
+
+ if (curr_entry->is_sub_map &&
+ curr_depth < user_max_depth) {
+ /*
+ * We're not as deep as we could be: we must have
+ * gone back up after not finding anything mapped
+ * below the original top-level map entry's.
+ * Let's move "curr_address" forward and recurse again.
+ */
+ user_address = curr_address;
+ goto recurse_again;
+ }
+
+ *nesting_depth = curr_depth;
+ *size = curr_max_above + curr_max_below;
+ *address = user_address + curr_skip - curr_max_below;
+
+// LP64todo: all the current tools are 32bit, obviously never worked for 64b
+// so probably should be a real 32b ID vs. ptr.
+// Current users just check for equality
+#define INFO_MAKE_OBJECT_ID(p) ((uint32_t)(uintptr_t)VM_KERNEL_ADDRPERM(p))
+
+ if (look_for_pages) {
+ submap_info->user_tag = VME_ALIAS(curr_entry);
+ submap_info->offset = VME_OFFSET(curr_entry);
+ submap_info->protection = curr_entry->protection;
+ submap_info->inheritance = curr_entry->inheritance;
+ submap_info->max_protection = curr_entry->max_protection;
+ submap_info->behavior = curr_entry->behavior;
+ submap_info->user_wired_count = curr_entry->user_wired_count;
+ submap_info->is_submap = curr_entry->is_sub_map;
+ submap_info->object_id = INFO_MAKE_OBJECT_ID(VME_OBJECT(curr_entry));
+ } else {
+ short_info->user_tag = VME_ALIAS(curr_entry);
+ short_info->offset = VME_OFFSET(curr_entry);
+ short_info->protection = curr_entry->protection;
+ short_info->inheritance = curr_entry->inheritance;
+ short_info->max_protection = curr_entry->max_protection;
+ short_info->behavior = curr_entry->behavior;
+ short_info->user_wired_count = curr_entry->user_wired_count;
+ short_info->is_submap = curr_entry->is_sub_map;
+ short_info->object_id = INFO_MAKE_OBJECT_ID(VME_OBJECT(curr_entry));
+ }
+
+ extended.pages_resident = 0;
+ extended.pages_swapped_out = 0;
+ extended.pages_shared_now_private = 0;
+ extended.pages_dirtied = 0;
+ extended.pages_reusable = 0;
+ extended.external_pager = 0;
+ extended.shadow_depth = 0;
+ extended.share_mode = SM_EMPTY;
+ extended.ref_count = 0;
+
+ if (not_in_kdp) {
+ if (!curr_entry->is_sub_map) {
+ vm_map_offset_t range_start, range_end;
+ range_start = MAX((curr_address - curr_max_below),
+ curr_entry->vme_start);
+ range_end = MIN((curr_address + curr_max_above),
+ curr_entry->vme_end);
+ vm_map_region_walk(curr_map,
+ range_start,
+ curr_entry,
+ (VME_OFFSET(curr_entry) +
+ (range_start -
+ curr_entry->vme_start)),
+ range_end - range_start,
+ &extended,
+ look_for_pages, VM_REGION_EXTENDED_INFO_COUNT);
+ if (extended.external_pager &&
+ extended.ref_count == 2 &&
+ extended.share_mode == SM_SHARED) {
+ extended.share_mode = SM_PRIVATE;
+ }
+ } else {
+ if (curr_entry->use_pmap) {
+ extended.share_mode = SM_TRUESHARED;
+ } else {
+ extended.share_mode = SM_PRIVATE;
+ }
+ extended.ref_count = os_ref_get_count(&VME_SUBMAP(curr_entry)->map_refcnt);
+ }
+ }
+
+ if (look_for_pages) {
+ submap_info->pages_resident = extended.pages_resident;
+ submap_info->pages_swapped_out = extended.pages_swapped_out;
+ submap_info->pages_shared_now_private =
+ extended.pages_shared_now_private;
+ submap_info->pages_dirtied = extended.pages_dirtied;
+ submap_info->external_pager = extended.external_pager;
+ submap_info->shadow_depth = extended.shadow_depth;
+ submap_info->share_mode = extended.share_mode;
+ submap_info->ref_count = extended.ref_count;
+
+ if (original_count >= VM_REGION_SUBMAP_INFO_V1_COUNT_64) {
+ submap_info->pages_reusable = extended.pages_reusable;
+ }
+ if (original_count >= VM_REGION_SUBMAP_INFO_V2_COUNT_64) {
+ submap_info->object_id_full = (vm_object_id_t) (VME_OBJECT(curr_entry) != NULL) ? VM_KERNEL_ADDRPERM(VME_OBJECT(curr_entry)) : 0ULL;
+ }
+ } else {
+ short_info->external_pager = extended.external_pager;
+ short_info->shadow_depth = extended.shadow_depth;
+ short_info->share_mode = extended.share_mode;
+ short_info->ref_count = extended.ref_count;
+ }
+
+ if (not_in_kdp) {
+ vm_map_unlock_read(curr_map);
+ }
+
+ return KERN_SUCCESS;
+}
+
+/*
+ * vm_region:
+ *
+ * User call to obtain information about a region in
+ * a task's address map. Currently, only one flavor is
+ * supported.
+ *
+ * XXX The reserved and behavior fields cannot be filled
+ * in until the vm merge from the IK is completed, and
+ * vm_reserve is implemented.
+ */
+
+kern_return_t
+vm_map_region(
+ vm_map_t map,
+ vm_map_offset_t *address, /* IN/OUT */
+ vm_map_size_t *size, /* OUT */
+ vm_region_flavor_t flavor, /* IN */
+ vm_region_info_t info, /* OUT */
+ mach_msg_type_number_t *count, /* IN/OUT */
+ mach_port_t *object_name) /* OUT */
+{
+ vm_map_entry_t tmp_entry;
+ vm_map_entry_t entry;
+ vm_map_offset_t start;
+
+ if (map == VM_MAP_NULL) {
+ return KERN_INVALID_ARGUMENT;
+ }
+
+ switch (flavor) {
+ case VM_REGION_BASIC_INFO:
+ /* legacy for old 32-bit objects info */
+ {
+ vm_region_basic_info_t basic;
+
+ if (*count < VM_REGION_BASIC_INFO_COUNT) {
+ return KERN_INVALID_ARGUMENT;
+ }
+
+ basic = (vm_region_basic_info_t) info;
+ *count = VM_REGION_BASIC_INFO_COUNT;
+
+ vm_map_lock_read(map);
+
+ start = *address;
+ if (!vm_map_lookup_entry(map, start, &tmp_entry)) {
+ if ((entry = tmp_entry->vme_next) == vm_map_to_entry(map)) {
+ vm_map_unlock_read(map);
+ return KERN_INVALID_ADDRESS;
+ }
+ } else {
+ entry = tmp_entry;
+ }
+
+ start = entry->vme_start;
+
+ basic->offset = (uint32_t)VME_OFFSET(entry);
+ basic->protection = entry->protection;
+ basic->inheritance = entry->inheritance;
+ basic->max_protection = entry->max_protection;
+ basic->behavior = entry->behavior;
+ basic->user_wired_count = entry->user_wired_count;
+ basic->reserved = entry->is_sub_map;
+ *address = start;
+ *size = (entry->vme_end - start);
+
+ if (object_name) {
+ *object_name = IP_NULL;
+ }
+ if (entry->is_sub_map) {
+ basic->shared = FALSE;
+ } else {
+ basic->shared = entry->is_shared;
+ }
+
+ vm_map_unlock_read(map);
+ return KERN_SUCCESS;
+ }
+
+ case VM_REGION_BASIC_INFO_64:
+ {
+ vm_region_basic_info_64_t basic;
+
+ if (*count < VM_REGION_BASIC_INFO_COUNT_64) {
+ return KERN_INVALID_ARGUMENT;
+ }
+
+ basic = (vm_region_basic_info_64_t) info;
+ *count = VM_REGION_BASIC_INFO_COUNT_64;
+
+ vm_map_lock_read(map);
+
+ start = *address;
+ if (!vm_map_lookup_entry(map, start, &tmp_entry)) {
+ if ((entry = tmp_entry->vme_next) == vm_map_to_entry(map)) {
+ vm_map_unlock_read(map);
+ return KERN_INVALID_ADDRESS;
+ }
+ } else {
+ entry = tmp_entry;
+ }
+
+ start = entry->vme_start;
+
+ basic->offset = VME_OFFSET(entry);
+ basic->protection = entry->protection;
+ basic->inheritance = entry->inheritance;
+ basic->max_protection = entry->max_protection;
+ basic->behavior = entry->behavior;
+ basic->user_wired_count = entry->user_wired_count;
+ basic->reserved = entry->is_sub_map;
+ *address = start;
+ *size = (entry->vme_end - start);
+
+ if (object_name) {
+ *object_name = IP_NULL;
+ }
+ if (entry->is_sub_map) {
+ basic->shared = FALSE;
+ } else {
+ basic->shared = entry->is_shared;
+ }
+
+ vm_map_unlock_read(map);
+ return KERN_SUCCESS;
+ }
+ case VM_REGION_EXTENDED_INFO:
+ if (*count < VM_REGION_EXTENDED_INFO_COUNT) {
+ return KERN_INVALID_ARGUMENT;
+ }
+ /*fallthru*/
+ case VM_REGION_EXTENDED_INFO__legacy:
+ if (*count < VM_REGION_EXTENDED_INFO_COUNT__legacy) {
+ return KERN_INVALID_ARGUMENT;
+ }
+
+ {
+ vm_region_extended_info_t extended;
+ mach_msg_type_number_t original_count;
+
+ extended = (vm_region_extended_info_t) info;
+
+ vm_map_lock_read(map);
+
+ start = *address;
+ if (!vm_map_lookup_entry(map, start, &tmp_entry)) {
+ if ((entry = tmp_entry->vme_next) == vm_map_to_entry(map)) {
+ vm_map_unlock_read(map);
+ return KERN_INVALID_ADDRESS;
+ }
+ } else {
+ entry = tmp_entry;
+ }
+ start = entry->vme_start;
+
+ extended->protection = entry->protection;
+ extended->user_tag = VME_ALIAS(entry);
+ extended->pages_resident = 0;
+ extended->pages_swapped_out = 0;
+ extended->pages_shared_now_private = 0;
+ extended->pages_dirtied = 0;
+ extended->external_pager = 0;
+ extended->shadow_depth = 0;
+
+ original_count = *count;
+ if (flavor == VM_REGION_EXTENDED_INFO__legacy) {
+ *count = VM_REGION_EXTENDED_INFO_COUNT__legacy;
+ } else {
+ extended->pages_reusable = 0;
+ *count = VM_REGION_EXTENDED_INFO_COUNT;
+ }
+
+ vm_map_region_walk(map, start, entry, VME_OFFSET(entry), entry->vme_end - start, extended, TRUE, *count);
+
+ if (extended->external_pager && extended->ref_count == 2 && extended->share_mode == SM_SHARED) {
+ extended->share_mode = SM_PRIVATE;
+ }
+
+ if (object_name) {
+ *object_name = IP_NULL;
+ }
+ *address = start;
+ *size = (entry->vme_end - start);
+
+ vm_map_unlock_read(map);
+ return KERN_SUCCESS;
+ }
+ case VM_REGION_TOP_INFO:
+ {
+ vm_region_top_info_t top;
+
+ if (*count < VM_REGION_TOP_INFO_COUNT) {
+ return KERN_INVALID_ARGUMENT;
+ }
+
+ top = (vm_region_top_info_t) info;
+ *count = VM_REGION_TOP_INFO_COUNT;
+
+ vm_map_lock_read(map);
+
+ start = *address;
+ if (!vm_map_lookup_entry(map, start, &tmp_entry)) {
+ if ((entry = tmp_entry->vme_next) == vm_map_to_entry(map)) {
+ vm_map_unlock_read(map);
+ return KERN_INVALID_ADDRESS;
+ }
+ } else {
+ entry = tmp_entry;
+ }
+ start = entry->vme_start;
+
+ top->private_pages_resident = 0;
+ top->shared_pages_resident = 0;
+
+ vm_map_region_top_walk(entry, top);
+
+ if (object_name) {
+ *object_name = IP_NULL;
+ }
+ *address = start;
+ *size = (entry->vme_end - start);
+
+ vm_map_unlock_read(map);
+ return KERN_SUCCESS;
+ }
+ default:
+ return KERN_INVALID_ARGUMENT;
+ }
+}
+
+#define OBJ_RESIDENT_COUNT(obj, entry_size) \
+ MIN((entry_size), \
+ ((obj)->all_reusable ? \
+ (obj)->wired_page_count : \
+ (obj)->resident_page_count - (obj)->reusable_page_count))
+
+void
+vm_map_region_top_walk(
+ vm_map_entry_t entry,
+ vm_region_top_info_t top)
+{
+ if (VME_OBJECT(entry) == 0 || entry->is_sub_map) {
+ top->share_mode = SM_EMPTY;
+ top->ref_count = 0;
+ top->obj_id = 0;
+ return;
+ }
+
+ {
+ struct vm_object *obj, *tmp_obj;
+ int ref_count;
+ uint32_t entry_size;
+
+ entry_size = (uint32_t) ((entry->vme_end - entry->vme_start) / PAGE_SIZE_64);
+
+ obj = VME_OBJECT(entry);
+
+ vm_object_lock(obj);
+
+ if ((ref_count = obj->ref_count) > 1 && obj->paging_in_progress) {
+ ref_count--;
+ }
+
+ assert(obj->reusable_page_count <= obj->resident_page_count);
+ if (obj->shadow) {
+ if (ref_count == 1) {
+ top->private_pages_resident =
+ OBJ_RESIDENT_COUNT(obj, entry_size);
+ } else {
+ top->shared_pages_resident =
+ OBJ_RESIDENT_COUNT(obj, entry_size);
+ }
+ top->ref_count = ref_count;
+ top->share_mode = SM_COW;
+
+ while ((tmp_obj = obj->shadow)) {
+ vm_object_lock(tmp_obj);
+ vm_object_unlock(obj);
+ obj = tmp_obj;
+
+ if ((ref_count = obj->ref_count) > 1 && obj->paging_in_progress) {
+ ref_count--;
+ }
+
+ assert(obj->reusable_page_count <= obj->resident_page_count);
+ top->shared_pages_resident +=
+ OBJ_RESIDENT_COUNT(obj, entry_size);
+ top->ref_count += ref_count - 1;
+ }
+ } else {
+ if (entry->superpage_size) {
+ top->share_mode = SM_LARGE_PAGE;
+ top->shared_pages_resident = 0;
+ top->private_pages_resident = entry_size;
+ } else if (entry->needs_copy) {
+ top->share_mode = SM_COW;
+ top->shared_pages_resident =
+ OBJ_RESIDENT_COUNT(obj, entry_size);
+ } else {
+ if (ref_count == 1 ||
+ (ref_count == 2 && obj->named)) {
+ top->share_mode = SM_PRIVATE;
+ top->private_pages_resident =
+ OBJ_RESIDENT_COUNT(obj,
+ entry_size);
+ } else {
+ top->share_mode = SM_SHARED;
+ top->shared_pages_resident =
+ OBJ_RESIDENT_COUNT(obj,
+ entry_size);
+ }
+ }
+ top->ref_count = ref_count;
+ }
+ /* XXX K64: obj_id will be truncated */
+ top->obj_id = (unsigned int) (uintptr_t)VM_KERNEL_ADDRPERM(obj);
+
+ vm_object_unlock(obj);
+ }
+}
+
+void
+vm_map_region_walk(
+ vm_map_t map,
+ vm_map_offset_t va,
+ vm_map_entry_t entry,
+ vm_object_offset_t offset,
+ vm_object_size_t range,
+ vm_region_extended_info_t extended,
+ boolean_t look_for_pages,
+ mach_msg_type_number_t count)
+{
+ struct vm_object *obj, *tmp_obj;
+ vm_map_offset_t last_offset;
+ int i;
+ int ref_count;
+ struct vm_object *shadow_object;
+ int shadow_depth;
+ boolean_t do_region_footprint;
+
+ do_region_footprint = task_self_region_footprint();
+
+ if ((VME_OBJECT(entry) == 0) ||
+ (entry->is_sub_map) ||
+ (VME_OBJECT(entry)->phys_contiguous &&
+ !entry->superpage_size)) {
+ extended->share_mode = SM_EMPTY;
+ extended->ref_count = 0;
+ return;
+ }
+
+ if (entry->superpage_size) {
+ extended->shadow_depth = 0;
+ extended->share_mode = SM_LARGE_PAGE;
+ extended->ref_count = 1;
+ extended->external_pager = 0;
+ extended->pages_resident = (unsigned int)(range >> PAGE_SHIFT);
+ extended->shadow_depth = 0;
+ return;
+ }
+
+ obj = VME_OBJECT(entry);
+
+ vm_object_lock(obj);
+
+ if ((ref_count = obj->ref_count) > 1 && obj->paging_in_progress) {
+ ref_count--;
+ }
+
+ if (look_for_pages) {
+ for (last_offset = offset + range;
+ offset < last_offset;
+ offset += PAGE_SIZE_64, va += PAGE_SIZE) {
+ if (do_region_footprint) {
+ int disp;
+
+ disp = 0;
+ if (map->has_corpse_footprint) {
+ /*
+ * Query the page info data we saved
+ * while forking the corpse.
+ */
+ vm_map_corpse_footprint_query_page_info(
+ map,
+ va,
+ &disp);
+ } else {
+ /*
+ * Query the pmap.
+ */
+ pmap_query_page_info(map->pmap,
+ va,
+ &disp);
+ }
+ if (disp & PMAP_QUERY_PAGE_PRESENT) {
+ if (!(disp & PMAP_QUERY_PAGE_ALTACCT)) {
+ extended->pages_resident++;
+ }
+ if (disp & PMAP_QUERY_PAGE_REUSABLE) {
+ extended->pages_reusable++;
+ } else if (!(disp & PMAP_QUERY_PAGE_INTERNAL) ||
+ (disp & PMAP_QUERY_PAGE_ALTACCT)) {
+ /* alternate accounting */
+ } else {
+ extended->pages_dirtied++;
+ }
+ } else if (disp & PMAP_QUERY_PAGE_COMPRESSED) {
+ if (disp & PMAP_QUERY_PAGE_COMPRESSED_ALTACCT) {
+ /* alternate accounting */
+ } else {
+ extended->pages_swapped_out++;
+ }
+ }
+ /* deal with alternate accounting */
+ if (obj->purgable == VM_PURGABLE_NONVOLATILE &&
+ /* && not tagged as no-footprint? */
+ VM_OBJECT_OWNER(obj) != NULL &&
+ VM_OBJECT_OWNER(obj)->map == map) {
+ if ((((va
+ - entry->vme_start
+ + VME_OFFSET(entry))
+ / PAGE_SIZE) <
+ (obj->resident_page_count +
+ vm_compressor_pager_get_count(obj->pager)))) {
+ /*
+ * Non-volatile purgeable object owned
+ * by this task: report the first
+ * "#resident + #compressed" pages as
+ * "resident" (to show that they
+ * contribute to the footprint) but not
+ * "dirty" (to avoid double-counting
+ * with the fake "non-volatile" region
+ * we'll report at the end of the
+ * address space to account for all
+ * (mapped or not) non-volatile memory
+ * owned by this task.
+ */
+ extended->pages_resident++;
+ }
+ } else if ((obj->purgable == VM_PURGABLE_VOLATILE ||
+ obj->purgable == VM_PURGABLE_EMPTY) &&
+ /* && not tagged as no-footprint? */
+ VM_OBJECT_OWNER(obj) != NULL &&
+ VM_OBJECT_OWNER(obj)->map == map) {
+ if ((((va
+ - entry->vme_start
+ + VME_OFFSET(entry))
+ / PAGE_SIZE) <
+ obj->wired_page_count)) {
+ /*
+ * Volatile|empty purgeable object owned
+ * by this task: report the first
+ * "#wired" pages as "resident" (to
+ * show that they contribute to the
+ * footprint) but not "dirty" (to avoid
+ * double-counting with the fake
+ * "non-volatile" region we'll report
+ * at the end of the address space to
+ * account for all (mapped or not)
+ * non-volatile memory owned by this
+ * task.
+ */
+ extended->pages_resident++;
+ }
+ } else if (obj->purgable != VM_PURGABLE_DENY) {
+ /*
+ * Pages from purgeable objects
+ * will be reported as dirty
+ * appropriately in an extra
+ * fake memory region at the end of
+ * the address space.
+ */
+ } else if (entry->iokit_acct) {
+ /*
+ * IOKit mappings are considered
+ * as fully dirty for footprint's
+ * sake.
+ */
+ extended->pages_dirtied++;
+ }
+ continue;
+ }
+
+ vm_map_region_look_for_page(map, va, obj,
+ offset, ref_count,
+ 0, extended, count);
+ }
+
+ if (do_region_footprint) {
+ goto collect_object_info;
+ }
+ } else {
+collect_object_info:
+ shadow_object = obj->shadow;
+ shadow_depth = 0;
+
+ if (!(obj->internal)) {
+ extended->external_pager = 1;
+ }
+
+ if (shadow_object != VM_OBJECT_NULL) {
+ vm_object_lock(shadow_object);
+ for (;
+ shadow_object != VM_OBJECT_NULL;
+ shadow_depth++) {
+ vm_object_t next_shadow;
+
+ if (!(shadow_object->internal)) {
+ extended->external_pager = 1;
+ }
+
+ next_shadow = shadow_object->shadow;
+ if (next_shadow) {
+ vm_object_lock(next_shadow);
+ }
+ vm_object_unlock(shadow_object);
+ shadow_object = next_shadow;
+ }
+ }
+ extended->shadow_depth = shadow_depth;
+ }
+
+ if (extended->shadow_depth || entry->needs_copy) {
+ extended->share_mode = SM_COW;
+ } else {
+ if (ref_count == 1) {
+ extended->share_mode = SM_PRIVATE;
+ } else {
+ if (obj->true_share) {
+ extended->share_mode = SM_TRUESHARED;
+ } else {
+ extended->share_mode = SM_SHARED;
+ }
+ }
+ }
+ extended->ref_count = ref_count - extended->shadow_depth;
+
+ for (i = 0; i < extended->shadow_depth; i++) {
+ if ((tmp_obj = obj->shadow) == 0) {
+ break;
+ }
+ vm_object_lock(tmp_obj);
+ vm_object_unlock(obj);
+
+ if ((ref_count = tmp_obj->ref_count) > 1 && tmp_obj->paging_in_progress) {
+ ref_count--;
+ }
+
+ extended->ref_count += ref_count;
+ obj = tmp_obj;
+ }
+ vm_object_unlock(obj);
+
+ if (extended->share_mode == SM_SHARED) {
+ vm_map_entry_t cur;
+ vm_map_entry_t last;
+ int my_refs;
+
+ obj = VME_OBJECT(entry);
+ last = vm_map_to_entry(map);
+ my_refs = 0;
+
+ if ((ref_count = obj->ref_count) > 1 && obj->paging_in_progress) {
+ ref_count--;
+ }
+ for (cur = vm_map_first_entry(map); cur != last; cur = cur->vme_next) {
+ my_refs += vm_map_region_count_obj_refs(cur, obj);
+ }
+
+ if (my_refs == ref_count) {
+ extended->share_mode = SM_PRIVATE_ALIASED;
+ } else if (my_refs > 1) {
+ extended->share_mode = SM_SHARED_ALIASED;
+ }
+ }
+}
+
+
+/* object is locked on entry and locked on return */
+
+
+static void
+vm_map_region_look_for_page(
+ __unused vm_map_t map,
+ __unused vm_map_offset_t va,
+ vm_object_t object,
+ vm_object_offset_t offset,
+ int max_refcnt,
+ int depth,
+ vm_region_extended_info_t extended,
+ mach_msg_type_number_t count)
+{
+ vm_page_t p;
+ vm_object_t shadow;
+ int ref_count;
+ vm_object_t caller_object;
+
+ shadow = object->shadow;
+ caller_object = object;
+
+
+ while (TRUE) {
+ if (!(object->internal)) {
+ extended->external_pager = 1;
+ }
+
+ if ((p = vm_page_lookup(object, offset)) != VM_PAGE_NULL) {
+ if (shadow && (max_refcnt == 1)) {
+ extended->pages_shared_now_private++;
+ }
+
+ if (!p->vmp_fictitious &&
+ (p->vmp_dirty || pmap_is_modified(VM_PAGE_GET_PHYS_PAGE(p)))) {
+ extended->pages_dirtied++;
+ } else if (count >= VM_REGION_EXTENDED_INFO_COUNT) {
+ if (p->vmp_reusable || object->all_reusable) {
+ extended->pages_reusable++;
+ }
+ }
+
+ extended->pages_resident++;
+
+ if (object != caller_object) {
+ vm_object_unlock(object);
+ }
+
+ return;
+ }
+ if (object->internal &&
+ object->alive &&
+ !object->terminating &&
+ object->pager_ready) {
+ if (VM_COMPRESSOR_PAGER_STATE_GET(object, offset)
+ == VM_EXTERNAL_STATE_EXISTS) {
+ /* the pager has that page */
+ extended->pages_swapped_out++;
+ if (object != caller_object) {
+ vm_object_unlock(object);
+ }
+ return;
+ }
+ }
+
+ if (shadow) {
+ vm_object_lock(shadow);
+
+ if ((ref_count = shadow->ref_count) > 1 && shadow->paging_in_progress) {
+ ref_count--;
+ }
+
+ if (++depth > extended->shadow_depth) {
+ extended->shadow_depth = depth;
+ }
+
+ if (ref_count > max_refcnt) {
+ max_refcnt = ref_count;
+ }
+
+ if (object != caller_object) {
+ vm_object_unlock(object);
+ }
+
+ offset = offset + object->vo_shadow_offset;
+ object = shadow;
+ shadow = object->shadow;
+ continue;
+ }
+ if (object != caller_object) {
+ vm_object_unlock(object);
+ }
+ break;
+ }
+}
+
+static int
+vm_map_region_count_obj_refs(
+ vm_map_entry_t entry,
+ vm_object_t object)
+{
+ int ref_count;
+ vm_object_t chk_obj;
+ vm_object_t tmp_obj;
+
+ if (VME_OBJECT(entry) == 0) {
+ return 0;
+ }
+
+ if (entry->is_sub_map) {
+ return 0;
+ } else {
+ ref_count = 0;
+
+ chk_obj = VME_OBJECT(entry);
+ vm_object_lock(chk_obj);
+
+ while (chk_obj) {
+ if (chk_obj == object) {
+ ref_count++;
+ }
+ tmp_obj = chk_obj->shadow;
+ if (tmp_obj) {
+ vm_object_lock(tmp_obj);
+ }
+ vm_object_unlock(chk_obj);
+
+ chk_obj = tmp_obj;
+ }
+ }
+ return ref_count;
+}
+
+
+/*
+ * Routine: vm_map_simplify
+ *
+ * Description:
+ * Attempt to simplify the map representation in
+ * the vicinity of the given starting address.
+ * Note:
+ * This routine is intended primarily to keep the
+ * kernel maps more compact -- they generally don't
+ * benefit from the "expand a map entry" technology
+ * at allocation time because the adjacent entry
+ * is often wired down.
+ */
+void
+vm_map_simplify_entry(
+ vm_map_t map,
+ vm_map_entry_t this_entry)
+{
+ vm_map_entry_t prev_entry;
+
+ counter(c_vm_map_simplify_entry_called++);
+
+ prev_entry = this_entry->vme_prev;
+
+ if ((this_entry != vm_map_to_entry(map)) &&
+ (prev_entry != vm_map_to_entry(map)) &&
+
+ (prev_entry->vme_end == this_entry->vme_start) &&
+
+ (prev_entry->is_sub_map == this_entry->is_sub_map) &&
+ (VME_OBJECT(prev_entry) == VME_OBJECT(this_entry)) &&
+ ((VME_OFFSET(prev_entry) + (prev_entry->vme_end -
+ prev_entry->vme_start))
+ == VME_OFFSET(this_entry)) &&
+
+ (prev_entry->behavior == this_entry->behavior) &&
+ (prev_entry->needs_copy == this_entry->needs_copy) &&
+ (prev_entry->protection == this_entry->protection) &&
+ (prev_entry->max_protection == this_entry->max_protection) &&
+ (prev_entry->inheritance == this_entry->inheritance) &&
+ (prev_entry->use_pmap == this_entry->use_pmap) &&
+ (VME_ALIAS(prev_entry) == VME_ALIAS(this_entry)) &&
+ (prev_entry->no_cache == this_entry->no_cache) &&
+ (prev_entry->permanent == this_entry->permanent) &&
+ (prev_entry->map_aligned == this_entry->map_aligned) &&
+ (prev_entry->zero_wired_pages == this_entry->zero_wired_pages) &&
+ (prev_entry->used_for_jit == this_entry->used_for_jit) &&
+ (prev_entry->pmap_cs_associated == this_entry->pmap_cs_associated) &&
+ /* from_reserved_zone: OK if that field doesn't match */
+ (prev_entry->iokit_acct == this_entry->iokit_acct) &&
+ (prev_entry->vme_resilient_codesign ==
+ this_entry->vme_resilient_codesign) &&
+ (prev_entry->vme_resilient_media ==
+ this_entry->vme_resilient_media) &&
+ (prev_entry->vme_no_copy_on_read == this_entry->vme_no_copy_on_read) &&
+
+ (prev_entry->wired_count == this_entry->wired_count) &&
+ (prev_entry->user_wired_count == this_entry->user_wired_count) &&
+
+ ((prev_entry->vme_atomic == FALSE) && (this_entry->vme_atomic == FALSE)) &&
+ (prev_entry->in_transition == FALSE) &&
+ (this_entry->in_transition == FALSE) &&
+ (prev_entry->needs_wakeup == FALSE) &&
+ (this_entry->needs_wakeup == FALSE) &&
+ (prev_entry->is_shared == FALSE) &&
+ (this_entry->is_shared == FALSE) &&
+ (prev_entry->superpage_size == FALSE) &&
+ (this_entry->superpage_size == FALSE)
+ ) {
+ vm_map_store_entry_unlink(map, prev_entry);
+ assert(prev_entry->vme_start < this_entry->vme_end);
+ if (prev_entry->map_aligned) {
+ assert(VM_MAP_PAGE_ALIGNED(prev_entry->vme_start,
+ VM_MAP_PAGE_MASK(map)));
+ }
+ this_entry->vme_start = prev_entry->vme_start;
+ VME_OFFSET_SET(this_entry, VME_OFFSET(prev_entry));
+
+ if (map->holelistenabled) {
+ vm_map_store_update_first_free(map, this_entry, TRUE);
+ }
+
+ if (prev_entry->is_sub_map) {
+ vm_map_deallocate(VME_SUBMAP(prev_entry));
+ } else {
+ vm_object_deallocate(VME_OBJECT(prev_entry));
+ }
+ vm_map_entry_dispose(map, prev_entry);
+ SAVE_HINT_MAP_WRITE(map, this_entry);
+ counter(c_vm_map_simplified++);
+ }
+}
+
+void
+vm_map_simplify(
+ vm_map_t map,
+ vm_map_offset_t start)
+{
+ vm_map_entry_t this_entry;
+
+ vm_map_lock(map);
+ if (vm_map_lookup_entry(map, start, &this_entry)) {
+ vm_map_simplify_entry(map, this_entry);
+ vm_map_simplify_entry(map, this_entry->vme_next);
+ }
+ counter(c_vm_map_simplify_called++);
+ vm_map_unlock(map);
+}
+
+static void
+vm_map_simplify_range(
+ vm_map_t map,
+ vm_map_offset_t start,
+ vm_map_offset_t end)
+{
+ vm_map_entry_t entry;
+
+ /*
+ * The map should be locked (for "write") by the caller.
+ */
+
+ if (start >= end) {
+ /* invalid address range */
+ return;
+ }
+
+ start = vm_map_trunc_page(start,
+ VM_MAP_PAGE_MASK(map));
+ end = vm_map_round_page(end,
+ VM_MAP_PAGE_MASK(map));
+
+ if (!vm_map_lookup_entry(map, start, &entry)) {
+ /* "start" is not mapped and "entry" ends before "start" */
+ if (entry == vm_map_to_entry(map)) {
+ /* start with first entry in the map */
+ entry = vm_map_first_entry(map);
+ } else {
+ /* start with next entry */
+ entry = entry->vme_next;
+ }
+ }
+
+ while (entry != vm_map_to_entry(map) &&
+ entry->vme_start <= end) {
+ /* try and coalesce "entry" with its previous entry */
+ vm_map_simplify_entry(map, entry);
+ entry = entry->vme_next;
+ }
+}
+
+
+/*
+ * Routine: vm_map_machine_attribute
+ * Purpose:
+ * Provide machine-specific attributes to mappings,
+ * such as cachability etc. for machines that provide
+ * them. NUMA architectures and machines with big/strange
+ * caches will use this.
+ * Note:
+ * Responsibilities for locking and checking are handled here,
+ * everything else in the pmap module. If any non-volatile
+ * information must be kept, the pmap module should handle
+ * it itself. [This assumes that attributes do not
+ * need to be inherited, which seems ok to me]
+ */
+kern_return_t
+vm_map_machine_attribute(
+ vm_map_t map,
+ vm_map_offset_t start,
+ vm_map_offset_t end,
+ vm_machine_attribute_t attribute,
+ vm_machine_attribute_val_t* value) /* IN/OUT */
+{
+ kern_return_t ret;
+ vm_map_size_t sync_size;
+ vm_map_entry_t entry;
+
+ if (start < vm_map_min(map) || end > vm_map_max(map)) {
+ return KERN_INVALID_ADDRESS;
+ }
+
+ /* Figure how much memory we need to flush (in page increments) */
+ sync_size = end - start;
+
+ vm_map_lock(map);
+
+ if (attribute != MATTR_CACHE) {
+ /* If we don't have to find physical addresses, we */
+ /* don't have to do an explicit traversal here. */
+ ret = pmap_attribute(map->pmap, start, end - start,
+ attribute, value);
+ vm_map_unlock(map);
+ return ret;
+ }
+
+ ret = KERN_SUCCESS; /* Assume it all worked */
+
+ while (sync_size) {
+ if (vm_map_lookup_entry(map, start, &entry)) {
+ vm_map_size_t sub_size;
+ if ((entry->vme_end - start) > sync_size) {
+ sub_size = sync_size;
+ sync_size = 0;
+ } else {
+ sub_size = entry->vme_end - start;
+ sync_size -= sub_size;
+ }
+ if (entry->is_sub_map) {
+ vm_map_offset_t sub_start;
+ vm_map_offset_t sub_end;
+
+ sub_start = (start - entry->vme_start)
+ + VME_OFFSET(entry);
+ sub_end = sub_start + sub_size;
+ vm_map_machine_attribute(
+ VME_SUBMAP(entry),
+ sub_start,
+ sub_end,
+ attribute, value);
+ } else {
+ if (VME_OBJECT(entry)) {
+ vm_page_t m;
+ vm_object_t object;
+ vm_object_t base_object;
+ vm_object_t last_object;
+ vm_object_offset_t offset;
+ vm_object_offset_t base_offset;
+ vm_map_size_t range;
+ range = sub_size;
+ offset = (start - entry->vme_start)
+ + VME_OFFSET(entry);
+ base_offset = offset;
+ object = VME_OBJECT(entry);
+ base_object = object;
+ last_object = NULL;
+
+ vm_object_lock(object);
+
+ while (range) {
+ m = vm_page_lookup(
+ object, offset);
+
+ if (m && !m->vmp_fictitious) {
+ ret =
+ pmap_attribute_cache_sync(
+ VM_PAGE_GET_PHYS_PAGE(m),
+ PAGE_SIZE,
+ attribute, value);
+ } else if (object->shadow) {
+ offset = offset + object->vo_shadow_offset;
+ last_object = object;
+ object = object->shadow;
+ vm_object_lock(last_object->shadow);
+ vm_object_unlock(last_object);
+ continue;
+ }
+ range -= PAGE_SIZE;
+
+ if (base_object != object) {
+ vm_object_unlock(object);
+ vm_object_lock(base_object);
+ object = base_object;
+ }
+ /* Bump to the next page */
+ base_offset += PAGE_SIZE;
+ offset = base_offset;
+ }
+ vm_object_unlock(object);
+ }
+ }
+ start += sub_size;
+ } else {
+ vm_map_unlock(map);
+ return KERN_FAILURE;
+ }
+ }
+
+ vm_map_unlock(map);
+
+ return ret;
+}
+
+/*
+ * vm_map_behavior_set:
+ *
+ * Sets the paging reference behavior of the specified address
+ * range in the target map. Paging reference behavior affects
+ * how pagein operations resulting from faults on the map will be
+ * clustered.
+ */
+kern_return_t
+vm_map_behavior_set(
+ vm_map_t map,
+ vm_map_offset_t start,
+ vm_map_offset_t end,
+ vm_behavior_t new_behavior)
+{
+ vm_map_entry_t entry;
+ vm_map_entry_t temp_entry;
+
+ if (start > end ||
+ start < vm_map_min(map) ||
+ end > vm_map_max(map)) {
+ return KERN_NO_SPACE;
+ }
+
+ switch (new_behavior) {
+ /*
+ * This first block of behaviors all set a persistent state on the specified
+ * memory range. All we have to do here is to record the desired behavior
+ * in the vm_map_entry_t's.
+ */
+
+ case VM_BEHAVIOR_DEFAULT:
+ case VM_BEHAVIOR_RANDOM:
+ case VM_BEHAVIOR_SEQUENTIAL:
+ case VM_BEHAVIOR_RSEQNTL:
+ case VM_BEHAVIOR_ZERO_WIRED_PAGES:
+ vm_map_lock(map);
+
+ /*
+ * The entire address range must be valid for the map.
+ * Note that vm_map_range_check() does a
+ * vm_map_lookup_entry() internally and returns the
+ * entry containing the start of the address range if
+ * the entire range is valid.
+ */
+ if (vm_map_range_check(map, start, end, &temp_entry)) {
+ entry = temp_entry;
+ vm_map_clip_start(map, entry, start);
+ } else {
+ vm_map_unlock(map);
+ return KERN_INVALID_ADDRESS;
+ }
+
+ while ((entry != vm_map_to_entry(map)) && (entry->vme_start < end)) {
+ vm_map_clip_end(map, entry, end);
+ if (entry->is_sub_map) {
+ assert(!entry->use_pmap);
+ }
+
+ if (new_behavior == VM_BEHAVIOR_ZERO_WIRED_PAGES) {
+ entry->zero_wired_pages = TRUE;
+ } else {
+ entry->behavior = new_behavior;
+ }
+ entry = entry->vme_next;
+ }
+
+ vm_map_unlock(map);
+ break;
+
+ /*
+ * The rest of these are different from the above in that they cause
+ * an immediate action to take place as opposed to setting a behavior that
+ * affects future actions.
+ */
+
+ case VM_BEHAVIOR_WILLNEED:
+ return vm_map_willneed(map, start, end);
+
+ case VM_BEHAVIOR_DONTNEED:
+ return vm_map_msync(map, start, end - start, VM_SYNC_DEACTIVATE | VM_SYNC_CONTIGUOUS);
+
+ case VM_BEHAVIOR_FREE:
+ return vm_map_msync(map, start, end - start, VM_SYNC_KILLPAGES | VM_SYNC_CONTIGUOUS);
+
+ case VM_BEHAVIOR_REUSABLE:
+ return vm_map_reusable_pages(map, start, end);
+
+ case VM_BEHAVIOR_REUSE:
+ return vm_map_reuse_pages(map, start, end);
+
+ case VM_BEHAVIOR_CAN_REUSE:
+ return vm_map_can_reuse(map, start, end);
+
+#if MACH_ASSERT
+ case VM_BEHAVIOR_PAGEOUT:
+ return vm_map_pageout(map, start, end);
+#endif /* MACH_ASSERT */
+
+ default:
+ return KERN_INVALID_ARGUMENT;
+ }
+
+ return KERN_SUCCESS;
+}
+
+
+/*
+ * Internals for madvise(MADV_WILLNEED) system call.
+ *
+ * The implementation is to do:-
+ * a) read-ahead if the mapping corresponds to a mapped regular file
+ * b) or, fault in the pages (zero-fill, decompress etc) if it's an anonymous mapping
+ */
+
+
+static kern_return_t
+vm_map_willneed(
+ vm_map_t map,
+ vm_map_offset_t start,
+ vm_map_offset_t end
+ )
+{
+ vm_map_entry_t entry;
+ vm_object_t object;
+ memory_object_t pager;
+ struct vm_object_fault_info fault_info = {};
+ kern_return_t kr;
+ vm_object_size_t len;
+ vm_object_offset_t offset;
+
+ fault_info.interruptible = THREAD_UNINT; /* ignored value */
+ fault_info.behavior = VM_BEHAVIOR_SEQUENTIAL;
+ fault_info.stealth = TRUE;
+
+ /*
+ * The MADV_WILLNEED operation doesn't require any changes to the
+ * vm_map_entry_t's, so the read lock is sufficient.
+ */
+
+ vm_map_lock_read(map);
+
+ /*
+ * The madvise semantics require that the address range be fully
+ * allocated with no holes. Otherwise, we're required to return
+ * an error.
+ */
+
+ if (!vm_map_range_check(map, start, end, &entry)) {
+ vm_map_unlock_read(map);
+ return KERN_INVALID_ADDRESS;
+ }
+
+ /*
+ * Examine each vm_map_entry_t in the range.
+ */
+ for (; entry != vm_map_to_entry(map) && start < end;) {
+ /*
+ * The first time through, the start address could be anywhere
+ * within the vm_map_entry we found. So adjust the offset to
+ * correspond. After that, the offset will always be zero to
+ * correspond to the beginning of the current vm_map_entry.
+ */
+ offset = (start - entry->vme_start) + VME_OFFSET(entry);
+
+ /*
+ * Set the length so we don't go beyond the end of the
+ * map_entry or beyond the end of the range we were given.
+ * This range could span also multiple map entries all of which
+ * map different files, so make sure we only do the right amount
+ * of I/O for each object. Note that it's possible for there
+ * to be multiple map entries all referring to the same object
+ * but with different page permissions, but it's not worth
+ * trying to optimize that case.
+ */
+ len = MIN(entry->vme_end - start, end - start);
+
+ if ((vm_size_t) len != len) {
+ /* 32-bit overflow */
+ len = (vm_size_t) (0 - PAGE_SIZE);
+ }
+ fault_info.cluster_size = (vm_size_t) len;
+ fault_info.lo_offset = offset;
+ fault_info.hi_offset = offset + len;
+ fault_info.user_tag = VME_ALIAS(entry);
+ fault_info.pmap_options = 0;
+ if (entry->iokit_acct ||
+ (!entry->is_sub_map && !entry->use_pmap)) {
+ fault_info.pmap_options |= PMAP_OPTIONS_ALT_ACCT;
+ }
+
+ /*
+ * If the entry is a submap OR there's no read permission
+ * to this mapping, then just skip it.
+ */
+ if ((entry->is_sub_map) || (entry->protection & VM_PROT_READ) == 0) {
+ entry = entry->vme_next;
+ start = entry->vme_start;
+ continue;
+ }
+
+ object = VME_OBJECT(entry);
+
+ if (object == NULL ||
+ (object && object->internal)) {
+ /*
+ * Memory range backed by anonymous memory.
+ */
+ vm_size_t region_size = 0, effective_page_size = 0;
+ vm_map_offset_t addr = 0, effective_page_mask = 0;
+
+ region_size = len;
+ addr = start;
+
+ effective_page_mask = MAX(vm_map_page_mask(current_map()), PAGE_MASK);
+ effective_page_size = effective_page_mask + 1;
+
+ vm_map_unlock_read(map);
+
+ while (region_size) {
+ vm_pre_fault(
+ vm_map_trunc_page(addr, effective_page_mask),
+ VM_PROT_READ | VM_PROT_WRITE);
+
+ region_size -= effective_page_size;
+ addr += effective_page_size;
+ }
+ } else {
+ /*
+ * Find the file object backing this map entry. If there is
+ * none, then we simply ignore the "will need" advice for this
+ * entry and go on to the next one.
+ */
+ if ((object = find_vnode_object(entry)) == VM_OBJECT_NULL) {
+ entry = entry->vme_next;
+ start = entry->vme_start;
+ continue;
+ }
+
+ vm_object_paging_begin(object);
+ pager = object->pager;
+ vm_object_unlock(object);
+
+ /*
+ * The data_request() could take a long time, so let's
+ * release the map lock to avoid blocking other threads.
+ */
+ vm_map_unlock_read(map);
+
+ /*
+ * Get the data from the object asynchronously.
+ *
+ * Note that memory_object_data_request() places limits on the
+ * amount of I/O it will do. Regardless of the len we
+ * specified, it won't do more than MAX_UPL_TRANSFER_BYTES and it
+ * silently truncates the len to that size. This isn't
+ * necessarily bad since madvise shouldn't really be used to
+ * page in unlimited amounts of data. Other Unix variants
+ * limit the willneed case as well. If this turns out to be an
+ * issue for developers, then we can always adjust the policy
+ * here and still be backwards compatible since this is all
+ * just "advice".
+ */
+ kr = memory_object_data_request(
+ pager,
+ offset + object->paging_offset,
+ 0, /* ignored */
+ VM_PROT_READ,
+ (memory_object_fault_info_t)&fault_info);
+
+ vm_object_lock(object);
+ vm_object_paging_end(object);
+ vm_object_unlock(object);
+
+ /*
+ * If we couldn't do the I/O for some reason, just give up on
+ * the madvise. We still return success to the user since
+ * madvise isn't supposed to fail when the advice can't be
+ * taken.
+ */
+
+ if (kr != KERN_SUCCESS) {
+ return KERN_SUCCESS;
+ }
+ }
+
+ start += len;
+ if (start >= end) {
+ /* done */
+ return KERN_SUCCESS;
+ }
+
+ /* look up next entry */
+ vm_map_lock_read(map);
+ if (!vm_map_lookup_entry(map, start, &entry)) {
+ /*
+ * There's a new hole in the address range.
+ */
+ vm_map_unlock_read(map);
+ return KERN_INVALID_ADDRESS;
+ }
+ }
+
+ vm_map_unlock_read(map);
+ return KERN_SUCCESS;
+}
+
+static boolean_t
+vm_map_entry_is_reusable(
+ vm_map_entry_t entry)
+{
+ /* Only user map entries */
+
+ vm_object_t object;
+
+ if (entry->is_sub_map) {
+ return FALSE;
+ }
+
+ switch (VME_ALIAS(entry)) {
+ case VM_MEMORY_MALLOC:
+ case VM_MEMORY_MALLOC_SMALL:
+ case VM_MEMORY_MALLOC_LARGE:
+ case VM_MEMORY_REALLOC:
+ case VM_MEMORY_MALLOC_TINY:
+ case VM_MEMORY_MALLOC_LARGE_REUSABLE:
+ case VM_MEMORY_MALLOC_LARGE_REUSED:
+ /*
+ * This is a malloc() memory region: check if it's still
+ * in its original state and can be re-used for more
+ * malloc() allocations.
+ */
+ break;
+ default:
+ /*
+ * Not a malloc() memory region: let the caller decide if
+ * it's re-usable.
+ */
+ return TRUE;
+ }
+
+ if (/*entry->is_shared ||*/
+ entry->is_sub_map ||
+ entry->in_transition ||
+ entry->protection != VM_PROT_DEFAULT ||
+ entry->max_protection != VM_PROT_ALL ||
+ entry->inheritance != VM_INHERIT_DEFAULT ||
+ entry->no_cache ||
+ entry->permanent ||
+ entry->superpage_size != FALSE ||
+ entry->zero_wired_pages ||
+ entry->wired_count != 0 ||
+ entry->user_wired_count != 0) {
+ return FALSE;
+ }
+
+ object = VME_OBJECT(entry);
+ if (object == VM_OBJECT_NULL) {
+ return TRUE;
+ }
+ if (
+#if 0
+ /*
+ * Let's proceed even if the VM object is potentially
+ * shared.
+ * We check for this later when processing the actual
+ * VM pages, so the contents will be safe if shared.
+ *
+ * But we can still mark this memory region as "reusable" to
+ * acknowledge that the caller did let us know that the memory
+ * could be re-used and should not be penalized for holding
+ * on to it. This allows its "resident size" to not include
+ * the reusable range.
+ */
+ object->ref_count == 1 &&
+#endif
+ object->wired_page_count == 0 &&
+ object->copy == VM_OBJECT_NULL &&
+ object->shadow == VM_OBJECT_NULL &&
+ object->internal &&
+ object->purgable == VM_PURGABLE_DENY &&
+ object->copy_strategy != MEMORY_OBJECT_COPY_DELAY &&
+ !object->true_share &&
+ object->wimg_bits == VM_WIMG_USE_DEFAULT &&
+ !object->code_signed) {
+ return TRUE;
+ }
+ return FALSE;
+}
+
+static kern_return_t
+vm_map_reuse_pages(
+ vm_map_t map,
+ vm_map_offset_t start,
+ vm_map_offset_t end)
+{
+ vm_map_entry_t entry;
+ vm_object_t object;
+ vm_object_offset_t start_offset, end_offset;
+
+ /*
+ * The MADV_REUSE operation doesn't require any changes to the
+ * vm_map_entry_t's, so the read lock is sufficient.
+ */
+
+ vm_map_lock_read(map);
+ assert(map->pmap != kernel_pmap); /* protect alias access */
+
+ /*
+ * The madvise semantics require that the address range be fully
+ * allocated with no holes. Otherwise, we're required to return
+ * an error.
+ */
+
+ if (!vm_map_range_check(map, start, end, &entry)) {
+ vm_map_unlock_read(map);
+ vm_page_stats_reusable.reuse_pages_failure++;
+ return KERN_INVALID_ADDRESS;
+ }
+
+ /*
+ * Examine each vm_map_entry_t in the range.
+ */
+ for (; entry != vm_map_to_entry(map) && entry->vme_start < end;
+ entry = entry->vme_next) {
+ /*
+ * Sanity check on the VM map entry.
+ */
+ if (!vm_map_entry_is_reusable(entry)) {
+ vm_map_unlock_read(map);
+ vm_page_stats_reusable.reuse_pages_failure++;
+ return KERN_INVALID_ADDRESS;
+ }
+
+ /*
+ * The first time through, the start address could be anywhere
+ * within the vm_map_entry we found. So adjust the offset to
+ * correspond.
+ */
+ if (entry->vme_start < start) {
+ start_offset = start - entry->vme_start;
+ } else {
+ start_offset = 0;
+ }
+ end_offset = MIN(end, entry->vme_end) - entry->vme_start;
+ start_offset += VME_OFFSET(entry);
+ end_offset += VME_OFFSET(entry);
+
+ assert(!entry->is_sub_map);
+ object = VME_OBJECT(entry);
+ if (object != VM_OBJECT_NULL) {
+ vm_object_lock(object);
+ vm_object_reuse_pages(object, start_offset, end_offset,
+ TRUE);
+ vm_object_unlock(object);
+ }
+
+ if (VME_ALIAS(entry) == VM_MEMORY_MALLOC_LARGE_REUSABLE) {
+ /*
+ * XXX
+ * We do not hold the VM map exclusively here.
+ * The "alias" field is not that critical, so it's
+ * safe to update it here, as long as it is the only
+ * one that can be modified while holding the VM map
+ * "shared".
+ */
+ VME_ALIAS_SET(entry, VM_MEMORY_MALLOC_LARGE_REUSED);
+ }
+ }
+
+ vm_map_unlock_read(map);
+ vm_page_stats_reusable.reuse_pages_success++;
+ return KERN_SUCCESS;
+}
+
+
+static kern_return_t
+vm_map_reusable_pages(
+ vm_map_t map,
+ vm_map_offset_t start,
+ vm_map_offset_t end)
+{
+ vm_map_entry_t entry;
+ vm_object_t object;
+ vm_object_offset_t start_offset, end_offset;
+ vm_map_offset_t pmap_offset;
+
+ /*
+ * The MADV_REUSABLE operation doesn't require any changes to the
+ * vm_map_entry_t's, so the read lock is sufficient.
+ */
+
+ vm_map_lock_read(map);
+ assert(map->pmap != kernel_pmap); /* protect alias access */
+
+ /*
+ * The madvise semantics require that the address range be fully
+ * allocated with no holes. Otherwise, we're required to return
+ * an error.
+ */
+
+ if (!vm_map_range_check(map, start, end, &entry)) {
+ vm_map_unlock_read(map);
+ vm_page_stats_reusable.reusable_pages_failure++;
+ return KERN_INVALID_ADDRESS;
+ }
+
+ /*
+ * Examine each vm_map_entry_t in the range.
+ */
+ for (; entry != vm_map_to_entry(map) && entry->vme_start < end;
+ entry = entry->vme_next) {
+ int kill_pages = 0;
+
+ /*
+ * Sanity check on the VM map entry.
+ */
+ if (!vm_map_entry_is_reusable(entry)) {
+ vm_map_unlock_read(map);
+ vm_page_stats_reusable.reusable_pages_failure++;
+ return KERN_INVALID_ADDRESS;
+ }
+
+ if (!(entry->protection & VM_PROT_WRITE) && !entry->used_for_jit) {
+ /* not writable: can't discard contents */
+ vm_map_unlock_read(map);
+ vm_page_stats_reusable.reusable_nonwritable++;
+ vm_page_stats_reusable.reusable_pages_failure++;
+ return KERN_PROTECTION_FAILURE;
+ }
+
+ /*
+ * The first time through, the start address could be anywhere
+ * within the vm_map_entry we found. So adjust the offset to
+ * correspond.
+ */
+ if (entry->vme_start < start) {
+ start_offset = start - entry->vme_start;
+ pmap_offset = start;
+ } else {
+ start_offset = 0;
+ pmap_offset = entry->vme_start;
+ }
+ end_offset = MIN(end, entry->vme_end) - entry->vme_start;
+ start_offset += VME_OFFSET(entry);
+ end_offset += VME_OFFSET(entry);
+
+ assert(!entry->is_sub_map);
+ object = VME_OBJECT(entry);
+ if (object == VM_OBJECT_NULL) {
+ continue;
+ }
+
+
+ vm_object_lock(object);
+ if (((object->ref_count == 1) ||
+ (object->copy_strategy != MEMORY_OBJECT_COPY_SYMMETRIC &&
+ object->copy == VM_OBJECT_NULL)) &&
+ object->shadow == VM_OBJECT_NULL &&
+ /*
+ * "iokit_acct" entries are billed for their virtual size
+ * (rather than for their resident pages only), so they
+ * wouldn't benefit from making pages reusable, and it
+ * would be hard to keep track of pages that are both
+ * "iokit_acct" and "reusable" in the pmap stats and
+ * ledgers.
+ */
+ !(entry->iokit_acct ||
+ (!entry->is_sub_map && !entry->use_pmap))) {
+ if (object->ref_count != 1) {
+ vm_page_stats_reusable.reusable_shared++;
+ }
+ kill_pages = 1;
+ } else {
+ kill_pages = -1;
+ }
+ if (kill_pages != -1) {
+ vm_object_deactivate_pages(object,
+ start_offset,
+ end_offset - start_offset,
+ kill_pages,
+ TRUE /*reusable_pages*/,
+ map->pmap,
+ pmap_offset);
+ } else {
+ vm_page_stats_reusable.reusable_pages_shared++;
+ }
+ vm_object_unlock(object);
+
+ if (VME_ALIAS(entry) == VM_MEMORY_MALLOC_LARGE ||
+ VME_ALIAS(entry) == VM_MEMORY_MALLOC_LARGE_REUSED) {
+ /*
+ * XXX
+ * We do not hold the VM map exclusively here.
+ * The "alias" field is not that critical, so it's
+ * safe to update it here, as long as it is the only
+ * one that can be modified while holding the VM map
+ * "shared".
+ */
+ VME_ALIAS_SET(entry, VM_MEMORY_MALLOC_LARGE_REUSABLE);
+ }
+ }
+
+ vm_map_unlock_read(map);
+ vm_page_stats_reusable.reusable_pages_success++;
+ return KERN_SUCCESS;
+}
+
+
+static kern_return_t
+vm_map_can_reuse(
+ vm_map_t map,
+ vm_map_offset_t start,
+ vm_map_offset_t end)
+{
+ vm_map_entry_t entry;
+
+ /*
+ * The MADV_REUSABLE operation doesn't require any changes to the
+ * vm_map_entry_t's, so the read lock is sufficient.
+ */
+
+ vm_map_lock_read(map);
+ assert(map->pmap != kernel_pmap); /* protect alias access */
+
+ /*
+ * The madvise semantics require that the address range be fully
+ * allocated with no holes. Otherwise, we're required to return
+ * an error.
+ */
+
+ if (!vm_map_range_check(map, start, end, &entry)) {
+ vm_map_unlock_read(map);
+ vm_page_stats_reusable.can_reuse_failure++;
+ return KERN_INVALID_ADDRESS;
+ }
+
+ /*
+ * Examine each vm_map_entry_t in the range.
+ */
+ for (; entry != vm_map_to_entry(map) && entry->vme_start < end;
+ entry = entry->vme_next) {
+ /*
+ * Sanity check on the VM map entry.
+ */
+ if (!vm_map_entry_is_reusable(entry)) {
+ vm_map_unlock_read(map);
+ vm_page_stats_reusable.can_reuse_failure++;
+ return KERN_INVALID_ADDRESS;
+ }
+ }
+
+ vm_map_unlock_read(map);
+ vm_page_stats_reusable.can_reuse_success++;
+ return KERN_SUCCESS;
+}
+
+
+#if MACH_ASSERT
+static kern_return_t
+vm_map_pageout(
+ vm_map_t map,
+ vm_map_offset_t start,
+ vm_map_offset_t end)
+{
+ vm_map_entry_t entry;
+
+ /*
+ * The MADV_PAGEOUT operation doesn't require any changes to the
+ * vm_map_entry_t's, so the read lock is sufficient.
+ */
+
+ vm_map_lock_read(map);
+
+ /*
+ * The madvise semantics require that the address range be fully
+ * allocated with no holes. Otherwise, we're required to return
+ * an error.
+ */
+
+ if (!vm_map_range_check(map, start, end, &entry)) {
+ vm_map_unlock_read(map);
+ return KERN_INVALID_ADDRESS;
+ }
+
+ /*
+ * Examine each vm_map_entry_t in the range.
+ */
+ for (; entry != vm_map_to_entry(map) && entry->vme_start < end;
+ entry = entry->vme_next) {
+ vm_object_t object;
+
+ /*
+ * Sanity check on the VM map entry.
+ */
+ if (entry->is_sub_map) {
+ vm_map_t submap;
+ vm_map_offset_t submap_start;
+ vm_map_offset_t submap_end;
+ vm_map_entry_t submap_entry;
+
+ submap = VME_SUBMAP(entry);
+ submap_start = VME_OFFSET(entry);
+ submap_end = submap_start + (entry->vme_end -
+ entry->vme_start);
+
+ vm_map_lock_read(submap);
+
+ if (!vm_map_range_check(submap,
+ submap_start,
+ submap_end,
+ &submap_entry)) {
+ vm_map_unlock_read(submap);
+ vm_map_unlock_read(map);
+ return KERN_INVALID_ADDRESS;
+ }
+
+ object = VME_OBJECT(submap_entry);
+ if (submap_entry->is_sub_map ||
+ object == VM_OBJECT_NULL ||
+ !object->internal) {
+ vm_map_unlock_read(submap);
+ continue;
+ }
+
+ vm_object_pageout(object);
+
+ vm_map_unlock_read(submap);
+ submap = VM_MAP_NULL;
+ submap_entry = VM_MAP_ENTRY_NULL;
+ continue;
+ }
+
+ object = VME_OBJECT(entry);
+ if (entry->is_sub_map ||
+ object == VM_OBJECT_NULL ||
+ !object->internal) {
+ continue;
+ }
+
+ vm_object_pageout(object);
+ }
+
+ vm_map_unlock_read(map);
+ return KERN_SUCCESS;
+}
+#endif /* MACH_ASSERT */
+
+
+/*
+ * Routine: vm_map_entry_insert
+ *
+ * Description: This routine inserts a new vm_entry in a locked map.
+ */
+vm_map_entry_t
+vm_map_entry_insert(
+ vm_map_t map,
+ vm_map_entry_t insp_entry,
+ vm_map_offset_t start,
+ vm_map_offset_t end,
+ vm_object_t object,
+ vm_object_offset_t offset,
+ boolean_t needs_copy,
+ boolean_t is_shared,
+ boolean_t in_transition,
+ vm_prot_t cur_protection,
+ vm_prot_t max_protection,
+ vm_behavior_t behavior,
+ vm_inherit_t inheritance,
+ unsigned wired_count,
+ boolean_t no_cache,
+ boolean_t permanent,
+ boolean_t no_copy_on_read,
+ unsigned int superpage_size,
+ boolean_t clear_map_aligned,
+ boolean_t is_submap,
+ boolean_t used_for_jit,
+ int alias)
+{
+ vm_map_entry_t new_entry;
+
+ assert(insp_entry != (vm_map_entry_t)0);
+ vm_map_lock_assert_exclusive(map);
+
+#if DEVELOPMENT || DEBUG
+ vm_object_offset_t end_offset = 0;
+ assertf(!os_add_overflow(end - start, offset, &end_offset), "size 0x%llx, offset 0x%llx caused overflow", (uint64_t)(end - start), offset);
+#endif /* DEVELOPMENT || DEBUG */
+
+ new_entry = vm_map_entry_create(map, !map->hdr.entries_pageable);
+
+ if (VM_MAP_PAGE_SHIFT(map) != PAGE_SHIFT) {
+ new_entry->map_aligned = TRUE;
+ } else {
+ new_entry->map_aligned = FALSE;
+ }
+ if (clear_map_aligned &&
+ (!VM_MAP_PAGE_ALIGNED(start, VM_MAP_PAGE_MASK(map)) ||
+ !VM_MAP_PAGE_ALIGNED(end, VM_MAP_PAGE_MASK(map)))) {
+ new_entry->map_aligned = FALSE;
+ }
+
+ new_entry->vme_start = start;
+ new_entry->vme_end = end;
+ assert(page_aligned(new_entry->vme_start));
+ assert(page_aligned(new_entry->vme_end));
+ if (new_entry->map_aligned) {
+ assert(VM_MAP_PAGE_ALIGNED(new_entry->vme_start,
+ VM_MAP_PAGE_MASK(map)));
+ assert(VM_MAP_PAGE_ALIGNED(new_entry->vme_end,
+ VM_MAP_PAGE_MASK(map)));
+ }
+ assert(new_entry->vme_start < new_entry->vme_end);
+
+ VME_OBJECT_SET(new_entry, object);
+ VME_OFFSET_SET(new_entry, offset);
+ new_entry->is_shared = is_shared;
+ new_entry->is_sub_map = is_submap;
+ new_entry->needs_copy = needs_copy;
+ new_entry->in_transition = in_transition;
+ new_entry->needs_wakeup = FALSE;
+ new_entry->inheritance = inheritance;
+ new_entry->protection = cur_protection;
+ new_entry->max_protection = max_protection;
+ new_entry->behavior = behavior;
+ new_entry->wired_count = wired_count;
+ new_entry->user_wired_count = 0;
+ if (is_submap) {
+ /*
+ * submap: "use_pmap" means "nested".
+ * default: false.
+ */
+ new_entry->use_pmap = FALSE;
+ } else {
+ /*
+ * object: "use_pmap" means "use pmap accounting" for footprint.
+ * default: true.
+ */
+ new_entry->use_pmap = TRUE;
+ }
+ VME_ALIAS_SET(new_entry, alias);
+ new_entry->zero_wired_pages = FALSE;
+ new_entry->no_cache = no_cache;
+ new_entry->permanent = permanent;
+ if (superpage_size) {
+ new_entry->superpage_size = TRUE;
+ } else {
+ new_entry->superpage_size = FALSE;
+ }
+ if (used_for_jit) {
+#if CONFIG_EMBEDDED
+ if (!(map->jit_entry_exists))
+#endif /* CONFIG_EMBEDDED */
+ {
+ new_entry->used_for_jit = TRUE;
+ map->jit_entry_exists = TRUE;
+ }
+ } else {
+ new_entry->used_for_jit = FALSE;
+ }
+ new_entry->pmap_cs_associated = FALSE;
+ new_entry->iokit_acct = FALSE;
+ new_entry->vme_resilient_codesign = FALSE;
+ new_entry->vme_resilient_media = FALSE;
+ new_entry->vme_atomic = FALSE;
+ new_entry->vme_no_copy_on_read = no_copy_on_read;
+
+ /*
+ * Insert the new entry into the list.
+ */
+
+ vm_map_store_entry_link(map, insp_entry, new_entry,
+ VM_MAP_KERNEL_FLAGS_NONE);
+ map->size += end - start;
+
+ /*
+ * Update the free space hint and the lookup hint.
+ */
+
+ SAVE_HINT_MAP_WRITE(map, new_entry);
+ return new_entry;
+}
+
+/*
+ * Routine: vm_map_remap_extract
+ *
+ * Descritpion: This routine returns a vm_entry list from a map.
+ */
+static kern_return_t
+vm_map_remap_extract(
+ vm_map_t map,
+ vm_map_offset_t addr,
+ vm_map_size_t size,
+ boolean_t copy,
+ struct vm_map_header *map_header,
+ vm_prot_t *cur_protection,
+ vm_prot_t *max_protection,
+ /* What, no behavior? */
+ vm_inherit_t inheritance,
+ boolean_t pageable,
+ boolean_t same_map,
+ vm_map_kernel_flags_t vmk_flags)
+{
+ kern_return_t result;
+ vm_map_size_t mapped_size;
+ vm_map_size_t tmp_size;
+ vm_map_entry_t src_entry; /* result of last map lookup */
+ vm_map_entry_t new_entry;
+ vm_object_offset_t offset;
+ vm_map_offset_t map_address;
+ vm_map_offset_t src_start; /* start of entry to map */
+ vm_map_offset_t src_end; /* end of region to be mapped */
+ vm_object_t object;
+ vm_map_version_t version;
+ boolean_t src_needs_copy;
+ boolean_t new_entry_needs_copy;
+ vm_map_entry_t saved_src_entry;
+ boolean_t src_entry_was_wired;
+ vm_prot_t max_prot_for_prot_copy;
+
+ assert(map != VM_MAP_NULL);
+ assert(size != 0);
+ assert(size == vm_map_round_page(size, PAGE_MASK));
+ assert(inheritance == VM_INHERIT_NONE ||
+ inheritance == VM_INHERIT_COPY ||
+ inheritance == VM_INHERIT_SHARE);
+
+ /*
+ * Compute start and end of region.
+ */
+ src_start = vm_map_trunc_page(addr, PAGE_MASK);
+ src_end = vm_map_round_page(src_start + size, PAGE_MASK);
+
+
+ /*
+ * Initialize map_header.
+ */
+ map_header->links.next = CAST_TO_VM_MAP_ENTRY(&map_header->links);
+ map_header->links.prev = CAST_TO_VM_MAP_ENTRY(&map_header->links);
+ map_header->nentries = 0;
+ map_header->entries_pageable = pageable;
+ map_header->page_shift = PAGE_SHIFT;
+
+ vm_map_store_init( map_header );
+
+ if (copy && vmk_flags.vmkf_remap_prot_copy) {
+ max_prot_for_prot_copy = *max_protection & VM_PROT_ALL;
+ } else {
+ max_prot_for_prot_copy = VM_PROT_NONE;
+ }
+ *cur_protection = VM_PROT_ALL;
+ *max_protection = VM_PROT_ALL;
+
+ map_address = 0;
+ mapped_size = 0;
+ result = KERN_SUCCESS;
+
+ /*
+ * The specified source virtual space might correspond to
+ * multiple map entries, need to loop on them.
+ */
+ vm_map_lock(map);
+ while (mapped_size != size) {
+ vm_map_size_t entry_size;
+
+ /*
+ * Find the beginning of the region.
+ */
+ if (!vm_map_lookup_entry(map, src_start, &src_entry)) {
+ result = KERN_INVALID_ADDRESS;
+ break;
+ }
+
+ if (src_start < src_entry->vme_start ||
+ (mapped_size && src_start != src_entry->vme_start)) {
+ result = KERN_INVALID_ADDRESS;
+ break;
+ }
+
+ tmp_size = size - mapped_size;
+ if (src_end > src_entry->vme_end) {
+ tmp_size -= (src_end - src_entry->vme_end);
+ }
+
+ entry_size = (vm_map_size_t)(src_entry->vme_end -
+ src_entry->vme_start);
+
+ if (src_entry->is_sub_map) {
+ vm_map_reference(VME_SUBMAP(src_entry));
+ object = VM_OBJECT_NULL;
+ } else {
+ object = VME_OBJECT(src_entry);
+ if (src_entry->iokit_acct) {
+ /*
+ * This entry uses "IOKit accounting".
+ */
+ } else if (object != VM_OBJECT_NULL &&
+ (object->purgable != VM_PURGABLE_DENY ||
+ object->vo_ledger_tag != VM_LEDGER_TAG_NONE)) {
+ /*
+ * Purgeable objects have their own accounting:
+ * no pmap accounting for them.
+ */
+ assertf(!src_entry->use_pmap,
+ "map=%p src_entry=%p [0x%llx:0x%llx] 0x%x/0x%x %d",
+ map,
+ src_entry,
+ (uint64_t)src_entry->vme_start,
+ (uint64_t)src_entry->vme_end,
+ src_entry->protection,
+ src_entry->max_protection,
+ VME_ALIAS(src_entry));
+ } else {
+ /*
+ * Not IOKit or purgeable:
+ * must be accounted by pmap stats.
+ */
+ assertf(src_entry->use_pmap,
+ "map=%p src_entry=%p [0x%llx:0x%llx] 0x%x/0x%x %d",
+ map,
+ src_entry,
+ (uint64_t)src_entry->vme_start,
+ (uint64_t)src_entry->vme_end,
+ src_entry->protection,
+ src_entry->max_protection,
+ VME_ALIAS(src_entry));
+ }
+
+ if (object == VM_OBJECT_NULL) {
+ object = vm_object_allocate(entry_size);
+ VME_OFFSET_SET(src_entry, 0);
+ VME_OBJECT_SET(src_entry, object);
+ assert(src_entry->use_pmap);
+ } else if (object->copy_strategy !=
+ MEMORY_OBJECT_COPY_SYMMETRIC) {
+ /*
+ * We are already using an asymmetric
+ * copy, and therefore we already have
+ * the right object.
+ */
+ assert(!src_entry->needs_copy);
+ } else if (src_entry->needs_copy || object->shadowed ||
+ (object->internal && !object->true_share &&
+ !src_entry->is_shared &&
+ object->vo_size > entry_size)) {
+ VME_OBJECT_SHADOW(src_entry, entry_size);
+ assert(src_entry->use_pmap);
+
+ if (!src_entry->needs_copy &&
+ (src_entry->protection & VM_PROT_WRITE)) {
+ vm_prot_t prot;
+
+ assert(!pmap_has_prot_policy(src_entry->protection));
+
+ prot = src_entry->protection & ~VM_PROT_WRITE;
+
+ if (override_nx(map,
+ VME_ALIAS(src_entry))
+ && prot) {
+ prot |= VM_PROT_EXECUTE;
+ }
+
+ assert(!pmap_has_prot_policy(prot));
+
+ if (map->mapped_in_other_pmaps) {
+ vm_object_pmap_protect(
+ VME_OBJECT(src_entry),
+ VME_OFFSET(src_entry),
+ entry_size,
+ PMAP_NULL,
+ src_entry->vme_start,
+ prot);
+ } else {
+ pmap_protect(vm_map_pmap(map),
+ src_entry->vme_start,
+ src_entry->vme_end,
+ prot);
+ }
+ }
+
+ object = VME_OBJECT(src_entry);
+ src_entry->needs_copy = FALSE;
+ }
+
+
+ vm_object_lock(object);
+ vm_object_reference_locked(object); /* object ref. for new entry */
+ if (object->copy_strategy ==
+ MEMORY_OBJECT_COPY_SYMMETRIC) {
+ object->copy_strategy =
+ MEMORY_OBJECT_COPY_DELAY;
+ }
+ vm_object_unlock(object);
+ }
+
+ offset = (VME_OFFSET(src_entry) +
+ (src_start - src_entry->vme_start));
+
+ new_entry = _vm_map_entry_create(map_header, !map_header->entries_pageable);
+ vm_map_entry_copy(new_entry, src_entry);
+ if (new_entry->is_sub_map) {
+ /* clr address space specifics */
+ new_entry->use_pmap = FALSE;
+ } else if (copy) {
+ /*
+ * We're dealing with a copy-on-write operation,
+ * so the resulting mapping should not inherit the
+ * original mapping's accounting settings.
+ * "use_pmap" should be reset to its default (TRUE)
+ * so that the new mapping gets accounted for in
+ * the task's memory footprint.
+ */
+ new_entry->use_pmap = TRUE;
+ }
+ /* "iokit_acct" was cleared in vm_map_entry_copy() */
+ assert(!new_entry->iokit_acct);
+
+ new_entry->map_aligned = FALSE;
+
+ new_entry->vme_start = map_address;
+ new_entry->vme_end = map_address + tmp_size;
+ assert(new_entry->vme_start < new_entry->vme_end);
+ if (copy && vmk_flags.vmkf_remap_prot_copy) {
+ /*
+ * Remapping for vm_map_protect(VM_PROT_COPY)
+ * to convert a read-only mapping into a
+ * copy-on-write version of itself but
+ * with write access:
+ * keep the original inheritance and add
+ * VM_PROT_WRITE to the max protection.
+ */
+ new_entry->inheritance = src_entry->inheritance;
+ new_entry->protection &= max_prot_for_prot_copy;
+ new_entry->max_protection |= VM_PROT_WRITE;
+ } else {
+ new_entry->inheritance = inheritance;
+ }
+ VME_OFFSET_SET(new_entry, offset);
+
+ /*
+ * The new region has to be copied now if required.
+ */
+RestartCopy:
+ if (!copy) {
+ if (src_entry->used_for_jit == TRUE) {
+ if (same_map) {
+ } else {
+#if CONFIG_EMBEDDED
+ /*
+ * Cannot allow an entry describing a JIT
+ * region to be shared across address spaces.
+ */
+ result = KERN_INVALID_ARGUMENT;
+ break;
+#endif /* CONFIG_EMBEDDED */
+ }
+ }
+
+ src_entry->is_shared = TRUE;
+ new_entry->is_shared = TRUE;
+ if (!(new_entry->is_sub_map)) {
+ new_entry->needs_copy = FALSE;
+ }
+ } else if (src_entry->is_sub_map) {
+ /* make this a COW sub_map if not already */
+ assert(new_entry->wired_count == 0);
+ new_entry->needs_copy = TRUE;
+ object = VM_OBJECT_NULL;
+ } else if (src_entry->wired_count == 0 &&
+ vm_object_copy_quickly(VME_OBJECT_PTR(new_entry),
+ VME_OFFSET(new_entry),
+ (new_entry->vme_end -
+ new_entry->vme_start),
+ &src_needs_copy,
+ &new_entry_needs_copy)) {
+ new_entry->needs_copy = new_entry_needs_copy;
+ new_entry->is_shared = FALSE;
+ assertf(new_entry->use_pmap, "map %p new_entry %p\n", map, new_entry);
+
+ /*
+ * Handle copy_on_write semantics.
+ */
+ if (src_needs_copy && !src_entry->needs_copy) {
+ vm_prot_t prot;
+
+ assert(!pmap_has_prot_policy(src_entry->protection));
+
+ prot = src_entry->protection & ~VM_PROT_WRITE;
+
+ if (override_nx(map,
+ VME_ALIAS(src_entry))
+ && prot) {
+ prot |= VM_PROT_EXECUTE;
+ }
+
+ assert(!pmap_has_prot_policy(prot));
+
+ vm_object_pmap_protect(object,
+ offset,
+ entry_size,
+ ((src_entry->is_shared
+ || map->mapped_in_other_pmaps) ?
+ PMAP_NULL : map->pmap),
+ src_entry->vme_start,
+ prot);
+
+ assert(src_entry->wired_count == 0);
+ src_entry->needs_copy = TRUE;
+ }
+ /*
+ * Throw away the old object reference of the new entry.
+ */
+ vm_object_deallocate(object);
+ } else {
+ new_entry->is_shared = FALSE;
+ assertf(new_entry->use_pmap, "map %p new_entry %p\n", map, new_entry);
+
+ src_entry_was_wired = (src_entry->wired_count > 0);
+ saved_src_entry = src_entry;
+ src_entry = VM_MAP_ENTRY_NULL;
+
+ /*
+ * The map can be safely unlocked since we
+ * already hold a reference on the object.
+ *
+ * Record the timestamp of the map for later
+ * verification, and unlock the map.
+ */
+ version.main_timestamp = map->timestamp;
+ vm_map_unlock(map); /* Increments timestamp once! */
+
+ /*
+ * Perform the copy.
+ */
+ if (src_entry_was_wired > 0) {
+ vm_object_lock(object);
+ result = vm_object_copy_slowly(
+ object,
+ offset,
+ (new_entry->vme_end -
+ new_entry->vme_start),
+ THREAD_UNINT,
+ VME_OBJECT_PTR(new_entry));
+
+ VME_OFFSET_SET(new_entry, 0);
+ new_entry->needs_copy = FALSE;
+ } else {
+ vm_object_offset_t new_offset;
+
+ new_offset = VME_OFFSET(new_entry);
+ result = vm_object_copy_strategically(
+ object,
+ offset,
+ (new_entry->vme_end -
+ new_entry->vme_start),
+ VME_OBJECT_PTR(new_entry),
+ &new_offset,
+ &new_entry_needs_copy);
+ if (new_offset != VME_OFFSET(new_entry)) {
+ VME_OFFSET_SET(new_entry, new_offset);
+ }
+
+ new_entry->needs_copy = new_entry_needs_copy;
+ }
+
+ /*
+ * Throw away the old object reference of the new entry.
+ */
+ vm_object_deallocate(object);
+
+ if (result != KERN_SUCCESS &&
+ result != KERN_MEMORY_RESTART_COPY) {
+ _vm_map_entry_dispose(map_header, new_entry);
+ vm_map_lock(map);
+ break;
+ }
+
+ /*
+ * Verify that the map has not substantially
+ * changed while the copy was being made.
+ */
+
+ vm_map_lock(map);
+ if (version.main_timestamp + 1 != map->timestamp) {
+ /*
+ * Simple version comparison failed.
+ *
+ * Retry the lookup and verify that the
+ * same object/offset are still present.
+ */
+ saved_src_entry = VM_MAP_ENTRY_NULL;
+ vm_object_deallocate(VME_OBJECT(new_entry));
+ _vm_map_entry_dispose(map_header, new_entry);
+ if (result == KERN_MEMORY_RESTART_COPY) {
+ result = KERN_SUCCESS;
+ }
+ continue;
+ }
+ /* map hasn't changed: src_entry is still valid */
+ src_entry = saved_src_entry;
+ saved_src_entry = VM_MAP_ENTRY_NULL;
+
+ if (result == KERN_MEMORY_RESTART_COPY) {
+ vm_object_reference(object);
+ goto RestartCopy;
+ }
+ }
+
+ _vm_map_store_entry_link(map_header,
+ map_header->links.prev, new_entry);
+
+ /*Protections for submap mapping are irrelevant here*/
+ if (!src_entry->is_sub_map) {
+ *cur_protection &= src_entry->protection;
+ *max_protection &= src_entry->max_protection;
+ }
+ map_address += tmp_size;
+ mapped_size += tmp_size;
+ src_start += tmp_size;
+ } /* end while */
+
+ vm_map_unlock(map);
+ if (result != KERN_SUCCESS) {
+ /*
+ * Free all allocated elements.
+ */
+ for (src_entry = map_header->links.next;
+ src_entry != CAST_TO_VM_MAP_ENTRY(&map_header->links);
+ src_entry = new_entry) {
+ new_entry = src_entry->vme_next;
+ _vm_map_store_entry_unlink(map_header, src_entry);
+ if (src_entry->is_sub_map) {
+ vm_map_deallocate(VME_SUBMAP(src_entry));
+ } else {
+ vm_object_deallocate(VME_OBJECT(src_entry));
+ }
+ _vm_map_entry_dispose(map_header, src_entry);
+ }
+ }
+ return result;
+}
+
+/*
+ * Routine: vm_remap
+ *
+ * Map portion of a task's address space.
+ * Mapped region must not overlap more than
+ * one vm memory object. Protections and
+ * inheritance attributes remain the same
+ * as in the original task and are out parameters.
+ * Source and Target task can be identical
+ * Other attributes are identical as for vm_map()
+ */
+kern_return_t
+vm_map_remap(
+ vm_map_t target_map,
+ vm_map_address_t *address,
+ vm_map_size_t size,
+ vm_map_offset_t mask,
+ int flags,
+ vm_map_kernel_flags_t vmk_flags,
+ vm_tag_t tag,
+ vm_map_t src_map,
+ vm_map_offset_t memory_address,
+ boolean_t copy,
+ vm_prot_t *cur_protection,
+ vm_prot_t *max_protection,
+ vm_inherit_t inheritance)
+{
+ kern_return_t result;
+ vm_map_entry_t entry;
+ vm_map_entry_t insp_entry = VM_MAP_ENTRY_NULL;
+ vm_map_entry_t new_entry;
+ struct vm_map_header map_header;
+ vm_map_offset_t offset_in_mapping;
+
+ if (target_map == VM_MAP_NULL) {
+ return KERN_INVALID_ARGUMENT;
+ }
+
+ switch (inheritance) {
+ case VM_INHERIT_NONE:
+ case VM_INHERIT_COPY:
+ case VM_INHERIT_SHARE:
+ if (size != 0 && src_map != VM_MAP_NULL) {
+ break;
+ }
+ /*FALL THRU*/
+ default:
+ return KERN_INVALID_ARGUMENT;
+ }
+
+ /*
+ * If the user is requesting that we return the address of the
+ * first byte of the data (rather than the base of the page),
+ * then we use different rounding semantics: specifically,
+ * we assume that (memory_address, size) describes a region
+ * all of whose pages we must cover, rather than a base to be truncated
+ * down and a size to be added to that base. So we figure out
+ * the highest page that the requested region includes and make
+ * sure that the size will cover it.
+ *
+ * The key example we're worried about it is of the form:
+ *
+ * memory_address = 0x1ff0, size = 0x20
+ *
+ * With the old semantics, we round down the memory_address to 0x1000
+ * and round up the size to 0x1000, resulting in our covering *only*
+ * page 0x1000. With the new semantics, we'd realize that the region covers
+ * 0x1ff0-0x2010, and compute a size of 0x2000. Thus, we cover both page
+ * 0x1000 and page 0x2000 in the region we remap.
+ */
+ if ((flags & VM_FLAGS_RETURN_DATA_ADDR) != 0) {
+ offset_in_mapping = memory_address - vm_map_trunc_page(memory_address, PAGE_MASK);
+ size = vm_map_round_page(memory_address + size - vm_map_trunc_page(memory_address, PAGE_MASK), PAGE_MASK);
+ } else {
+ size = vm_map_round_page(size, PAGE_MASK);
+ }
+ if (size == 0) {
+ return KERN_INVALID_ARGUMENT;
+ }
+
+ if (flags & VM_FLAGS_RESILIENT_MEDIA) {
+ /* must be copy-on-write to be "media resilient" */
+ if (!copy) {
+ return KERN_INVALID_ARGUMENT;
+ }
+ }
+
+ result = vm_map_remap_extract(src_map, memory_address,
+ size, copy, &map_header,
+ cur_protection,
+ max_protection,
+ inheritance,
+ target_map->hdr.entries_pageable,
+ src_map == target_map,
+ vmk_flags);
+
+ if (result != KERN_SUCCESS) {
+ return result;
+ }
+
+ /*
+ * Allocate/check a range of free virtual address
+ * space for the target
+ */
+ *address = vm_map_trunc_page(*address,
+ VM_MAP_PAGE_MASK(target_map));
+ vm_map_lock(target_map);
+ result = vm_map_remap_range_allocate(target_map, address, size,
+ mask, flags, vmk_flags, tag,
+ &insp_entry);
+
+ for (entry = map_header.links.next;
+ entry != CAST_TO_VM_MAP_ENTRY(&map_header.links);
+ entry = new_entry) {
+ new_entry = entry->vme_next;
+ _vm_map_store_entry_unlink(&map_header, entry);
+ if (result == KERN_SUCCESS) {
+ if (flags & VM_FLAGS_RESILIENT_CODESIGN) {
+ /* no codesigning -> read-only access */
+ entry->max_protection = VM_PROT_READ;
+ entry->protection = VM_PROT_READ;
+ entry->vme_resilient_codesign = TRUE;
+ }
+ entry->vme_start += *address;
+ entry->vme_end += *address;
+ assert(!entry->map_aligned);
+ if ((flags & VM_FLAGS_RESILIENT_MEDIA) &&
+ !entry->is_sub_map &&
+ (VME_OBJECT(entry) == VM_OBJECT_NULL ||
+ VME_OBJECT(entry)->internal)) {
+ entry->vme_resilient_media = TRUE;
+ }
+ vm_map_store_entry_link(target_map, insp_entry, entry,
+ vmk_flags);
+ insp_entry = entry;
+ } else {
+ if (!entry->is_sub_map) {
+ vm_object_deallocate(VME_OBJECT(entry));
+ } else {
+ vm_map_deallocate(VME_SUBMAP(entry));
+ }
+ _vm_map_entry_dispose(&map_header, entry);
+ }
+ }
+
+ if (flags & VM_FLAGS_RESILIENT_CODESIGN) {
+ *cur_protection = VM_PROT_READ;
+ *max_protection = VM_PROT_READ;
+ }
+
+ if (target_map->disable_vmentry_reuse == TRUE) {
+ assert(!target_map->is_nested_map);
+ if (target_map->highest_entry_end < insp_entry->vme_end) {
+ target_map->highest_entry_end = insp_entry->vme_end;
+ }
+ }
+
+ if (result == KERN_SUCCESS) {
+ target_map->size += size;
+ SAVE_HINT_MAP_WRITE(target_map, insp_entry);
+
+#if PMAP_CS
+ if (*max_protection & VM_PROT_EXECUTE) {
+ vm_map_address_t region_start = 0, region_size = 0;
+ struct pmap_cs_code_directory *region_cd = NULL;
+ vm_map_address_t base = 0;
+ struct pmap_cs_lookup_results results = {};
+ vm_map_size_t page_addr = vm_map_trunc_page(memory_address, PAGE_MASK);
+ vm_map_size_t assoc_size = vm_map_round_page(memory_address + size - page_addr, PAGE_MASK);
+
+ pmap_cs_lookup(src_map->pmap, memory_address, &results);
+ region_size = results.region_size;
+ region_start = results.region_start;
+ region_cd = results.region_cd_entry;
+ base = results.base;
+
+ if (region_cd != NULL && (page_addr != region_start || assoc_size != region_size)) {
+ *cur_protection = VM_PROT_READ;
+ *max_protection = VM_PROT_READ;
+ printf("mismatched remap of executable range 0x%llx-0x%llx to 0x%llx, "
+ "region_start 0x%llx, region_size 0x%llx, cd_entry %sNULL, making non-executable.\n",
+ page_addr, page_addr + assoc_size, *address,
+ region_start, region_size,
+ region_cd != NULL ? "not " : "" // Don't leak kernel slide
+ );
+ }
+ }
+#endif
+ }
+ vm_map_unlock(target_map);
+
+ if (result == KERN_SUCCESS && target_map->wiring_required) {
+ result = vm_map_wire_kernel(target_map, *address,
+ *address + size, *cur_protection, VM_KERN_MEMORY_MLOCK,
+ TRUE);
+ }
+
+ /*
+ * If requested, return the address of the data pointed to by the
+ * request, rather than the base of the resulting page.
+ */
+ if ((flags & VM_FLAGS_RETURN_DATA_ADDR) != 0) {
+ *address += offset_in_mapping;
+ }
+
+ return result;
+}
+
+/*
+ * Routine: vm_map_remap_range_allocate
+ *
+ * Description:
+ * Allocate a range in the specified virtual address map.
+ * returns the address and the map entry just before the allocated
+ * range
+ *
+ * Map must be locked.
+ */
+
+static kern_return_t
+vm_map_remap_range_allocate(
+ vm_map_t map,
+ vm_map_address_t *address, /* IN/OUT */
+ vm_map_size_t size,
+ vm_map_offset_t mask,
+ int flags,
+ vm_map_kernel_flags_t vmk_flags,
+ __unused vm_tag_t tag,
+ vm_map_entry_t *map_entry) /* OUT */
+{
+ vm_map_entry_t entry;
+ vm_map_offset_t start;
+ vm_map_offset_t end;
+ vm_map_offset_t desired_empty_end;
+ kern_return_t kr;
+ vm_map_entry_t hole_entry;
+
+StartAgain:;
+
+ start = *address;
+
+ if (flags & VM_FLAGS_ANYWHERE) {
+ if (flags & VM_FLAGS_RANDOM_ADDR) {
+ /*
+ * Get a random start address.
+ */
+ kr = vm_map_random_address_for_size(map, address, size);
+ if (kr != KERN_SUCCESS) {
+ return kr;
+ }
+ start = *address;
+ }
+
+ /*
+ * Calculate the first possible address.
projects / apple / xnu.git / blobdiff