-/*
- * vm_map_copyin_page_list:
- *
- * This is a variant of vm_map_copyin that copies in a list of pages.
- * If steal_pages is TRUE, the pages are only in the returned list.
- * If steal_pages is FALSE, the pages are busy and still in their
- * objects. A continuation may be returned if not all the pages fit:
- * the recipient of this copy_result must be prepared to deal with it.
- */
-
-kern_return_t
-vm_map_copyin_page_list(
- vm_map_t src_map,
- vm_offset_t src_addr,
- vm_size_t len,
- int options,
- vm_map_copy_t *copy_result, /* OUT */
- boolean_t is_cont)
-{
- vm_map_entry_t src_entry;
- vm_page_t m;
- vm_offset_t src_start;
- vm_offset_t src_end;
- vm_size_t src_size;
- register vm_object_t src_object;
- register vm_object_offset_t src_offset;
- vm_object_offset_t src_last_offset;
- register vm_map_copy_t copy; /* Resulting copy */
- kern_return_t result = KERN_SUCCESS;
- boolean_t need_map_lookup;
- vm_map_copyin_args_t cont_args;
- kern_return_t error_code;
- vm_prot_t prot;
- boolean_t wired;
- boolean_t no_zero_fill;
-
- submap_map_t *parent_maps = NULL;
- vm_map_t base_map = src_map;
-
- prot = (options & VM_MAP_COPYIN_OPT_VM_PROT);
- no_zero_fill = (options & VM_MAP_COPYIN_OPT_NO_ZERO_FILL);
-
- /*
- * If steal_pages is FALSE, this leaves busy pages in
- * the object. A continuation must be used if src_destroy
- * is true in this case (!steal_pages && src_destroy).
- *
- * XXX Still have a more general problem of what happens
- * XXX if the same page occurs twice in a list. Deadlock
- * XXX can happen if vm_fault_page was called. A
- * XXX possible solution is to use a continuation if vm_fault_page
- * XXX is called and we cross a map entry boundary.
- */
-
- /*
- * Check for copies of zero bytes.
- */
-
- if (len == 0) {
- *copy_result = VM_MAP_COPY_NULL;
- return(KERN_SUCCESS);
- }
-
- /*
- * Compute start and end of region
- */
-
- src_start = trunc_page(src_addr);
- src_end = round_page(src_addr + len);
-
- /*
- * If the region is not page aligned, override the no_zero_fill
- * argument.
- */
-
- if (options & VM_MAP_COPYIN_OPT_NO_ZERO_FILL) {
- if (!page_aligned(src_addr) || !page_aligned(src_addr +len))
- options &= ~VM_MAP_COPYIN_OPT_NO_ZERO_FILL;
- }
-
- /*
- * Check that the end address doesn't overflow
- */
-
- if (src_end <= src_start && (src_end < src_start || src_start != 0)) {
- return KERN_INVALID_ADDRESS;
- }
-
- /*
- * Allocate a header element for the page list.
- *
- * Record original offset and size, as caller may not
- * be page-aligned.
- */
-
- copy = (vm_map_copy_t) zalloc(vm_map_copy_zone);
- copy->type = VM_MAP_COPY_PAGE_LIST;
- copy->cpy_npages = 0;
- copy->cpy_page_loose = FALSE;
- copy->offset = src_addr;
- copy->size = len;
- copy->cpy_cont = VM_MAP_COPY_CONT_NULL;
- copy->cpy_cont_args = VM_MAP_COPYIN_ARGS_NULL;
-
- /*
- * Find the beginning of the region.
- */
-
-do_map_lookup:
-
- vm_map_lock(src_map);
-
- if (!vm_map_lookup_entry(src_map, src_start, &src_entry)) {
- result = KERN_INVALID_ADDRESS;
- goto error;
- }
- need_map_lookup = FALSE;
-
- /*
- * Go through entries until we get to the end.
- */
-
- while (TRUE) {
- if ((src_entry->protection & prot) != prot) {
- result = KERN_PROTECTION_FAILURE;
- goto error;
- }
-
- /* translate down through submaps to find the target entry */
- while(src_entry->is_sub_map) {
- vm_size_t submap_len;
- submap_map_t *ptr;
-
- ptr = (submap_map_t *)kalloc(sizeof(submap_map_t));
- ptr->next = parent_maps;
- parent_maps = ptr;
- ptr->parent_map = src_map;
- ptr->base_start = src_start;
- ptr->base_end = src_end;
- submap_len = src_entry->vme_end - src_entry->vme_start;
- if(submap_len > (src_end-src_start))
- submap_len = src_end-src_start;
- ptr->base_start += submap_len;
-
- src_start -= src_entry->vme_start;
- src_start += src_entry->offset;
- src_end = src_start + submap_len;
- src_map = src_entry->object.sub_map;
- vm_map_lock(src_map);
- vm_map_unlock(ptr->parent_map);
- if (!vm_map_lookup_entry(
- src_map, src_start, &src_entry)) {
- result = KERN_INVALID_ADDRESS;
- goto error;
- }
- vm_map_clip_start(src_map, src_entry, src_start);
- }
-
- wired = (src_entry->wired_count != 0);
-
- if (src_end > src_entry->vme_end)
- src_size = src_entry->vme_end - src_start;
- else
- src_size = src_end - src_start;
-
- src_object = src_entry->object.vm_object;
-
- /*
- * If src_object is NULL, allocate it now;
- * we're going to fault on it shortly.
- */
- if (src_object == VM_OBJECT_NULL) {
- src_object = vm_object_allocate((vm_size_t)
- src_entry->vme_end -
- src_entry->vme_start);
- src_entry->object.vm_object = src_object;
- }
- else if (src_entry->needs_copy && (prot & VM_PROT_WRITE)) {
- vm_object_shadow(
- &src_entry->object.vm_object,
- &src_entry->offset,
- (vm_size_t) (src_entry->vme_end -
- src_entry->vme_start));
-
- src_entry->needs_copy = FALSE;
-
- /* reset src_object */
- src_object = src_entry->object.vm_object;
- }
-
- /*
- * calculate src_offset now, since vm_object_shadow
- * may have changed src_entry->offset.
- */
- src_offset = src_entry->offset + (src_start - src_entry->vme_start);
-
- /*
- * Iterate over pages. Fault in ones that aren't present.
- */
- src_last_offset = src_offset + src_size;
- for (; (src_offset < src_last_offset);
- src_offset += PAGE_SIZE_64, src_start += PAGE_SIZE) {
-
- if (copy->cpy_npages == VM_MAP_COPY_PAGE_LIST_MAX) {
- vm_offset_t src_delta;
-make_continuation:
- /*
- * At this point we have the max number of
- * pages busy for this thread that we're
- * willing to allow. Stop here and record
- * arguments for the remainder. Note:
- * this means that this routine isn't atomic,
- * but that's the breaks. Note that only
- * the first vm_map_copy_t that comes back
- * from this routine has the right offset
- * and size; those from continuations are
- * page rounded, and short by the amount
- * already done.
- *
- * Reset src_end so the src_destroy
- * code at the bottom doesn't do
- * something stupid.
- */
-
- src_delta = src_end - src_start;
- while (src_map != base_map) {
- submap_map_t *ptr;
-
- if(!need_map_lookup) {
- vm_map_unlock(src_map);
- }
- ptr = parent_maps;
- assert(ptr != NULL);
- parent_maps = parent_maps->next;
- src_map = ptr->parent_map;
- src_start = ptr->base_start - src_delta;
- src_delta = ptr->base_end - src_start;
- kfree((vm_offset_t)ptr, sizeof(submap_map_t));
-
- need_map_lookup = TRUE;
- }
- src_end = src_start;
-
-
- cont_args = (vm_map_copyin_args_t)
- kalloc(sizeof(vm_map_copyin_args_data_t));
- cont_args->map = src_map;
- vm_map_reference(src_map);
- cont_args->src_addr = src_start;
- cont_args->src_len = len - (src_start - src_addr);
- if (options & VM_MAP_COPYIN_OPT_SRC_DESTROY) {
- cont_args->destroy_addr = cont_args->src_addr;
- cont_args->destroy_len = cont_args->src_len;
- } else {
- cont_args->destroy_addr = (vm_offset_t) 0;
- cont_args->destroy_len = (vm_offset_t) 0;
- }
- cont_args->options = options;
-
- copy->cpy_cont_args = cont_args;
- copy->cpy_cont = vm_map_copyin_page_list_cont;
-
- break;
- }
-
- /*
- * Try to find the page of data. Have to
- * fault it in if there's no page, or something
- * going on with the page, or the object has
- * a copy object.
- */
- vm_object_lock(src_object);
- vm_object_paging_begin(src_object);
- if (((m = vm_page_lookup(src_object, src_offset)) !=
- VM_PAGE_NULL) &&
- !m->busy && !m->fictitious && !m->unusual &&
- ((prot & VM_PROT_WRITE) == 0 ||
- (m->object->copy == VM_OBJECT_NULL))) {
-
- if (!m->absent &&
- !(options & VM_MAP_COPYIN_OPT_STEAL_PAGES)) {
-
- /*
- * The page is present and will not be
- * replaced, prep it. Thus allowing
- * mutiple access on this page
- */
- kern_return_t kr;
-
- kr = vm_page_prep(m);
- assert(kr == KERN_SUCCESS);
- kr = vm_page_pin(m);
- assert(kr == KERN_SUCCESS);
- } else {
- /*
- * This is the page. Mark it busy
- * and keep the paging reference on
- * the object whilst we do our thing.
- */
-
- m->busy = TRUE;
- }
- } else {
- vm_prot_t result_prot;
- vm_page_t top_page;
- kern_return_t kr;
- boolean_t data_supply;
-
- /*
- * Have to fault the page in; must
- * unlock the map to do so. While
- * the map is unlocked, anything
- * can happen, we must lookup the
- * map entry before continuing.
- */
- vm_map_unlock(src_map);
- need_map_lookup = TRUE;
- data_supply = src_object->silent_overwrite &&
- (prot & VM_PROT_WRITE) &&
- src_start >= src_addr &&
- src_start + PAGE_SIZE <=
- src_addr + len;
-
-retry:
- result_prot = prot;
-
- XPR(XPR_VM_FAULT,
- "vm_map_copyin_page_list -> vm_fault_page\n",
- 0,0,0,0,0);
- kr = vm_fault_page(src_object, src_offset,
- prot, FALSE, THREAD_UNINT,
- src_entry->offset,
- src_entry->offset +
- (src_entry->vme_end -
- src_entry->vme_start),
- VM_BEHAVIOR_SEQUENTIAL,
- &result_prot, &m, &top_page,
- (int *)0,
- &error_code,
- options & VM_MAP_COPYIN_OPT_NO_ZERO_FILL,
- data_supply);
- /*
- * Cope with what happened.
- */
- switch (kr) {
- case VM_FAULT_SUCCESS:
-
- /*
- * If we lost write access,
- * try again.
- */
- if ((prot & VM_PROT_WRITE) &&
- !(result_prot & VM_PROT_WRITE)) {
- vm_object_lock(src_object);
- vm_object_paging_begin(src_object);
- goto retry;
- }
- break;
- case VM_FAULT_MEMORY_SHORTAGE:
- VM_PAGE_WAIT();
- /* fall thru */
- case VM_FAULT_INTERRUPTED: /* ??? */
- case VM_FAULT_RETRY:
- vm_object_lock(src_object);
- vm_object_paging_begin(src_object);
- goto retry;
- case VM_FAULT_FICTITIOUS_SHORTAGE:
- vm_page_more_fictitious();
- vm_object_lock(src_object);
- vm_object_paging_begin(src_object);
- goto retry;
- case VM_FAULT_MEMORY_ERROR:
- /*
- * Something broke. If this
- * is a continuation, return
- * a partial result if possible,
- * else fail the whole thing.
- * In the continuation case, the
- * next continuation call will
- * get this error if it persists.
- */
- vm_map_lock(src_map);
- if (is_cont &&
- copy->cpy_npages != 0)
- goto make_continuation;
-
- result = error_code ? error_code : KERN_MEMORY_ERROR;
- goto error;
- }
-
- if (top_page != VM_PAGE_NULL) {
- vm_object_lock(src_object);
- VM_PAGE_FREE(top_page);
- vm_object_paging_end(src_object);
- vm_object_unlock(src_object);
- }
-
- }
-
- /*
- * The page is busy, its object is locked, and
- * we have a paging reference on it. Either
- * the map is locked, or need_map_lookup is
- * TRUE.
- */
-
- /*
- * Put the page in the page list.
- */
- copy->cpy_page_list[copy->cpy_npages++] = m;
- vm_object_unlock(m->object);
-
- /*
- * Pmap enter support. Only used for
- * device I/O for colocated server.
- *
- * WARNING: This code assumes that this
- * option is only used for well behaved
- * memory. If the mapping has changed,
- * the following code will make mistakes.
- *
- * XXXO probably ought to do pmap_extract first,
- * XXXO to avoid needless pmap_enter, but this
- * XXXO can't detect protection mismatch??
- */
-
- if (options & VM_MAP_COPYIN_OPT_PMAP_ENTER) {
- /*
- * XXX Only used on kernel map.
- * XXX Must not remove VM_PROT_WRITE on
- * XXX an I/O only requiring VM_PROT_READ
- * XXX as another I/O may be active on same page
- * XXX assume that if mapping exists, it must
- * XXX have the equivalent of at least VM_PROT_READ,
- * XXX but don't assume it has VM_PROT_WRITE as the
- * XXX pmap might not all the rights of the object
- */
- assert(vm_map_pmap(src_map) == kernel_pmap);
-
- if ((prot & VM_PROT_WRITE) ||
- (pmap_extract(vm_map_pmap(src_map),
- src_start) != m->phys_addr))
-
- PMAP_ENTER(vm_map_pmap(src_map), src_start,
- m, prot, wired);
- }
- if(need_map_lookup) {
- need_map_lookup = FALSE;
- vm_map_lock(src_map);
- if (!vm_map_lookup_entry(src_map, src_start, &src_entry)) {
- result = KERN_INVALID_ADDRESS;
- goto error;
- }
- }
- }
-
- /*
- * Verify that there are no gaps in the region
- */
- src_start = src_entry->vme_end;
- if (src_start < src_end) {
- src_entry = src_entry->vme_next;
- if (need_map_lookup) {
- need_map_lookup = FALSE;
- vm_map_lock(src_map);
- if(!vm_map_lookup_entry(src_map,
- src_start, &src_entry)) {
- result = KERN_INVALID_ADDRESS;
- goto error;
- }
- } else if (src_entry->vme_start != src_start) {
- result = KERN_INVALID_ADDRESS;
- goto error;
- }
- }
-
- /*
- * DETERMINE whether the entire region
- * has been copied.
- */
-
- while ((src_start >= src_end) && (src_end != 0)) {
- if (src_map != base_map) {
- submap_map_t *ptr;
-
- ptr = parent_maps;
- assert(ptr != NULL);
- parent_maps = parent_maps->next;
- src_start = ptr->base_start;
- src_end = ptr->base_end;
- if(need_map_lookup) {
- need_map_lookup = FALSE;
- }
- else {
- vm_map_unlock(src_map);
- }
- src_map = ptr->parent_map;
- vm_map_lock(src_map);
- if((src_start < src_end) &&
- (!vm_map_lookup_entry(ptr->parent_map,
- src_start, &src_entry))) {
- result = KERN_INVALID_ADDRESS;
- kfree((vm_offset_t)ptr, sizeof(submap_map_t));
- goto error;
- }
- kfree((vm_offset_t)ptr, sizeof(submap_map_t));
- } else
- break;
- }
- if ((src_start >= src_end) && (src_end != 0)) {
- if (need_map_lookup)
- vm_map_lock(src_map);
- break;
- }
-
- }
-
- /*
- * If steal_pages is true, make sure all
- * pages in the copy are not in any object
- * We try to remove them from the original
- * object, but we may have to copy them.
- *
- * At this point every page in the list is busy
- * and holds a paging reference to its object.
- * When we're done stealing, every page is busy,
- * and in no object (m->tabled == FALSE).
- */
- src_start = trunc_page(src_addr);
- if (options & VM_MAP_COPYIN_OPT_STEAL_PAGES) {
- register int i;
- vm_offset_t page_vaddr;
- vm_offset_t unwire_end;
- vm_offset_t map_entry_end;
- boolean_t share_map = FALSE;
-
- unwire_end = src_start;
- map_entry_end = src_start;
- for (i = 0; i < copy->cpy_npages; i++) {
-
-
- /*
- * Remove the page from its object if it
- * can be stolen. It can be stolen if:
- *
- * (1) The source is being destroyed,
- * the object is internal (hence
- * temporary), and not shared.
- * (2) The page is not precious.
- *
- * The not shared check consists of two
- * parts: (a) there are no objects that
- * shadow this object. (b) it is not the
- * object in any shared map entries (i.e.,
- * use_shared_copy is not set).
- *
- * The first check (a) means that we can't
- * steal pages from objects that are not
- * at the top of their shadow chains. This
- * should not be a frequent occurrence.
- *
- * Stealing wired pages requires telling the
- * pmap module to let go of them.
- *
- * NOTE: stealing clean pages from objects
- * whose mappings survive requires a call to
- * the pmap module. Maybe later.
- */
- m = copy->cpy_page_list[i];
- src_object = m->object;
- vm_object_lock(src_object);
-
- page_vaddr = src_start + (i * PAGE_SIZE);
- if(page_vaddr > map_entry_end) {
- if (!vm_map_lookup_entry(src_map, page_vaddr, &src_entry))
- share_map = TRUE;
- else if (src_entry->is_sub_map) {
- map_entry_end = src_entry->vme_end;
- share_map = TRUE;
- } else {
- map_entry_end = src_entry->vme_end;
- share_map = FALSE;
- }
- }
-
-
- if ((options & VM_MAP_COPYIN_OPT_SRC_DESTROY) &&
- src_object->internal &&
- !src_object->true_share &&
- (!src_object->shadowed) &&
- (src_object->copy_strategy ==
- MEMORY_OBJECT_COPY_SYMMETRIC) &&
- !m->precious &&
- !share_map) {
-
- if (m->wire_count > 0) {
-
- assert(m->wire_count == 1);
- /*
- * In order to steal a wired
- * page, we have to unwire it
- * first. We do this inline
- * here because we have the page.
- *
- * Step 1: Unwire the map entry.
- * Also tell the pmap module
- * that this piece of the
- * pmap is pageable.
- */
- vm_object_unlock(src_object);
- if (page_vaddr >= unwire_end) {
- if (!vm_map_lookup_entry(src_map,
- page_vaddr, &src_entry))
- panic("vm_map_copyin_page_list: missing wired map entry");
-
- vm_map_clip_start(src_map, src_entry,
- page_vaddr);
- vm_map_clip_end(src_map, src_entry,
- src_start + src_size);
-
-/* revisit why this assert fails CDY
- assert(src_entry->wired_count > 0);
-*/
- src_entry->wired_count = 0;
- src_entry->user_wired_count = 0;
- unwire_end = src_entry->vme_end;
- pmap_pageable(vm_map_pmap(src_map),
- page_vaddr, unwire_end, TRUE);
- }
-
- /*
- * Step 2: Unwire the page.
- * pmap_remove handles this for us.
- */
- vm_object_lock(src_object);
- }
-
- /*
- * Don't need to remove the mapping;
- * vm_map_delete will handle it.
- *
- * Steal the page. Setting the wire count
- * to zero is vm_page_unwire without
- * activating the page.
- */
- vm_page_lock_queues();
- vm_page_remove(m);
- if (m->wire_count > 0) {
- m->wire_count = 0;
- vm_page_wire_count--;
- } else {
- VM_PAGE_QUEUES_REMOVE(m);
- }
- vm_page_unlock_queues();
- } else {
- /*
- * Have to copy this page. Have to
- * unlock the map while copying,
- * hence no further page stealing.
- * Hence just copy all the pages.
- * Unlock the map while copying;
- * This means no further page stealing.
- */
- vm_object_unlock(src_object);
- vm_map_unlock(src_map);
- vm_map_copy_steal_pages(copy);
- vm_map_lock(src_map);
- break;
- }
-
- vm_object_paging_end(src_object);
- vm_object_unlock(src_object);
- }
-
- copy->cpy_page_loose = TRUE;
-
- /*
- * If the source should be destroyed, do it now, since the
- * copy was successful.
- */
-
- if (options & VM_MAP_COPYIN_OPT_SRC_DESTROY) {
- (void) vm_map_delete(src_map, src_start,
- src_end, VM_MAP_NO_FLAGS);
- }
- } else {
- /*
- * Not stealing pages leaves busy or prepped pages in the map.
- * This will cause source destruction to hang. Use
- * a continuation to prevent this.
- */
- if ((options & VM_MAP_COPYIN_OPT_SRC_DESTROY) &&
- !vm_map_copy_has_cont(copy)) {
- cont_args = (vm_map_copyin_args_t)
- kalloc(sizeof(vm_map_copyin_args_data_t));
- vm_map_reference(src_map);
- cont_args->map = src_map;
- cont_args->src_addr = (vm_offset_t) 0;
- cont_args->src_len = (vm_size_t) 0;
- cont_args->destroy_addr = src_start;
- cont_args->destroy_len = src_end - src_start;
- cont_args->options = options;
-
- copy->cpy_cont_args = cont_args;
- copy->cpy_cont = vm_map_copyin_page_list_cont;
- }
- }
-
- vm_map_unlock(src_map);
-
- *copy_result = copy;
- return(result);
-
-error:
- {
- submap_map_t *ptr;
-
- vm_map_unlock(src_map);
- vm_map_copy_discard(copy);
-
- for(ptr = parent_maps; ptr != NULL; ptr = parent_maps) {
- parent_maps=parent_maps->next;
- kfree((vm_offset_t)ptr, sizeof(submap_map_t));
- }
- return(result);
- }
-}
-
-void
-vm_map_fork_share(
- vm_map_t old_map,
- vm_map_entry_t old_entry,
- vm_map_t new_map)
-{
- vm_object_t object;
- vm_map_entry_t new_entry;
- kern_return_t result;
-
- /*
- * New sharing code. New map entry
- * references original object. Internal
- * objects use asynchronous copy algorithm for
- * future copies. First make sure we have
- * the right object. If we need a shadow,
- * or someone else already has one, then
- * make a new shadow and share it.
- */
-
- object = old_entry->object.vm_object;
- if (old_entry->is_sub_map) {
- assert(old_entry->wired_count == 0);
-#ifndef i386
- if(old_entry->use_pmap) {
projects / apple / xnu.git / blobdiff