[apple/xnu.git] / osfmk / vm / vm_map.c

/*
 * Copyright (c) 2000 Apple Computer, Inc. All rights reserved.
 *
 * @APPLE_LICENSE_HEADER_START@
 * 
 * The contents of this file constitute Original Code as defined in and
 * are subject to the Apple Public Source License Version 1.1 (the
 * "License").  You may not use this file except in compliance with the
 * License.  Please obtain a copy of the License at
 * http://www.apple.com/publicsource and read it before using this file.
 * 
 * This Original Code and all software distributed under the License are
 * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
 * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
 * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
 * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
 * License for the specific language governing rights and limitations
 * under the License.
 * 
 * @APPLE_LICENSE_HEADER_END@
 */
/*
 * @OSF_COPYRIGHT@
 */
/* 
 * Mach Operating System
 * Copyright (c) 1991,1990,1989,1988,1987 Carnegie Mellon University
 * All Rights Reserved.
 * 
 * Permission to use, copy, modify and distribute this software and its
 * documentation is hereby granted, provided that both the copyright
 * notice and this permission notice appear in all copies of the
 * software, derivative works or modified versions, and any portions
 * thereof, and that both notices appear in supporting documentation.
 * 
 * CARNEGIE MELLON ALLOWS FREE USE OF THIS SOFTWARE IN ITS "AS IS"
 * CONDITION.  CARNEGIE MELLON DISCLAIMS ANY LIABILITY OF ANY KIND FOR
 * ANY DAMAGES WHATSOEVER RESULTING FROM THE USE OF THIS SOFTWARE.
 * 
 * Carnegie Mellon requests users of this software to return to
 * 
 *  Software Distribution Coordinator  or  Software.Distribution@CS.CMU.EDU
 *  School of Computer Science
 *  Carnegie Mellon University
 *  Pittsburgh PA 15213-3890
 * 
 * any improvements or extensions that they make and grant Carnegie Mellon
 * the rights to redistribute these changes.
 */
/*
 */
/*
 *      File:   vm/vm_map.c
 *      Author: Avadis Tevanian, Jr., Michael Wayne Young
 *      Date:   1985
 *
 *      Virtual memory mapping module.
 */

#include <cpus.h>
#include <task_swapper.h>
#include <mach_assert.h>

#include <mach/kern_return.h>
#include <mach/port.h>
#include <mach/vm_attributes.h>
#include <mach/vm_param.h>
#include <mach/vm_behavior.h>
#include <kern/assert.h>
#include <kern/counters.h>
#include <kern/zalloc.h>
#include <vm/vm_init.h>
#include <vm/vm_fault.h>
#include <vm/vm_map.h>
#include <vm/vm_object.h>
#include <vm/vm_page.h>
#include <vm/vm_kern.h>
#include <ipc/ipc_port.h>
#include <kern/sched_prim.h>
#include <kern/misc_protos.h>
#include <mach/vm_map_server.h>
#include <mach/mach_host_server.h>
#include <ddb/tr.h>
#include <kern/xpr.h>

/* Internal prototypes
 */
extern boolean_t vm_map_range_check(
                                vm_map_t        map,
                                vm_offset_t     start,
                                vm_offset_t     end,
                                vm_map_entry_t  *entry);

extern vm_map_entry_t   _vm_map_entry_create(
                                struct vm_map_header    *map_header);

extern void             _vm_map_entry_dispose(
                                struct vm_map_header    *map_header,
                                vm_map_entry_t          entry);

extern void             vm_map_pmap_enter(
                                vm_map_t                map,
                                vm_offset_t             addr,
                                vm_offset_t             end_addr,
                                vm_object_t             object,
                                vm_object_offset_t      offset,
                                vm_prot_t               protection);

extern void             _vm_map_clip_end(
                                struct vm_map_header    *map_header,
                                vm_map_entry_t          entry,
                                vm_offset_t             end);

extern void             vm_map_entry_delete(
                                vm_map_t        map,
                                vm_map_entry_t  entry);

extern kern_return_t    vm_map_delete(
                                vm_map_t        map,
                                vm_offset_t     start,
                                vm_offset_t     end,
                                int             flags);

extern void             vm_map_copy_steal_pages(
                                vm_map_copy_t   copy);

extern kern_return_t    vm_map_copy_overwrite_unaligned(
                                vm_map_t        dst_map,
                                vm_map_entry_t  entry,
                                vm_map_copy_t   copy,
                                vm_offset_t     start);

extern kern_return_t    vm_map_copy_overwrite_aligned(
                                vm_map_t        dst_map,
                                vm_map_entry_t  tmp_entry,
                                vm_map_copy_t   copy,
                                vm_offset_t     start,
                                pmap_t          pmap);

extern kern_return_t    vm_map_copyin_kernel_buffer(
                                vm_map_t        src_map,
                                vm_offset_t     src_addr,
                                vm_size_t       len,
                                boolean_t       src_destroy,
                                vm_map_copy_t   *copy_result);  /* OUT */

extern kern_return_t    vm_map_copyout_kernel_buffer(
                                vm_map_t        map,
                                vm_offset_t     *addr,  /* IN/OUT */
                                vm_map_copy_t   copy,
                                boolean_t       overwrite);

extern kern_return_t    vm_map_copyin_page_list_cont(
                                vm_map_copyin_args_t    cont_args,
                                vm_map_copy_t           *copy_result); /* OUT */

extern void             vm_map_fork_share(
                                vm_map_t        old_map,
                                vm_map_entry_t  old_entry,
                                vm_map_t        new_map);

extern boolean_t        vm_map_fork_copy(
                                vm_map_t        old_map,
                                vm_map_entry_t  *old_entry_p,
                                vm_map_t        new_map);

extern kern_return_t    vm_remap_range_allocate(
                                vm_map_t        map,
                                vm_offset_t     *address,       /* IN/OUT */
                                vm_size_t       size,
                                vm_offset_t     mask,
                                boolean_t       anywhere,
                                vm_map_entry_t  *map_entry);    /* OUT */

extern void             _vm_map_clip_start(
                                struct vm_map_header    *map_header,
                                vm_map_entry_t          entry,
                                vm_offset_t             start);

void                    vm_region_top_walk(
                                vm_map_entry_t             entry,
                                vm_region_top_info_t       top);

void                    vm_region_walk(
                                vm_map_entry_t             entry,
                                vm_region_extended_info_t  extended,
                                vm_object_offset_t         offset,
                                vm_offset_t                range,
                                vm_map_t                   map,
                                vm_offset_t                va);

/*
 * Macros to copy a vm_map_entry. We must be careful to correctly
 * manage the wired page count. vm_map_entry_copy() creates a new
 * map entry to the same memory - the wired count in the new entry
 * must be set to zero. vm_map_entry_copy_full() creates a new
 * entry that is identical to the old entry.  This preserves the
 * wire count; it's used for map splitting and zone changing in
 * vm_map_copyout.
 */
#define vm_map_entry_copy(NEW,OLD) \
MACRO_BEGIN                                     \
                *(NEW) = *(OLD);                \
                (NEW)->is_shared = FALSE;       \
                (NEW)->needs_wakeup = FALSE;    \
                (NEW)->in_transition = FALSE;   \
                (NEW)->wired_count = 0;         \
                (NEW)->user_wired_count = 0;    \
MACRO_END

#define vm_map_entry_copy_full(NEW,OLD)        (*(NEW) = *(OLD))

/*
 *      Virtual memory maps provide for the mapping, protection,
 *      and sharing of virtual memory objects.  In addition,
 *      this module provides for an efficient virtual copy of
 *      memory from one map to another.
 *
 *      Synchronization is required prior to most operations.
 *
 *      Maps consist of an ordered doubly-linked list of simple
 *      entries; a single hint is used to speed up lookups.
 *
 *      Sharing maps have been deleted from this version of Mach.
 *      All shared objects are now mapped directly into the respective
 *      maps.  This requires a change in the copy on write strategy;
 *      the asymmetric (delayed) strategy is used for shared temporary
 *      objects instead of the symmetric (shadow) strategy.  All maps
 *      are now "top level" maps (either task map, kernel map or submap
 *      of the kernel map).  
 *
 *      Since portions of maps are specified by start/end addreses,
 *      which may not align with existing map entries, all
 *      routines merely "clip" entries to these start/end values.
 *      [That is, an entry is split into two, bordering at a
 *      start or end value.]  Note that these clippings may not
 *      always be necessary (as the two resulting entries are then
 *      not changed); however, the clipping is done for convenience.
 *      No attempt is currently made to "glue back together" two
 *      abutting entries.
 *
 *      The symmetric (shadow) copy strategy implements virtual copy
 *      by copying VM object references from one map to
 *      another, and then marking both regions as copy-on-write.
 *      It is important to note that only one writeable reference
 *      to a VM object region exists in any map when this strategy
 *      is used -- this means that shadow object creation can be
 *      delayed until a write operation occurs.  The symmetric (delayed)
 *      strategy allows multiple maps to have writeable references to
 *      the same region of a vm object, and hence cannot delay creating
 *      its copy objects.  See vm_object_copy_quickly() in vm_object.c.
 *      Copying of permanent objects is completely different; see
 *      vm_object_copy_strategically() in vm_object.c.
 */

zone_t          vm_map_zone;            /* zone for vm_map structures */
zone_t          vm_map_entry_zone;      /* zone for vm_map_entry structures */
zone_t          vm_map_kentry_zone;     /* zone for kernel entry structures */
zone_t          vm_map_copy_zone;       /* zone for vm_map_copy structures */


/*
 *      Placeholder object for submap operations.  This object is dropped
 *      into the range by a call to vm_map_find, and removed when
 *      vm_map_submap creates the submap.
 */

vm_object_t     vm_submap_object;

/*
 *      vm_map_init:
 *
 *      Initialize the vm_map module.  Must be called before
 *      any other vm_map routines.
 *
 *      Map and entry structures are allocated from zones -- we must
 *      initialize those zones.
 *
 *      There are three zones of interest:
 *
 *      vm_map_zone:            used to allocate maps.
 *      vm_map_entry_zone:      used to allocate map entries.
 *      vm_map_kentry_zone:     used to allocate map entries for the kernel.
 *
 *      The kernel allocates map entries from a special zone that is initially
 *      "crammed" with memory.  It would be difficult (perhaps impossible) for
 *      the kernel to allocate more memory to a entry zone when it became
 *      empty since the very act of allocating memory implies the creation
 *      of a new entry.
 */

vm_offset_t     map_data;
vm_size_t       map_data_size;
vm_offset_t     kentry_data;
vm_size_t       kentry_data_size;
int             kentry_count = 2048;            /* to init kentry_data_size */

/*
 *      Threshold for aggressive (eager) page map entering for vm copyout
 *      operations.  Any copyout larger will NOT be aggressively entered.
 */
vm_size_t vm_map_aggressive_enter_max;          /* set by bootstrap */

void
vm_map_init(
        void)
{
        vm_map_zone = zinit((vm_size_t) sizeof(struct vm_map), 40*1024,
                                        PAGE_SIZE, "maps");

        vm_map_entry_zone = zinit((vm_size_t) sizeof(struct vm_map_entry),
                                        1024*1024, PAGE_SIZE*5,
                                        "non-kernel map entries");

        vm_map_kentry_zone = zinit((vm_size_t) sizeof(struct vm_map_entry),
                                        kentry_data_size, kentry_data_size,
                                        "kernel map entries");

        vm_map_copy_zone = zinit((vm_size_t) sizeof(struct vm_map_copy),
                                        16*1024, PAGE_SIZE, "map copies");

        /*
         *      Cram the map and kentry zones with initial data.
         *      Set kentry_zone non-collectible to aid zone_gc().
         */
        zone_change(vm_map_zone, Z_COLLECT, FALSE);
        zone_change(vm_map_kentry_zone, Z_COLLECT, FALSE);
        zone_change(vm_map_kentry_zone, Z_EXPAND, FALSE);
        zcram(vm_map_zone, map_data, map_data_size);
        zcram(vm_map_kentry_zone, kentry_data, kentry_data_size);
}

void
vm_map_steal_memory(
        void)
{
        map_data_size = round_page(10 * sizeof(struct vm_map));
        map_data = pmap_steal_memory(map_data_size);

#if 0
        /*
         * Limiting worst case: vm_map_kentry_zone needs to map each "available"
         * physical page (i.e. that beyond the kernel image and page tables)
         * individually; we guess at most one entry per eight pages in the
         * real world. This works out to roughly .1 of 1% of physical memory,
         * or roughly 1900 entries (64K) for a 64M machine with 4K pages.
         */
#endif
        kentry_count = pmap_free_pages() / 8;


        kentry_data_size =
                round_page(kentry_count * sizeof(struct vm_map_entry));
        kentry_data = pmap_steal_memory(kentry_data_size);
}

/*
 *      vm_map_create:
 *
 *      Creates and returns a new empty VM map with
 *      the given physical map structure, and having
 *      the given lower and upper address bounds.
 */
vm_map_t
vm_map_create(
        pmap_t          pmap,
        vm_offset_t     min,
        vm_offset_t     max,
        boolean_t       pageable)
{
        register vm_map_t       result;

        result = (vm_map_t) zalloc(vm_map_zone);
        if (result == VM_MAP_NULL)
                panic("vm_map_create");

        vm_map_first_entry(result) = vm_map_to_entry(result);
        vm_map_last_entry(result)  = vm_map_to_entry(result);
        result->hdr.nentries = 0;
        result->hdr.entries_pageable = pageable;

        result->size = 0;
        result->ref_count = 1;
#if     TASK_SWAPPER
        result->res_count = 1;
        result->sw_state = MAP_SW_IN;
#endif  /* TASK_SWAPPER */
        result->pmap = pmap;
        result->min_offset = min;
        result->max_offset = max;
        result->wiring_required = FALSE;
        result->no_zero_fill = FALSE;
        result->wait_for_space = FALSE;
        result->first_free = vm_map_to_entry(result);
        result->hint = vm_map_to_entry(result);
        vm_map_lock_init(result);
        mutex_init(&result->s_lock, ETAP_VM_RESULT);

        return(result);
}

/*
 *      vm_map_entry_create:    [ internal use only ]
 *
 *      Allocates a VM map entry for insertion in the
 *      given map (or map copy).  No fields are filled.
 */
#define vm_map_entry_create(map) \
            _vm_map_entry_create(&(map)->hdr)

#define vm_map_copy_entry_create(copy) \
            _vm_map_entry_create(&(copy)->cpy_hdr)

vm_map_entry_t
_vm_map_entry_create(
        register struct vm_map_header   *map_header)
{
        register zone_t zone;
        register vm_map_entry_t entry;

        if (map_header->entries_pageable)
            zone = vm_map_entry_zone;
        else
            zone = vm_map_kentry_zone;

        entry = (vm_map_entry_t) zalloc(zone);
        if (entry == VM_MAP_ENTRY_NULL)
                panic("vm_map_entry_create");

        return(entry);
}

/*
 *      vm_map_entry_dispose:   [ internal use only ]
 *
 *      Inverse of vm_map_entry_create.
 */
#define vm_map_entry_dispose(map, entry)                        \
MACRO_BEGIN                                                     \
        if((entry) == (map)->first_free)                        \
                (map)->first_free = vm_map_to_entry(map);       \
        if((entry) == (map)->hint)                              \
                (map)->hint = vm_map_to_entry(map);             \
        _vm_map_entry_dispose(&(map)->hdr, (entry));            \
MACRO_END

#define vm_map_copy_entry_dispose(map, entry) \
        _vm_map_entry_dispose(&(copy)->cpy_hdr, (entry))

void
_vm_map_entry_dispose(
        register struct vm_map_header   *map_header,
        register vm_map_entry_t         entry)
{
        register zone_t         zone;

        if (map_header->entries_pageable)
            zone = vm_map_entry_zone;
        else
            zone = vm_map_kentry_zone;

        zfree(zone, (vm_offset_t) entry);
}

boolean_t first_free_is_valid(vm_map_t map);    /* forward */
boolean_t first_free_check = FALSE;
boolean_t
first_free_is_valid(
        vm_map_t        map)
{
        vm_map_entry_t  entry, next;

        if (!first_free_check)
                return TRUE;
                
        entry = vm_map_to_entry(map);
        next = entry->vme_next;
        while (trunc_page(next->vme_start) == trunc_page(entry->vme_end) ||
               (trunc_page(next->vme_start) == trunc_page(entry->vme_start) &&
                next != vm_map_to_entry(map))) {
                entry = next;
                next = entry->vme_next;
                if (entry == vm_map_to_entry(map))
                        break;
        }
        if (map->first_free != entry) {
                printf("Bad first_free for map 0x%x: 0x%x should be 0x%x\n",
                       map, map->first_free, entry);
                return FALSE;
        }
        return TRUE;
}

/*
 *      UPDATE_FIRST_FREE:
 *
 *      Updates the map->first_free pointer to the
 *      entry immediately before the first hole in the map.
 *      The map should be locked.
 */
#define UPDATE_FIRST_FREE(map, new_first_free)                          \
MACRO_BEGIN                                                             \
        vm_map_t        UFF_map;                                        \
        vm_map_entry_t  UFF_first_free;                                 \
        vm_map_entry_t  UFF_next_entry;                                 \
        UFF_map = (map);                                                \
        UFF_first_free = (new_first_free);                              \
        UFF_next_entry = UFF_first_free->vme_next;                      \
        while (trunc_page(UFF_next_entry->vme_start) ==                 \
               trunc_page(UFF_first_free->vme_end) ||                   \
               (trunc_page(UFF_next_entry->vme_start) ==                \
                trunc_page(UFF_first_free->vme_start) &&                \
                UFF_next_entry != vm_map_to_entry(UFF_map))) {          \
                UFF_first_free = UFF_next_entry;                        \
                UFF_next_entry = UFF_first_free->vme_next;              \
                if (UFF_first_free == vm_map_to_entry(UFF_map))         \
                        break;                                          \
        }                                                               \
        UFF_map->first_free = UFF_first_free;                           \
        assert(first_free_is_valid(UFF_map));                           \
MACRO_END

/*
 *      vm_map_entry_{un,}link:
 *
 *      Insert/remove entries from maps (or map copies).
 */
#define vm_map_entry_link(map, after_where, entry)                      \
MACRO_BEGIN                                                             \
        vm_map_t VMEL_map;                                              \
        vm_map_entry_t VMEL_entry;                                      \
        VMEL_map = (map);                                               \
        VMEL_entry = (entry);                                           \
        _vm_map_entry_link(&VMEL_map->hdr, after_where, VMEL_entry);    \
        UPDATE_FIRST_FREE(VMEL_map, VMEL_map->first_free);              \
MACRO_END


#define vm_map_copy_entry_link(copy, after_where, entry)                \
        _vm_map_entry_link(&(copy)->cpy_hdr, after_where, (entry))

#define _vm_map_entry_link(hdr, after_where, entry)                     \
        MACRO_BEGIN                                                     \
        (hdr)->nentries++;                                              \
        (entry)->vme_prev = (after_where);                              \
        (entry)->vme_next = (after_where)->vme_next;                    \
        (entry)->vme_prev->vme_next = (entry)->vme_next->vme_prev = (entry); \
        MACRO_END

#define vm_map_entry_unlink(map, entry)                                 \
MACRO_BEGIN                                                             \
        vm_map_t VMEU_map;                                              \
        vm_map_entry_t VMEU_entry;                                      \
        vm_map_entry_t VMEU_first_free;                                 \
        VMEU_map = (map);                                               \
        VMEU_entry = (entry);                                           \
        if (VMEU_entry->vme_start <= VMEU_map->first_free->vme_start)   \
                VMEU_first_free = VMEU_entry->vme_prev;                 \
        else                                                            \
                VMEU_first_free = VMEU_map->first_free;                 \
        _vm_map_entry_unlink(&VMEU_map->hdr, VMEU_entry);               \
        UPDATE_FIRST_FREE(VMEU_map, VMEU_first_free);                   \
MACRO_END

#define vm_map_copy_entry_unlink(copy, entry)                           \
        _vm_map_entry_unlink(&(copy)->cpy_hdr, (entry))

#define _vm_map_entry_unlink(hdr, entry)                                \
        MACRO_BEGIN                                                     \
        (hdr)->nentries--;                                              \
        (entry)->vme_next->vme_prev = (entry)->vme_prev;                \
        (entry)->vme_prev->vme_next = (entry)->vme_next;                \
        MACRO_END

/*
 *      kernel_vm_map_reference:
 *
 *      kernel internal export version for iokit and bsd components
 *      in lieu of component interface semantics.
 *
 */
void
kernel_vm_map_reference(
        register vm_map_t       map)
{
        if (map == VM_MAP_NULL)
                return;

        mutex_lock(&map->s_lock);
#if     TASK_SWAPPER
        assert(map->res_count > 0);
        assert(map->ref_count >= map->res_count);
        map->res_count++;
#endif
        map->ref_count++;
        mutex_unlock(&map->s_lock);
}

#if     MACH_ASSERT && TASK_SWAPPER
/*
 *      vm_map_reference:
 *
 *      Adds valid reference and residence counts to the given map.
 *      The map must be in memory (i.e. non-zero residence count).
 *
 */
void
vm_map_reference(
        register vm_map_t       map)
{
        if (map == VM_MAP_NULL)
                return;

        mutex_lock(&map->s_lock);
        assert(map->res_count > 0);
        assert(map->ref_count >= map->res_count);
        map->ref_count++;
        map->res_count++;
        mutex_unlock(&map->s_lock);
}

/*
 *      vm_map_res_reference:
 *
 *      Adds another valid residence count to the given map.
 *
 *      Map is locked so this function can be called from
 *      vm_map_swapin.
 *
 */
void vm_map_res_reference(register vm_map_t map)
{
        /* assert map is locked */
        assert(map->res_count >= 0);
        assert(map->ref_count >= map->res_count);
        if (map->res_count == 0) {
                mutex_unlock(&map->s_lock);
                vm_map_lock(map);
                vm_map_swapin(map);
                mutex_lock(&map->s_lock);
                ++map->res_count;
                vm_map_unlock(map);
        } else
                ++map->res_count;
}

/*
 *      vm_map_reference_swap:
 *
 *      Adds valid reference and residence counts to the given map.
 *
 *      The map may not be in memory (i.e. zero residence count).
 *
 */
void vm_map_reference_swap(register vm_map_t map)
{
        assert(map != VM_MAP_NULL);
        mutex_lock(&map->s_lock);
        assert(map->res_count >= 0);
        assert(map->ref_count >= map->res_count);
        map->ref_count++;
        vm_map_res_reference(map);
        mutex_unlock(&map->s_lock);
}

/*
 *      vm_map_res_deallocate:
 *
 *      Decrement residence count on a map; possibly causing swapout.
 *
 *      The map must be in memory (i.e. non-zero residence count).
 *
 *      The map is locked, so this function is callable from vm_map_deallocate.
 *
 */
void vm_map_res_deallocate(register vm_map_t map)
{
        assert(map->res_count > 0);
        if (--map->res_count == 0) {
                mutex_unlock(&map->s_lock);
                vm_map_lock(map);
                vm_map_swapout(map);
                vm_map_unlock(map);
                mutex_lock(&map->s_lock);
        }
        assert(map->ref_count >= map->res_count);
}
#endif  /* MACH_ASSERT && TASK_SWAPPER */

/*
 *      vm_map_deallocate:
 *
 *      Removes a reference from the specified map,
 *      destroying it if no references remain.
 *      The map should not be locked.
 */
void
vm_map_deallocate(
        register vm_map_t       map)
{
        unsigned int            ref;

        if (map == VM_MAP_NULL)
                return;

        mutex_lock(&map->s_lock);
        ref = --map->ref_count;
        if (ref > 0) {
                vm_map_res_deallocate(map);
                mutex_unlock(&map->s_lock);
                return;
        }
        assert(map->ref_count == 0);
        mutex_unlock(&map->s_lock);

#if     TASK_SWAPPER
        /*
         * The map residence count isn't decremented here because
         * the vm_map_delete below will traverse the entire map, 
         * deleting entries, and the residence counts on objects
         * and sharing maps will go away then.
         */
#endif

        vm_map_destroy(map);
}

/*
 *      vm_map_destroy:
 *
 *      Actually destroy a map.
 */
void
vm_map_destroy(
        register vm_map_t       map)
{
        vm_map_lock(map);
        (void) vm_map_delete(map, map->min_offset,
                             map->max_offset, VM_MAP_NO_FLAGS);
        vm_map_unlock(map);

        pmap_destroy(map->pmap);

        zfree(vm_map_zone, (vm_offset_t) map);
}

#if     TASK_SWAPPER
/*
 * vm_map_swapin/vm_map_swapout
 *
 * Swap a map in and out, either referencing or releasing its resources.  
 * These functions are internal use only; however, they must be exported
 * because they may be called from macros, which are exported.
 *
 * In the case of swapout, there could be races on the residence count, 
 * so if the residence count is up, we return, assuming that a 
 * vm_map_deallocate() call in the near future will bring us back.
 *
 * Locking:
 *      -- We use the map write lock for synchronization among races.
 *      -- The map write lock, and not the simple s_lock, protects the
 *         swap state of the map.
 *      -- If a map entry is a share map, then we hold both locks, in
 *         hierarchical order.
 *
 * Synchronization Notes:
 *      1) If a vm_map_swapin() call happens while swapout in progress, it
 *      will block on the map lock and proceed when swapout is through.
 *      2) A vm_map_reference() call at this time is illegal, and will
 *      cause a panic.  vm_map_reference() is only allowed on resident
 *      maps, since it refuses to block.
 *      3) A vm_map_swapin() call during a swapin will block, and 
 *      proceeed when the first swapin is done, turning into a nop.
 *      This is the reason the res_count is not incremented until
 *      after the swapin is complete.
 *      4) There is a timing hole after the checks of the res_count, before
 *      the map lock is taken, during which a swapin may get the lock
 *      before a swapout about to happen.  If this happens, the swapin
 *      will detect the state and increment the reference count, causing
 *      the swapout to be a nop, thereby delaying it until a later 
 *      vm_map_deallocate.  If the swapout gets the lock first, then 
 *      the swapin will simply block until the swapout is done, and 
 *      then proceed.
 *
 * Because vm_map_swapin() is potentially an expensive operation, it
 * should be used with caution.
 *
 * Invariants:
 *      1) A map with a residence count of zero is either swapped, or
 *         being swapped.
 *      2) A map with a non-zero residence count is either resident,
 *         or being swapped in.
 */

int vm_map_swap_enable = 1;

void vm_map_swapin (vm_map_t map)
{
        register vm_map_entry_t entry;
        
        if (!vm_map_swap_enable)        /* debug */
                return;

        /*
         * Map is locked
         * First deal with various races.
         */
        if (map->sw_state == MAP_SW_IN)
                /* 
                 * we raced with swapout and won.  Returning will incr.
                 * the res_count, turning the swapout into a nop.
                 */
                return;

        /*
         * The residence count must be zero.  If we raced with another
         * swapin, the state would have been IN; if we raced with a
         * swapout (after another competing swapin), we must have lost
         * the race to get here (see above comment), in which case
         * res_count is still 0.
         */
        assert(map->res_count == 0);

        /*
         * There are no intermediate states of a map going out or
         * coming in, since the map is locked during the transition.
         */
        assert(map->sw_state == MAP_SW_OUT);

        /*
         * We now operate upon each map entry.  If the entry is a sub- 
         * or share-map, we call vm_map_res_reference upon it.
         * If the entry is an object, we call vm_object_res_reference
         * (this may iterate through the shadow chain).
         * Note that we hold the map locked the entire time,
         * even if we get back here via a recursive call in
         * vm_map_res_reference.
         */
        entry = vm_map_first_entry(map);

        while (entry != vm_map_to_entry(map)) {
                if (entry->object.vm_object != VM_OBJECT_NULL) {
                        if (entry->is_sub_map) {
                                vm_map_t lmap = entry->object.sub_map;
                                mutex_lock(&lmap->s_lock);
                                vm_map_res_reference(lmap);
                                mutex_unlock(&lmap->s_lock);
                        } else {
                                vm_object_t object = entry->object.vm_object;
                                vm_object_lock(object);
                                /*
                                 * This call may iterate through the
                                 * shadow chain.
                                 */
                                vm_object_res_reference(object);
                                vm_object_unlock(object);
                        }
                }
                entry = entry->vme_next;
        }
        assert(map->sw_state == MAP_SW_OUT);
        map->sw_state = MAP_SW_IN;
}

void vm_map_swapout(vm_map_t map)
{
        register vm_map_entry_t entry;
        
        /*
         * Map is locked
         * First deal with various races.
         * If we raced with a swapin and lost, the residence count
         * will have been incremented to 1, and we simply return.
         */
        mutex_lock(&map->s_lock);
        if (map->res_count != 0) {
                mutex_unlock(&map->s_lock);
                return;
        }
        mutex_unlock(&map->s_lock);

        /*
         * There are no intermediate states of a map going out or
         * coming in, since the map is locked during the transition.
         */
        assert(map->sw_state == MAP_SW_IN);

        if (!vm_map_swap_enable)
                return;

        /*
         * We now operate upon each map entry.  If the entry is a sub- 
         * or share-map, we call vm_map_res_deallocate upon it.
         * If the entry is an object, we call vm_object_res_deallocate
         * (this may iterate through the shadow chain).
         * Note that we hold the map locked the entire time,
         * even if we get back here via a recursive call in
         * vm_map_res_deallocate.
         */
        entry = vm_map_first_entry(map);

        while (entry != vm_map_to_entry(map)) {
                if (entry->object.vm_object != VM_OBJECT_NULL) {
                        if (entry->is_sub_map) {
                                vm_map_t lmap = entry->object.sub_map;
                                mutex_lock(&lmap->s_lock);
                                vm_map_res_deallocate(lmap);
                                mutex_unlock(&lmap->s_lock);
                        } else {
                                vm_object_t object = entry->object.vm_object;
                                vm_object_lock(object);
                                /*
                                 * This call may take a long time, 
                                 * since it could actively push 
                                 * out pages (if we implement it 
                                 * that way).
                                 */
                                vm_object_res_deallocate(object);
                                vm_object_unlock(object);
                        }
                }
                entry = entry->vme_next;
        }
        assert(map->sw_state == MAP_SW_IN);
        map->sw_state = MAP_SW_OUT;
}

#endif  /* TASK_SWAPPER */


/*
 *      SAVE_HINT:
 *
 *      Saves the specified entry as the hint for
 *      future lookups.  Performs necessary interlocks.
 */
#define SAVE_HINT(map,value) \
                mutex_lock(&(map)->s_lock); \
                (map)->hint = (value); \
                mutex_unlock(&(map)->s_lock);

/*
 *      vm_map_lookup_entry:    [ internal use only ]
 *
 *      Finds the map entry containing (or
 *      immediately preceding) the specified address
 *      in the given map; the entry is returned
 *      in the "entry" parameter.  The boolean
 *      result indicates whether the address is
 *      actually contained in the map.
 */
boolean_t
vm_map_lookup_entry(
        register vm_map_t       map,
        register vm_offset_t    address,
        vm_map_entry_t          *entry)         /* OUT */
{
        register vm_map_entry_t         cur;
        register vm_map_entry_t         last;

        /*
         *      Start looking either from the head of the
         *      list, or from the hint.
         */

        mutex_lock(&map->s_lock);
        cur = map->hint;
        mutex_unlock(&map->s_lock);

        if (cur == vm_map_to_entry(map))
                cur = cur->vme_next;

        if (address >= cur->vme_start) {
                /*
                 *      Go from hint to end of list.
                 *
                 *      But first, make a quick check to see if
                 *      we are already looking at the entry we
                 *      want (which is usually the case).
                 *      Note also that we don't need to save the hint
                 *      here... it is the same hint (unless we are
                 *      at the header, in which case the hint didn't
                 *      buy us anything anyway).
                 */
                last = vm_map_to_entry(map);
                if ((cur != last) && (cur->vme_end > address)) {
                        *entry = cur;
                        return(TRUE);
                }
        }
        else {
                /*
                 *      Go from start to hint, *inclusively*
                 */
                last = cur->vme_next;
                cur = vm_map_first_entry(map);
        }

        /*
         *      Search linearly
         */

        while (cur != last) {
                if (cur->vme_end > address) {
                        if (address >= cur->vme_start) {
                                /*
                                 *      Save this lookup for future
                                 *      hints, and return
                                 */

                                *entry = cur;
                                SAVE_HINT(map, cur);
                                return(TRUE);
                        }
                        break;
                }
                cur = cur->vme_next;
        }
        *entry = cur->vme_prev;
        SAVE_HINT(map, *entry);
        return(FALSE);
}

/*
 *      Routine:        vm_map_find_space
 *      Purpose:
 *              Allocate a range in the specified virtual address map,
 *              returning the entry allocated for that range.
 *              Used by kmem_alloc, etc.
 *
 *              The map must be NOT be locked. It will be returned locked
 *              on KERN_SUCCESS, unlocked on failure.
 *
 *              If an entry is allocated, the object/offset fields
 *              are initialized to zero.
 */
kern_return_t
vm_map_find_space(
        register vm_map_t       map,
        vm_offset_t             *address,       /* OUT */
        vm_size_t               size,
        vm_offset_t             mask,
        vm_map_entry_t          *o_entry)       /* OUT */
{
        register vm_map_entry_t entry, new_entry;
        register vm_offset_t    start;
        register vm_offset_t    end;

        new_entry = vm_map_entry_create(map);

        /*
         *      Look for the first possible address; if there's already
         *      something at this address, we have to start after it.
         */

        vm_map_lock(map);

        assert(first_free_is_valid(map));
        if ((entry = map->first_free) == vm_map_to_entry(map))
                start = map->min_offset;
        else
                start = entry->vme_end;

        /*
         *      In any case, the "entry" always precedes
         *      the proposed new region throughout the loop:
         */

        while (TRUE) {
                register vm_map_entry_t next;

                /*
                 *      Find the end of the proposed new region.
                 *      Be sure we didn't go beyond the end, or
                 *      wrap around the address.
                 */

                end = ((start + mask) & ~mask);
                if (end < start) {
                        vm_map_entry_dispose(map, new_entry);
                        vm_map_unlock(map);
                        return(KERN_NO_SPACE);
                }
                start = end;
                end += size;

                if ((end > map->max_offset) || (end < start)) {
                        vm_map_entry_dispose(map, new_entry);
                        vm_map_unlock(map);
                        return(KERN_NO_SPACE);
                }

                /*
                 *      If there are no more entries, we must win.
                 */

                next = entry->vme_next;
                if (next == vm_map_to_entry(map))
                        break;

                /*
                 *      If there is another entry, it must be
                 *      after the end of the potential new region.
                 */

                if (next->vme_start >= end)
                        break;

                /*
                 *      Didn't fit -- move to the next entry.
                 */

                entry = next;
                start = entry->vme_end;
        }

        /*
         *      At this point,
         *              "start" and "end" should define the endpoints of the
         *                      available new range, and
         *              "entry" should refer to the region before the new
         *                      range, and
         *
         *              the map should be locked.
         */

        *address = start;

        new_entry->vme_start = start;
        new_entry->vme_end = end;
        assert(page_aligned(new_entry->vme_start));
        assert(page_aligned(new_entry->vme_end));

        new_entry->is_shared = FALSE;
        new_entry->is_sub_map = FALSE;
        new_entry->use_pmap = FALSE;
        new_entry->object.vm_object = VM_OBJECT_NULL;
        new_entry->offset = (vm_object_offset_t) 0;

        new_entry->needs_copy = FALSE;

        new_entry->inheritance = VM_INHERIT_DEFAULT;
        new_entry->protection = VM_PROT_DEFAULT;
        new_entry->max_protection = VM_PROT_ALL;
        new_entry->behavior = VM_BEHAVIOR_DEFAULT;
        new_entry->wired_count = 0;
        new_entry->user_wired_count = 0;

        new_entry->in_transition = FALSE;
        new_entry->needs_wakeup = FALSE;

        /*
         *      Insert the new entry into the list
         */

        vm_map_entry_link(map, entry, new_entry);

        map->size += size;

        /*
         *      Update the lookup hint
         */
        SAVE_HINT(map, new_entry);

        *o_entry = new_entry;
        return(KERN_SUCCESS);
}

int vm_map_pmap_enter_print = FALSE;
int vm_map_pmap_enter_enable = FALSE;

/*
 *      Routine:        vm_map_pmap_enter
 *
 *      Description:
 *              Force pages from the specified object to be entered into
 *              the pmap at the specified address if they are present.
 *              As soon as a page not found in the object the scan ends.
 *
 *      Returns:
 *              Nothing.  
 *
 *      In/out conditions:
 *              The source map should not be locked on entry.
 */
void
vm_map_pmap_enter(
        vm_map_t                map,
        register vm_offset_t    addr,
        register vm_offset_t    end_addr,
        register vm_object_t    object,
        vm_object_offset_t      offset,
        vm_prot_t               protection)
{
        while (addr < end_addr) {
                register vm_page_t      m;

                vm_object_lock(object);
                vm_object_paging_begin(object);

                m = vm_page_lookup(object, offset);
                if (m == VM_PAGE_NULL || m->busy ||
                    (m->unusual && ( m->error || m->restart || m->absent ||
                                    protection & m->page_lock))) {

                        vm_object_paging_end(object);
                        vm_object_unlock(object);
                        return;
                }

                assert(!m->fictitious); /* XXX is this possible ??? */

                if (vm_map_pmap_enter_print) {
                        printf("vm_map_pmap_enter:");
                        printf("map: %x, addr: %x, object: %x, offset: %x\n",
                                map, addr, object, offset);
                }

                m->busy = TRUE;
                vm_object_unlock(object);

                PMAP_ENTER(map->pmap, addr, m,
                           protection, FALSE);

                vm_object_lock(object);
                PAGE_WAKEUP_DONE(m);
                vm_page_lock_queues();
                if (!m->active && !m->inactive)
                    vm_page_activate(m);
                vm_page_unlock_queues();
                vm_object_paging_end(object);
                vm_object_unlock(object);

                offset += PAGE_SIZE_64;
                addr += PAGE_SIZE;
        }
}

/*
 *      Routine:        vm_map_enter
 *
 *      Description:
 *              Allocate a range in the specified virtual address map.
 *              The resulting range will refer to memory defined by
 *              the given memory object and offset into that object.
 *
 *              Arguments are as defined in the vm_map call.
 */
kern_return_t
vm_map_enter(
        register vm_map_t       map,
        vm_offset_t             *address,       /* IN/OUT */
        vm_size_t               size,
        vm_offset_t             mask,
        int                     flags,
        vm_object_t             object,
        vm_object_offset_t      offset,
        boolean_t               needs_copy,
        vm_prot_t               cur_protection,
        vm_prot_t               max_protection,
        vm_inherit_t            inheritance)
{
        vm_map_entry_t          entry;
        register vm_offset_t    start;
        register vm_offset_t    end;
        kern_return_t           result = KERN_SUCCESS;

        boolean_t               anywhere = VM_FLAGS_ANYWHERE & flags;
        char                    alias;

        VM_GET_FLAGS_ALIAS(flags, alias);

#define RETURN(value)   { result = value; goto BailOut; }

        assert(page_aligned(*address));
        assert(page_aligned(size));
 StartAgain: ;

        start = *address;

        if (anywhere) {
                vm_map_lock(map);

                /*
                 *      Calculate the first possible address.
                 */

                if (start < map->min_offset)
                        start = map->min_offset;
                if (start > map->max_offset)
                        RETURN(KERN_NO_SPACE);

                /*
                 *      Look for the first possible address;
                 *      if there's already something at this
                 *      address, we have to start after it.
                 */

                assert(first_free_is_valid(map));
                if (start == map->min_offset) {
                        if ((entry = map->first_free) != vm_map_to_entry(map))
                                start = entry->vme_end;
                } else {
                        vm_map_entry_t  tmp_entry;
                        if (vm_map_lookup_entry(map, start, &tmp_entry))
                                start = tmp_entry->vme_end;
                        entry = tmp_entry;
                }

                /*
                 *      In any case, the "entry" always precedes
                 *      the proposed new region throughout the
                 *      loop:
                 */

                while (TRUE) {
                        register vm_map_entry_t next;

                        /*
                         *      Find the end of the proposed new region.
                         *      Be sure we didn't go beyond the end, or
                         *      wrap around the address.
                         */

                        end = ((start + mask) & ~mask);
                        if (end < start)
                                RETURN(KERN_NO_SPACE);
                        start = end;
                        end += size;

                        if ((end > map->max_offset) || (end < start)) {
                                if (map->wait_for_space) {
                                        if (size <= (map->max_offset -
                                                     map->min_offset)) {
                                                assert_wait((event_t)map,
                                                            THREAD_ABORTSAFE);
                                                vm_map_unlock(map);
                                                thread_block((void (*)(void))0);
                                                goto StartAgain;
                                        }
                                }
                                RETURN(KERN_NO_SPACE);
                        }

                        /*
                         *      If there are no more entries, we must win.
                         */

                        next = entry->vme_next;
                        if (next == vm_map_to_entry(map))
                                break;

                        /*
                         *      If there is another entry, it must be
                         *      after the end of the potential new region.
                         */

                        if (next->vme_start >= end)
                                break;

                        /*
                         *      Didn't fit -- move to the next entry.
                         */

                        entry = next;
                        start = entry->vme_end;
                }
                *address = start;
        } else {
                vm_map_entry_t          temp_entry;

                /*
                 *      Verify that:
                 *              the address doesn't itself violate
                 *              the mask requirement.
                 */

                vm_map_lock(map);
                if ((start & mask) != 0)
                        RETURN(KERN_NO_SPACE);

                /*
                 *      ...     the address is within bounds
                 */

                end = start + size;

                if ((start < map->min_offset) ||
                    (end > map->max_offset) ||
                    (start >= end)) {
                        RETURN(KERN_INVALID_ADDRESS);
                }

                /*
                 *      ...     the starting address isn't allocated
                 */

                if (vm_map_lookup_entry(map, start, &temp_entry))
                        RETURN(KERN_NO_SPACE);

                entry = temp_entry;

                /*
                 *      ...     the next region doesn't overlap the
                 *              end point.
                 */

                if ((entry->vme_next != vm_map_to_entry(map)) &&
                    (entry->vme_next->vme_start < end))
                        RETURN(KERN_NO_SPACE);
        }

        /*
         *      At this point,
         *              "start" and "end" should define the endpoints of the
         *                      available new range, and
         *              "entry" should refer to the region before the new
         *                      range, and
         *
         *              the map should be locked.
         */

        /*
         *      See whether we can avoid creating a new entry (and object) by
         *      extending one of our neighbors.  [So far, we only attempt to
         *      extend from below.]
         */

        if ((object == VM_OBJECT_NULL) &&
            (entry != vm_map_to_entry(map)) &&
            (entry->vme_end == start) &&
            (!entry->is_shared) &&
            (!entry->is_sub_map) &&
            (entry->alias == alias) &&
            (entry->inheritance == inheritance) &&
            (entry->protection == cur_protection) &&
            (entry->max_protection == max_protection) &&
            (entry->behavior == VM_BEHAVIOR_DEFAULT) &&
            (entry->in_transition == 0) &&
            (entry->wired_count == 0)) { /* implies user_wired_count == 0 */
                if (vm_object_coalesce(entry->object.vm_object,
                                VM_OBJECT_NULL,
                                entry->offset,
                                (vm_object_offset_t) 0,
                                (vm_size_t)(entry->vme_end - entry->vme_start),
                                (vm_size_t)(end - entry->vme_end))) {

                        /*
                         *      Coalesced the two objects - can extend
                         *      the previous map entry to include the
                         *      new range.
                         */
                        map->size += (end - entry->vme_end);
                        entry->vme_end = end;
                        UPDATE_FIRST_FREE(map, map->first_free);
                        RETURN(KERN_SUCCESS);
                }
        }

        /*
         *      Create a new entry
         */

        { /**/
        register vm_map_entry_t new_entry;

        new_entry = vm_map_entry_insert(map, entry, start, end, object,
                                        offset, needs_copy, FALSE, FALSE,
                                        cur_protection, max_protection,
                                        VM_BEHAVIOR_DEFAULT, inheritance, 0);
        new_entry->alias = alias;
        vm_map_unlock(map);

        /*      Wire down the new entry if the user
         *      requested all new map entries be wired.
         */
        if (map->wiring_required) {
                result = vm_map_wire(map, start, end,
                                    new_entry->protection, TRUE);
                return(result);
        }

        if ((object != VM_OBJECT_NULL) &&
            (vm_map_pmap_enter_enable) &&
            (!anywhere)  &&
            (!needs_copy) && 
            (size < (128*1024))) {
                vm_map_pmap_enter(map, start, end, 
                                  object, offset, cur_protection);
        }

        return(result);
        } /**/

 BailOut: ;
        vm_map_unlock(map);
        return(result);

#undef  RETURN
}

/*
 *      vm_map_clip_start:      [ internal use only ]
 *
 *      Asserts that the given entry begins at or after
 *      the specified address; if necessary,
 *      it splits the entry into two.
 */
#ifndef i386
#define vm_map_clip_start(map, entry, startaddr)                        \
MACRO_BEGIN                                                             \
        vm_map_t VMCS_map;                                              \
        vm_map_entry_t VMCS_entry;                                      \
        vm_offset_t VMCS_startaddr;                                     \
        VMCS_map = (map);                                               \
        VMCS_entry = (entry);                                           \
        VMCS_startaddr = (startaddr);                                   \
        if (VMCS_startaddr > VMCS_entry->vme_start) {                   \
                if(entry->use_pmap) {                                   \
                        vm_offset_t     pmap_base_addr;                 \
                        vm_offset_t     pmap_end_addr;                  \
                                                                        \
                        pmap_base_addr = 0xF0000000 & entry->vme_start; \
                        pmap_end_addr = (pmap_base_addr + 0x10000000) - 1; \
                        pmap_unnest(map->pmap, pmap_base_addr,          \
                                (pmap_end_addr - pmap_base_addr) + 1);  \
                        entry->use_pmap = FALSE;                        \
                }                                                       \
                _vm_map_clip_start(&VMCS_map->hdr,VMCS_entry,VMCS_startaddr);\
        }                                                               \
        UPDATE_FIRST_FREE(VMCS_map, VMCS_map->first_free);              \
MACRO_END
#else
#define vm_map_clip_start(map, entry, startaddr)                        \
MACRO_BEGIN                                                             \
        vm_map_t VMCS_map;                                              \
        vm_map_entry_t VMCS_entry;                                      \
        vm_offset_t VMCS_startaddr;                                     \
        VMCS_map = (map);                                               \
        VMCS_entry = (entry);                                           \
        VMCS_startaddr = (startaddr);                                   \
        if (VMCS_startaddr > VMCS_entry->vme_start) {                   \
                _vm_map_clip_start(&VMCS_map->hdr,VMCS_entry,VMCS_startaddr);\
        }                                                               \
        UPDATE_FIRST_FREE(VMCS_map, VMCS_map->first_free);              \
MACRO_END
#endif

#define vm_map_copy_clip_start(copy, entry, startaddr) \
        MACRO_BEGIN \
        if ((startaddr) > (entry)->vme_start) \
                _vm_map_clip_start(&(copy)->cpy_hdr,(entry),(startaddr)); \
        MACRO_END

/*
 *      This routine is called only when it is known that
 *      the entry must be split.
 */
void
_vm_map_clip_start(
        register struct vm_map_header   *map_header,
        register vm_map_entry_t         entry,
        register vm_offset_t            start)
{
        register vm_map_entry_t new_entry;

        /*
         *      Split off the front portion --
         *      note that we must insert the new
         *      entry BEFORE this one, so that
         *      this entry has the specified starting
         *      address.
         */

        new_entry = _vm_map_entry_create(map_header);
        vm_map_entry_copy_full(new_entry, entry);

        new_entry->vme_end = start;
        entry->offset += (start - entry->vme_start);
        entry->vme_start = start;

        _vm_map_entry_link(map_header, entry->vme_prev, new_entry);

        if (entry->is_sub_map)
                vm_map_reference(new_entry->object.sub_map);
        else
                vm_object_reference(new_entry->object.vm_object);
}


/*
 *      vm_map_clip_end:        [ internal use only ]
 *
 *      Asserts that the given entry ends at or before
 *      the specified address; if necessary,
 *      it splits the entry into two.
 */
#ifndef i386
#define vm_map_clip_end(map, entry, endaddr)                            \
MACRO_BEGIN                                                             \
        vm_map_t VMCE_map;                                              \
        vm_map_entry_t VMCE_entry;                                      \
        vm_offset_t VMCE_endaddr;                                       \
        VMCE_map = (map);                                               \
        VMCE_entry = (entry);                                           \
        VMCE_endaddr = (endaddr);                                       \
        if (VMCE_endaddr < VMCE_entry->vme_end) {                       \
                if(entry->use_pmap) {                                   \
                        vm_offset_t     pmap_base_addr;                 \
                        vm_offset_t     pmap_end_addr;                  \
                                                                        \
                        pmap_base_addr = 0xF0000000 & entry->vme_start; \
                        pmap_end_addr = (pmap_base_addr + 0x10000000) - 1; \
                        pmap_unnest(map->pmap, pmap_base_addr,          \
                                (pmap_end_addr - pmap_base_addr) + 1);  \
                        entry->use_pmap = FALSE;                        \
                }                                                       \
                _vm_map_clip_end(&VMCE_map->hdr,VMCE_entry,VMCE_endaddr); \
        }                                                               \
        UPDATE_FIRST_FREE(VMCE_map, VMCE_map->first_free);              \
MACRO_END
#else
#define vm_map_clip_end(map, entry, endaddr)                            \
MACRO_BEGIN                                                             \
        vm_map_t VMCE_map;                                              \
        vm_map_entry_t VMCE_entry;                                      \
        vm_offset_t VMCE_endaddr;                                       \
        VMCE_map = (map);                                               \
        VMCE_entry = (entry);                                           \
        VMCE_endaddr = (endaddr);                                       \
        if (VMCE_endaddr < VMCE_entry->vme_end) {                       \
                _vm_map_clip_end(&VMCE_map->hdr,VMCE_entry,VMCE_endaddr); \
        }                                                               \
        UPDATE_FIRST_FREE(VMCE_map, VMCE_map->first_free);              \
MACRO_END
#endif

#define vm_map_copy_clip_end(copy, entry, endaddr) \
        MACRO_BEGIN \
        if ((endaddr) < (entry)->vme_end) \
                _vm_map_clip_end(&(copy)->cpy_hdr,(entry),(endaddr)); \
        MACRO_END

/*
 *      This routine is called only when it is known that
 *      the entry must be split.
 */
void
_vm_map_clip_end(
        register struct vm_map_header   *map_header,
        register vm_map_entry_t         entry,
        register vm_offset_t            end)
{
        register vm_map_entry_t new_entry;

        /*
         *      Create a new entry and insert it
         *      AFTER the specified entry
         */

        new_entry = _vm_map_entry_create(map_header);
        vm_map_entry_copy_full(new_entry, entry);

        new_entry->vme_start = entry->vme_end = end;
        new_entry->offset += (end - entry->vme_start);

        _vm_map_entry_link(map_header, entry, new_entry);

        if (entry->is_sub_map)
                vm_map_reference(new_entry->object.sub_map);
        else
                vm_object_reference(new_entry->object.vm_object);
}


/*
 *      VM_MAP_RANGE_CHECK:     [ internal use only ]
 *
 *      Asserts that the starting and ending region
 *      addresses fall within the valid range of the map.
 */
#define VM_MAP_RANGE_CHECK(map, start, end)             \
                {                                       \
                if (start < vm_map_min(map))            \
                        start = vm_map_min(map);        \
                if (end > vm_map_max(map))              \
                        end = vm_map_max(map);          \
                if (start > end)                        \
                        start = end;                    \
                }

/*
 *      vm_map_range_check:     [ internal use only ]
 *      
 *      Check that the region defined by the specified start and
 *      end addresses are wholly contained within a single map
 *      entry or set of adjacent map entries of the spacified map,
 *      i.e. the specified region contains no unmapped space.
 *      If any or all of the region is unmapped, FALSE is returned.
 *      Otherwise, TRUE is returned and if the output argument 'entry'
 *      is not NULL it points to the map entry containing the start
 *      of the region.
 *
 *      The map is locked for reading on entry and is left locked.
 */
boolean_t
vm_map_range_check(
        register vm_map_t       map,
        register vm_offset_t    start,
        register vm_offset_t    end,
        vm_map_entry_t          *entry)
{
        vm_map_entry_t          cur;
        register vm_offset_t    prev;

        /*
         *      Basic sanity checks first
         */
        if (start < vm_map_min(map) || end > vm_map_max(map) || start > end)
                return (FALSE);

        /*
         *      Check first if the region starts within a valid
         *      mapping for the map.
         */
        if (!vm_map_lookup_entry(map, start, &cur))
                return (FALSE);

        /*
         *      Optimize for the case that the region is contained 
         *      in a single map entry.
         */
        if (entry != (vm_map_entry_t *) NULL)
                *entry = cur;
        if (end <= cur->vme_end)
                return (TRUE);

        /*
         *      If the region is not wholly contained within a
         *      single entry, walk the entries looking for holes.
         */
        prev = cur->vme_end;
        cur = cur->vme_next;
        while ((cur != vm_map_to_entry(map)) && (prev == cur->vme_start)) {
                if (end <= cur->vme_end)
                        return (TRUE);
                prev = cur->vme_end;
                cur = cur->vme_next;
        }
        return (FALSE);
}

/*
 *      vm_map_submap:          [ kernel use only ]
 *
 *      Mark the given range as handled by a subordinate map.
 *
 *      This range must have been created with vm_map_find using
 *      the vm_submap_object, and no other operations may have been
 *      performed on this range prior to calling vm_map_submap.
 *
 *      Only a limited number of operations can be performed
 *      within this rage after calling vm_map_submap:
 *              vm_fault
 *      [Don't try vm_map_copyin!]
 *
 *      To remove a submapping, one must first remove the
 *      range from the superior map, and then destroy the
 *      submap (if desired).  [Better yet, don't try it.]
 */
kern_return_t
vm_map_submap(
        register vm_map_t       map,
        register vm_offset_t    start,
        register vm_offset_t    end,
        vm_map_t                submap,
        vm_offset_t             offset,
        boolean_t               use_pmap)
{
        vm_map_entry_t          entry;
        register kern_return_t  result = KERN_INVALID_ARGUMENT;
        register vm_object_t    object;

        vm_map_lock(map);

        VM_MAP_RANGE_CHECK(map, start, end);

        if (vm_map_lookup_entry(map, start, &entry)) {
                vm_map_clip_start(map, entry, start);
        }
        else
                entry = entry->vme_next;

        if(entry == vm_map_to_entry(map)) {
                vm_map_unlock(map);
                return KERN_INVALID_ARGUMENT;
        }

        vm_map_clip_end(map, entry, end);

        if ((entry->vme_start == start) && (entry->vme_end == end) &&
            (!entry->is_sub_map) &&
            ((object = entry->object.vm_object) == vm_submap_object) &&
            (object->resident_page_count == 0) &&
            (object->copy == VM_OBJECT_NULL) &&
            (object->shadow == VM_OBJECT_NULL) &&
            (!object->pager_created)) {
                entry->offset = (vm_object_offset_t)offset;
                entry->object.vm_object = VM_OBJECT_NULL;
                vm_object_deallocate(object);
                entry->is_sub_map = TRUE;
                vm_map_reference(entry->object.sub_map = submap);
#ifndef i386
                if ((use_pmap) && (offset == 0)) {
                        /* nest if platform code will allow */
                        result = pmap_nest(map->pmap, (entry->object.sub_map)->pmap, 
                                                        start, end - start);
                        if(result)
                                panic("pmap_nest failed!");
                        entry->use_pmap = TRUE;
                }
#endif
#ifdef i386
                pmap_remove(map->pmap, start, end);
#endif
                result = KERN_SUCCESS;
        }
        vm_map_unlock(map);

        return(result);
}

/*
 *      vm_map_protect:
 *
 *      Sets the protection of the specified address
 *      region in the target map.  If "set_max" is
 *      specified, the maximum protection is to be set;
 *      otherwise, only the current protection is affected.
 */
kern_return_t
vm_map_protect(
        register vm_map_t       map,
        register vm_offset_t    start,
        register vm_offset_t    end,
        register vm_prot_t      new_prot,
        register boolean_t      set_max)
{
        register vm_map_entry_t         current;
        register vm_offset_t            prev;
        vm_map_entry_t                  entry;
        vm_prot_t                       new_max;
        boolean_t                       clip;

        XPR(XPR_VM_MAP,
                "vm_map_protect, 0x%X start 0x%X end 0x%X, new 0x%X %d",
                (integer_t)map, start, end, new_prot, set_max);

        vm_map_lock(map);

        /*
         *      Lookup the entry.  If it doesn't start in a valid
         *      entry, return an error.  Remember if we need to
         *      clip the entry.  We don't do it here because we don't
         *      want to make any changes until we've scanned the 
         *      entire range below for address and protection
         *      violations.
         */
        if (!(clip = vm_map_lookup_entry(map, start, &entry))) {
                vm_map_unlock(map);
                return(KERN_INVALID_ADDRESS);
        }

        /*
         *      Make a first pass to check for protection and address
         *      violations.
         */

        current = entry;
        prev = current->vme_start;
        while ((current != vm_map_to_entry(map)) &&
               (current->vme_start < end)) {

                /*
                 * If there is a hole, return an error.
                 */
                if (current->vme_start != prev) {
                        vm_map_unlock(map);
                        return(KERN_INVALID_ADDRESS);
                }

                new_max = current->max_protection;
                if(new_prot & VM_PROT_COPY) {
                        new_max |= VM_PROT_WRITE;
                        if ((new_prot & (new_max | VM_PROT_COPY)) != new_prot) {
                                vm_map_unlock(map);
                                return(KERN_PROTECTION_FAILURE);
                        }
                } else {
                        if ((new_prot & new_max) != new_prot) {
                                vm_map_unlock(map);
                                return(KERN_PROTECTION_FAILURE);
                        }
                }

                prev = current->vme_end;
                current = current->vme_next;
        }
        if (end > prev) {
                vm_map_unlock(map);
                return(KERN_INVALID_ADDRESS);
        }

        /*
         *      Go back and fix up protections.
         *      Clip to start here if the range starts within
         *      the entry.
         */

        current = entry;
        if (clip) {
                vm_map_clip_start(map, entry, start);
        }
        while ((current != vm_map_to_entry(map)) &&
               (current->vme_start < end)) {

                vm_prot_t       old_prot;

                vm_map_clip_end(map, current, end);

                old_prot = current->protection;

                if(new_prot & VM_PROT_COPY) {
                        /* caller is asking specifically to copy the      */
                        /* mapped data, this implies that max protection  */
                        /* will include write.  Caller must be prepared   */
                        /* for loss of shared memory communication in the */
                        /* target area after taking this step */
                        current->needs_copy = TRUE;
                        current->max_protection |= VM_PROT_WRITE;
                }

                if (set_max)
                        current->protection =
                                (current->max_protection = 
                                        new_prot & ~VM_PROT_COPY) &
                                        old_prot;
                else
                        current->protection = new_prot & ~VM_PROT_COPY;

                /*
                 *      Update physical map if necessary.
                 *      If the request is to turn off write protection, 
                 *      we won't do it for real (in pmap). This is because 
                 *      it would cause copy-on-write to fail.  We've already 
                 *      set, the new protection in the map, so if a 
                 *      write-protect fault occurred, it will be fixed up 
                 *      properly, COW or not.
                 */
                /* the 256M hack for existing hardware limitations */
                if (current->protection != old_prot) {
                   if(current->is_sub_map && current->use_pmap) {
                        vm_offset_t     pmap_base_addr;
                        vm_offset_t     pmap_end_addr;
                        vm_map_entry_t  local_entry;

                        pmap_base_addr = 0xF0000000 & current->vme_start;
                        pmap_end_addr = (pmap_base_addr + 0x10000000) - 1;
#ifndef i386
                        if(!vm_map_lookup_entry(map, 
                                        pmap_base_addr, &local_entry))
                           panic("vm_map_protect: nested pmap area is missing");
                           while ((local_entry != vm_map_to_entry(map)) &&
                                  (local_entry->vme_start < pmap_end_addr)) {
                                local_entry->use_pmap = FALSE;
                                local_entry = local_entry->vme_next;
                           }
                           pmap_unnest(map->pmap, pmap_base_addr,
                                        (pmap_end_addr - pmap_base_addr) + 1);
#endif
                   }
                   if (!(current->protection & VM_PROT_WRITE)) {
                        /* Look one level in we support nested pmaps */
                        /* from mapped submaps which are direct entries */
                        /* in our map */
                        if(current->is_sub_map && current->use_pmap) {
                                pmap_protect(current->object.sub_map->pmap, 
                                        current->vme_start,
                                        current->vme_end,
                                        current->protection);
                        } else {
                                pmap_protect(map->pmap, current->vme_start,
                                        current->vme_end,
                                        current->protection);
                        }
                   }
                }
                current = current->vme_next;
        }

        vm_map_unlock(map);
        return(KERN_SUCCESS);
}

/*
 *      vm_map_inherit:
 *
 *      Sets the inheritance of the specified address
 *      range in the target map.  Inheritance
 *      affects how the map will be shared with
 *      child maps at the time of vm_map_fork.
 */
kern_return_t
vm_map_inherit(
        register vm_map_t       map,
        register vm_offset_t    start,
        register vm_offset_t    end,
        register vm_inherit_t   new_inheritance)
{
        register vm_map_entry_t entry;
        vm_map_entry_t  temp_entry;

        vm_map_lock(map);

        VM_MAP_RANGE_CHECK(map, start, end);

        if (vm_map_lookup_entry(map, start, &temp_entry)) {
                entry = temp_entry;
                vm_map_clip_start(map, entry, start);
        }
        else {
                temp_entry = temp_entry->vme_next;
                entry = temp_entry;
        }

        /* first check entire range for submaps which can't support the */
        /* given inheritance. */
        while ((entry != vm_map_to_entry(map)) && (entry->vme_start < end)) {
                if(entry->is_sub_map) {
                        if(new_inheritance == VM_INHERIT_COPY)
                                return(KERN_INVALID_ARGUMENT);
                }

                entry = entry->vme_next;
        }

        entry = temp_entry;

        while ((entry != vm_map_to_entry(map)) && (entry->vme_start < end)) {
                vm_map_clip_end(map, entry, end);

                entry->inheritance = new_inheritance;

                entry = entry->vme_next;
        }

        vm_map_unlock(map);
        return(KERN_SUCCESS);
}

/*
 *      vm_map_wire:
 *
 *      Sets the pageability of the specified address range in the
 *      target map as wired.  Regions specified as not pageable require
 *      locked-down physical memory and physical page maps.  The
 *      access_type variable indicates types of accesses that must not
 *      generate page faults.  This is checked against protection of
 *      memory being locked-down.
 *
 *      The map must not be locked, but a reference must remain to the
 *      map throughout the call.
 */
kern_return_t
vm_map_wire_nested(
        register vm_map_t       map,
        register vm_offset_t    start,
        register vm_offset_t    end,
        register vm_prot_t      access_type,
        boolean_t               user_wire,
        pmap_t                  map_pmap)
{
        register vm_map_entry_t entry;
        struct vm_map_entry     *first_entry, tmp_entry;
        vm_map_t                pmap_map;
        register vm_offset_t    s,e;
        kern_return_t           rc;
        boolean_t               need_wakeup;
        boolean_t               main_map = FALSE;
        unsigned int            last_timestamp;
        vm_size_t               size;

        vm_map_lock(map);
        if(map_pmap == NULL)
                main_map = TRUE;
        last_timestamp = map->timestamp;

        VM_MAP_RANGE_CHECK(map, start, end);
        assert(page_aligned(start));
        assert(page_aligned(end));

        if (vm_map_lookup_entry(map, start, &first_entry)) {
                entry = first_entry;
                /* vm_map_clip_start will be done later. */
        } else {
                /* Start address is not in map */
                vm_map_unlock(map);
                return(KERN_INVALID_ADDRESS);
        }

        s=start;
        need_wakeup = FALSE;
        while ((entry != vm_map_to_entry(map)) && (entry->vme_start < end)) {
                /*
                 * If another thread is wiring/unwiring this entry then
                 * block after informing other thread to wake us up.
                 */
                if (entry->in_transition) {
                        /*
                         * We have not clipped the entry.  Make sure that
                         * the start address is in range so that the lookup
                         * below will succeed.
                         */
                        s = entry->vme_start < start? start: entry->vme_start;

                        entry->needs_wakeup = TRUE;

                        /*
                         * wake up anybody waiting on entries that we have
                         * already wired.
                         */
                        if (need_wakeup) {
                                vm_map_entry_wakeup(map);
                                need_wakeup = FALSE;
                        }
                        /*
                         * User wiring is interruptible
                         */
                        vm_map_entry_wait(map, 
                                          (user_wire) ? THREAD_ABORTSAFE :
                                                        THREAD_UNINT);
                        if (user_wire && current_thread()->wait_result ==
                                                        THREAD_INTERRUPTED) {
                                /*
                                 * undo the wirings we have done so far
                                 * We do not clear the needs_wakeup flag,
                                 * because we cannot tell if we were the
                                 * only one waiting.
                                 */
                                vm_map_unwire(map, start, s, user_wire);
                                return(KERN_FAILURE);
                        }

                        vm_map_lock(map);
                        /*
                         * Cannot avoid a lookup here. reset timestamp.
                         */
                        last_timestamp = map->timestamp;

                        /*
                         * The entry could have been clipped, look it up again.
                         * Worse that can happen is, it may not exist anymore.
                         */
                        if (!vm_map_lookup_entry(map, s, &first_entry)) {
                                if (!user_wire)
                                        panic("vm_map_wire: re-lookup failed");

                                /*
                                 * User: undo everything upto the previous
                                 * entry.  let vm_map_unwire worry about
                                 * checking the validity of the range.
                                 */
                                vm_map_unlock(map);
                                vm_map_unwire(map, start, s, user_wire);
                                return(KERN_FAILURE);
                        }
                        entry = first_entry;
                        continue;
                }
                
                if(entry->is_sub_map) {
                        vm_offset_t     sub_start;
                        vm_offset_t     sub_end;
                        vm_offset_t     local_end;
                        pmap_t          pmap;
                        
                        vm_map_clip_start(map, entry, start);
                        vm_map_clip_end(map, entry, end);

                        sub_start += entry->offset;
                        sub_end = entry->vme_end - entry->vme_start;
                        sub_end += entry->offset;
                        
                        local_end = entry->vme_end;
                        if(map_pmap == NULL) {
                                if(entry->use_pmap) {
                                        pmap = entry->object.sub_map->pmap;
                                } else {
                                        pmap = map->pmap;
                                }
                                if (entry->wired_count) {
                                        if (entry->wired_count 
                                                        >= MAX_WIRE_COUNT)
                                        panic("vm_map_wire: too many wirings");

                                        if (user_wire &&
                                                entry->user_wired_count 
                                                        >= MAX_WIRE_COUNT) {
                                           vm_map_unlock(map);
                                           vm_map_unwire(map, start,
                                                entry->vme_start, user_wire);
                                           return(KERN_FAILURE);
                                        }
                                        if (!user_wire || 
                                              (entry->user_wired_count++ == 0))
                                                entry->wired_count++;
                                        entry = entry->vme_next;
                                        continue;

                                } else {
                                        vm_object_t             object;
                                        vm_object_offset_t      offset_hi;
                                        vm_object_offset_t      offset_lo;
                                        vm_object_offset_t      offset;
                                        vm_prot_t               prot;
                                        boolean_t               wired;
                                        vm_behavior_t           behavior;
                                        vm_offset_t             local_start;
                                        vm_map_entry_t          local_entry;
                                        vm_map_version_t         version;
                                        vm_map_t                lookup_map;

                                        /* call vm_map_lookup_locked to */
                                        /* cause any needs copy to be   */
                                        /* evaluated */
                                        local_start = entry->vme_start;
                                        lookup_map = map;
                                        vm_map_lock_write_to_read(map);
                                        if(vm_map_lookup_locked(
                                                &lookup_map, local_start, 
                                                VM_PROT_WRITE,
                                                &version, &object,
                                                &offset, &prot, &wired,
                                                &behavior, &offset_lo,
                                                &offset_hi, &pmap_map)) {
                                                
                                                vm_map_unlock(lookup_map);
                                                vm_map_unwire(map, start,
                                                   entry->vme_start, user_wire);
                                                return(KERN_FAILURE);
                                        }
                                        if(pmap_map != lookup_map)
                                                vm_map_unlock(pmap_map);
                                        if(lookup_map != map) {
                                                vm_map_unlock(lookup_map);
                                                vm_map_lock(map);
                                        } else {
                                                vm_map_unlock(map);
                                                vm_map_lock(map);
                                        }
                                        last_timestamp = 
                                                version.main_timestamp;
                                        vm_object_unlock(object);
                                        if (vm_map_lookup_entry(map, 
                                                local_start, &local_entry)) {
                                                vm_map_unlock(map);
                                                vm_map_unwire(map, start,
                                                   entry->vme_start, user_wire);
                                                return(KERN_FAILURE);
                                        }
                                        /* did we have a change of type? */
                                        if (!local_entry->is_sub_map)
                                                continue;
                                        entry = local_entry;
                                        if (user_wire)
                                                entry->user_wired_count++;
                                        entry->wired_count++;

                                        entry->in_transition = TRUE;

                                        vm_map_unlock(map);
                                        rc = vm_map_wire_nested(
                                                entry->object.sub_map, 
                                                sub_start, sub_end,
                                                access_type, 
                                                user_wire, pmap);
                                        vm_map_lock(map);
                                        last_timestamp = map->timestamp;
                                }
                        } else {
                                vm_map_unlock(map);
                                rc = vm_map_wire_nested(entry->object.sub_map, 
                                                sub_start, sub_end,
                                                access_type, 
                                                user_wire, pmap);
                                vm_map_lock(map);
                                last_timestamp = map->timestamp;
                        }
                        s = entry->vme_start;
                        e = entry->vme_end;
                        if (last_timestamp+1 != map->timestamp) {
                        /*
                         * Find the entry again.  It could have been clipped
                         * after we unlocked the map.
                         */
                                if (!vm_map_lookup_entry(map, local_end,
                                                         &first_entry))
                                        panic("vm_map_wire: re-lookup failed");

                                entry = first_entry;
                        }

                        last_timestamp = map->timestamp;
                        while ((entry != vm_map_to_entry(map)) &&
                                     (entry->vme_start < e)) {
                                assert(entry->in_transition);
                                entry->in_transition = FALSE;
                                if (entry->needs_wakeup) {
                                        entry->needs_wakeup = FALSE;
                                        need_wakeup = TRUE;
                                }
                                if (rc != KERN_SUCCESS) {/* from vm_*_wire */
                                   if(main_map) {
                                        if (user_wire)
                                                entry->user_wired_count--;
                                        entry->wired_count--;
                                   }
                                }
                                entry = entry->vme_next;
                        }
                        if (rc != KERN_SUCCESS) {       /* from vm_*_wire */
                                vm_map_unlock(map);
                                if (need_wakeup)
                                        vm_map_entry_wakeup(map);
                                /*
                                 * undo everything upto the previous entry.
                                 */
                                (void)vm_map_unwire(map, start, s, user_wire);
                                return rc;
                        }
                        continue;
                }

                /*
                 * If this entry is already wired then increment
                 * the appropriate wire reference count.
                 */
                if (entry->wired_count && main_map) {
                        /* sanity check: wired_count is a short */
                        if (entry->wired_count >= MAX_WIRE_COUNT)
                                panic("vm_map_wire: too many wirings");

                        if (user_wire &&
                            entry->user_wired_count >= MAX_WIRE_COUNT) {
                                vm_map_unlock(map);
                                vm_map_unwire(map, start,
                                                entry->vme_start, user_wire);
                                return(KERN_FAILURE);
                        }
                        /*
                         * entry is already wired down, get our reference
                         * after clipping to our range.
                         */
                        vm_map_clip_start(map, entry, start);
                        vm_map_clip_end(map, entry, end);
                        if (!user_wire || (entry->user_wired_count++ == 0))
                                entry->wired_count++;

                        entry = entry->vme_next;
                        continue;
                }

                /*
                 * Unwired entry or wire request transmitted via submap
                 */


                /*
                 * Perform actions of vm_map_lookup that need the write
                 * lock on the map: create a shadow object for a
                 * copy-on-write region, or an object for a zero-fill
                 * region.
                 */
                size = entry->vme_end - entry->vme_start;
                /*
                 * If wiring a copy-on-write page, we need to copy it now
                 * even if we're only (currently) requesting read access.
                 * This is aggressive, but once it's wired we can't move it.
                 */
                if (entry->needs_copy) {
                        vm_object_shadow(&entry->object.vm_object,
                                         &entry->offset, size);
                        entry->needs_copy = FALSE;
                } else if (entry->object.vm_object == VM_OBJECT_NULL) {
                        entry->object.vm_object = vm_object_allocate(size);
                        entry->offset = (vm_object_offset_t)0;
                }

                vm_map_clip_start(map, entry, start);
                vm_map_clip_end(map, entry, end);

                s = entry->vme_start;
                e = entry->vme_end;

                /*
                 * Check for holes and protection mismatch.
                 * Holes: Next entry should be contiguous unless this
                 *        is the end of the region.
                 * Protection: Access requested must be allowed, unless
                 *      wiring is by protection class
                 */
                if ((((entry->vme_end < end) &&
                     ((entry->vme_next == vm_map_to_entry(map)) ||
                      (entry->vme_next->vme_start > entry->vme_end))) ||
                     ((entry->protection & access_type) != access_type))) {
                        /*
                         * Found a hole or protection problem.
                         * Unwire the region we wired so far.
                         */
                        if (start != entry->vme_start) {
                                vm_map_unlock(map);
                                vm_map_unwire(map, start, s, user_wire);
                        } else {
                                vm_map_unlock(map);
                        }
                        return((entry->protection&access_type) != access_type?
                                KERN_PROTECTION_FAILURE: KERN_INVALID_ADDRESS);
                }

                assert(entry->wired_count == 0 && entry->user_wired_count == 0);

                if(main_map) {
                        if (user_wire)
                                entry->user_wired_count++;
                        entry->wired_count++;
                }

                entry->in_transition = TRUE;

                /*
                 * This entry might get split once we unlock the map.
                 * In vm_fault_wire(), we need the current range as
                 * defined by this entry.  In order for this to work
                 * along with a simultaneous clip operation, we make a
                 * temporary copy of this entry and use that for the
                 * wiring.  Note that the underlying objects do not
                 * change during a clip.
                 */
                tmp_entry = *entry;

                /*
                 * The in_transition state guarentees that the entry
                 * (or entries for this range, if split occured) will be
                 * there when the map lock is acquired for the second time.
                 */
                vm_map_unlock(map);
                if(map_pmap)
                        rc = vm_fault_wire(map, &tmp_entry, map_pmap);
                else
                        rc = vm_fault_wire(map, &tmp_entry, map->pmap);
                vm_map_lock(map);

                if (last_timestamp+1 != map->timestamp) {
                        /*
                         * Find the entry again.  It could have been clipped
                         * after we unlocked the map.
                         */
                        if (!vm_map_lookup_entry(map, tmp_entry.vme_start,
                                                                &first_entry))
                                panic("vm_map_wire: re-lookup failed");

                        entry = first_entry;
                }

                last_timestamp = map->timestamp;

                while ((entry != vm_map_to_entry(map)) &&
                       (entry->vme_start < tmp_entry.vme_end)) {
                        assert(entry->in_transition);
                        entry->in_transition = FALSE;
                        if (entry->needs_wakeup) {
                                entry->needs_wakeup = FALSE;
                                need_wakeup = TRUE;
                        }
                        if (rc != KERN_SUCCESS) {       /* from vm_*_wire */
                                if(main_map) {
                                        if (user_wire)
                                                entry->user_wired_count--;
                                        entry->wired_count--;
                                }
                        }
                        entry = entry->vme_next;
                }

                if (rc != KERN_SUCCESS) {               /* from vm_*_wire */
                        vm_map_unlock(map);
                        if (need_wakeup)
                                vm_map_entry_wakeup(map);
                        /*
                         * undo everything upto the previous entry.
                         */
                        (void)vm_map_unwire(map, start, s, user_wire);
                        return rc;
                }
        } /* end while loop through map entries */
        vm_map_unlock(map);

        /*
         * wake up anybody waiting on entries we wired.
         */
        if (need_wakeup)
                vm_map_entry_wakeup(map);

        return(KERN_SUCCESS);

}

kern_return_t
vm_map_wire(
        register vm_map_t       map,
        register vm_offset_t    start,
        register vm_offset_t    end,
        register vm_prot_t      access_type,
        boolean_t               user_wire)
{

        kern_return_t   kret;

#ifdef ppc
        /*
         * the calls to mapping_prealloc and mapping_relpre
         * (along with the VM_MAP_RANGE_CHECK to insure a
         * resonable range was passed in) are
         * currently necessary because
         * we haven't enabled kernel pre-emption
         * and/or the pmap_enter cannot purge and re-use
         * existing mappings
         */
        VM_MAP_RANGE_CHECK(map, start, end);
        mapping_prealloc(end - start);
#endif
        kret = vm_map_wire_nested(map, start, end, access_type, 
                                                user_wire, (pmap_t)NULL);
#ifdef ppc
        mapping_relpre();
#endif
        return kret;
}

/*
 *      vm_map_unwire:
 *
 *      Sets the pageability of the specified address range in the target
 *      as pageable.  Regions specified must have been wired previously.
 *
 *      The map must not be locked, but a reference must remain to the map
 *      throughout the call.
 *
 *      Kernel will panic on failures.  User unwire ignores holes and
 *      unwired and intransition entries to avoid losing memory by leaving
 *      it unwired.
 */
kern_return_t
vm_map_unwire_nested(
        register vm_map_t       map,
        register vm_offset_t    start,
        register vm_offset_t    end,
        boolean_t               user_wire,
        pmap_t                  map_pmap)
{
        register vm_map_entry_t entry;
        struct vm_map_entry     *first_entry, tmp_entry;
        boolean_t               need_wakeup;
        boolean_t               main_map = FALSE;
        unsigned int            last_timestamp;

        vm_map_lock(map);
        if(map_pmap == NULL)
                main_map = TRUE;
        last_timestamp = map->timestamp;

        VM_MAP_RANGE_CHECK(map, start, end);
        assert(page_aligned(start));
        assert(page_aligned(end));

        if (vm_map_lookup_entry(map, start, &first_entry)) {
                entry = first_entry;
                /*      vm_map_clip_start will be done later. */
        }
        else {
                /*      Start address is not in map. */
                vm_map_unlock(map);
                return(KERN_INVALID_ADDRESS);
        }

        need_wakeup = FALSE;
        while ((entry != vm_map_to_entry(map)) && (entry->vme_start < end)) {
                if (entry->in_transition) {
                        /*
                         * 1)
                         * Another thread is wiring down this entry. Note
                         * that if it is not for the other thread we would
                         * be unwiring an unwired entry.  This is not
                         * permitted.  If we wait, we will be unwiring memory
                         * we did not wire.
                         *
                         * 2)
                         * Another thread is unwiring this entry.  We did not
                         * have a reference to it, because if we did, this
                         * entry will not be getting unwired now.
                         */
                        if (!user_wire)
                                panic("vm_map_unwire: in_transition entry");

                        entry = entry->vme_next;
                        continue;
                }

                if(entry->is_sub_map) {
                        vm_offset_t     sub_start;
                        vm_offset_t     sub_end;
                        vm_offset_t     local_end;
                        pmap_t          pmap;
                        

                        vm_map_clip_start(map, entry, start);
                        vm_map_clip_end(map, entry, end);

                        sub_start = entry->offset;
                        sub_end = entry->vme_end - entry->vme_start;
                        sub_end += entry->offset;
                        local_end = entry->vme_end;
                        if(map_pmap == NULL) {
                           if(entry->use_pmap) {
                                        pmap = entry->object.sub_map->pmap;
                           } else {
                                        pmap = map->pmap;
                           }
                           if (entry->wired_count == 0 ||
                               (user_wire && entry->user_wired_count == 0)) {
                                if (!user_wire)
                                   panic("vm_map_unwire: entry is unwired");
                              entry = entry->vme_next;
                              continue;
                           }

                           /*
                            * Check for holes
                            * Holes: Next entry should be contiguous unless
                            * this is the end of the region.
                            */
                           if (((entry->vme_end < end) && 
                                ((entry->vme_next == vm_map_to_entry(map)) ||
                                (entry->vme_next->vme_start 
                                                > entry->vme_end)))) {
                                if (!user_wire)
                                  panic("vm_map_unwire: non-contiguous region");
/*
                                entry = entry->vme_next;
                                continue;
*/
                           }

                           if (!user_wire || (--entry->user_wired_count == 0))
                                entry->wired_count--;

                           if (entry->wired_count != 0) {
                                entry = entry->vme_next;
                                continue;
                           }

                           entry->in_transition = TRUE;
                           tmp_entry = *entry;/* see comment in vm_map_wire() */

                           /*
                            * We can unlock the map now. The in_transition state
                            * guarantees existance of the entry.
                            */
                           vm_map_unlock(map);
                           vm_map_unwire_nested(entry->object.sub_map, 
                                        sub_start, sub_end, user_wire, pmap);
                           vm_map_lock(map);

                           if (last_timestamp+1 != map->timestamp) {
                                /*
                                 * Find the entry again.  It could have been 
                                 * clipped or deleted after we unlocked the map.
                                 */
                                if (!vm_map_lookup_entry(map, 
                                                tmp_entry.vme_start,
                                                        &first_entry)) {
                                        if (!user_wire)
                                          panic("vm_map_unwire: re-lookup failed");
                                        entry = first_entry->vme_next;
                                } else
                                        entry = first_entry;
                           }
                           last_timestamp = map->timestamp;

                           /*
                            * clear transition bit for all constituent entries
                            * that were in the original entry (saved in 
                            * tmp_entry).  Also check for waiters.
                            */
                           while ((entry != vm_map_to_entry(map)) &&
                                (entry->vme_start < tmp_entry.vme_end)) {
                                assert(entry->in_transition);
                                entry->in_transition = FALSE;
                                if (entry->needs_wakeup) {
                                        entry->needs_wakeup = FALSE;
                                        need_wakeup = TRUE;
                                }
                                entry = entry->vme_next;
                           }
                           continue;
                        } else {
                           vm_map_unlock(map);
                           vm_map_unwire_nested(entry->object.sub_map, 
                                        sub_start, sub_end, user_wire, pmap);
                           vm_map_lock(map);

                           if (last_timestamp+1 != map->timestamp) {
                                /*
                                 * Find the entry again.  It could have been 
                                 * clipped or deleted after we unlocked the map.
                                 */
                                if (!vm_map_lookup_entry(map, 
                                                tmp_entry.vme_start,
                                                        &first_entry)) {
                                        if (!user_wire)
                                          panic("vm_map_unwire: re-lookup failed");
                                        entry = first_entry->vme_next;
                                } else
                                        entry = first_entry;
                           }
                           last_timestamp = map->timestamp;
                        }
                }


                if (main_map && (entry->wired_count == 0 ||
                   (user_wire && entry->user_wired_count == 0))) {
                        if (!user_wire)
                                panic("vm_map_unwire: entry is unwired");

                        entry = entry->vme_next;
                        continue;
                }
                
                assert(entry->wired_count > 0 &&
                        (!user_wire || entry->user_wired_count > 0));

                vm_map_clip_start(map, entry, start);
                vm_map_clip_end(map, entry, end);

                /*
                 * Check for holes
                 * Holes: Next entry should be contiguous unless
                 *        this is the end of the region.
                 */
                if (((entry->vme_end < end) && 
                    ((entry->vme_next == vm_map_to_entry(map)) ||
                     (entry->vme_next->vme_start > entry->vme_end)))) {

                        if (!user_wire)
                                panic("vm_map_unwire: non-contiguous region");
                        entry = entry->vme_next;
                        continue;
                }

                if(main_map) {
                   if (!user_wire || (--entry->user_wired_count == 0))
                        entry->wired_count--;

                   if (entry->wired_count != 0) {
                        entry = entry->vme_next;
                        continue;
                   }
                }

                entry->in_transition = TRUE;
                tmp_entry = *entry;     /* see comment in vm_map_wire() */

                /*
                 * We can unlock the map now. The in_transition state
                 * guarantees existance of the entry.
                 */
                vm_map_unlock(map);
                if(map_pmap) {
                        vm_fault_unwire(map, &tmp_entry, FALSE, map_pmap);
                } else {
                        vm_fault_unwire(map, &tmp_entry, FALSE, map->pmap);
                }
                vm_map_lock(map);

                if (last_timestamp+1 != map->timestamp) {
                        /*
                         * Find the entry again.  It could have been clipped
                         * or deleted after we unlocked the map.
                         */
                        if (!vm_map_lookup_entry(map, tmp_entry.vme_start,
                                                                &first_entry)) {
                                if (!user_wire)
                                       panic("vm_map_unwire: re-lookup failed");
                                entry = first_entry->vme_next;
                        } else
                                entry = first_entry;
                }
                last_timestamp = map->timestamp;

                /*
                 * clear transition bit for all constituent entries that
                 * were in the original entry (saved in tmp_entry).  Also
                 * check for waiters.
                 */
                while ((entry != vm_map_to_entry(map)) &&
                       (entry->vme_start < tmp_entry.vme_end)) {
                        assert(entry->in_transition);
                        entry->in_transition = FALSE;
                        if (entry->needs_wakeup) {
                                entry->needs_wakeup = FALSE;
                                need_wakeup = TRUE;
                        }
                        entry = entry->vme_next;
                }
        }
        vm_map_unlock(map);
        /*
         * wake up anybody waiting on entries that we have unwired.
         */
        if (need_wakeup)
                vm_map_entry_wakeup(map);
        return(KERN_SUCCESS);

}

kern_return_t
vm_map_unwire(
        register vm_map_t       map,
        register vm_offset_t    start,
        register vm_offset_t    end,
        boolean_t               user_wire)
{
        return vm_map_unwire_nested(map, start, end, user_wire, (pmap_t)NULL);
}


/*
 *      vm_map_entry_delete:    [ internal use only ]
 *
 *      Deallocate the given entry from the target map.
 */             
void
vm_map_entry_delete(
        register vm_map_t       map,
        register vm_map_entry_t entry)
{
        register vm_offset_t    s, e;
        register vm_object_t    object;
        register vm_map_t       submap;
        extern vm_object_t      kernel_object;

        s = entry->vme_start;
        e = entry->vme_end;
        assert(page_aligned(s));
        assert(page_aligned(e));
        assert(entry->wired_count == 0);
        assert(entry->user_wired_count == 0);

        if (entry->is_sub_map) {
                object = NULL;
                submap = entry->object.sub_map;
        } else {
                submap = NULL;
                object = entry->object.vm_object;
        }

        vm_map_entry_unlink(map, entry);
        map->size -= e - s;

        vm_map_entry_dispose(map, entry);

        vm_map_unlock(map);
        /*
         *      Deallocate the object only after removing all
         *      pmap entries pointing to its pages.
         */
        if (submap)
                vm_map_deallocate(submap);
        else
                vm_object_deallocate(object);

}

void
vm_map_submap_pmap_clean(
        vm_map_t        map,
        vm_offset_t     start,
        vm_offset_t     end,
        vm_map_t        sub_map,
        vm_offset_t     offset)
{
        vm_offset_t     submap_start;
        vm_offset_t     submap_end;
        vm_offset_t     addr;
        vm_size_t       remove_size;
        vm_map_entry_t  entry;

        submap_end = offset + (end - start);
        submap_start = offset;
        if(vm_map_lookup_entry(sub_map, offset, &entry)) {
                        
                remove_size = (entry->vme_end - entry->vme_start);
                if(offset > entry->vme_start)
                        remove_size -= offset - entry->vme_start;
                        

                if(submap_end < entry->vme_end) {
                        remove_size -=
                                entry->vme_end - submap_end;
                }
                if(entry->is_sub_map) {
                        vm_map_submap_pmap_clean(
                                sub_map,
                                start,
                                start + remove_size,
                                entry->object.sub_map,
                                entry->offset);
                } else {
                        pmap_remove(map->pmap, start, start + remove_size);
                }
        }

        entry = entry->vme_next;
        
        while((entry != vm_map_to_entry(sub_map)) 
                        && (entry->vme_start < submap_end)) {
                remove_size = (entry->vme_end - entry->vme_start); 
                if(submap_end < entry->vme_end) {
                        remove_size -= entry->vme_end - submap_end;
                }
                if(entry->is_sub_map) {
                        vm_map_submap_pmap_clean(
                                sub_map,
                                (start + entry->vme_start) - offset,
                                ((start + entry->vme_start) - offset) + remove_size,
                                entry->object.sub_map,
                                entry->offset);
                } else {
                        pmap_remove(map->pmap, 
                                (start + entry->vme_start) - offset,
                                ((start + entry->vme_start) - offset) + remove_size);
                }
                entry = entry->vme_next;
        } 
        return;
}

/*
 *      vm_map_delete:  [ internal use only ]
 *
 *      Deallocates the given address range from the target map.
 *      Removes all user wirings. Unwires one kernel wiring if
 *      VM_MAP_REMOVE_KUNWIRE is set.  Waits for kernel wirings to go
 *      away if VM_MAP_REMOVE_WAIT_FOR_KWIRE is set.  Sleeps
 *      interruptibly if VM_MAP_REMOVE_INTERRUPTIBLE is set.
 *
 *      This routine is called with map locked and leaves map locked.
 */
kern_return_t
vm_map_delete(
        register vm_map_t       map,
        vm_offset_t             start,
        register vm_offset_t    end,
        int                     flags)
{
        vm_map_entry_t          entry, next;
        struct   vm_map_entry   *first_entry, tmp_entry;
        register vm_offset_t    s, e;
        register vm_object_t    object;
        boolean_t               need_wakeup;
        unsigned int            last_timestamp = ~0; /* unlikely value */
        int                     interruptible;
        extern vm_map_t         kernel_map;

        interruptible = (flags & VM_MAP_REMOVE_INTERRUPTIBLE) ? 
                        THREAD_ABORTSAFE : THREAD_UNINT;

        /*
         * All our DMA I/O operations in IOKit are currently done by
         * wiring through the map entries of the task requesting the I/O.
         * Because of this, we must always wait for kernel wirings
         * to go away on the entries before deleting them.
         *
         * Any caller who wants to actually remove a kernel wiring
         * should explicitly set the VM_MAP_REMOVE_KUNWIRE flag to
         * properly remove one wiring instead of blasting through
         * them all.
         */
        flags |= VM_MAP_REMOVE_WAIT_FOR_KWIRE;

        /*
         *      Find the start of the region, and clip it
         */
        if (vm_map_lookup_entry(map, start, &first_entry)) {
                entry = first_entry;
                vm_map_clip_start(map, entry, start);

                /*
                 *      Fix the lookup hint now, rather than each
                 *      time through the loop.
                 */
                SAVE_HINT(map, entry->vme_prev);
        } else {
                entry = first_entry->vme_next;
        }

        need_wakeup = FALSE;
        /*
         *      Step through all entries in this region
         */
        while ((entry != vm_map_to_entry(map)) && (entry->vme_start < end)) {

                vm_map_clip_end(map, entry, end);
                if (entry->in_transition) {
                        /*
                         * Another thread is wiring/unwiring this entry.
                         * Let the other thread know we are waiting.
                         */
                        s = entry->vme_start;
                        entry->needs_wakeup = TRUE;

                        /*
                         * wake up anybody waiting on entries that we have
                         * already unwired/deleted.
                         */
                        if (need_wakeup) {
                                vm_map_entry_wakeup(map);
                                need_wakeup = FALSE;
                        }

                        vm_map_entry_wait(map, interruptible);

                        if (interruptible &&
                           current_thread()->wait_result == THREAD_INTERRUPTED)
                                /*
                                 * We do not clear the needs_wakeup flag,
                                 * since we cannot tell if we were the only one.
                                 */
                                return KERN_ABORTED;

                        vm_map_lock(map);
                        /*
                         * Cannot avoid a lookup here. reset timestamp.
                         */
                        last_timestamp = map->timestamp;

                        /*
                         * The entry could have been clipped or it
                         * may not exist anymore.  Look it up again.
                         */
                        if (!vm_map_lookup_entry(map, s, &first_entry)) {
                                assert((map != kernel_map) && 
                                       (!entry->is_sub_map));
                                /*
                                 * User: use the next entry
                                 */
                                entry = first_entry->vme_next;
                        } else {
                                entry = first_entry;
                                SAVE_HINT(map, entry->vme_prev);
                        }
                        continue;
                } /* end in_transition */

                if (entry->wired_count) {
                        /*
                         *      Remove a kernel wiring if requested or if
                         *      there are user wirings.
                         */
                        if ((flags & VM_MAP_REMOVE_KUNWIRE) || 
                           (entry->user_wired_count > 0))
                                entry->wired_count--;

                        /* remove all user wire references */
                        entry->user_wired_count = 0;

                        if (entry->wired_count != 0) {
                                assert((map != kernel_map) && 
                                       (!entry->is_sub_map));
                                /*
                                 * Cannot continue.  Typical case is when
                                 * a user thread has physical io pending on
                                 * on this page.  Either wait for the
                                 * kernel wiring to go away or return an
                                 * error.
                                 */
                                if (flags & VM_MAP_REMOVE_WAIT_FOR_KWIRE) {

                                        s = entry->vme_start;
                                        entry->needs_wakeup = TRUE;
                                        vm_map_entry_wait(map, interruptible);

                                        if (interruptible &&
                                            current_thread()->wait_result == 
                                                        THREAD_INTERRUPTED)
                                                /*
                                                 * We do not clear the 
                                                 * needs_wakeup flag, since we 
                                                 * cannot tell if we were the 
                                                 * only one.
                                                 */
                                                return KERN_ABORTED;

                                        vm_map_lock(map);
                                        /*
                                         * Cannot avoid a lookup here. reset 
                                         * timestamp.
                                         */
                                        last_timestamp = map->timestamp;

                                        /*
                                         * The entry could have been clipped or
                                         * it may not exist anymore.  Look it
                                         * up again.
                                         */
                                        if (!vm_map_lookup_entry(map, s, 
                                                                &first_entry)) {
                                                assert((map != kernel_map) && 
                                                (!entry->is_sub_map));
                                                /*
                                                 * User: use the next entry
                                                 */
                                                entry = first_entry->vme_next;
                                        } else {
                                                entry = first_entry;
                                                SAVE_HINT(map, entry->vme_prev);
                                        }
                                        continue;
                                }
                                else {
                                        return KERN_FAILURE;
                                }
                        }

                        entry->in_transition = TRUE;
                        /*
                         * copy current entry.  see comment in vm_map_wire()
                         */
                        tmp_entry = *entry;
                        s = entry->vme_start;
                        e = entry->vme_end;

                        /*
                         * We can unlock the map now. The in_transition
                         * state guarentees existance of the entry.
                         */
                        vm_map_unlock(map);
                        vm_fault_unwire(map, &tmp_entry,
                                tmp_entry.object.vm_object == kernel_object,
                                map->pmap);
                        vm_map_lock(map);

                        if (last_timestamp+1 != map->timestamp) {
                                /*
                                 * Find the entry again.  It could have
                                 * been clipped after we unlocked the map.
                                 */
                                if (!vm_map_lookup_entry(map, s, &first_entry)){
                                        assert((map != kernel_map) && 
                                               (!entry->is_sub_map));
                                        first_entry = first_entry->vme_next;
                                } else {
                                        SAVE_HINT(map, entry->vme_prev);
                                }
                        } else {
                                SAVE_HINT(map, entry->vme_prev);
                                first_entry = entry;
                        }

                        last_timestamp = map->timestamp;

                        entry = first_entry;
                        while ((entry != vm_map_to_entry(map)) &&
                               (entry->vme_start < tmp_entry.vme_end)) {
                                assert(entry->in_transition);
                                entry->in_transition = FALSE;
                                if (entry->needs_wakeup) {
                                        entry->needs_wakeup = FALSE;
                                        need_wakeup = TRUE;
                                }
                                entry = entry->vme_next;
                        }
                        /*
                         * We have unwired the entry(s).  Go back and
                         * delete them.
                         */
                        entry = first_entry;
                        continue;
                }

                /* entry is unwired */
                assert(entry->wired_count == 0);
                assert(entry->user_wired_count == 0);

                if ((!entry->is_sub_map &&
                    entry->object.vm_object != kernel_object) ||
                    entry->is_sub_map) {
                        if(entry->is_sub_map) {
                           if(entry->use_pmap) {
#ifndef i386
                                pmap_unnest(map->pmap, entry->vme_start,
                                        entry->vme_end - entry->vme_start);
#endif
                           } else {
                                vm_map_submap_pmap_clean(
                                        map, entry->vme_start, entry->vme_end,
                                        entry->object.sub_map,
                                        entry->offset);
                           }
                        } else {
                                pmap_remove(map->pmap, 
                                        entry->vme_start, entry->vme_end);
                        }
                }

                next = entry->vme_next;
                s = next->vme_start;
                last_timestamp = map->timestamp;
                vm_map_entry_delete(map, entry);
                /* vm_map_entry_delete unlocks the map */
                vm_map_lock(map);
                entry = next;

                if(entry == vm_map_to_entry(map)) {
                        break;
                }
                if (last_timestamp+1 != map->timestamp) {
                        /*
                         * we are responsible for deleting everything
                         * from the give space, if someone has interfered
                         * we pick up where we left off, back fills should
                         * be all right for anyone except map_delete and
                         * we have to assume that the task has been fully
                         * disabled before we get here
                         */
                        if (!vm_map_lookup_entry(map, s, &entry)){
                                entry = entry->vme_next;
                        } else {
                                 SAVE_HINT(map, entry->vme_prev);
                        }
                        /* 
                         * others can not only allocate behind us, we can 
                         * also see coalesce while we don't have the map lock 
                         */
                        if(entry == vm_map_to_entry(map)) {
                                break;
                        }
                        vm_map_clip_start(map, entry, s);
                }
                last_timestamp = map->timestamp;
        }

        if (map->wait_for_space)
                thread_wakeup((event_t) map);
        /*
         * wake up anybody waiting on entries that we have already deleted.
         */
        if (need_wakeup)
                vm_map_entry_wakeup(map);

        return KERN_SUCCESS;
}

/*
 *      vm_map_remove:
 *
 *      Remove the given address range from the target map.
 *      This is the exported form of vm_map_delete.
 */
kern_return_t
vm_map_remove(
        register vm_map_t       map,
        register vm_offset_t    start,
        register vm_offset_t    end,
        register boolean_t      flags)
{
        register kern_return_t  result;

        vm_map_lock(map);
        VM_MAP_RANGE_CHECK(map, start, end);
        result = vm_map_delete(map, start, end, flags);
        vm_map_unlock(map);

        return(result);
}


/*
 *      vm_map_copy_steal_pages:
 *
 *      Steal all the pages from a vm_map_copy page_list by copying ones
 *      that have not already been stolen.
 */
void
vm_map_copy_steal_pages(
        vm_map_copy_t   copy)
{
        register vm_page_t      m, new_m;
        register int            i;
        vm_object_t             object;

        assert(copy->type == VM_MAP_COPY_PAGE_LIST);
        for (i = 0; i < copy->cpy_npages; i++) {

                /*
                 *      If the page is not tabled, then it's already stolen.
                 */
                m = copy->cpy_page_list[i];
                if (!m->tabled)
                        continue;

                /*
                 *      Page was not stolen,  get a new
                 *      one and do the copy now.
                 */
                while ((new_m = vm_page_grab()) == VM_PAGE_NULL) {
                        VM_PAGE_WAIT();
                }

                vm_page_gobble(new_m); /* mark as consumed internally */
                vm_page_copy(m, new_m);

                object = m->object;
                vm_object_lock(object);
                vm_page_lock_queues();
                if (!m->active && !m->inactive)
                        vm_page_activate(m);
                vm_page_unlock_queues();
                PAGE_WAKEUP_DONE(m);
                vm_object_paging_end(object);
                vm_object_unlock(object);

                copy->cpy_page_list[i] = new_m;
        }
        copy->cpy_page_loose = TRUE;
}

/*
 *      vm_map_copy_page_discard:
 *
 *      Get rid of the pages in a page_list copy.  If the pages are
 *      stolen, they are freed.  If the pages are not stolen, they
 *      are unbusied, and associated state is cleaned up.
 */
void
vm_map_copy_page_discard(
        vm_map_copy_t   copy)
{
        assert(copy->type == VM_MAP_COPY_PAGE_LIST);
        while (copy->cpy_npages > 0) {
                vm_page_t       m;

                if ((m = copy->cpy_page_list[--(copy->cpy_npages)]) !=
                    VM_PAGE_NULL) {

                        /*
                         *      If it's not in the table, then it's
                         *      a stolen page that goes back
                         *      to the free list.  Else it belongs
                         *      to some object, and we hold a
                         *      paging reference on that object.
                         */
                        if (!m->tabled) {
                                VM_PAGE_FREE(m);
                        }
                        else {
                                vm_object_t     object;

                                object = m->object;

                                vm_object_lock(object);
                                vm_page_lock_queues();
                                if (!m->active && !m->inactive)
                                        vm_page_activate(m);
                                vm_page_unlock_queues();

                                if ((!m->busy)) {
                                    kern_return_t kr;
                                    kr = vm_page_unpin(m);
                                    assert(kr == KERN_SUCCESS);
                                } else {
                                    PAGE_WAKEUP_DONE(m);
                                }
                                vm_object_paging_end(object);
                                vm_object_unlock(object);
                        }
                }
        }
}

/*
 *      Routine:        vm_map_copy_discard
 *
 *      Description:
 *              Dispose of a map copy object (returned by
 *              vm_map_copyin).
 */
void
vm_map_copy_discard(
        vm_map_copy_t   copy)
{
        TR_DECL("vm_map_copy_discard");

/*      tr3("enter: copy 0x%x type %d", copy, copy->type);*/
free_next_copy:
        if (copy == VM_MAP_COPY_NULL)
                return;

        switch (copy->type) {
        case VM_MAP_COPY_ENTRY_LIST:
                while (vm_map_copy_first_entry(copy) !=
                                        vm_map_copy_to_entry(copy)) {
                        vm_map_entry_t  entry = vm_map_copy_first_entry(copy);

                        vm_map_copy_entry_unlink(copy, entry);
                        vm_object_deallocate(entry->object.vm_object);
                        vm_map_copy_entry_dispose(copy, entry);
                }
                break;
        case VM_MAP_COPY_OBJECT:
                vm_object_deallocate(copy->cpy_object);
                break;
        case VM_MAP_COPY_PAGE_LIST:

                /*
                 *      To clean this up, we have to unbusy all the pages
                 *      and release the paging references in their objects.
                 */
                if (copy->cpy_npages > 0)
                        vm_map_copy_page_discard(copy);

                /*
                 *      If there's a continuation, abort it.  The
                 *      abort routine releases any storage.
                 */
                if (vm_map_copy_has_cont(copy)) {

                        assert(vm_map_copy_cont_is_valid(copy));
                        /*
                         *      Special case: recognize
                         *      vm_map_copy_discard_cont and optimize
                         *      here to avoid tail recursion.
                         */
                        if (copy->cpy_cont == vm_map_copy_discard_cont) {
                                register vm_map_copy_t  new_copy;

                                new_copy = (vm_map_copy_t) copy->cpy_cont_args;
                                zfree(vm_map_copy_zone, (vm_offset_t) copy);
                                copy = new_copy;
                                goto free_next_copy;
                        } else {
                                vm_map_copy_abort_cont(copy);
                        }
                }

                break;

        case VM_MAP_COPY_KERNEL_BUFFER:

                /*
                 * The vm_map_copy_t and possibly the data buffer were
                 * allocated by a single call to kalloc(), i.e. the
                 * vm_map_copy_t was not allocated out of the zone.
                 */
                kfree((vm_offset_t) copy, copy->cpy_kalloc_size);
                return;
        }
        zfree(vm_map_copy_zone, (vm_offset_t) copy);
}

/*
 *      Routine:        vm_map_copy_copy
 *
 *      Description:
 *                      Move the information in a map copy object to
 *                      a new map copy object, leaving the old one
 *                      empty.
 *
 *                      This is used by kernel routines that need
 *                      to look at out-of-line data (in copyin form)
 *                      before deciding whether to return SUCCESS.
 *                      If the routine returns FAILURE, the original
 *                      copy object will be deallocated; therefore,
 *                      these routines must make a copy of the copy
 *                      object and leave the original empty so that
 *                      deallocation will not fail.
 */
vm_map_copy_t
vm_map_copy_copy(
        vm_map_copy_t   copy)
{
        vm_map_copy_t   new_copy;

        if (copy == VM_MAP_COPY_NULL)
                return VM_MAP_COPY_NULL;

        /*
         * Allocate a new copy object, and copy the information
         * from the old one into it.
         */

        new_copy = (vm_map_copy_t) zalloc(vm_map_copy_zone);
        *new_copy = *copy;

        if (copy->type == VM_MAP_COPY_ENTRY_LIST) {
                /*
                 * The links in the entry chain must be
                 * changed to point to the new copy object.
                 */
                vm_map_copy_first_entry(copy)->vme_prev
                        = vm_map_copy_to_entry(new_copy);
                vm_map_copy_last_entry(copy)->vme_next
                        = vm_map_copy_to_entry(new_copy);
        }

        /*
         * Change the old copy object into one that contains
         * nothing to be deallocated.
         */
        copy->type = VM_MAP_COPY_OBJECT;
        copy->cpy_object = VM_OBJECT_NULL;

        /*
         * Return the new object.
         */
        return new_copy;
}

/*
 *      Routine:        vm_map_copy_discard_cont
 *
 *      Description:
 *              A version of vm_map_copy_discard that can be called
 *              as a continuation from a vm_map_copy page list.
 */
kern_return_t
vm_map_copy_discard_cont(
        vm_map_copyin_args_t    cont_args,
        vm_map_copy_t           *copy_result)   /* OUT */
{
        vm_map_copy_discard((vm_map_copy_t) cont_args);
        if (copy_result != (vm_map_copy_t *)0)
                *copy_result = VM_MAP_COPY_NULL;
        return(KERN_SUCCESS);
}

kern_return_t
vm_map_overwrite_submap_recurse(
        vm_map_t        dst_map,
        vm_offset_t     dst_addr,
        vm_size_t       dst_size)
{
        vm_offset_t     dst_end;
        vm_map_entry_t  tmp_entry;
        vm_map_entry_t  entry;
        kern_return_t   result;
        boolean_t       encountered_sub_map = FALSE;



        /*
         *      Verify that the destination is all writeable
         *      initially.  We have to trunc the destination
         *      address and round the copy size or we'll end up
         *      splitting entries in strange ways.
         */

        dst_end = round_page(dst_addr + dst_size);

start_pass_1:
        vm_map_lock(dst_map);
        if (!vm_map_lookup_entry(dst_map, dst_addr, &tmp_entry)) {
                vm_map_unlock(dst_map);
                return(KERN_INVALID_ADDRESS);
        }

        vm_map_clip_start(dst_map, tmp_entry, trunc_page(dst_addr));

        for (entry = tmp_entry;;) {
                vm_map_entry_t  next;

                next = entry->vme_next;
                while(entry->is_sub_map) {
                        vm_offset_t     sub_start;
                        vm_offset_t     sub_end;
                        vm_offset_t     local_end;

                        if (entry->in_transition) {
                        /*
                         * Say that we are waiting, and wait for entry.
                         */
                                entry->needs_wakeup = TRUE;
                                vm_map_entry_wait(dst_map, THREAD_UNINT);

                                goto start_pass_1;
                        }

                        encountered_sub_map = TRUE;
                        sub_start = entry->offset;

                        if(entry->vme_end < dst_end)
                                sub_end = entry->vme_end;
                        else 
                                sub_end = dst_end;
                        sub_end -= entry->vme_start;
                        sub_end += entry->offset;
                        local_end = entry->vme_end;
                        vm_map_unlock(dst_map);
                        
                        result = vm_map_overwrite_submap_recurse(
                                        entry->object.sub_map,
                                        sub_start,
                                        sub_end - sub_start);

                        if(result != KERN_SUCCESS)
                                return result;
                        if (dst_end <= entry->vme_end)
                                return KERN_SUCCESS;
                        vm_map_lock(dst_map);
                        if(!vm_map_lookup_entry(dst_map, local_end, 
                                                &tmp_entry)) {
                                vm_map_unlock(dst_map);
                                return(KERN_INVALID_ADDRESS);
                        }
                        entry = tmp_entry;
                        next = entry->vme_next;
                }

                if ( ! (entry->protection & VM_PROT_WRITE)) {
                        vm_map_unlock(dst_map);
                        return(KERN_PROTECTION_FAILURE);
                }

                /*
                 *      If the entry is in transition, we must wait
                 *      for it to exit that state.  Anything could happen
                 *      when we unlock the map, so start over.
                 */
                if (entry->in_transition) {

                        /*
                         * Say that we are waiting, and wait for entry.
                         */
                        entry->needs_wakeup = TRUE;
                        vm_map_entry_wait(dst_map, THREAD_UNINT);

                        goto start_pass_1;
                }

/*
 *              our range is contained completely within this map entry
 */
                if (dst_end <= entry->vme_end) {
                        vm_map_unlock(dst_map);
                        return KERN_SUCCESS;
                }
/*
 *              check that range specified is contiguous region
 */
                if ((next == vm_map_to_entry(dst_map)) ||
                    (next->vme_start != entry->vme_end)) {
                        vm_map_unlock(dst_map);
                        return(KERN_INVALID_ADDRESS);
                }

                /*
                 *      Check for permanent objects in the destination.
                 */
                if ((entry->object.vm_object != VM_OBJECT_NULL) &&
                           ((!entry->object.vm_object->internal) ||
                           (entry->object.vm_object->true_share))) {
                        if(encountered_sub_map) {
                                vm_map_unlock(dst_map);
                                return(KERN_FAILURE);
                        }
                }


                entry = next;
        }/* for */
        vm_map_unlock(dst_map);
        return(KERN_SUCCESS);
}

/*
 *      Routine:        vm_map_copy_overwrite
 *
 *      Description:
 *              Copy the memory described by the map copy
 *              object (copy; returned by vm_map_copyin) onto
 *              the specified destination region (dst_map, dst_addr).
 *              The destination must be writeable.
 *
 *              Unlike vm_map_copyout, this routine actually
 *              writes over previously-mapped memory.  If the
 *              previous mapping was to a permanent (user-supplied)
 *              memory object, it is preserved.
 *
 *              The attributes (protection and inheritance) of the
 *              destination region are preserved.
 *
 *              If successful, consumes the copy object.
 *              Otherwise, the caller is responsible for it.
 *
 *      Implementation notes:
 *              To overwrite aligned temporary virtual memory, it is
 *              sufficient to remove the previous mapping and insert
 *              the new copy.  This replacement is done either on
 *              the whole region (if no permanent virtual memory
 *              objects are embedded in the destination region) or
 *              in individual map entries.
 *
 *              To overwrite permanent virtual memory , it is necessary
 *              to copy each page, as the external memory management
 *              interface currently does not provide any optimizations.
 *
 *              Unaligned memory also has to be copied.  It is possible
 *              to use 'vm_trickery' to copy the aligned data.  This is
 *              not done but not hard to implement.
 *
 *              Once a page of permanent memory has been overwritten,
 *              it is impossible to interrupt this function; otherwise,
 *              the call would be neither atomic nor location-independent.
 *              The kernel-state portion of a user thread must be
 *              interruptible.
 *
 *              It may be expensive to forward all requests that might
 *              overwrite permanent memory (vm_write, vm_copy) to
 *              uninterruptible kernel threads.  This routine may be
 *              called by interruptible threads; however, success is
 *              not guaranteed -- if the request cannot be performed
 *              atomically and interruptibly, an error indication is
 *              returned.
 */

kern_return_t
vm_map_copy_overwrite_nested(
        vm_map_t        dst_map,
        vm_offset_t     dst_addr,
        vm_map_copy_t   copy,
        boolean_t       interruptible,
        pmap_t          pmap)
{
        vm_offset_t     dst_end;
        vm_map_entry_t  tmp_entry;
        vm_map_entry_t  entry;
        kern_return_t   kr;
        boolean_t       aligned = TRUE;
        boolean_t       contains_permanent_objects = FALSE;
        boolean_t       encountered_sub_map = FALSE;
        vm_offset_t     base_addr;
        vm_size_t       copy_size;
        vm_size_t       total_size;


        /*
         *      Check for null copy object.
         */

        if (copy == VM_MAP_COPY_NULL)
                return(KERN_SUCCESS);

        /*
         *      Check for special kernel buffer allocated
         *      by new_ipc_kmsg_copyin.
         */

        if (copy->type == VM_MAP_COPY_KERNEL_BUFFER) {
                return(vm_map_copyout_kernel_buffer(dst_map, &dst_addr, 
                                                    copy, TRUE));
        }

        /*
         *      Only works for entry lists at the moment.  Will
         *      support page lists later.
         */

        assert(copy->type == VM_MAP_COPY_ENTRY_LIST);

        if (copy->size == 0) {
                vm_map_copy_discard(copy);
                return(KERN_SUCCESS);
        }

        /*
         *      Verify that the destination is all writeable
         *      initially.  We have to trunc the destination
         *      address and round the copy size or we'll end up
         *      splitting entries in strange ways.
         */

        if (!page_aligned(copy->size) ||
                !page_aligned (copy->offset) ||
                !page_aligned (dst_addr))
        {
                aligned = FALSE;
                dst_end = round_page(dst_addr + copy->size);
        } else {
                dst_end = dst_addr + copy->size;
        }

start_pass_1:
        vm_map_lock(dst_map);
        if (!vm_map_lookup_entry(dst_map, dst_addr, &tmp_entry)) {
                vm_map_unlock(dst_map);
                return(KERN_INVALID_ADDRESS);
        }
        vm_map_clip_start(dst_map, tmp_entry, trunc_page(dst_addr));
        for (entry = tmp_entry;;) {
                vm_map_entry_t  next = entry->vme_next;

                while(entry->is_sub_map) {
                        vm_offset_t     sub_start;
                        vm_offset_t     sub_end;
                        vm_offset_t     local_end;

                        if (entry->in_transition) {

                        /*
                         * Say that we are waiting, and wait for entry.
                         */
                                entry->needs_wakeup = TRUE;
                                vm_map_entry_wait(dst_map, THREAD_UNINT);

                                goto start_pass_1;
                        }

                        local_end = entry->vme_end;
                        if (!(entry->needs_copy)) {
                                /* if needs_copy we are a COW submap */
                                /* in such a case we just replace so */
                                /* there is no need for the follow-  */
                                /* ing check.                        */
                                encountered_sub_map = TRUE;
                                sub_start = entry->offset;

                                if(entry->vme_end < dst_end)
                                        sub_end = entry->vme_end;
                                else 
                                        sub_end = dst_end;
                                sub_end -= entry->vme_start;
                                sub_end += entry->offset;
                                vm_map_unlock(dst_map);
                        
                                kr = vm_map_overwrite_submap_recurse(
                                        entry->object.sub_map,
                                        sub_start,
                                        sub_end - sub_start);
                                if(kr != KERN_SUCCESS)
                                        return kr;
                                vm_map_lock(dst_map);
                        }

                        if (dst_end <= entry->vme_end)
                                goto start_overwrite;
                        if(!vm_map_lookup_entry(dst_map, local_end, 
                                                &entry)) {
                                vm_map_unlock(dst_map);
                                return(KERN_INVALID_ADDRESS);
                        }
                        next = entry->vme_next;
                }

                if ( ! (entry->protection & VM_PROT_WRITE)) {
                        vm_map_unlock(dst_map);
                        return(KERN_PROTECTION_FAILURE);
                }

                /*
                 *      If the entry is in transition, we must wait
                 *      for it to exit that state.  Anything could happen
                 *      when we unlock the map, so start over.
                 */
                if (entry->in_transition) {

                        /*
                         * Say that we are waiting, and wait for entry.
                         */
                        entry->needs_wakeup = TRUE;
                        vm_map_entry_wait(dst_map, THREAD_UNINT);

                        goto start_pass_1;
                }

/*
 *              our range is contained completely within this map entry
 */
                if (dst_end <= entry->vme_end)
                        break;
/*
 *              check that range specified is contiguous region
 */
                if ((next == vm_map_to_entry(dst_map)) ||
                    (next->vme_start != entry->vme_end)) {
                        vm_map_unlock(dst_map);
                        return(KERN_INVALID_ADDRESS);
                }


                /*
                 *      Check for permanent objects in the destination.
                 */
                if ((entry->object.vm_object != VM_OBJECT_NULL) &&
                           ((!entry->object.vm_object->internal) ||
                           (entry->object.vm_object->true_share))) {
                        contains_permanent_objects = TRUE;
                }

                entry = next;
        }/* for */

start_overwrite:
        /*
         *      If there are permanent objects in the destination, then
         *      the copy cannot be interrupted.
         */

        if (interruptible && contains_permanent_objects) {
                vm_map_unlock(dst_map);
                return(KERN_FAILURE);   /* XXX */
        }

        /*
         *
         *      Make a second pass, overwriting the data
         *      At the beginning of each loop iteration,
         *      the next entry to be overwritten is "tmp_entry"
         *      (initially, the value returned from the lookup above),
         *      and the starting address expected in that entry
         *      is "start".
         */

        total_size = copy->size;
        if(encountered_sub_map) {
                copy_size = 0;
                /* re-calculate tmp_entry since we've had the map */
                /* unlocked */
                if (!vm_map_lookup_entry( dst_map, dst_addr, &tmp_entry)) {
                        vm_map_unlock(dst_map);
                        return(KERN_INVALID_ADDRESS);
                }
        } else {
                copy_size = copy->size;
        }
        
        base_addr = dst_addr;
        while(TRUE) {
                /* deconstruct the copy object and do in parts */
                /* only in sub_map, interruptable case */
                vm_map_entry_t  copy_entry;
                vm_map_entry_t  previous_prev;
                vm_map_entry_t  next_copy;
                int             nentries;
                int             remaining_entries;
                int             new_offset;
        
                for (entry = tmp_entry; copy_size == 0;) {
                        vm_map_entry_t  next;

                        next = entry->vme_next;

                        /* tmp_entry and base address are moved along */
                        /* each time we encounter a sub-map.  Otherwise */
                        /* entry can outpase tmp_entry, and the copy_size */
                        /* may reflect the distance between them */
                        /* if the current entry is found to be in transition */
                        /* we will start over at the beginning or the last */
                        /* encounter of a submap as dictated by base_addr */
                        /* we will zero copy_size accordingly. */
                        if (entry->in_transition) {
                                /*
                                 * Say that we are waiting, and wait for entry.
                                 */
                                entry->needs_wakeup = TRUE;
                                vm_map_entry_wait(dst_map, THREAD_UNINT);

                                vm_map_lock(dst_map);
                                if(!vm_map_lookup_entry(dst_map, base_addr, 
                                                                &tmp_entry)) {
                                        vm_map_unlock(dst_map);
                                        return(KERN_INVALID_ADDRESS);
                                }
                                copy_size = 0;
                                entry = tmp_entry;
                                continue;
                        }
                        if(entry->is_sub_map) {
                                vm_offset_t     sub_start;
                                vm_offset_t     sub_end;
                                vm_offset_t     local_end;

                                if (entry->needs_copy) {
                                        /* if this is a COW submap */
                                        /* just back the range with a */
                                        /* anonymous entry */
                                        if(entry->vme_end < dst_end)
                                                sub_end = entry->vme_end;
                                        else 
                                                sub_end = dst_end;
                                        if(entry->vme_start < base_addr)
                                                sub_start = base_addr;
                                        else 
                                                sub_start = entry->vme_start;
                                        vm_map_clip_end(
                                                dst_map, entry, sub_end);
                                        vm_map_clip_start(
                                                dst_map, entry, sub_start);
                                        entry->is_sub_map = FALSE;
                                        vm_map_deallocate(
                                                entry->object.sub_map);
                                        entry->object.sub_map = NULL;
                                        entry->is_shared = FALSE;
                                        entry->needs_copy = FALSE;
                                        entry->offset = 0;
                                        entry->protection = VM_PROT_ALL;
                                        entry->max_protection = VM_PROT_ALL;
                                        entry->wired_count = 0;
                                        entry->user_wired_count = 0;
                                        if(entry->inheritance 
                                                        == VM_INHERIT_SHARE) 
                                           entry->inheritance = VM_INHERIT_COPY;
                                        continue;
                                }
                                /* first take care of any non-sub_map */
                                /* entries to send */
                                if(base_addr < entry->vme_start) {
                                        /* stuff to send */
                                        copy_size = 
                                                entry->vme_start - base_addr;
                                        break;
                                }
                                sub_start = entry->offset;

                                if(entry->vme_end < dst_end)
                                        sub_end = entry->vme_end;
                                else 
                                        sub_end = dst_end;
                                sub_end -= entry->vme_start;
                                sub_end += entry->offset;
                                local_end = entry->vme_end;
                                vm_map_unlock(dst_map);
                                copy_size = sub_end - sub_start;

                                /* adjust the copy object */
                                if (total_size > copy_size) {
                                        vm_size_t       local_size = 0;
                                        vm_size_t       entry_size;

                                   nentries = 1;
                                   new_offset = copy->offset;
                                   copy_entry = vm_map_copy_first_entry(copy);
                                   while(copy_entry != 
                                                 vm_map_copy_to_entry(copy)){
                                       entry_size = copy_entry->vme_end - 
                                                      copy_entry->vme_start;
                                       if((local_size < copy_size) &&
                                                ((local_size + entry_size) 
                                                 >= copy_size)) {
                                          vm_map_copy_clip_end(copy, 
                                                 copy_entry, 
                                                 copy_entry->vme_start +
                                                 (copy_size - local_size));
                                          entry_size = copy_entry->vme_end - 
                                                         copy_entry->vme_start;
                                          local_size += entry_size;
                                          new_offset += entry_size;
                                       }
                                       if(local_size >= copy_size) {
                                          next_copy = copy_entry->vme_next;
                                          copy_entry->vme_next = 
                                                    vm_map_copy_to_entry(copy);
                                          previous_prev = 
                                                   copy->cpy_hdr.links.prev;
                                          copy->cpy_hdr.links.prev = copy_entry;
                                          copy->size = copy_size;
                                          remaining_entries = 
                                                        copy->cpy_hdr.nentries;
                                          remaining_entries -= nentries;
                                          copy->cpy_hdr.nentries = nentries;
                                          break;
                                       } else {
                                          local_size += entry_size;
                                          new_offset += entry_size;
                                          nentries++;
                                       }
                                       copy_entry = copy_entry->vme_next;
                                   }
                                }
                        
                                if((entry->use_pmap) && (pmap == NULL)) {
                                        kr = vm_map_copy_overwrite_nested(
                                                entry->object.sub_map,
                                                sub_start,
                                                copy,
                                                interruptible, 
                                                entry->object.sub_map->pmap);
                                } else if (pmap != NULL) {
                                        kr = vm_map_copy_overwrite_nested(
                                                entry->object.sub_map,
                                                sub_start,
                                                copy,
                                                interruptible, pmap);
                                } else {
                                        kr = vm_map_copy_overwrite_nested(
                                                entry->object.sub_map,
                                                sub_start,
                                                copy,
                                                interruptible,
                                                dst_map->pmap);
                                }
                                if(kr != KERN_SUCCESS) {
                                        if(next_copy != NULL) {
                                           copy->cpy_hdr.nentries += 
                                                           remaining_entries;
                                           copy->cpy_hdr.links.prev->vme_next = 
                                                           next_copy;
                                           copy->cpy_hdr.links.prev 
                                                           = previous_prev;
                                           copy->size = total_size;
                                        }
                                        return kr;
                                }
                                if (dst_end <= local_end) {
                                        return(KERN_SUCCESS);
                                }
                                /* otherwise copy no longer exists, it was */
                                /* destroyed after successful copy_overwrite */
                                copy = (vm_map_copy_t) 
                                                zalloc(vm_map_copy_zone);
                                vm_map_copy_first_entry(copy) =
                                   vm_map_copy_last_entry(copy) =
                                   vm_map_copy_to_entry(copy);
                                copy->type = VM_MAP_COPY_ENTRY_LIST;
                                copy->offset = new_offset;

                                total_size -= copy_size;
                                copy_size = 0;
                                /* put back remainder of copy in container */
                                if(next_copy != NULL) {
                                   copy->cpy_hdr.nentries = remaining_entries;
                                   copy->cpy_hdr.links.next = next_copy;
                                   copy->cpy_hdr.links.prev = previous_prev;
                                   copy->size = total_size;
                                   next_copy->vme_prev = 
                                                 vm_map_copy_to_entry(copy);
                                   next_copy = NULL;
                                }
                                base_addr = local_end;
                                vm_map_lock(dst_map);
                                if(!vm_map_lookup_entry(dst_map, 
                                                local_end, &tmp_entry)) {
                                        vm_map_unlock(dst_map);
                                        return(KERN_INVALID_ADDRESS);
                                }
                                entry = tmp_entry;
                                continue;
                        } 
                        if (dst_end <= entry->vme_end) {
                                copy_size = dst_end - base_addr;
                                break;
                        }

                        if ((next == vm_map_to_entry(dst_map)) ||
                                    (next->vme_start != entry->vme_end)) {
                                vm_map_unlock(dst_map);
                                return(KERN_INVALID_ADDRESS);
                        }

                        entry = next;
                }/* for */

                next_copy = NULL;
                nentries = 1;

                /* adjust the copy object */
                if (total_size > copy_size) {
                        vm_size_t       local_size = 0;
                        vm_size_t       entry_size;

                        new_offset = copy->offset;
                        copy_entry = vm_map_copy_first_entry(copy);
                        while(copy_entry != vm_map_copy_to_entry(copy)) {
                                entry_size = copy_entry->vme_end - 
                                                copy_entry->vme_start;
                                if((local_size < copy_size) &&
                                                ((local_size + entry_size) 
                                                >= copy_size)) {
                                        vm_map_copy_clip_end(copy, copy_entry, 
                                                copy_entry->vme_start +
                                                (copy_size - local_size));
                                        entry_size = copy_entry->vme_end - 
                                                   copy_entry->vme_start;
                                        local_size += entry_size;
                                        new_offset += entry_size;
                                }
                                if(local_size >= copy_size) {
                                        next_copy = copy_entry->vme_next;
                                        copy_entry->vme_next = 
                                                vm_map_copy_to_entry(copy);
                                        previous_prev = 
                                                copy->cpy_hdr.links.prev;
                                        copy->cpy_hdr.links.prev = copy_entry;
                                        copy->size = copy_size;
                                        remaining_entries = 
                                                copy->cpy_hdr.nentries;
                                        remaining_entries -= nentries;
                                        copy->cpy_hdr.nentries = nentries;
                                        break;
                                } else {
                                        local_size += entry_size;
                                        new_offset += entry_size;
                                        nentries++;
                                }
                                copy_entry = copy_entry->vme_next;
                        }
                }

                if (aligned) {
                        pmap_t  local_pmap;

                        if(pmap)
                                local_pmap = pmap;
                        else
                                local_pmap = dst_map->pmap;

                        if ((kr =  vm_map_copy_overwrite_aligned( 
                                dst_map, tmp_entry, copy,
                                base_addr, local_pmap)) != KERN_SUCCESS) {
                                if(next_copy != NULL) {
                                        copy->cpy_hdr.nentries += 
                                                           remaining_entries;
                                        copy->cpy_hdr.links.prev->vme_next = 
                                                           next_copy;
                                        copy->cpy_hdr.links.prev = 
                                                        previous_prev;
                                        copy->size += copy_size;
                                }
                                return kr;
                        }
                        vm_map_unlock(dst_map);
                } else {
                /*
                 * Performance gain:
                 *
                 * if the copy and dst address are misaligned but the same
                 * offset within the page we can copy_not_aligned the
                 * misaligned parts and copy aligned the rest.  If they are
                 * aligned but len is unaligned we simply need to copy
                 * the end bit unaligned.  We'll need to split the misaligned
                 * bits of the region in this case !
                 */
                /* ALWAYS UNLOCKS THE dst_map MAP */
                        if ((kr =  vm_map_copy_overwrite_unaligned( dst_map,
                                tmp_entry, copy, base_addr)) != KERN_SUCCESS) {
                                if(next_copy != NULL) {
                                        copy->cpy_hdr.nentries +=
                                                             remaining_entries;
                                        copy->cpy_hdr.links.prev->vme_next = 
                                                             next_copy;
                                        copy->cpy_hdr.links.prev = 
                                                previous_prev;
                                        copy->size += copy_size;
                                }
                                return kr;
                        }
                }
                total_size -= copy_size;
                if(total_size == 0)
                        break;
                base_addr += copy_size;
                copy_size = 0;
                copy->offset = new_offset;
                if(next_copy != NULL) {
                        copy->cpy_hdr.nentries = remaining_entries;
                        copy->cpy_hdr.links.next = next_copy;
                        copy->cpy_hdr.links.prev = previous_prev;
                        next_copy->vme_prev = vm_map_copy_to_entry(copy);
                        copy->size = total_size;
                }
                vm_map_lock(dst_map);
                while(TRUE) {
                        if (!vm_map_lookup_entry(dst_map, 
                                                base_addr, &tmp_entry)) {
                                vm_map_unlock(dst_map);
                                return(KERN_INVALID_ADDRESS);
                        }
                        if (tmp_entry->in_transition) {
                                entry->needs_wakeup = TRUE;
                                vm_map_entry_wait(dst_map, THREAD_UNINT);
                        } else {
                                break;
                        }
                }
                vm_map_clip_start(dst_map, tmp_entry, trunc_page(base_addr));

                entry = tmp_entry;
        } /* while */

        /*
         *      Throw away the vm_map_copy object
         */
        vm_map_copy_discard(copy);

        return(KERN_SUCCESS);
}/* vm_map_copy_overwrite */

kern_return_t
vm_map_copy_overwrite(
        vm_map_t        dst_map,
        vm_offset_t     dst_addr,
        vm_map_copy_t   copy,
        boolean_t       interruptible)
{
        return vm_map_copy_overwrite_nested(
                        dst_map, dst_addr, copy, interruptible, (pmap_t) NULL);
}


/*
 *      Routine: vm_map_copy_overwrite_unaligned
 *
 *      Decription:
 *      Physically copy unaligned data
 *
 *      Implementation:
 *      Unaligned parts of pages have to be physically copied.  We use
 *      a modified form of vm_fault_copy (which understands none-aligned
 *      page offsets and sizes) to do the copy.  We attempt to copy as
 *      much memory in one go as possibly, however vm_fault_copy copies
 *      within 1 memory object so we have to find the smaller of "amount left"
 *      "source object data size" and "target object data size".  With
 *      unaligned data we don't need to split regions, therefore the source
 *      (copy) object should be one map entry, the target range may be split
 *      over multiple map entries however.  In any event we are pessimistic
 *      about these assumptions.
 *
 *      Assumptions:
 *      dst_map is locked on entry and is return locked on success,
 *      unlocked on error.
 */

kern_return_t
vm_map_copy_overwrite_unaligned(
        vm_map_t        dst_map,
        vm_map_entry_t  entry,
        vm_map_copy_t   copy,
        vm_offset_t     start)
{
        vm_map_entry_t          copy_entry = vm_map_copy_first_entry(copy);
        vm_map_version_t        version;
        vm_object_t             dst_object;
        vm_object_offset_t      dst_offset;
        vm_object_offset_t      src_offset;
        vm_object_offset_t      entry_offset;
        vm_offset_t             entry_end;
        vm_size_t               src_size,
                                dst_size,
                                copy_size,
                                amount_left;
        kern_return_t           kr = KERN_SUCCESS;

        vm_map_lock_write_to_read(dst_map);

        src_offset = copy->offset - trunc_page_64(copy->offset);
        amount_left = copy->size;
/*
 *      unaligned so we never clipped this entry, we need the offset into
 *      the vm_object not just the data.
 */     
        while (amount_left > 0) {

                if (entry == vm_map_to_entry(dst_map)) {
                        vm_map_unlock_read(dst_map);
                        return KERN_INVALID_ADDRESS;
                }

                /* "start" must be within the current map entry */
                assert ((start>=entry->vme_start) && (start<entry->vme_end));

                dst_offset = start - entry->vme_start;

                dst_size = entry->vme_end - start;

                src_size = copy_entry->vme_end -
                        (copy_entry->vme_start + src_offset);

                if (dst_size < src_size) {
/*
 *                      we can only copy dst_size bytes before
 *                      we have to get the next destination entry
 */
                        copy_size = dst_size;
                } else {
/*
 *                      we can only copy src_size bytes before
 *                      we have to get the next source copy entry
 */
                        copy_size = src_size;
                }

                if (copy_size > amount_left) {
                        copy_size = amount_left;
                }
/*
 *              Entry needs copy, create a shadow shadow object for
 *              Copy on write region.
 */
                if (entry->needs_copy &&
                         ((entry->protection & VM_PROT_WRITE) != 0))
                {
                        if (vm_map_lock_read_to_write(dst_map)) {
                                vm_map_lock_read(dst_map);
                                goto RetryLookup;
                        }
                        vm_object_shadow(&entry->object.vm_object,
                                        &entry->offset,
                                        (vm_size_t)(entry->vme_end
                                                - entry->vme_start));
                        entry->needs_copy = FALSE;
                        vm_map_lock_write_to_read(dst_map);
                }
                dst_object = entry->object.vm_object;
/*
 *              unlike with the virtual (aligned) copy we're going
 *              to fault on it therefore we need a target object.
 */
                if (dst_object == VM_OBJECT_NULL) {
                        if (vm_map_lock_read_to_write(dst_map)) {
                                vm_map_lock_read(dst_map);
                                goto RetryLookup;
                        }
                        dst_object = vm_object_allocate((vm_size_t)
                                        entry->vme_end - entry->vme_start);
                        entry->object.vm_object = dst_object;
                        entry->offset = 0;
                        vm_map_lock_write_to_read(dst_map);
                }
/*
 *              Take an object reference and unlock map. The "entry" may
 *              disappear or change when the map is unlocked.
 */
                vm_object_reference(dst_object);
                version.main_timestamp = dst_map->timestamp;
                entry_offset = entry->offset;
                entry_end = entry->vme_end;
                vm_map_unlock_read(dst_map);
/*
 *              Copy as much as possible in one pass
 */
                kr = vm_fault_copy(
                        copy_entry->object.vm_object,
                        copy_entry->offset + src_offset,
                        &copy_size,
                        dst_object,
                        entry_offset + dst_offset,
                        dst_map,
                        &version,
                        THREAD_UNINT );

                start += copy_size;
                src_offset += copy_size;
                amount_left -= copy_size;
/*
 *              Release the object reference
 */
                vm_object_deallocate(dst_object);
/*
 *              If a hard error occurred, return it now
 */
                if (kr != KERN_SUCCESS)
                        return kr;

                if ((copy_entry->vme_start + src_offset) == copy_entry->vme_end
                        || amount_left == 0)
                {
/*
 *                      all done with this copy entry, dispose.
 */
                        vm_map_copy_entry_unlink(copy, copy_entry);
                        vm_object_deallocate(copy_entry->object.vm_object);
                        vm_map_copy_entry_dispose(copy, copy_entry);

                        if ((copy_entry = vm_map_copy_first_entry(copy))
                                == vm_map_copy_to_entry(copy) && amount_left) {
/*
 *                              not finished copying but run out of source
 */
                                return KERN_INVALID_ADDRESS;
                        }
                        src_offset = 0;
                }

                if (amount_left == 0)
                        return KERN_SUCCESS;

                vm_map_lock_read(dst_map);
                if (version.main_timestamp == dst_map->timestamp) {
                        if (start == entry_end) {
/*
 *                              destination region is split.  Use the version
 *                              information to avoid a lookup in the normal
 *                              case.
 */
                                entry = entry->vme_next;
/*
 *                              should be contiguous. Fail if we encounter
 *                              a hole in the destination.
 */
                                if (start != entry->vme_start) {
                                        vm_map_unlock_read(dst_map);
                                        return KERN_INVALID_ADDRESS ;
                                }
                        }
                } else {
/*
 *                      Map version check failed.
 *                      we must lookup the entry because somebody
 *                      might have changed the map behind our backs.
 */
RetryLookup:
                        if (!vm_map_lookup_entry(dst_map, start, &entry))
                        {
                                vm_map_unlock_read(dst_map);
                                return KERN_INVALID_ADDRESS ;
                        }
                }
        }/* while */

        /* NOTREACHED ?? */
        vm_map_unlock_read(dst_map);

        return KERN_SUCCESS;
}/* vm_map_copy_overwrite_unaligned */

/*
 *      Routine:        vm_map_copy_overwrite_aligned
 *
 *      Description:
 *      Does all the vm_trickery possible for whole pages.
 *
 *      Implementation:
 *
 *      If there are no permanent objects in the destination,
 *      and the source and destination map entry zones match,
 *      and the destination map entry is not shared,
 *      then the map entries can be deleted and replaced
 *      with those from the copy.  The following code is the
 *      basic idea of what to do, but there are lots of annoying
 *      little details about getting protection and inheritance
 *      right.  Should add protection, inheritance, and sharing checks
 *      to the above pass and make sure that no wiring is involved.
 */

kern_return_t
vm_map_copy_overwrite_aligned(
        vm_map_t        dst_map,
        vm_map_entry_t  tmp_entry,
        vm_map_copy_t   copy,
        vm_offset_t     start,
        pmap_t          pmap)
{
        vm_object_t     object;
        vm_map_entry_t  copy_entry;
        vm_size_t       copy_size;
        vm_size_t       size;
        vm_map_entry_t  entry;
                
        while ((copy_entry = vm_map_copy_first_entry(copy))
                != vm_map_copy_to_entry(copy))
        {
                copy_size = (copy_entry->vme_end - copy_entry->vme_start);
                
                entry = tmp_entry;
                if (entry == vm_map_to_entry(dst_map)) {
                        vm_map_unlock(dst_map);
                        return KERN_INVALID_ADDRESS;
                }
                size = (entry->vme_end - entry->vme_start);
                /*
                 *      Make sure that no holes popped up in the
                 *      address map, and that the protection is
                 *      still valid, in case the map was unlocked
                 *      earlier.
                 */

                if ((entry->vme_start != start) || ((entry->is_sub_map)
                                && !entry->needs_copy)) {
                        vm_map_unlock(dst_map);
                        return(KERN_INVALID_ADDRESS);
                }
                assert(entry != vm_map_to_entry(dst_map));

                /*
                 *      Check protection again
                 */

                if ( ! (entry->protection & VM_PROT_WRITE)) {
                        vm_map_unlock(dst_map);
                        return(KERN_PROTECTION_FAILURE);
                }

                /*
                 *      Adjust to source size first
                 */

                if (copy_size < size) {
                        vm_map_clip_end(dst_map, entry, entry->vme_start + copy_size);
                        size = copy_size;
                }

                /*
                 *      Adjust to destination size
                 */

                if (size < copy_size) {
                        vm_map_copy_clip_end(copy, copy_entry,
                                copy_entry->vme_start + size);
                        copy_size = size;
                }

                assert((entry->vme_end - entry->vme_start) == size);
                assert((tmp_entry->vme_end - tmp_entry->vme_start) == size);
                assert((copy_entry->vme_end - copy_entry->vme_start) == size);

                /*
                 *      If the destination contains temporary unshared memory,
                 *      we can perform the copy by throwing it away and
                 *      installing the source data.
                 */

                object = entry->object.vm_object;
                if ((!entry->is_shared && 
                    ((object == VM_OBJECT_NULL) || 
                    (object->internal && !object->true_share))) ||
                    entry->needs_copy) {
                        vm_object_t     old_object = entry->object.vm_object;
                        vm_object_offset_t      old_offset = entry->offset;
                        vm_object_offset_t      offset;

                        /*
                         * Ensure that the source and destination aren't
                         * identical
                         */
                        if (old_object == copy_entry->object.vm_object &&
                            old_offset == copy_entry->offset) {
                                vm_map_copy_entry_unlink(copy, copy_entry);
                                vm_map_copy_entry_dispose(copy, copy_entry);

                                if (old_object != VM_OBJECT_NULL)
                                        vm_object_deallocate(old_object);

                                start = tmp_entry->vme_end;
                                tmp_entry = tmp_entry->vme_next;
                                continue;
                        }

                        if (old_object != VM_OBJECT_NULL) {
                                if(entry->is_sub_map) {
                                   if(entry->use_pmap) {
#ifndef i386
                                      pmap_unnest(dst_map->pmap, 
                                        entry->vme_start,
                                        entry->vme_end - entry->vme_start);
#endif
                                   } else {
                                      vm_map_submap_pmap_clean(
                                        dst_map, entry->vme_start, 
                                        entry->vme_end,
                                        entry->object.sub_map,
                                        entry->offset);
                                   }
                                   vm_map_deallocate(
                                                entry->object.sub_map);
                                } else {
                                        vm_object_pmap_protect(
                                                old_object,
                                                old_offset,
                                                size,
                                                pmap,
                                                tmp_entry->vme_start,
                                                VM_PROT_NONE);

                                        vm_object_deallocate(old_object);
                                }
                        }

                        entry->is_sub_map = FALSE;
                        entry->object = copy_entry->object;
                        object = entry->object.vm_object;
                        entry->needs_copy = copy_entry->needs_copy;
                        entry->wired_count = 0;
                        entry->user_wired_count = 0;
                        offset = entry->offset = copy_entry->offset;

                        vm_map_copy_entry_unlink(copy, copy_entry);
                        vm_map_copy_entry_dispose(copy, copy_entry);
#if BAD_OPTIMIZATION
                        /*
                         * if we turn this optimization back on
                         * we need to revisit our use of pmap mappings
                         * large copies will cause us to run out and panic
                         * this optimization only saved on average 2 us per page if ALL
                         * the pages in the source were currently mapped
                         * and ALL the pages in the dest were touched, if there were fewer
                         * than 2/3 of the pages touched, this optimization actually cost more cycles
                         */

                        /*
                         * Try to aggressively enter physical mappings
                         * (but avoid uninstantiated objects)
                         */
                        if (object != VM_OBJECT_NULL) {
                            vm_offset_t va = entry->vme_start;

                            while (va < entry->vme_end) {
                                register vm_page_t      m;
                                vm_prot_t               prot;

                                /*
                                 * Look for the page in the top object
                                 */
                                prot = entry->protection;
                                vm_object_lock(object);
                                vm_object_paging_begin(object);

                                if ((m = vm_page_lookup(object,offset)) !=
                                    VM_PAGE_NULL && !m->busy && 
                                    !m->fictitious &&
                                    (!m->unusual || (!m->error &&
                                        !m->restart && !m->absent &&
                                         (prot & m->page_lock) == 0))) {
                                        
                                        m->busy = TRUE;
                                        vm_object_unlock(object);
                                        
                                        /* 
                                         * Honor COW obligations
                                         */
                                        if (entry->needs_copy)
                                                prot &= ~VM_PROT_WRITE;

                                        PMAP_ENTER(pmap, va, m,
                                                   prot, FALSE);
                
                                        vm_object_lock(object);
                                        vm_page_lock_queues();
                                        if (!m->active && !m->inactive)
                                                vm_page_activate(m);
                                        vm_page_unlock_queues();
                                         PAGE_WAKEUP_DONE(m);
                                }
                                vm_object_paging_end(object);
                                vm_object_unlock(object);

                                offset += PAGE_SIZE_64;
                                va += PAGE_SIZE;
                            } /* end while (va < entry->vme_end) */
                        } /* end if (object) */
#endif
                        /*
                         *      Set up for the next iteration.  The map
                         *      has not been unlocked, so the next
                         *      address should be at the end of this
                         *      entry, and the next map entry should be
                         *      the one following it.
                         */

                        start = tmp_entry->vme_end;
                        tmp_entry = tmp_entry->vme_next;
                } else {
                        vm_map_version_t        version;
                        vm_object_t             dst_object = entry->object.vm_object;
                        vm_object_offset_t      dst_offset = entry->offset;
                        kern_return_t           r;

                        /*
                         *      Take an object reference, and record
                         *      the map version information so that the
                         *      map can be safely unlocked.
                         */

                        vm_object_reference(dst_object);

                        version.main_timestamp = dst_map->timestamp;

                        vm_map_unlock(dst_map);

                        /*
                         *      Copy as much as possible in one pass
                         */

                        copy_size = size;
                        r = vm_fault_copy(
                                        copy_entry->object.vm_object,
                                        copy_entry->offset,
                                        &copy_size,
                                        dst_object,
                                        dst_offset,
                                        dst_map,
                                        &version,
                                        THREAD_UNINT );

                        /*
                         *      Release the object reference
                         */

                        vm_object_deallocate(dst_object);

                        /*
                         *      If a hard error occurred, return it now
                         */

                        if (r != KERN_SUCCESS)
                                return(r);

                        if (copy_size != 0) {
                                /*
                                 *      Dispose of the copied region
                                 */

                                vm_map_copy_clip_end(copy, copy_entry,
                                        copy_entry->vme_start + copy_size);
                                vm_map_copy_entry_unlink(copy, copy_entry);
                                vm_object_deallocate(copy_entry->object.vm_object);
                                vm_map_copy_entry_dispose(copy, copy_entry);
                        }

                        /*
                         *      Pick up in the destination map where we left off.
                         *
                         *      Use the version information to avoid a lookup
                         *      in the normal case.
                         */

                        start += copy_size;
                        vm_map_lock(dst_map);
                        if ((version.main_timestamp + 1) == dst_map->timestamp) {
                                /* We can safely use saved tmp_entry value */

                                vm_map_clip_end(dst_map, tmp_entry, start);
                                tmp_entry = tmp_entry->vme_next;
                        } else {
                                /* Must do lookup of tmp_entry */

                                if (!vm_map_lookup_entry(dst_map, start, &tmp_entry)) {
                                        vm_map_unlock(dst_map);
                                        return(KERN_INVALID_ADDRESS);
                                }
                                vm_map_clip_start(dst_map, tmp_entry, start);
                        }
                }
        }/* while */

        return(KERN_SUCCESS);
}/* vm_map_copy_overwrite_aligned */

/*
 *      Routine:        vm_map_copyin_kernel_buffer
 *
 *      Description:
 *              Copy in data to a kernel buffer from space in the
 *              source map. The original space may be otpionally
 *              deallocated.
 *
 *              If successful, returns a new copy object.
 */
kern_return_t
vm_map_copyin_kernel_buffer(
        vm_map_t        src_map,
        vm_offset_t     src_addr,
        vm_size_t       len,
        boolean_t       src_destroy,
        vm_map_copy_t   *copy_result)
{
        boolean_t flags;
        vm_map_copy_t copy;
        vm_size_t kalloc_size = sizeof(struct vm_map_copy) + len;

        copy = (vm_map_copy_t) kalloc(kalloc_size);
        if (copy == VM_MAP_COPY_NULL) {
                return KERN_RESOURCE_SHORTAGE;
        }
        copy->type = VM_MAP_COPY_KERNEL_BUFFER;
        copy->size = len;
        copy->offset = 0;
        copy->cpy_kdata = (vm_offset_t) (copy + 1);
        copy->cpy_kalloc_size = kalloc_size;

        if (src_map == kernel_map) {
                bcopy((char *)src_addr, (char *)copy->cpy_kdata, len);
                flags = VM_MAP_REMOVE_KUNWIRE | VM_MAP_REMOVE_WAIT_FOR_KWIRE |
                        VM_MAP_REMOVE_INTERRUPTIBLE;
        } else {
                kern_return_t kr;
                kr = copyinmap(src_map, src_addr, copy->cpy_kdata, len);
                if (kr != KERN_SUCCESS) {
                        kfree((vm_offset_t)copy, kalloc_size);
                        return kr;
                }
                flags = VM_MAP_REMOVE_WAIT_FOR_KWIRE |
                        VM_MAP_REMOVE_INTERRUPTIBLE;
        }
        if (src_destroy) {
                (void) vm_map_remove(src_map, trunc_page(src_addr), 
                                     round_page(src_addr + len),
                                     flags);
        }
        *copy_result = copy;
        return KERN_SUCCESS;
}

/*
 *      Routine:        vm_map_copyout_kernel_buffer
 *
 *      Description:
 *              Copy out data from a kernel buffer into space in the
 *              destination map. The space may be otpionally dynamically
 *              allocated.
 *
 *              If successful, consumes the copy object.
 *              Otherwise, the caller is responsible for it.
 */
kern_return_t
vm_map_copyout_kernel_buffer(
        vm_map_t        map,
        vm_offset_t     *addr,  /* IN/OUT */
        vm_map_copy_t   copy,
        boolean_t       overwrite)
{
        kern_return_t kr = KERN_SUCCESS;
        thread_act_t thr_act = current_act();

        if (!overwrite) {

                /*
                 * Allocate space in the target map for the data
                 */
                *addr = 0;
                kr = vm_map_enter(map, 
                                  addr, 
                                  round_page(copy->size),
                                  (vm_offset_t) 0, 
                                  TRUE,
                                  VM_OBJECT_NULL, 
                                  (vm_object_offset_t) 0, 
                                  FALSE,
                                  VM_PROT_DEFAULT, 
                                  VM_PROT_ALL,
                                  VM_INHERIT_DEFAULT);
                if (kr != KERN_SUCCESS)
                        return(kr);
        }

        /*
         * Copyout the data from the kernel buffer to the target map.
         */     
        if (thr_act->map == map) {
        
                /*
                 * If the target map is the current map, just do
                 * the copy.
                 */
                if (copyout((char *)copy->cpy_kdata, (char *)*addr,
                                copy->size)) {
                        kr = KERN_INVALID_ADDRESS;
                }
        }
        else {
                vm_map_t oldmap;

                /*
                 * If the target map is another map, assume the
                 * target's address space identity for the duration
                 * of the copy.
                 */
                vm_map_reference(map);
                oldmap = vm_map_switch(map);

                if (copyout((char *)copy->cpy_kdata, (char *)*addr,
                                copy->size)) {
                        kr = KERN_INVALID_ADDRESS;
                }
        
                (void) vm_map_switch(oldmap);
                vm_map_deallocate(map);
        }

        kfree((vm_offset_t)copy, copy->cpy_kalloc_size);

        return(kr);
}
                
/*
 *      Macro:          vm_map_copy_insert
 *      
 *      Description:
 *              Link a copy chain ("copy") into a map at the
 *              specified location (after "where").
 *      Side effects:
 *              The copy chain is destroyed.
 *      Warning:
 *              The arguments are evaluated multiple times.
 */
#define vm_map_copy_insert(map, where, copy)                            \
MACRO_BEGIN                                                             \
        vm_map_t VMCI_map;                                              \
        vm_map_entry_t VMCI_where;                                      \
        vm_map_copy_t VMCI_copy;                                        \
        VMCI_map = (map);                                               \
        VMCI_where = (where);                                           \
        VMCI_copy = (copy);                                             \
        ((VMCI_where->vme_next)->vme_prev = vm_map_copy_last_entry(VMCI_copy))\
                ->vme_next = (VMCI_where->vme_next);                    \
        ((VMCI_where)->vme_next = vm_map_copy_first_entry(VMCI_copy))   \
                ->vme_prev = VMCI_where;                                \
        VMCI_map->hdr.nentries += VMCI_copy->cpy_hdr.nentries;          \
        UPDATE_FIRST_FREE(VMCI_map, VMCI_map->first_free);              \
        zfree(vm_map_copy_zone, (vm_offset_t) VMCI_copy);               \
MACRO_END

/*
 *      Routine:        vm_map_copyout
 *
 *      Description:
 *              Copy out a copy chain ("copy") into newly-allocated
 *              space in the destination map.
 *
 *              If successful, consumes the copy object.
 *              Otherwise, the caller is responsible for it.
 */
kern_return_t
vm_map_copyout(
        register vm_map_t       dst_map,
        vm_offset_t             *dst_addr,      /* OUT */
        register vm_map_copy_t  copy)
{
        vm_size_t               size;
        vm_size_t               adjustment;
        vm_offset_t             start;
        vm_object_offset_t      vm_copy_start;
        vm_map_entry_t          last;
        register
        vm_map_entry_t          entry;

        /*
         *      Check for null copy object.
         */

        if (copy == VM_MAP_COPY_NULL) {
                *dst_addr = 0;
                return(KERN_SUCCESS);
        }

        /*
         *      Check for special copy object, created
         *      by vm_map_copyin_object.
         */

        if (copy->type == VM_MAP_COPY_OBJECT) {
                vm_object_t             object = copy->cpy_object;
                kern_return_t           kr;
                vm_object_offset_t      offset;

                offset = trunc_page_64(copy->offset);
                size = round_page(copy->size + 
                                (vm_size_t)(copy->offset - offset));
                *dst_addr = 0;
                kr = vm_map_enter(dst_map, dst_addr, size,
                                  (vm_offset_t) 0, TRUE,
                                  object, offset, FALSE,
                                  VM_PROT_DEFAULT, VM_PROT_ALL,
                                  VM_INHERIT_DEFAULT);
                if (kr != KERN_SUCCESS)
                        return(kr);
                /* Account for non-pagealigned copy object */
                *dst_addr += (vm_offset_t)(copy->offset - offset);
                zfree(vm_map_copy_zone, (vm_offset_t) copy);
                return(KERN_SUCCESS);
        }

        /*
         *      Check for special kernel buffer allocated
         *      by new_ipc_kmsg_copyin.
         */

        if (copy->type == VM_MAP_COPY_KERNEL_BUFFER) {
                return(vm_map_copyout_kernel_buffer(dst_map, dst_addr, 
                                                    copy, FALSE));
        }

        if (copy->type == VM_MAP_COPY_PAGE_LIST)
                return(vm_map_copyout_page_list(dst_map, dst_addr, copy));

        /*
         *      Find space for the data
         */

        vm_copy_start = trunc_page_64(copy->offset);
        size =  round_page((vm_size_t)copy->offset + copy->size) 
                                                        - vm_copy_start;

 StartAgain: ;

        vm_map_lock(dst_map);
        assert(first_free_is_valid(dst_map));
        start = ((last = dst_map->first_free) == vm_map_to_entry(dst_map)) ?
                vm_map_min(dst_map) : last->vme_end;

        while (TRUE) {
                vm_map_entry_t  next = last->vme_next;
                vm_offset_t     end = start + size;

                if ((end > dst_map->max_offset) || (end < start)) {
                        if (dst_map->wait_for_space) {
                                if (size <= (dst_map->max_offset - dst_map->min_offset)) {
                                        assert_wait((event_t) dst_map,
                                                    THREAD_INTERRUPTIBLE);
                                        vm_map_unlock(dst_map);
                                        thread_block((void (*)(void))0);
                                        goto StartAgain;
                                }
                        }
                        vm_map_unlock(dst_map);
                        return(KERN_NO_SPACE);
                }

                if ((next == vm_map_to_entry(dst_map)) ||
                    (next->vme_start >= end))
                        break;

                last = next;
                start = last->vme_end;
        }

        /*
         *      Since we're going to just drop the map
         *      entries from the copy into the destination
         *      map, they must come from the same pool.
         */

        if (copy->cpy_hdr.entries_pageable != dst_map->hdr.entries_pageable) {
            /*
             * Mismatches occur when dealing with the default
             * pager.
             */
            zone_t              old_zone;
            vm_map_entry_t      next, new;

            /*
             * Find the zone that the copies were allocated from
             */
            old_zone = (copy->cpy_hdr.entries_pageable)
                        ? vm_map_entry_zone
                        : vm_map_kentry_zone;
            entry = vm_map_copy_first_entry(copy);

            /*
             * Reinitialize the copy so that vm_map_copy_entry_link
             * will work.
             */
            copy->cpy_hdr.nentries = 0;
            copy->cpy_hdr.entries_pageable = dst_map->hdr.entries_pageable;
            vm_map_copy_first_entry(copy) =
             vm_map_copy_last_entry(copy) =
                vm_map_copy_to_entry(copy);

            /*
             * Copy each entry.
             */
            while (entry != vm_map_copy_to_entry(copy)) {
                new = vm_map_copy_entry_create(copy);
                vm_map_entry_copy_full(new, entry);
                new->use_pmap = FALSE;  /* clr address space specifics */
                vm_map_copy_entry_link(copy,
                                vm_map_copy_last_entry(copy),
                                new);
                next = entry->vme_next;
                zfree(old_zone, (vm_offset_t) entry);
                entry = next;
            }
        }

        /*
         *      Adjust the addresses in the copy chain, and
         *      reset the region attributes.
         */

        adjustment = start - vm_copy_start;
        for (entry = vm_map_copy_first_entry(copy);
             entry != vm_map_copy_to_entry(copy);
             entry = entry->vme_next) {
                entry->vme_start += adjustment;
                entry->vme_end += adjustment;

                entry->inheritance = VM_INHERIT_DEFAULT;
                entry->protection = VM_PROT_DEFAULT;
                entry->max_protection = VM_PROT_ALL;
                entry->behavior = VM_BEHAVIOR_DEFAULT;

                /*
                 * If the entry is now wired,
                 * map the pages into the destination map.
                 */
                if (entry->wired_count != 0) {
                    register vm_offset_t va;
                    vm_object_offset_t   offset;
                    register vm_object_t object;

                    object = entry->object.vm_object;
                    offset = entry->offset;
                    va = entry->vme_start;

                    pmap_pageable(dst_map->pmap,
                                  entry->vme_start,
                                  entry->vme_end,
                                  TRUE);

                    while (va < entry->vme_end) {
                        register vm_page_t      m;

                        /*
                         * Look up the page in the object.
                         * Assert that the page will be found in the
                         * top object:
                         * either
                         *      the object was newly created by
                         *      vm_object_copy_slowly, and has
                         *      copies of all of the pages from
                         *      the source object
                         * or
                         *      the object was moved from the old
                         *      map entry; because the old map
                         *      entry was wired, all of the pages
                         *      were in the top-level object.
                         *      (XXX not true if we wire pages for
                         *       reading)
                         */
                        vm_object_lock(object);
                        vm_object_paging_begin(object);

                        m = vm_page_lookup(object, offset);
                        if (m == VM_PAGE_NULL || m->wire_count == 0 ||
                            m->absent)
                            panic("vm_map_copyout: wiring 0x%x", m);

                        m->busy = TRUE;
                        vm_object_unlock(object);

                        PMAP_ENTER(dst_map->pmap, va, m,
                                   entry->protection, TRUE);

                        vm_object_lock(object);
                        PAGE_WAKEUP_DONE(m);
                        /* the page is wired, so we don't have to activate */
                        vm_object_paging_end(object);
                        vm_object_unlock(object);

                        offset += PAGE_SIZE_64;
                        va += PAGE_SIZE;
                    }
                }
                else if (size <= vm_map_aggressive_enter_max) {

                        register vm_offset_t    va;
                        vm_object_offset_t      offset;
                        register vm_object_t    object;
                        vm_prot_t               prot;

                        object = entry->object.vm_object;
                        if (object != VM_OBJECT_NULL) {

                                offset = entry->offset;
                                va = entry->vme_start;
                                while (va < entry->vme_end) {
                                        register vm_page_t      m;
                                    
                                        /*
                                         * Look up the page in the object.
                                         * Assert that the page will be found
                                         * in the top object if at all...
                                         */
                                        vm_object_lock(object);
                                        vm_object_paging_begin(object);

                                        if (((m = vm_page_lookup(object,
                                                                 offset))
                                             != VM_PAGE_NULL) &&
                                            !m->busy && !m->fictitious &&
                                            !m->absent && !m->error) {
                                                m->busy = TRUE;
                                                vm_object_unlock(object);

                                                /* honor cow obligations */
                                                prot = entry->protection;
                                                if (entry->needs_copy)
                                                        prot &= ~VM_PROT_WRITE;

                                                PMAP_ENTER(dst_map->pmap, va, 
                                                           m, prot, FALSE);

                                                vm_object_lock(object);
                                                vm_page_lock_queues();
                                                if (!m->active && !m->inactive)
                                                        vm_page_activate(m);
                                                vm_page_unlock_queues();
                                                PAGE_WAKEUP_DONE(m);
                                        }
                                        vm_object_paging_end(object);
                                        vm_object_unlock(object);

                                        offset += PAGE_SIZE_64;
                                        va += PAGE_SIZE;
                                }
                        }
                }
        }

        /*
         *      Correct the page alignment for the result
         */

        *dst_addr = start + (copy->offset - vm_copy_start);

        /*
         *      Update the hints and the map size
         */

        SAVE_HINT(dst_map, vm_map_copy_last_entry(copy));

        dst_map->size += size;

        /*
         *      Link in the copy
         */

        vm_map_copy_insert(dst_map, last, copy);

        vm_map_unlock(dst_map);

        /*
         * XXX  If wiring_required, call vm_map_pageable
         */

        return(KERN_SUCCESS);
}

boolean_t       vm_map_aggressive_enter;        /* not used yet */

/*
 *
 *      vm_map_copyout_page_list:
 *
 *      Version of vm_map_copyout() for page list vm map copies.
 *
 */
kern_return_t
vm_map_copyout_page_list(
        register vm_map_t       dst_map,
        vm_offset_t             *dst_addr,      /* OUT */
        register vm_map_copy_t  copy)
{
        vm_size_t               size;
        vm_offset_t             start;
        vm_offset_t             end;
        vm_object_offset_t      offset;
        vm_map_entry_t          last;
        register
        vm_object_t             object;
        vm_page_t               *page_list, m;
        vm_map_entry_t          entry;
        vm_object_offset_t      old_last_offset;
        boolean_t               cont_invoked, needs_wakeup;
        kern_return_t           result = KERN_SUCCESS;
        vm_map_copy_t           orig_copy;
        vm_object_offset_t      dst_offset;
        boolean_t               must_wire;
        boolean_t               aggressive_enter;

        /*
         *      Check for null copy object.
         */

        if (copy == VM_MAP_COPY_NULL) {
                *dst_addr = 0;
                return(KERN_SUCCESS);
        }

        assert(copy->type == VM_MAP_COPY_PAGE_LIST);

        /*
         *      Make sure the pages are stolen, because we are
         *      going to put them in a new object.  Assume that
         *      all pages are identical to first in this regard.
         */

        page_list = &copy->cpy_page_list[0];
        if (!copy->cpy_page_loose)
                vm_map_copy_steal_pages(copy);

        /*
         *      Find space for the data
         */

        size =  round_page_64(copy->offset + (vm_object_offset_t)copy->size) -
                trunc_page_64(copy->offset);
StartAgain:
        vm_map_lock(dst_map);
        must_wire = dst_map->wiring_required;

        assert(first_free_is_valid(dst_map));
        last = dst_map->first_free;
        if (last == vm_map_to_entry(dst_map)) {
                start = vm_map_min(dst_map);
        } else {
                start = last->vme_end;
        }

        while (TRUE) {
                vm_map_entry_t next = last->vme_next;
                end = start + size;

                if ((end > dst_map->max_offset) || (end < start)) {
                        if (dst_map->wait_for_space) {
                                if (size <= (dst_map->max_offset -
                                             dst_map->min_offset)) {
                                        assert_wait((event_t) dst_map,
                                                    THREAD_INTERRUPTIBLE);
                                        vm_map_unlock(dst_map);
                                        thread_block((void (*)(void))0);
                                        goto StartAgain;
                                }
                        }
                        vm_map_unlock(dst_map);
                        return(KERN_NO_SPACE);
                }

                if ((next == vm_map_to_entry(dst_map)) ||
                    (next->vme_start >= end)) {
                        break;
                }

                last = next;
                start = last->vme_end;
        }

        /*
         *      See whether we can avoid creating a new entry (and object) by
         *      extending one of our neighbors.  [So far, we only attempt to
         *      extend from below.]
         *
         *      The code path below here is a bit twisted.  If any of the
         *      extension checks fails, we branch to create_object.  If
         *      it all works, we fall out the bottom and goto insert_pages.
         */
        if (last == vm_map_to_entry(dst_map) ||
            last->vme_end != start ||
            last->is_shared != FALSE ||
            last->is_sub_map != FALSE ||
            last->inheritance != VM_INHERIT_DEFAULT ||
            last->protection != VM_PROT_DEFAULT ||
            last->max_protection != VM_PROT_ALL ||
            last->behavior != VM_BEHAVIOR_DEFAULT ||
            last->in_transition ||
            (must_wire ? (last->wired_count != 1 ||
                    last->user_wired_count != 0) :
                (last->wired_count != 0))) {
                    goto create_object;
        }
        
        /*
         * If this entry needs an object, make one.
         */
        if (last->object.vm_object == VM_OBJECT_NULL) {
                object = vm_object_allocate(
                        (vm_size_t)(last->vme_end - last->vme_start + size));
                last->object.vm_object = object;
                last->offset = 0;
        }
        else {
            vm_object_offset_t  prev_offset = last->offset;
            vm_size_t           prev_size = start - last->vme_start;
            vm_size_t           new_size;

            /*
             *  This is basically vm_object_coalesce.
             */

            object = last->object.vm_object;
            vm_object_lock(object);

           /*
            *  Try to collapse the object first
            */
           vm_object_collapse(object);


            /*
             *  Can't coalesce if pages not mapped to
             *  last may be in use anyway:
             *  . more than one reference
             *  . paged out
             *  . shadows another object
             *  . has a copy elsewhere
             *  . paging references (pages might be in page-list)
             */

            if ((object->ref_count > 1) ||
                object->pager_created ||
                (object->shadow != VM_OBJECT_NULL) ||
                (object->copy != VM_OBJECT_NULL) ||
                (object->paging_in_progress != 0)) {
                    vm_object_unlock(object);
                    goto create_object;
            }

            /*
             *  Extend the object if necessary.  Don't have to call
             *  vm_object_page_remove because the pages aren't mapped,
             *  and vm_page_replace will free up any old ones it encounters.
             */
            new_size = prev_offset + prev_size + size;
            if (new_size > object->size) {
#if     MACH_PAGEMAP
                    /*
                     *  We cannot extend an object that has existence info,
                     *  since the existence info might then fail to cover
                     *  the entire object.
                     *
                     *  This assertion must be true because the object
                     *  has no pager, and we only create existence info
                     *  for objects with pagers.
                     */
                    assert(object->existence_map == VM_EXTERNAL_NULL);
#endif  /* MACH_PAGEMAP */
                    object->size = new_size;
            }
            vm_object_unlock(object);
        }

        /*
         *      Coalesced the two objects - can extend
         *      the previous map entry to include the
         *      new range.
         */
        dst_map->size += size;
        last->vme_end = end;
        UPDATE_FIRST_FREE(dst_map, dst_map->first_free);

        SAVE_HINT(dst_map, last);

        goto insert_pages;

create_object:

        /*
         *      Create object
         */
        object = vm_object_allocate(size);

        /*
         *      Create entry
         */
        last = vm_map_entry_insert(dst_map, last, start, start + size,
                                   object, 0, FALSE, FALSE, TRUE,
                                   VM_PROT_DEFAULT, VM_PROT_ALL,
                                   VM_BEHAVIOR_DEFAULT,
                                   VM_INHERIT_DEFAULT, (must_wire ? 1 : 0));

        /*
         *      Transfer pages into new object.  
         *      Scan page list in vm_map_copy.
         */
insert_pages:
        dst_offset = copy->offset & PAGE_MASK_64;
        cont_invoked = FALSE;
        orig_copy = copy;
        last->in_transition = TRUE;
        old_last_offset = last->offset
            + (start - last->vme_start);

        aggressive_enter = (size <= vm_map_aggressive_enter_max);

        for (offset = 0; offset < size; offset += PAGE_SIZE_64) {
                m = *page_list;
                assert(m && !m->tabled);

                /*
                 *      Must clear busy bit in page before inserting it.
                 *      Ok to skip wakeup logic because nobody else
                 *      can possibly know about this page.  Also set
                 *      dirty bit on the assumption that the page is
                 *      not a page of zeros.
                 */

                m->busy = FALSE;
                m->dirty = TRUE;
                vm_object_lock(object);
                vm_page_lock_queues();
                vm_page_replace(m, object, old_last_offset + offset);
                if (must_wire) {
                        vm_page_wire(m);
                } else if (aggressive_enter) {
                        vm_page_activate(m);
                }
                vm_page_unlock_queues();
                vm_object_unlock(object);

                if (aggressive_enter || must_wire) {
                        PMAP_ENTER(dst_map->pmap,
                                   last->vme_start + m->offset - last->offset,
                                   m, last->protection, must_wire);
                }

                *page_list++ = VM_PAGE_NULL;
                assert(copy != VM_MAP_COPY_NULL);
                assert(copy->type == VM_MAP_COPY_PAGE_LIST);
                if (--(copy->cpy_npages) == 0 &&
                    vm_map_copy_has_cont(copy)) {
                        vm_map_copy_t   new_copy;

                        /*
                         *      Ok to unlock map because entry is
                         *      marked in_transition.
                         */
                        cont_invoked = TRUE;
                        vm_map_unlock(dst_map);
                        vm_map_copy_invoke_cont(copy, &new_copy, &result);

                        if (result == KERN_SUCCESS) {

                                /*
                                 *      If we got back a copy with real pages,
                                 *      steal them now.  Either all of the
                                 *      pages in the list are tabled or none
                                 *      of them are; mixtures are not possible.
                                 *
                                 *      Save original copy for consume on
                                 *      success logic at end of routine.
                                 */
                                if (copy != orig_copy)
                                        vm_map_copy_discard(copy);

                                if ((copy = new_copy) != VM_MAP_COPY_NULL) {
                                        page_list = &copy->cpy_page_list[0];
                                        if (!copy->cpy_page_loose)
                                                vm_map_copy_steal_pages(copy);
                                }
                        }
                        else {
                                /*
                                 *      Continuation failed.
                                 */
                                vm_map_lock(dst_map);
                                goto error;
                        }

                        vm_map_lock(dst_map);
                }
        }

        *dst_addr = start + dst_offset;
        
        /*
         *      Clear the in transition bits.  This is easy if we
         *      didn't have a continuation.
         */
error:
        needs_wakeup = FALSE;
        if (!cont_invoked) {
                /*
                 *      We didn't unlock the map, so nobody could
                 *      be waiting.
                 */
                last->in_transition = FALSE;
                assert(!last->needs_wakeup);
        }
        else {
                if (!vm_map_lookup_entry(dst_map, start, &entry))
                        panic("vm_map_copyout_page_list: missing entry");

                /*
                 * Clear transition bit for all constituent entries that
                 * were in the original entry.  Also check for waiters.
                 */
                while ((entry != vm_map_to_entry(dst_map)) &&
                       (entry->vme_start < end)) {
                        assert(entry->in_transition);
                        entry->in_transition = FALSE;
                        if (entry->needs_wakeup) {
                                entry->needs_wakeup = FALSE;
                                needs_wakeup = TRUE;
                        }
                        entry = entry->vme_next;
                }
        }
        
        if (result != KERN_SUCCESS)
                (void) vm_map_delete(dst_map, start, end, VM_MAP_NO_FLAGS);

        vm_map_unlock(dst_map);

        if (needs_wakeup)
                vm_map_entry_wakeup(dst_map);

        /*
         *      Consume on success logic.
         */
        if (copy != VM_MAP_COPY_NULL && copy != orig_copy) {
                zfree(vm_map_copy_zone, (vm_offset_t) copy);
        }
        if (result == KERN_SUCCESS) {
                assert(orig_copy != VM_MAP_COPY_NULL);
                assert(orig_copy->type == VM_MAP_COPY_PAGE_LIST);
                zfree(vm_map_copy_zone, (vm_offset_t) orig_copy);
        }
        
        return(result);
}

/*
 *      Routine:        vm_map_copyin
 *
 *      Description:
 *              Copy the specified region (src_addr, len) from the
 *              source address space (src_map), possibly removing
 *              the region from the source address space (src_destroy).
 *
 *      Returns:
 *              A vm_map_copy_t object (copy_result), suitable for
 *              insertion into another address space (using vm_map_copyout),
 *              copying over another address space region (using
 *              vm_map_copy_overwrite).  If the copy is unused, it
 *              should be destroyed (using vm_map_copy_discard).
 *
 *      In/out conditions:
 *              The source map should not be locked on entry.
 */

typedef struct submap_map {
        vm_map_t        parent_map;
        vm_offset_t     base_start;
        vm_offset_t     base_end;
        struct submap_map *next;
} submap_map_t;

kern_return_t
vm_map_copyin_common(
        vm_map_t        src_map,
        vm_offset_t     src_addr,
        vm_size_t       len,
        boolean_t       src_destroy,
        boolean_t       src_volatile,
        vm_map_copy_t   *copy_result,   /* OUT */
        boolean_t       use_maxprot)
{
        extern int      msg_ool_size_small;

        vm_map_entry_t  tmp_entry;      /* Result of last map lookup --
                                         * in multi-level lookup, this
                                         * entry contains the actual
                                         * vm_object/offset.
                                         */
        register
        vm_map_entry_t  new_entry = VM_MAP_ENTRY_NULL;  /* Map entry for copy */

        vm_offset_t     src_start;      /* Start of current entry --
                                         * where copy is taking place now
                                         */
        vm_offset_t     src_end;        /* End of entire region to be
                                         * copied */
        vm_offset_t     base_start;     /* submap fields to save offsets */
                                        /* in original map */
        vm_offset_t     base_end;
        vm_map_t        base_map=src_map;
        vm_map_entry_t  base_entry;
        boolean_t       map_share=FALSE;
        submap_map_t    *parent_maps = NULL;

        register
        vm_map_copy_t   copy;           /* Resulting copy */
        vm_offset_t     copy_addr;

        /*
         *      Check for copies of zero bytes.
         */

        if (len == 0) {
                *copy_result = VM_MAP_COPY_NULL;
                return(KERN_SUCCESS);
        }

        /*
         * If the copy is sufficiently small, use a kernel buffer instead
         * of making a virtual copy.  The theory being that the cost of
         * setting up VM (and taking C-O-W faults) dominates the copy costs
         * for small regions.
         */
        if ((len < msg_ool_size_small) && !use_maxprot)
          return vm_map_copyin_kernel_buffer(src_map, src_addr, len,
                                             src_destroy, copy_result);

        /*
         *      Compute start and end of region
         */

        src_start = trunc_page(src_addr);
        src_end = round_page(src_addr + len);

        XPR(XPR_VM_MAP, "vm_map_copyin_common map 0x%x addr 0x%x len 0x%x dest %d\n", (natural_t)src_map, src_addr, len, src_destroy, 0);

        /*
         *      Check that the end address doesn't overflow
         */

        if (src_end <= src_start)
                if ((src_end < src_start) || (src_start != 0))
                        return(KERN_INVALID_ADDRESS);

        /*
         *      Allocate a header element for the list.
         *
         *      Use the start and end in the header to 
         *      remember the endpoints prior to rounding.
         */

        copy = (vm_map_copy_t) zalloc(vm_map_copy_zone);
        vm_map_copy_first_entry(copy) =
         vm_map_copy_last_entry(copy) = vm_map_copy_to_entry(copy);
        copy->type = VM_MAP_COPY_ENTRY_LIST;
        copy->cpy_hdr.nentries = 0;
        copy->cpy_hdr.entries_pageable = TRUE;

        copy->offset = src_addr;
        copy->size = len;
        
        new_entry = vm_map_copy_entry_create(copy);

#define RETURN(x)                                               \
        MACRO_BEGIN                                             \
        vm_map_unlock(src_map);                                 \
        if (new_entry != VM_MAP_ENTRY_NULL)                     \
                vm_map_copy_entry_dispose(copy,new_entry);      \
        vm_map_copy_discard(copy);                              \
        {                                                       \
                submap_map_t    *ptr;                           \
                                                                \
                for(ptr = parent_maps; ptr != NULL; ptr = parent_maps) { \
                        parent_maps=parent_maps->next;          \
                        kfree((vm_offset_t)ptr, sizeof(submap_map_t));  \
                }                                               \
        }                                                       \
        MACRO_RETURN(x);                                        \
        MACRO_END

        /*
         *      Find the beginning of the region.
         */

        vm_map_lock(src_map);

        if (!vm_map_lookup_entry(src_map, src_start, &tmp_entry))
                RETURN(KERN_INVALID_ADDRESS);
        if(!tmp_entry->is_sub_map) {
                vm_map_clip_start(src_map, tmp_entry, src_start);
        }
        /* set for later submap fix-up */
        copy_addr = src_start;

        /*
         *      Go through entries until we get to the end.
         */

        while (TRUE) {
                register
                vm_map_entry_t  src_entry = tmp_entry;  /* Top-level entry */
                vm_size_t       src_size;               /* Size of source
                                                         * map entry (in both
                                                         * maps)
                                                         */

                register
                vm_object_t             src_object;     /* Object to copy */
                vm_object_offset_t      src_offset;

                boolean_t       src_needs_copy;         /* Should source map
                                                         * be made read-only
                                                         * for copy-on-write?
                                                         */

                boolean_t       new_entry_needs_copy;   /* Will new entry be COW? */

                boolean_t       was_wired;              /* Was source wired? */
                vm_map_version_t version;               /* Version before locks
                                                         * dropped to make copy
                                                         */
                kern_return_t   result;                 /* Return value from
                                                         * copy_strategically.
                                                         */
                while(tmp_entry->is_sub_map) {
                        vm_size_t submap_len;
                        submap_map_t *ptr;

                        ptr = (submap_map_t *)kalloc(sizeof(submap_map_t));
                        ptr->next = parent_maps;
                        parent_maps = ptr;
                        ptr->parent_map = src_map;
                        ptr->base_start = src_start;
                        ptr->base_end = src_end;
                        submap_len = tmp_entry->vme_end - src_start;
                        if(submap_len > (src_end-src_start))
                                submap_len = src_end-src_start;
                        ptr->base_start += submap_len;
        
                        src_start -= tmp_entry->vme_start;
                        src_start += tmp_entry->offset;
                        src_end = src_start + submap_len;
                        src_map = tmp_entry->object.sub_map;
                        vm_map_lock(src_map);
                        vm_map_unlock(ptr->parent_map);
                        if (!vm_map_lookup_entry(
                                        src_map, src_start, &tmp_entry))
                                RETURN(KERN_INVALID_ADDRESS);
                        map_share = TRUE;
                        if(!tmp_entry->is_sub_map)
                           vm_map_clip_start(src_map, tmp_entry, src_start);
                        src_entry = tmp_entry;
                }
                /*
                 *      Create a new address map entry to hold the result. 
                 *      Fill in the fields from the appropriate source entries.
                 *      We must unlock the source map to do this if we need
                 *      to allocate a map entry.
                 */
                if (new_entry == VM_MAP_ENTRY_NULL) {
                    version.main_timestamp = src_map->timestamp;
                    vm_map_unlock(src_map);

                    new_entry = vm_map_copy_entry_create(copy);

                    vm_map_lock(src_map);
                    if ((version.main_timestamp + 1) != src_map->timestamp) {
                        if (!vm_map_lookup_entry(src_map, src_start,
                                        &tmp_entry)) {
                                RETURN(KERN_INVALID_ADDRESS);
                        }
                        vm_map_clip_start(src_map, tmp_entry, src_start);
                        continue; /* restart w/ new tmp_entry */
                    }
                }

                /*
                 *      Verify that the region can be read.
                 */
                if (((src_entry->protection & VM_PROT_READ) == VM_PROT_NONE &&
                        !use_maxprot) ||
                    (src_entry->max_protection & VM_PROT_READ) == 0)
                        RETURN(KERN_PROTECTION_FAILURE);

                /*
                 *      Clip against the endpoints of the entire region.
                 */

                vm_map_clip_end(src_map, src_entry, src_end);

                src_size = src_entry->vme_end - src_start;
                src_object = src_entry->object.vm_object;
                src_offset = src_entry->offset;
                was_wired = (src_entry->wired_count != 0);

                vm_map_entry_copy(new_entry, src_entry);
                new_entry->use_pmap = FALSE; /* clr address space specifics */

                /*
                 *      Attempt non-blocking copy-on-write optimizations.
                 */

                if (src_destroy && 
                    (src_object == VM_OBJECT_NULL || 
                    (src_object->internal && !src_object->true_share
                    && !map_share))) {
                    /*
                     * If we are destroying the source, and the object
                     * is internal, we can move the object reference
                     * from the source to the copy.  The copy is
                     * copy-on-write only if the source is.
                     * We make another reference to the object, because
                     * destroying the source entry will deallocate it.
                     */
                    vm_object_reference(src_object);

                    /*
                     * Copy is always unwired.  vm_map_copy_entry
                     * set its wired count to zero.
                     */

                    goto CopySuccessful;
                }


RestartCopy:
                XPR(XPR_VM_MAP, "vm_map_copyin_common src_obj 0x%x ent 0x%x obj 0x%x was_wired %d\n",
                    src_object, new_entry, new_entry->object.vm_object,
                    was_wired, 0);
                if (!was_wired &&
                    vm_object_copy_quickly(
                                &new_entry->object.vm_object,
                                src_offset,
                                src_size,
                                &src_needs_copy,
                                &new_entry_needs_copy)) {

                        new_entry->needs_copy = new_entry_needs_copy;

                        /*
                         *      Handle copy-on-write obligations
                         */

                        if (src_needs_copy && !tmp_entry->needs_copy) {
                                if (tmp_entry->is_shared  || 
                                     tmp_entry->object.vm_object->true_share ||
                                     map_share) {
                                        /* dec ref gained in copy_quickly */
                                        vm_object_lock(src_object);
                                        src_object->ref_count--; 
                                        vm_object_res_deallocate(src_object);
                                        vm_object_unlock(src_object);
                                        new_entry->object.vm_object = 
                                                vm_object_copy_delayed(
                                                        src_object,
                                                        src_offset,     
                                                        src_size);
                                } else {
                                        vm_object_pmap_protect(
                                                src_object,
                                                src_offset,
                                                src_size,
                                                (src_entry->is_shared ? 
                                                        PMAP_NULL
                                                        : src_map->pmap),
                                                src_entry->vme_start,
                                                src_entry->protection &
                                                        ~VM_PROT_WRITE);

                                        tmp_entry->needs_copy = TRUE;
                                }
                        }

                        /*
                         *      The map has never been unlocked, so it's safe
                         *      to move to the next entry rather than doing
                         *      another lookup.
                         */

                        goto CopySuccessful;
                }

                new_entry->needs_copy = FALSE;

                /*
                 *      Take an object reference, so that we may
                 *      release the map lock(s).
                 */

                assert(src_object != VM_OBJECT_NULL);
                vm_object_reference(src_object);

                /*
                 *      Record the timestamp for later verification.
                 *      Unlock the map.
                 */

                version.main_timestamp = src_map->timestamp;
                vm_map_unlock(src_map);

                /*
                 *      Perform the copy
                 */

                if (was_wired) {
                        vm_object_lock(src_object);
                        result = vm_object_copy_slowly(
                                        src_object,
                                        src_offset,
                                        src_size,
                                        THREAD_UNINT,
                                        &new_entry->object.vm_object);
                        new_entry->offset = 0;
                        new_entry->needs_copy = FALSE;
                } else {
                        result = vm_object_copy_strategically(src_object,
                                src_offset,
                                src_size,
                                &new_entry->object.vm_object,
                                &new_entry->offset,
                                &new_entry_needs_copy);

                        new_entry->needs_copy = new_entry_needs_copy;
                        
                }

                if (result != KERN_SUCCESS &&
                    result != KERN_MEMORY_RESTART_COPY) {
                        vm_map_lock(src_map);
                        RETURN(result);
                }

                /*
                 *      Throw away the extra reference
                 */

                vm_object_deallocate(src_object);

                /*
                 *      Verify that the map has not substantially
                 *      changed while the copy was being made.
                 */

                vm_map_lock(src_map);   /* Increments timestamp once! */

                if ((version.main_timestamp + 1) == src_map->timestamp)
                        goto VerificationSuccessful;

                /*
                 *      Simple version comparison failed.
                 *
                 *      Retry the lookup and verify that the
                 *      same object/offset are still present.
                 *
                 *      [Note: a memory manager that colludes with
                 *      the calling task can detect that we have
                 *      cheated.  While the map was unlocked, the
                 *      mapping could have been changed and restored.]
                 */

                if (!vm_map_lookup_entry(src_map, src_start, &tmp_entry)) {
                        RETURN(KERN_INVALID_ADDRESS);
                }

                src_entry = tmp_entry;
                vm_map_clip_start(src_map, src_entry, src_start);

                if ((src_entry->protection & VM_PROT_READ == VM_PROT_NONE &&
                        !use_maxprot) ||
                    src_entry->max_protection & VM_PROT_READ == 0)
                        goto VerificationFailed;

                if (src_entry->vme_end < new_entry->vme_end)
                        src_size = (new_entry->vme_end = src_entry->vme_end) - src_start;

                if ((src_entry->object.vm_object != src_object) ||
                    (src_entry->offset != src_offset) ) {

                        /*
                         *      Verification failed.
                         *
                         *      Start over with this top-level entry.
                         */

                 VerificationFailed: ;

                        vm_object_deallocate(new_entry->object.vm_object);
                        tmp_entry = src_entry;
                        continue;
                }

                /*
                 *      Verification succeeded.
                 */

         VerificationSuccessful: ;

                if (result == KERN_MEMORY_RESTART_COPY)
                        goto RestartCopy;

                /*
                 *      Copy succeeded.
                 */

         CopySuccessful: ;

                /*
                 *      Link in the new copy entry.
                 */

                vm_map_copy_entry_link(copy, vm_map_copy_last_entry(copy),
                                       new_entry);
                
                /*
                 *      Determine whether the entire region
                 *      has been copied.
                 */
                src_start = new_entry->vme_end;
                new_entry = VM_MAP_ENTRY_NULL;
                while ((src_start >= src_end) && (src_end != 0)) {
                        if (src_map != base_map) {
                                submap_map_t    *ptr;

                                ptr = parent_maps;
                                assert(ptr != NULL);
                                parent_maps = parent_maps->next;
                                vm_map_lock(ptr->parent_map);
                                vm_map_unlock(src_map);
                                src_map = ptr->parent_map;
                                src_start = ptr->base_start;
                                src_end = ptr->base_end;
                                if ((src_end > src_start) &&
                                              !vm_map_lookup_entry(
                                              src_map, src_start, &tmp_entry))
                                        RETURN(KERN_INVALID_ADDRESS);
                                kfree((vm_offset_t)ptr, sizeof(submap_map_t));
                                if(parent_maps == NULL)
                                        map_share = FALSE;
                                src_entry = tmp_entry->vme_prev;
                        } else
                                break;
                }
                if ((src_start >= src_end) && (src_end != 0))
                        break;

                /*
                 *      Verify that there are no gaps in the region
                 */

                tmp_entry = src_entry->vme_next;
                if ((tmp_entry->vme_start != src_start) || 
                                (tmp_entry == vm_map_to_entry(src_map)))
                        RETURN(KERN_INVALID_ADDRESS);
        }

        /*
         * If the source should be destroyed, do it now, since the
         * copy was successful. 
         */
        if (src_destroy) {
                (void) vm_map_delete(src_map,
                                     trunc_page(src_addr),
                                     src_end,
                                     (src_map == kernel_map) ?
                                        VM_MAP_REMOVE_KUNWIRE :
                                        VM_MAP_NO_FLAGS);
        }

        vm_map_unlock(src_map);

        /* Fix-up start and end points in copy.  This is necessary */
        /* when the various entries in the copy object were picked */
        /* up from different sub-maps */

        tmp_entry = vm_map_copy_first_entry(copy);
        while (tmp_entry != vm_map_copy_to_entry(copy)) {
                tmp_entry->vme_end = copy_addr + 
                        (tmp_entry->vme_end - tmp_entry->vme_start);
                tmp_entry->vme_start = copy_addr;
                copy_addr += tmp_entry->vme_end - tmp_entry->vme_start;
                tmp_entry = (struct vm_map_entry *)tmp_entry->vme_next;
        }

        *copy_result = copy;
        return(KERN_SUCCESS);

#undef  RETURN
}

/*
 *      vm_map_copyin_object:
 *
 *      Create a copy object from an object.
 *      Our caller donates an object reference.
 */

kern_return_t
vm_map_copyin_object(
        vm_object_t             object,
        vm_object_offset_t      offset, /* offset of region in object */
        vm_object_size_t        size,   /* size of region in object */
        vm_map_copy_t   *copy_result)   /* OUT */
{
        vm_map_copy_t   copy;           /* Resulting copy */

        /*
         *      We drop the object into a special copy object
         *      that contains the object directly.
         */

        copy = (vm_map_copy_t) zalloc(vm_map_copy_zone);
        copy->type = VM_MAP_COPY_OBJECT;
        copy->cpy_object = object;
        copy->cpy_index = 0;
        copy->offset = offset;
        copy->size = size;

        *copy_result = copy;
        return(KERN_SUCCESS);
}

/*
 *      vm_map_copyin_page_list_cont:
 *
 *      Continuation routine for vm_map_copyin_page_list.
 *      
 *      If vm_map_copyin_page_list can't fit the entire vm range
 *      into a single page list object, it creates a continuation.
 *      When the target of the operation has used the pages in the
 *      initial page list, it invokes the continuation, which calls
 *      this routine.  If an error happens, the continuation is aborted
 *      (abort arg to this routine is TRUE).  To avoid deadlocks, the
 *      pages are discarded from the initial page list before invoking
 *      the continuation.
 *
 *      NOTE: This is not the same sort of continuation used by
 *      the scheduler.
 */

kern_return_t
vm_map_copyin_page_list_cont(
        vm_map_copyin_args_t    cont_args,
        vm_map_copy_t           *copy_result)   /* OUT */
{
        kern_return_t   result = KERN_SUCCESS;
        register boolean_t      abort, src_destroy, src_destroy_only;

        /*
         *      Check for cases that only require memory destruction.
         */
        abort = (copy_result == (vm_map_copy_t *) 0);
        src_destroy = (cont_args->destroy_len != (vm_size_t) 0);
        src_destroy_only = (cont_args->src_len == (vm_size_t) 0);

        if (abort || src_destroy_only) {
                if (src_destroy)
                        result = vm_map_remove(cont_args->map,
                            cont_args->destroy_addr,
                            cont_args->destroy_addr + cont_args->destroy_len,
                            VM_MAP_NO_FLAGS);
                if (!abort)
                        *copy_result = VM_MAP_COPY_NULL;
        }
        else {
                result = vm_map_copyin_page_list(cont_args->map,
                        cont_args->src_addr, cont_args->src_len,
                        cont_args->options, copy_result, TRUE);

                if (src_destroy &&
                    (cont_args->options & VM_MAP_COPYIN_OPT_STEAL_PAGES) &&
                    vm_map_copy_has_cont(*copy_result)) {
                            vm_map_copyin_args_t        new_args;
                            /*
                             *  Transfer old destroy info.
                             */
                            new_args = (vm_map_copyin_args_t)
                                        (*copy_result)->cpy_cont_args;
                            new_args->destroy_addr = cont_args->destroy_addr;
                            new_args->destroy_len = cont_args->destroy_len;
                }
        }
        
        vm_map_deallocate(cont_args->map);
        kfree((vm_offset_t)cont_args, sizeof(vm_map_copyin_args_data_t));

        return(result);
}

/*
 *      vm_map_copyin_page_list:
 *
 *      This is a variant of vm_map_copyin that copies in a list of pages.
 *      If steal_pages is TRUE, the pages are only in the returned list.
 *      If steal_pages is FALSE, the pages are busy and still in their
 *      objects.  A continuation may be returned if not all the pages fit:
 *      the recipient of this copy_result must be prepared to deal with it.
 */

kern_return_t
vm_map_copyin_page_list(
    vm_map_t            src_map,
    vm_offset_t         src_addr,
    vm_size_t           len,
    int                 options,
    vm_map_copy_t       *copy_result,   /* OUT */
    boolean_t           is_cont)
{
    vm_map_entry_t              src_entry;
    vm_page_t                   m;
    vm_offset_t                 src_start;
    vm_offset_t                 src_end;
    vm_size_t                   src_size;
    register vm_object_t        src_object;
    register vm_object_offset_t src_offset;
    vm_object_offset_t          src_last_offset;
    register vm_map_copy_t      copy;           /* Resulting copy */
    kern_return_t               result = KERN_SUCCESS;
    boolean_t                   need_map_lookup;
    vm_map_copyin_args_t        cont_args;
    kern_return_t               error_code;
    vm_prot_t                   prot;
    boolean_t                   wired;
    boolean_t                   no_zero_fill;

    submap_map_t        *parent_maps = NULL;
    vm_map_t            base_map = src_map;

    prot = (options & VM_MAP_COPYIN_OPT_VM_PROT);
    no_zero_fill = (options & VM_MAP_COPYIN_OPT_NO_ZERO_FILL);
    
    /*
     *  If steal_pages is FALSE, this leaves busy pages in
     *  the object.  A continuation must be used if src_destroy
     *  is true in this case (!steal_pages && src_destroy).
     *
     * XXX      Still have a more general problem of what happens
     * XXX      if the same page occurs twice in a list.  Deadlock
     * XXX      can happen if vm_fault_page was called.  A
     * XXX      possible solution is to use a continuation if vm_fault_page
     * XXX      is called and we cross a map entry boundary.
     */

    /*
     *  Check for copies of zero bytes.
     */

    if (len == 0) {
        *copy_result = VM_MAP_COPY_NULL;
        return(KERN_SUCCESS);
    }

    /*
     *  Compute start and end of region
     */

    src_start = trunc_page(src_addr);
    src_end = round_page(src_addr + len);

    /*
     * If the region is not page aligned, override the no_zero_fill
     * argument.
     */

    if (options & VM_MAP_COPYIN_OPT_NO_ZERO_FILL) {
        if (!page_aligned(src_addr) || !page_aligned(src_addr +len))
            options &= ~VM_MAP_COPYIN_OPT_NO_ZERO_FILL;
    }

    /*
     *  Check that the end address doesn't overflow
     */

    if (src_end <= src_start && (src_end < src_start || src_start != 0)) {
        return KERN_INVALID_ADDRESS;
    }

    /*
     *  Allocate a header element for the page list.
     *
     *  Record original offset and size, as caller may not
     *      be page-aligned.
     */

    copy = (vm_map_copy_t) zalloc(vm_map_copy_zone);
    copy->type = VM_MAP_COPY_PAGE_LIST;
    copy->cpy_npages = 0;
    copy->cpy_page_loose = FALSE;
    copy->offset = src_addr;
    copy->size = len;
    copy->cpy_cont = VM_MAP_COPY_CONT_NULL;
    copy->cpy_cont_args = VM_MAP_COPYIN_ARGS_NULL;
        
    /*
     *  Find the beginning of the region.
     */

do_map_lookup:

    vm_map_lock(src_map);

    if (!vm_map_lookup_entry(src_map, src_start, &src_entry)) {
        result = KERN_INVALID_ADDRESS;
        goto error;
    }
    need_map_lookup = FALSE;

    /*
     *  Go through entries until we get to the end.
     */

    while (TRUE) {
        if ((src_entry->protection & prot) != prot) {
            result = KERN_PROTECTION_FAILURE;
            goto error;
        }

        /* translate down through submaps to find the target entry */
        while(src_entry->is_sub_map) {
                vm_size_t submap_len;
                submap_map_t *ptr;

                ptr = (submap_map_t *)kalloc(sizeof(submap_map_t));
                ptr->next = parent_maps;
                parent_maps = ptr;
                ptr->parent_map = src_map;
                ptr->base_start = src_start;
                ptr->base_end = src_end;
                submap_len = src_entry->vme_end - src_entry->vme_start;
                if(submap_len > (src_end-src_start))
                                submap_len = src_end-src_start;
                ptr->base_start += submap_len;
        
                src_start -= src_entry->vme_start;
                src_start += src_entry->offset;
                src_end = src_start + submap_len;
                src_map = src_entry->object.sub_map;
                vm_map_lock(src_map);
                vm_map_unlock(ptr->parent_map);
                if (!vm_map_lookup_entry(
                                src_map, src_start, &src_entry)) {
                        result = KERN_INVALID_ADDRESS;
                        goto error;
                }
                vm_map_clip_start(src_map, src_entry, src_start);
        }

        wired = (src_entry->wired_count != 0);

        if (src_end > src_entry->vme_end)
            src_size = src_entry->vme_end - src_start;
        else
            src_size = src_end - src_start;

        src_object = src_entry->object.vm_object;

        /*
         *      If src_object is NULL, allocate it now;
         *      we're going to fault on it shortly.
         */
        if (src_object == VM_OBJECT_NULL) {
            src_object = vm_object_allocate((vm_size_t)
                                            src_entry->vme_end -
                                            src_entry->vme_start);
            src_entry->object.vm_object = src_object;
        }
        else if (src_entry->needs_copy && (prot & VM_PROT_WRITE)) {
            vm_object_shadow(
                             &src_entry->object.vm_object,
                             &src_entry->offset,
                             (vm_size_t) (src_entry->vme_end -
                                          src_entry->vme_start));

            src_entry->needs_copy = FALSE;

            /* reset src_object */
            src_object = src_entry->object.vm_object;
        }

        /*
         * calculate src_offset now, since vm_object_shadow
         * may have changed src_entry->offset.
         */
        src_offset = src_entry->offset + (src_start - src_entry->vme_start);

        /*
         * Iterate over pages.  Fault in ones that aren't present.
         */
        src_last_offset = src_offset + src_size;
        for (; (src_offset < src_last_offset);
             src_offset += PAGE_SIZE_64, src_start += PAGE_SIZE) {

            if (copy->cpy_npages == VM_MAP_COPY_PAGE_LIST_MAX) {
                vm_offset_t     src_delta;
make_continuation:
                /*
                 * At this point we have the max number of
                 * pages busy for this thread that we're
                 * willing to allow.  Stop here and record
                 * arguments for the remainder.  Note:
                 * this means that this routine isn't atomic,
                 * but that's the breaks.  Note that only
                 * the first vm_map_copy_t that comes back
                 * from this routine has the right offset
                 * and size; those from continuations are
                 * page rounded, and short by the amount
                 * already done.
                 *
                 * Reset src_end so the src_destroy
                 * code at the bottom doesn't do
                 * something stupid.
                 */

                src_delta = src_end - src_start;
                while (src_map != base_map) {
                        submap_map_t *ptr;

                        if(!need_map_lookup) {
                                vm_map_unlock(src_map);
                        }
                        ptr = parent_maps;
                        assert(ptr != NULL);
                        parent_maps = parent_maps->next;
                        src_map = ptr->parent_map;
                        src_start = ptr->base_start - src_delta;
                        src_delta = ptr->base_end - src_start;
                        kfree((vm_offset_t)ptr, sizeof(submap_map_t));

                        need_map_lookup = TRUE;
                }
                src_end = src_start;


                cont_args = (vm_map_copyin_args_t) 
                            kalloc(sizeof(vm_map_copyin_args_data_t));
                cont_args->map = src_map;
                vm_map_reference(src_map);
                cont_args->src_addr = src_start;
                cont_args->src_len = len - (src_start - src_addr);
                if (options & VM_MAP_COPYIN_OPT_SRC_DESTROY) {
                    cont_args->destroy_addr = cont_args->src_addr;
                    cont_args->destroy_len = cont_args->src_len;
                } else {
                    cont_args->destroy_addr = (vm_offset_t) 0;
                    cont_args->destroy_len = (vm_offset_t) 0;
                }
                cont_args->options = options;
                
                copy->cpy_cont_args = cont_args;
                copy->cpy_cont = vm_map_copyin_page_list_cont;
                
                break;
            }

            /*
             *  Try to find the page of data.  Have to
             *  fault it in if there's no page, or something
             *  going on with the page, or the object has
             *  a copy object.
             */
            vm_object_lock(src_object);
            vm_object_paging_begin(src_object);
            if (((m = vm_page_lookup(src_object, src_offset)) !=
                 VM_PAGE_NULL) && 
                !m->busy && !m->fictitious && !m->unusual &&
                ((prot & VM_PROT_WRITE) == 0 ||
                 (m->object->copy == VM_OBJECT_NULL))) {
              
                if (!m->absent &&
                    !(options & VM_MAP_COPYIN_OPT_STEAL_PAGES)) {

                        /* 
                         * The page is present and will not be
                         * replaced, prep it. Thus allowing
                         * mutiple access on this page 
                         */
                        kern_return_t kr;

                        kr = vm_page_prep(m);
                        assert(kr == KERN_SUCCESS);
                        kr = vm_page_pin(m);
                        assert(kr == KERN_SUCCESS);
                } else {
                        /*
                         *      This is the page.  Mark it busy
                         *      and keep the paging reference on
                         *      the object whilst we do our thing.
                         */
        
                        m->busy = TRUE;
                }
            } else {
                vm_prot_t       result_prot;
                vm_page_t       top_page;
                kern_return_t   kr;
                boolean_t       data_supply;
                                
                /*
                 *      Have to fault the page in; must
                 *      unlock the map to do so.  While
                 *      the map is unlocked, anything
                 *      can happen, we must lookup the
                 *      map entry before continuing.
                 */
                vm_map_unlock(src_map);
                need_map_lookup = TRUE;
                data_supply = src_object->silent_overwrite &&
                  (prot & VM_PROT_WRITE) &&
                    src_start >= src_addr &&
                      src_start + PAGE_SIZE <=
                        src_addr + len;

retry:
                result_prot = prot;
                                
                XPR(XPR_VM_FAULT,
                    "vm_map_copyin_page_list -> vm_fault_page\n",
                    0,0,0,0,0);
                kr = vm_fault_page(src_object, src_offset,
                                   prot, FALSE, THREAD_UNINT,
                                   src_entry->offset,
                                   src_entry->offset +
                                   (src_entry->vme_end -
                                    src_entry->vme_start),
                                   VM_BEHAVIOR_SEQUENTIAL,
                                   &result_prot, &m, &top_page,
                                   (int *)0,
                                   &error_code,
                                   options & VM_MAP_COPYIN_OPT_NO_ZERO_FILL,
                                   data_supply);
                /*
                 *      Cope with what happened.
                 */
                switch (kr) {
                case VM_FAULT_SUCCESS:

                    /*
                     *  If we lost write access,
                     *  try again.
                     */
                    if ((prot & VM_PROT_WRITE) &&
                        !(result_prot & VM_PROT_WRITE)) {
                        vm_object_lock(src_object);
                        vm_object_paging_begin(src_object);
                        goto retry;
                    }
                    break;
                case VM_FAULT_MEMORY_SHORTAGE:
                    VM_PAGE_WAIT();
                    /* fall thru */
                case VM_FAULT_INTERRUPTED: /* ??? */
                case VM_FAULT_RETRY:
                    vm_object_lock(src_object);
                    vm_object_paging_begin(src_object);
                    goto retry;
                case VM_FAULT_FICTITIOUS_SHORTAGE:
                    vm_page_more_fictitious();
                    vm_object_lock(src_object);
                    vm_object_paging_begin(src_object);
                    goto retry;
                case VM_FAULT_MEMORY_ERROR:
                    /*
                     * Something broke.  If this
                     * is a continuation, return
                     * a partial result if possible,
                     * else fail the whole thing.
                     * In the continuation case, the
                     * next continuation call will
                     * get this error if it persists.
                     */
                    vm_map_lock(src_map);
                    if (is_cont &&
                        copy->cpy_npages != 0)
                        goto make_continuation;

                    result = error_code ? error_code : KERN_MEMORY_ERROR;
                    goto error;
                }
                                
                if (top_page != VM_PAGE_NULL) {
                    vm_object_lock(src_object);
                    VM_PAGE_FREE(top_page);
                    vm_object_paging_end(src_object);
                    vm_object_unlock(src_object);
                }

            }

            /*
             * The page is busy, its object is locked, and
             * we have a paging reference on it.  Either
             * the map is locked, or need_map_lookup is
             * TRUE.
             */

            /*
             * Put the page in the page list.
             */
            copy->cpy_page_list[copy->cpy_npages++] = m;
            vm_object_unlock(m->object);

            /*
             * Pmap enter support.  Only used for
             * device I/O for colocated server.
             *
             * WARNING:  This code assumes that this
             * option is only used for well behaved
             * memory.  If the mapping has changed,
             * the following code will make mistakes.
             *
             * XXXO probably ought to do pmap_extract first,
             * XXXO to avoid needless pmap_enter, but this
             * XXXO can't detect protection mismatch??
             */

            if (options & VM_MAP_COPYIN_OPT_PMAP_ENTER) {
                /*
                 * XXX  Only used on kernel map.
                 * XXX  Must not remove VM_PROT_WRITE on
                 * XXX  an I/O only requiring VM_PROT_READ
                 * XXX  as another I/O may be active on same page
                 * XXX  assume that if mapping exists, it must
                 * XXX  have the equivalent of at least VM_PROT_READ,
                 * XXX  but don't assume it has VM_PROT_WRITE as the
                 * XXX  pmap might not all the rights of the object
                 */
                assert(vm_map_pmap(src_map) == kernel_pmap);
              
                if ((prot & VM_PROT_WRITE) ||
                    (pmap_extract(vm_map_pmap(src_map),
                                  src_start) != m->phys_addr))

                    PMAP_ENTER(vm_map_pmap(src_map), src_start,
                               m, prot, wired);
            }
            if(need_map_lookup) {
                need_map_lookup = FALSE;
                vm_map_lock(src_map);
                if (!vm_map_lookup_entry(src_map, src_start, &src_entry)) {
                   result = KERN_INVALID_ADDRESS;
                   goto error;
                }
           }
        }
                        
        /*
         *      Verify that there are no gaps in the region
         */
        src_start = src_entry->vme_end;
        if (src_start < src_end) {
                src_entry = src_entry->vme_next;
                if (need_map_lookup) {
                        need_map_lookup = FALSE;
                        vm_map_lock(src_map);
                        if(!vm_map_lookup_entry(src_map, 
                                                src_start, &src_entry)) {
                                result = KERN_INVALID_ADDRESS;
                                goto error;
                        }
                } else if (src_entry->vme_start != src_start) {
                        result = KERN_INVALID_ADDRESS;
                        goto error;
                }
        }

        /*
         *      DETERMINE whether the entire region
         *      has been copied.
         */

        while ((src_start >= src_end) && (src_end != 0)) {
                if (src_map != base_map) {
                        submap_map_t    *ptr;

                        ptr = parent_maps;
                        assert(ptr != NULL);
                        parent_maps = parent_maps->next;
                        src_start = ptr->base_start;
                        src_end = ptr->base_end;
                        if(need_map_lookup) {
                                need_map_lookup = FALSE;
                        }
                        else {
                                vm_map_unlock(src_map);
                        }
                        src_map = ptr->parent_map;
                        vm_map_lock(src_map);
                        if((src_start < src_end) &&
                                        (!vm_map_lookup_entry(ptr->parent_map, 
                                        src_start, &src_entry))) {
                                result = KERN_INVALID_ADDRESS;
                                kfree((vm_offset_t)ptr, sizeof(submap_map_t));
                                goto error;
                        }
                        kfree((vm_offset_t)ptr, sizeof(submap_map_t));
                } else
                        break;
        }
        if ((src_start >= src_end) && (src_end != 0)) {
                if (need_map_lookup)
                        vm_map_lock(src_map);
                break;
        }

    }

    /*
     * If steal_pages is true, make sure all
     * pages in the copy are not in any object
     * We try to remove them from the original
     * object, but we may have to copy them.
     *
     * At this point every page in the list is busy
     * and holds a paging reference to its object.
     * When we're done stealing, every page is busy,
     * and in no object (m->tabled == FALSE).
     */
    src_start = trunc_page(src_addr);
    if (options & VM_MAP_COPYIN_OPT_STEAL_PAGES) {
        register int    i;
        vm_offset_t     page_vaddr;
        vm_offset_t     unwire_end;
        vm_offset_t     map_entry_end;
        boolean_t       share_map = FALSE;

        unwire_end = src_start;
        map_entry_end = src_start;
        for (i = 0; i < copy->cpy_npages; i++) {
          
          
            /*
             * Remove the page from its object if it
             * can be stolen.  It can be stolen if:
             *
             * (1) The source is being destroyed, 
             *       the object is internal (hence
             *       temporary), and not shared.
             * (2) The page is not precious.
             *
             * The not shared check consists of two
             * parts:  (a) there are no objects that
             * shadow this object.  (b) it is not the
             * object in any shared map entries (i.e.,
             * use_shared_copy is not set).
             *
             * The first check (a) means that we can't
             * steal pages from objects that are not
             * at the top of their shadow chains.  This
             * should not be a frequent occurrence.
             *
             * Stealing wired pages requires telling the
             * pmap module to let go of them.
             * 
             * NOTE: stealing clean pages from objects
             *          whose mappings survive requires a call to
             * the pmap module.  Maybe later.
             */
            m = copy->cpy_page_list[i];
            src_object = m->object;
            vm_object_lock(src_object);

            page_vaddr = src_start + (i * PAGE_SIZE);
            if(page_vaddr > map_entry_end) {
                if (!vm_map_lookup_entry(src_map, page_vaddr, &src_entry))
                        share_map = TRUE;
                else if (src_entry->is_sub_map)  {
                        map_entry_end = src_entry->vme_end;
                        share_map = TRUE;
                } else {
                        map_entry_end = src_entry->vme_end;
                        share_map = FALSE;
                }
            }
            

            if ((options & VM_MAP_COPYIN_OPT_SRC_DESTROY) &&
                src_object->internal &&
                !src_object->true_share &&
                (!src_object->shadowed) &&
                (src_object->copy_strategy ==
                 MEMORY_OBJECT_COPY_SYMMETRIC) &&
                !m->precious &&
                !share_map) {
                
                if (m->wire_count > 0) {

                    assert(m->wire_count == 1);
                    /*
                     * In order to steal a wired
                     * page, we have to unwire it
                     * first.  We do this inline
                     * here because we have the page.
                     *
                     * Step 1: Unwire the map entry.
                     *  Also tell the pmap module
                     *  that this piece of the
                     *  pmap is pageable.
                     */
                    vm_object_unlock(src_object);
                    if (page_vaddr >= unwire_end) {
                        if (!vm_map_lookup_entry(src_map,
                                                 page_vaddr, &src_entry))
                            panic("vm_map_copyin_page_list: missing wired map entry");

                        vm_map_clip_start(src_map, src_entry,
                                          page_vaddr);
                        vm_map_clip_end(src_map, src_entry,
                                        src_start + src_size);

/*  revisit why this assert fails CDY
                        assert(src_entry->wired_count > 0);
*/
                        src_entry->wired_count = 0;
                        src_entry->user_wired_count = 0;
                        unwire_end = src_entry->vme_end;
                        pmap_pageable(vm_map_pmap(src_map),
                                      page_vaddr, unwire_end, TRUE);
                    }

                    /*
                     * Step 2: Unwire the page.
                     * pmap_remove handles this for us.
                     */
                    vm_object_lock(src_object);
                }

                /*
                 * Don't need to remove the mapping;
                 * vm_map_delete will handle it.
                 * 
                 * Steal the page.  Setting the wire count
                 * to zero is vm_page_unwire without
                 * activating the page.
                 */
                vm_page_lock_queues();
                vm_page_remove(m);
                if (m->wire_count > 0) {
                    m->wire_count = 0;
                    vm_page_wire_count--;
                } else {
                    VM_PAGE_QUEUES_REMOVE(m);
                }
                vm_page_unlock_queues();
            } else {
                /*
                 * Have to copy this page.  Have to
                 * unlock the map while copying,
                 * hence no further page stealing.
                 * Hence just copy all the pages.
                 * Unlock the map while copying;
                 * This means no further page stealing.
                 */
                vm_object_unlock(src_object);
                vm_map_unlock(src_map);
                vm_map_copy_steal_pages(copy);
                vm_map_lock(src_map);
                break;
            }

            vm_object_paging_end(src_object);
            vm_object_unlock(src_object);
        }

        copy->cpy_page_loose = TRUE;

        /*
         * If the source should be destroyed, do it now, since the
         * copy was successful.
         */

        if (options & VM_MAP_COPYIN_OPT_SRC_DESTROY) {
            (void) vm_map_delete(src_map, src_start,
                                 src_end, VM_MAP_NO_FLAGS);
        }
    } else {
        /*
         * Not stealing pages leaves busy or prepped pages in the map.
         * This will cause source destruction to hang.  Use
         * a continuation to prevent this.
         */
        if ((options & VM_MAP_COPYIN_OPT_SRC_DESTROY) &&
            !vm_map_copy_has_cont(copy)) {
            cont_args = (vm_map_copyin_args_t) 
                        kalloc(sizeof(vm_map_copyin_args_data_t));
            vm_map_reference(src_map);
            cont_args->map = src_map;
            cont_args->src_addr = (vm_offset_t) 0;
            cont_args->src_len = (vm_size_t) 0;
            cont_args->destroy_addr = src_start;
            cont_args->destroy_len = src_end - src_start;
            cont_args->options = options;

            copy->cpy_cont_args = cont_args;
            copy->cpy_cont = vm_map_copyin_page_list_cont;
        }
    }

    vm_map_unlock(src_map);

    *copy_result = copy;
    return(result);

error:
    {
        submap_map_t    *ptr;

        vm_map_unlock(src_map);
        vm_map_copy_discard(copy);

        for(ptr = parent_maps; ptr != NULL; ptr = parent_maps) {
                parent_maps=parent_maps->next;
                kfree((vm_offset_t)ptr, sizeof(submap_map_t));
        }
        return(result);
     }
}

void
vm_map_fork_share(
        vm_map_t        old_map,
        vm_map_entry_t  old_entry,
        vm_map_t        new_map)
{
        vm_object_t     object;
        vm_map_entry_t  new_entry;
        kern_return_t   result;

        /*
         *      New sharing code.  New map entry
         *      references original object.  Internal
         *      objects use asynchronous copy algorithm for
         *      future copies.  First make sure we have
         *      the right object.  If we need a shadow,
         *      or someone else already has one, then
         *      make a new shadow and share it.
         */
        
        object = old_entry->object.vm_object;
        if (old_entry->is_sub_map) {
                assert(old_entry->wired_count == 0);
#ifndef i386
                if(old_entry->use_pmap) {
                        result = pmap_nest(new_map->pmap, 
                                (old_entry->object.sub_map)->pmap, 
                                old_entry->vme_start,
                                old_entry->vme_end - old_entry->vme_start);
                        if(result)
                                panic("vm_map_fork_share: pmap_nest failed!");
                }
#endif
        } else if (object == VM_OBJECT_NULL) {
                object = vm_object_allocate((vm_size_t)(old_entry->vme_end -
                                                        old_entry->vme_start));
                old_entry->offset = 0;
                old_entry->object.vm_object = object;
                assert(!old_entry->needs_copy);
        } else if (object->copy_strategy !=
                 MEMORY_OBJECT_COPY_SYMMETRIC) {
                
                /*
                 *      We are already using an asymmetric
                 *      copy, and therefore we already have
                 *      the right object.
                 */
                
                assert(! old_entry->needs_copy);
        }
        else if (old_entry->needs_copy ||       /* case 1 */
                 object->shadowed ||            /* case 2 */
                 (!object->true_share &&        /* case 3 */
                 !old_entry->is_shared &&
                 (object->size >
                  (vm_size_t)(old_entry->vme_end -
                              old_entry->vme_start)))) {
                
                /*
                 *      We need to create a shadow.
                 *      There are three cases here.
                 *      In the first case, we need to
                 *      complete a deferred symmetrical
                 *      copy that we participated in.
                 *      In the second and third cases,
                 *      we need to create the shadow so
                 *      that changes that we make to the
                 *      object do not interfere with
                 *      any symmetrical copies which
                 *      have occured (case 2) or which
                 *      might occur (case 3).
                 *
                 *      The first case is when we had
                 *      deferred shadow object creation
                 *      via the entry->needs_copy mechanism.
                 *      This mechanism only works when
                 *      only one entry points to the source
                 *      object, and we are about to create
                 *      a second entry pointing to the
                 *      same object. The problem is that
                 *      there is no way of mapping from
                 *      an object to the entries pointing
                 *      to it. (Deferred shadow creation
                 *      works with one entry because occurs
                 *      at fault time, and we walk from the
                 *      entry to the object when handling
                 *      the fault.)
                 *
                 *      The second case is when the object
                 *      to be shared has already been copied
                 *      with a symmetric copy, but we point
                 *      directly to the object without
                 *      needs_copy set in our entry. (This
                 *      can happen because different ranges
                 *      of an object can be pointed to by
                 *      different entries. In particular,
                 *      a single entry pointing to an object
                 *      can be split by a call to vm_inherit,
                 *      which, combined with task_create, can
                 *      result in the different entries
                 *      having different needs_copy values.)
                 *      The shadowed flag in the object allows
                 *      us to detect this case. The problem
                 *      with this case is that if this object
                 *      has or will have shadows, then we
                 *      must not perform an asymmetric copy
                 *      of this object, since such a copy
                 *      allows the object to be changed, which
                 *      will break the previous symmetrical
                 *      copies (which rely upon the object
                 *      not changing). In a sense, the shadowed
                 *      flag says "don't change this object".
                 *      We fix this by creating a shadow
                 *      object for this object, and sharing
                 *      that. This works because we are free
                 *      to change the shadow object (and thus
                 *      to use an asymmetric copy strategy);
                 *      this is also semantically correct,
                 *      since this object is temporary, and
                 *      therefore a copy of the object is
                 *      as good as the object itself. (This
                 *      is not true for permanent objects,
                 *      since the pager needs to see changes,
                 *      which won't happen if the changes
                 *      are made to a copy.)
                 *
                 *      The third case is when the object
                 *      to be shared has parts sticking
                 *      outside of the entry we're working
                 *      with, and thus may in the future
                 *      be subject to a symmetrical copy.
                 *      (This is a preemptive version of
                 *      case 2.)
                 */
                
                assert(!(object->shadowed && old_entry->is_shared));
                vm_object_shadow(&old_entry->object.vm_object,
                                 &old_entry->offset,
                                 (vm_size_t) (old_entry->vme_end -
                                              old_entry->vme_start));
                
                /*
                 *      If we're making a shadow for other than
                 *      copy on write reasons, then we have
                 *      to remove write permission.
                 */

/* CDY FIX this! page_protect! */
                if (!old_entry->needs_copy &&
                    (old_entry->protection & VM_PROT_WRITE)) {
                        if(old_entry->is_sub_map && old_entry->use_pmap) {
                                pmap_protect(old_entry->object.sub_map->pmap,
                                     old_entry->vme_start,
                                     old_entry->vme_end,
                                     old_entry->protection & ~VM_PROT_WRITE);
                        } else {
                                pmap_protect(vm_map_pmap(old_map),
                                     old_entry->vme_start,
                                     old_entry->vme_end,
                                     old_entry->protection & ~VM_PROT_WRITE);
                        }
                }
                
                old_entry->needs_copy = FALSE;
                object = old_entry->object.vm_object;
        }
        
        /*
         *      If object was using a symmetric copy strategy,
         *      change its copy strategy to the default
         *      asymmetric copy strategy, which is copy_delay
         *      in the non-norma case and copy_call in the
         *      norma case. Bump the reference count for the
         *      new entry.
         */
        
        if(old_entry->is_sub_map) {
                vm_map_lock(old_entry->object.sub_map);
                vm_map_reference(old_entry->object.sub_map);
                vm_map_unlock(old_entry->object.sub_map);
        } else {
                vm_object_lock(object);
                object->ref_count++;
                vm_object_res_reference(object);
                if (object->copy_strategy == MEMORY_OBJECT_COPY_SYMMETRIC) {
                        object->copy_strategy = MEMORY_OBJECT_COPY_DELAY;
                }
                vm_object_unlock(object);
        }
        
        /*
         *      Clone the entry, using object ref from above.
         *      Mark both entries as shared.
         */
        
        new_entry = vm_map_entry_create(new_map);
        vm_map_entry_copy(new_entry, old_entry);
        old_entry->is_shared = TRUE;
        new_entry->is_shared = TRUE;
        
        /*
         *      Insert the entry into the new map -- we
         *      know we're inserting at the end of the new
         *      map.
         */
        
        vm_map_entry_link(new_map, vm_map_last_entry(new_map), new_entry);
        
        /*
         *      Update the physical map
         */
        
        if (old_entry->is_sub_map) {
                /* Bill Angell pmap support goes here */
        } else {
                pmap_copy(new_map->pmap, old_map->pmap, new_entry->vme_start,
                  old_entry->vme_end - old_entry->vme_start,
                  old_entry->vme_start);
        }
}

boolean_t
vm_map_fork_copy(
        vm_map_t        old_map,
        vm_map_entry_t  *old_entry_p,
        vm_map_t        new_map)
{
        vm_map_entry_t old_entry = *old_entry_p;
        vm_size_t entry_size = old_entry->vme_end - old_entry->vme_start;
        vm_offset_t start = old_entry->vme_start;
        vm_map_copy_t copy;
        vm_map_entry_t last = vm_map_last_entry(new_map);

        vm_map_unlock(old_map);
        /*
         *      Use maxprot version of copyin because we
         *      care about whether this memory can ever
         *      be accessed, not just whether it's accessible
         *      right now.
         */
        if (vm_map_copyin_maxprot(old_map, start, entry_size, FALSE, &copy)
            != KERN_SUCCESS) {
                /*
                 *      The map might have changed while it
                 *      was unlocked, check it again.  Skip
                 *      any blank space or permanently
                 *      unreadable region.
                 */
                vm_map_lock(old_map);
                if (!vm_map_lookup_entry(old_map, start, &last) ||
                    last->max_protection & VM_PROT_READ ==
                                         VM_PROT_NONE) {
                        last = last->vme_next;
                }
                *old_entry_p = last;

                /*
                 * XXX  For some error returns, want to
                 * XXX  skip to the next element.  Note
                 *      that INVALID_ADDRESS and
                 *      PROTECTION_FAILURE are handled above.
                 */
                
                return FALSE;
        }
        
        /*
         *      Insert the copy into the new map
         */
        
        vm_map_copy_insert(new_map, last, copy);
        
        /*
         *      Pick up the traversal at the end of
         *      the copied region.
         */
        
        vm_map_lock(old_map);
        start += entry_size;
        if (! vm_map_lookup_entry(old_map, start, &last)) {
                last = last->vme_next;
        } else {
                vm_map_clip_start(old_map, last, start);
        }
        *old_entry_p = last;

        return TRUE;
}

/*
 *      vm_map_fork:
 *
 *      Create and return a new map based on the old
 *      map, according to the inheritance values on the
 *      regions in that map.
 *
 *      The source map must not be locked.
 */
vm_map_t
vm_map_fork(
        vm_map_t        old_map)
{
        pmap_t          new_pmap = pmap_create((vm_size_t) 0);
        vm_map_t        new_map;
        vm_map_entry_t  old_entry;
        vm_size_t       new_size = 0, entry_size;
        vm_map_entry_t  new_entry;
        boolean_t       src_needs_copy;
        boolean_t       new_entry_needs_copy;

        vm_map_reference_swap(old_map);
        vm_map_lock(old_map);

        new_map = vm_map_create(new_pmap,
                        old_map->min_offset,
                        old_map->max_offset,
                        old_map->hdr.entries_pageable);

        for (
            old_entry = vm_map_first_entry(old_map);
            old_entry != vm_map_to_entry(old_map);
            ) {

                entry_size = old_entry->vme_end - old_entry->vme_start;

                switch (old_entry->inheritance) {
                case VM_INHERIT_NONE:
                        break;

                case VM_INHERIT_SHARE:
                        vm_map_fork_share(old_map, old_entry, new_map);
                        new_size += entry_size;
                        break;

                case VM_INHERIT_COPY:

                        /*
                         *      Inline the copy_quickly case;
                         *      upon failure, fall back on call
                         *      to vm_map_fork_copy.
                         */

                        if(old_entry->is_sub_map)
                                break;
                        if (old_entry->wired_count != 0) {
                                goto slow_vm_map_fork_copy;
                        }

                        new_entry = vm_map_entry_create(new_map);
                        vm_map_entry_copy(new_entry, old_entry);
                        /* clear address space specifics */
                        new_entry->use_pmap = FALSE;

                        if (! vm_object_copy_quickly(
                                                &new_entry->object.vm_object,
                                                old_entry->offset,
                                                (old_entry->vme_end -
                                                        old_entry->vme_start),
                                                &src_needs_copy,
                                                &new_entry_needs_copy)) {
                                vm_map_entry_dispose(new_map, new_entry);
                                goto slow_vm_map_fork_copy;
                        }

                        /*
                         *      Handle copy-on-write obligations
                         */
                        
                        if (src_needs_copy && !old_entry->needs_copy) {
                                vm_object_pmap_protect(
                                        old_entry->object.vm_object,
                                        old_entry->offset,
                                        (old_entry->vme_end -
                                                        old_entry->vme_start),
                                        ((old_entry->is_shared 
                                                || old_entry->is_sub_map)
                                                        ? PMAP_NULL :
                                                        old_map->pmap),
                                        old_entry->vme_start,
                                        old_entry->protection & ~VM_PROT_WRITE);

                                old_entry->needs_copy = TRUE;
                        }
                        new_entry->needs_copy = new_entry_needs_copy;
                        
                        /*
                         *      Insert the entry at the end
                         *      of the map.
                         */
                        
                        vm_map_entry_link(new_map, vm_map_last_entry(new_map),
                                          new_entry);
                        new_size += entry_size;
                        break;

                slow_vm_map_fork_copy:
                        if (vm_map_fork_copy(old_map, &old_entry, new_map)) {
                                new_size += entry_size;
                        }
                        continue;
                }
                old_entry = old_entry->vme_next;
        }

        new_map->size = new_size;
        vm_map_unlock(old_map);
        vm_map_deallocate(old_map);

        return(new_map);
}


/*
 *      vm_map_lookup_locked:
 *
 *      Finds the VM object, offset, and
 *      protection for a given virtual address in the
 *      specified map, assuming a page fault of the
 *      type specified.
 *
 *      Returns the (object, offset, protection) for
 *      this address, whether it is wired down, and whether
 *      this map has the only reference to the data in question.
 *      In order to later verify this lookup, a "version"
 *      is returned.
 *
 *      The map MUST be locked by the caller and WILL be
 *      locked on exit.  In order to guarantee the
 *      existence of the returned object, it is returned
 *      locked.
 *
 *      If a lookup is requested with "write protection"
 *      specified, the map may be changed to perform virtual
 *      copying operations, although the data referenced will
 *      remain the same.
 */
kern_return_t
vm_map_lookup_locked(
        vm_map_t                *var_map,       /* IN/OUT */
        register vm_offset_t    vaddr,
        register vm_prot_t      fault_type,
        vm_map_version_t        *out_version,   /* OUT */
        vm_object_t             *object,        /* OUT */
        vm_object_offset_t      *offset,        /* OUT */
        vm_prot_t               *out_prot,      /* OUT */
        boolean_t               *wired,         /* OUT */
        int                     *behavior,      /* OUT */
        vm_object_offset_t      *lo_offset,     /* OUT */
        vm_object_offset_t      *hi_offset,     /* OUT */
        vm_map_t                *pmap_map)
{
        vm_map_entry_t                  entry;
        register vm_map_t               map = *var_map;
        vm_map_t                        old_map = *var_map;
        vm_map_t                        cow_sub_map_parent = VM_MAP_NULL;
        vm_offset_t                     cow_parent_vaddr;
        vm_offset_t                     old_start;
        vm_offset_t                     old_end;
        register vm_prot_t              prot;

        *pmap_map = map;
        RetryLookup: ;

        /*
         *      If the map has an interesting hint, try it before calling
         *      full blown lookup routine.
         */

        mutex_lock(&map->s_lock);
        entry = map->hint;
        mutex_unlock(&map->s_lock);

        if ((entry == vm_map_to_entry(map)) ||
            (vaddr < entry->vme_start) || (vaddr >= entry->vme_end)) {
                vm_map_entry_t  tmp_entry;

                /*
                 *      Entry was either not a valid hint, or the vaddr
                 *      was not contained in the entry, so do a full lookup.
                 */
                if (!vm_map_lookup_entry(map, vaddr, &tmp_entry)) {
                        if((cow_sub_map_parent) && (cow_sub_map_parent != map))
                                vm_map_unlock(cow_sub_map_parent);
                        if((*pmap_map != map) 
                                        && (*pmap_map != cow_sub_map_parent))
                                vm_map_unlock(*pmap_map);
                        return KERN_INVALID_ADDRESS;
                }

                entry = tmp_entry;
        }
        if(map == old_map) {
                old_start = entry->vme_start;
                old_end = entry->vme_end;
        }

        /*
         *      Handle submaps.  Drop lock on upper map, submap is
         *      returned locked.
         */

submap_recurse:
        if (entry->is_sub_map) {
                vm_offset_t             local_vaddr;
                vm_offset_t             end_delta;
                vm_offset_t             start_delta; 
                vm_offset_t             object_start_delta; 
                vm_map_entry_t          submap_entry;
                boolean_t               mapped_needs_copy=FALSE;

                local_vaddr = vaddr;

                if ((!entry->needs_copy) && (entry->use_pmap)) {
                        /* if pmap_map equals map we unlock below */
                        if ((*pmap_map != map) && 
                                        (*pmap_map != cow_sub_map_parent))
                                vm_map_unlock(*pmap_map);
                        *pmap_map = entry->object.sub_map;
                }

                if(entry->needs_copy) {
                        if (!mapped_needs_copy) {
                                if (vm_map_lock_read_to_write(map)) {
                                        vm_map_lock_read(map);
                                        if(*pmap_map == entry->object.sub_map)
                                                *pmap_map = map;
                                        goto RetryLookup;
                                }
                                vm_map_lock_read(entry->object.sub_map);
                                cow_sub_map_parent = map;
                                /* reset base to map before cow object */
                                /* this is the map which will accept   */
                                /* the new cow object */
                                old_start = entry->vme_start;
                                old_end = entry->vme_end;
                                cow_parent_vaddr = vaddr;
                                mapped_needs_copy = TRUE;
                        } else {
                                vm_map_lock_read(entry->object.sub_map);
                                if((cow_sub_map_parent != map) &&
                                                        (*pmap_map != map))
                                        vm_map_unlock(map);
                        }
                } else {
                        vm_map_lock_read(entry->object.sub_map);
                        /* leave map locked if it is a target */
                        /* cow sub_map above otherwise, just  */
                        /* follow the maps down to the object */
                        /* here we unlock knowing we are not  */
                        /* revisiting the map.  */
                        if((*pmap_map != map) && (map != cow_sub_map_parent))
                                vm_map_unlock_read(map);
                }

                *var_map = map = entry->object.sub_map;

                /* calculate the offset in the submap for vaddr */
                local_vaddr = (local_vaddr - entry->vme_start) + entry->offset;

RetrySubMap:
                if(!vm_map_lookup_entry(map, local_vaddr, &submap_entry)) {
                        if((cow_sub_map_parent) && (cow_sub_map_parent != map)){
                                vm_map_unlock(cow_sub_map_parent);
                        }
                        if((*pmap_map != map) 
                                        && (*pmap_map != cow_sub_map_parent)) {
                                vm_map_unlock(*pmap_map);
                        }
                        *pmap_map = map;
                        return KERN_INVALID_ADDRESS;
                }
                /* find the attenuated shadow of the underlying object */
                /* on our target map */

                /* in english the submap object may extend beyond the     */
                /* region mapped by the entry or, may only fill a portion */
                /* of it.  For our purposes, we only care if the object   */
                /* doesn't fill.  In this case the area which will        */
                /* ultimately be clipped in the top map will only need    */
                /* to be as big as the portion of the underlying entry    */
                /* which is mapped */
                start_delta = submap_entry->vme_start > entry->offset ?
                            submap_entry->vme_start - entry->offset : 0;

                end_delta = 
                   (entry->offset + start_delta + (old_end - old_start)) <=
                        submap_entry->vme_end ?
                                0 : (entry->offset + 
                                        (old_end - old_start))
                                        - submap_entry->vme_end; 

                old_start += start_delta;
                old_end -= end_delta;

                if(submap_entry->is_sub_map) {
                        entry = submap_entry;
                        vaddr = local_vaddr;
                        goto submap_recurse;
                }

                if(((fault_type & VM_PROT_WRITE) && cow_sub_map_parent)) {

                        vm_object_t     copy_object;
                        vm_offset_t     local_start;
                        vm_offset_t     local_end;

                        if (vm_map_lock_read_to_write(map)) {
                                vm_map_lock_read(map);
                                old_start -= start_delta;
                                old_end += end_delta;
                                goto RetrySubMap;
                        }
                        if (submap_entry->object.vm_object == VM_OBJECT_NULL) {
                                submap_entry->object.vm_object = 
                                        vm_object_allocate(
                                                (vm_size_t)
                                                (submap_entry->vme_end 
                                                - submap_entry->vme_start));
                                        submap_entry->offset = 0;
                        }
                        local_start =  local_vaddr - 
                                        (cow_parent_vaddr - old_start);
                        local_end = local_vaddr + 
                                        (old_end - cow_parent_vaddr);
                        vm_map_clip_start(map, submap_entry, local_start);
                        vm_map_clip_end(map, submap_entry, local_end);

                        /* This is the COW case, lets connect */
                        /* an entry in our space to the underlying */
                        /* object in the submap, bypassing the  */
                        /* submap. */
                                
                        /* set up shadow object */
                        copy_object = submap_entry->object.vm_object;
                        submap_entry->object.vm_object->shadowed = TRUE;
                        submap_entry->needs_copy = TRUE;
                        vm_object_pmap_protect(submap_entry->object.vm_object,
                                        submap_entry->offset,
                                        submap_entry->vme_end - 
                                                submap_entry->vme_start,
                                        submap_entry->is_shared ?
                                                PMAP_NULL : map->pmap,
                                        submap_entry->vme_start,
                                        submap_entry->protection &
                                                ~VM_PROT_WRITE);
                        

                        /* This works diffently than the   */
                        /* normal submap case. We go back  */
                        /* to the parent of the cow map and*/
                        /* clip out the target portion of  */
                        /* the sub_map, substituting the   */
                        /* new copy object,                */

                        vm_map_unlock(map);
                        local_start = old_start;
                        local_end = old_end;
                        map = cow_sub_map_parent;
                        *var_map = cow_sub_map_parent;
                        vaddr = cow_parent_vaddr;
                        cow_sub_map_parent = NULL;

                        if(!vm_map_lookup_entry(map, 
                                        vaddr, &entry)) {
                                        vm_object_deallocate(
                                                        copy_object);
                                        vm_map_lock_write_to_read(map);
                                        return KERN_INVALID_ADDRESS;
                        }
                                        
                        /* clip out the portion of space */
                        /* mapped by the sub map which   */
                        /* corresponds to the underlying */
                        /* object */
                        vm_map_clip_start(map, entry, local_start);
                        vm_map_clip_end(map, entry, local_end);


                        /* substitute copy object for */
                        /* shared map entry           */
                        vm_map_deallocate(entry->object.sub_map);
                        entry->is_sub_map = FALSE;
                        vm_object_reference(copy_object);
                        entry->object.vm_object = copy_object;
                        entry->offset = submap_entry->offset;

                        entry->protection |= VM_PROT_WRITE;
                        entry->max_protection |= VM_PROT_WRITE;
                        entry->needs_copy = TRUE;
                        if(entry->inheritance == VM_INHERIT_SHARE) 
                        entry->inheritance = VM_INHERIT_COPY;
                        if (map != old_map)
                                entry->is_shared = TRUE;

                        vm_map_lock_write_to_read(map);
                } else {
                        if((cow_sub_map_parent)
                                        && (cow_sub_map_parent != *pmap_map)
                                        && (cow_sub_map_parent != map)) {
                                vm_map_unlock(cow_sub_map_parent);
                        }
                        entry = submap_entry;
                        vaddr = local_vaddr;
                }
        }
                
        /*
         *      Check whether this task is allowed to have
         *      this page.
         */

        prot = entry->protection;
        if ((fault_type & (prot)) != fault_type) {
          if (*pmap_map != map) {
                vm_map_unlock(*pmap_map);
          }
          *pmap_map = map;
          return KERN_PROTECTION_FAILURE;
        }

        /*
         *      If this page is not pageable, we have to get
         *      it for all possible accesses.
         */

        if (*wired = (entry->wired_count != 0))
                prot = fault_type = entry->protection;

        /*
         *      If the entry was copy-on-write, we either ...
         */

        if (entry->needs_copy) {
                /*
                 *      If we want to write the page, we may as well
                 *      handle that now since we've got the map locked.
                 *
                 *      If we don't need to write the page, we just
                 *      demote the permissions allowed.
                 */

                if (fault_type & VM_PROT_WRITE || *wired) {
                        /*
                         *      Make a new object, and place it in the
                         *      object chain.  Note that no new references
                         *      have appeared -- one just moved from the
                         *      map to the new object.
                         */

                        if (vm_map_lock_read_to_write(map)) {
                                vm_map_lock_read(map);
                                goto RetryLookup;
                        }
                        vm_object_shadow(&entry->object.vm_object,
                                         &entry->offset,
                                         (vm_size_t) (entry->vme_end -
                                                      entry->vme_start));

                        entry->object.vm_object->shadowed = TRUE;
                        entry->needs_copy = FALSE;
                        vm_map_lock_write_to_read(map);
                }
                else {
                        /*
                         *      We're attempting to read a copy-on-write
                         *      page -- don't allow writes.
                         */

                        prot &= (~VM_PROT_WRITE);
                }
        }

        /*
         *      Create an object if necessary.
         */
        if (entry->object.vm_object == VM_OBJECT_NULL) {

                if (vm_map_lock_read_to_write(map)) {
                        vm_map_lock_read(map);
                        goto RetryLookup;
                }

                entry->object.vm_object = vm_object_allocate(
                        (vm_size_t)(entry->vme_end - entry->vme_start));
                entry->offset = 0;
                vm_map_lock_write_to_read(map);
        }

        /*
         *      Return the object/offset from this entry.  If the entry
         *      was copy-on-write or empty, it has been fixed up.  Also
         *      return the protection.
         */

        *offset = (vaddr - entry->vme_start) + entry->offset;
        *object = entry->object.vm_object;
        *out_prot = prot;
        *behavior = entry->behavior;
        *lo_offset = entry->offset;
        *hi_offset = (entry->vme_end - entry->vme_start) + entry->offset;

        /*
         *      Lock the object to prevent it from disappearing
         */

        vm_object_lock(*object);

        /*
         *      Save the version number
         */

        out_version->main_timestamp = map->timestamp;

        return KERN_SUCCESS;
}


/*
 *      vm_map_verify:
 *
 *      Verifies that the map in question has not changed
 *      since the given version.  If successful, the map
 *      will not change until vm_map_verify_done() is called.
 */
boolean_t
vm_map_verify(
        register vm_map_t               map,
        register vm_map_version_t       *version)       /* REF */
{
        boolean_t       result;

        vm_map_lock_read(map);
        result = (map->timestamp == version->main_timestamp);

        if (!result)
                vm_map_unlock_read(map);

        return(result);
}

/*
 *      vm_map_verify_done:
 *
 *      Releases locks acquired by a vm_map_verify.
 *
 *      This is now a macro in vm/vm_map.h.  It does a
 *      vm_map_unlock_read on the map.
 */


/*
 *      vm_region:
 *
 *      User call to obtain information about a region in
 *      a task's address map. Currently, only one flavor is
 *      supported.
 *
 *      XXX The reserved and behavior fields cannot be filled
 *          in until the vm merge from the IK is completed, and
 *          vm_reserve is implemented.
 *
 *      XXX Dependency: syscall_vm_region() also supports only one flavor.
 */

kern_return_t
vm_region(
        vm_map_t                 map,
        vm_offset_t             *address,               /* IN/OUT */
        vm_size_t               *size,                  /* OUT */
        vm_region_flavor_t       flavor,                /* IN */
        vm_region_info_t         info,                  /* OUT */
        mach_msg_type_number_t  *count,                 /* IN/OUT */
        ipc_port_t              *object_name)           /* OUT */
{
        vm_map_entry_t          tmp_entry;
        register
        vm_map_entry_t          entry;
        register
        vm_offset_t             start;
        vm_region_basic_info_t  basic;
        vm_region_extended_info_t       extended;
        vm_region_top_info_t    top;

        if (map == VM_MAP_NULL) 
                return(KERN_INVALID_ARGUMENT);

        switch (flavor) {
        
        case VM_REGION_BASIC_INFO:
        {
            if (*count < VM_REGION_BASIC_INFO_COUNT)
                return(KERN_INVALID_ARGUMENT);

            basic = (vm_region_basic_info_t) info;
            *count = VM_REGION_BASIC_INFO_COUNT;

            vm_map_lock_read(map);

            start = *address;
            if (!vm_map_lookup_entry(map, start, &tmp_entry)) {
                if ((entry = tmp_entry->vme_next) == vm_map_to_entry(map)) {
                        vm_map_unlock_read(map);
                        return(KERN_INVALID_ADDRESS);
                }
            } else {
                entry = tmp_entry;
            }

            start = entry->vme_start;

            basic->offset = entry->offset;
            basic->protection = entry->protection;
            basic->inheritance = entry->inheritance;
            basic->max_protection = entry->max_protection;
            basic->behavior = entry->behavior;
            basic->user_wired_count = entry->user_wired_count;
            basic->reserved = entry->is_sub_map;
            *address = start;
            *size = (entry->vme_end - start);

            if (object_name) *object_name = IP_NULL;
            if (entry->is_sub_map) {
                basic->shared = FALSE;
            } else {
                basic->shared = entry->is_shared;
            }

            vm_map_unlock_read(map);
            return(KERN_SUCCESS);
        }
        case VM_REGION_EXTENDED_INFO:
        {

            if (*count < VM_REGION_EXTENDED_INFO_COUNT)
                return(KERN_INVALID_ARGUMENT);

            extended = (vm_region_extended_info_t) info;
            *count = VM_REGION_EXTENDED_INFO_COUNT;

            vm_map_lock_read(map);

            start = *address;
            if (!vm_map_lookup_entry(map, start, &tmp_entry)) {
                if ((entry = tmp_entry->vme_next) == vm_map_to_entry(map)) {
                        vm_map_unlock_read(map);
                        return(KERN_INVALID_ADDRESS);
                }
            } else {
                entry = tmp_entry;
            }
            start = entry->vme_start;

            extended->protection = entry->protection;
            extended->user_tag = entry->alias;
            extended->pages_resident = 0;
            extended->pages_swapped_out = 0;
            extended->pages_shared_now_private = 0;
            extended->pages_referenced = 0;
            extended->external_pager = 0;
            extended->shadow_depth = 0;

            vm_region_walk(entry, extended, entry->offset, entry->vme_end - start, map, start);

            if (extended->external_pager && extended->ref_count == 2 && extended->share_mode == SM_SHARED)
                    extended->share_mode = SM_PRIVATE;

            if (object_name)
                *object_name = IP_NULL;
            *address = start;
            *size = (entry->vme_end - start);

            vm_map_unlock_read(map);
            return(KERN_SUCCESS);
        }
        case VM_REGION_TOP_INFO:
        {   

            if (*count < VM_REGION_TOP_INFO_COUNT)
                return(KERN_INVALID_ARGUMENT);

            top = (vm_region_top_info_t) info;
            *count = VM_REGION_TOP_INFO_COUNT;

            vm_map_lock_read(map);

            start = *address;
            if (!vm_map_lookup_entry(map, start, &tmp_entry)) {
                if ((entry = tmp_entry->vme_next) == vm_map_to_entry(map)) {
                        vm_map_unlock_read(map);
                        return(KERN_INVALID_ADDRESS);
                }
            } else {
                entry = tmp_entry;

            }
            start = entry->vme_start;

            top->private_pages_resident = 0;
            top->shared_pages_resident = 0;

            vm_region_top_walk(entry, top);

            if (object_name)
                *object_name = IP_NULL;
            *address = start;
            *size = (entry->vme_end - start);

            vm_map_unlock_read(map);
            return(KERN_SUCCESS);
        }
        default:
            return(KERN_INVALID_ARGUMENT);
        }
}

/*
 *      vm_region_recurse: A form of vm_region which follows the
 *      submaps in a target map
 *
 */

kern_return_t
vm_region_recurse(
        vm_map_t                 map,
        vm_offset_t             *address,               /* IN/OUT */
        vm_size_t               *size,                  /* OUT */
        natural_t               *nesting_depth,         /* IN/OUT */
        vm_region_recurse_info_t info,                  /* IN/OUT */
        mach_msg_type_number_t  *count)                 /* IN/OUT */
{
        vm_map_entry_t          tmp_entry;
        register
        vm_map_entry_t          entry;
        register
        vm_offset_t             start;

        unsigned int                    recurse_count;
        vm_map_t                        submap;
        vm_map_t                        base_map;
        vm_map_entry_t                  base_entry;
        vm_offset_t                     base_next;
        vm_offset_t                     base_addr;
        vm_offset_t                     baddr_start_delta;
        vm_region_submap_info_t         submap_info;
        vm_region_extended_info_data_t  extended;

        if (map == VM_MAP_NULL) 
                return(KERN_INVALID_ARGUMENT);

        submap_info = (vm_region_submap_info_t) info;
        *count = VM_REGION_SUBMAP_INFO_COUNT;

        if (*count < VM_REGION_SUBMAP_INFO_COUNT)
                return(KERN_INVALID_ARGUMENT);

        start = *address;
        base_map = map;
        recurse_count = *nesting_depth;

LOOKUP_NEXT_BASE_ENTRY:
        vm_map_lock_read(map);
        if (!vm_map_lookup_entry(map, start, &tmp_entry)) {
                if ((entry = tmp_entry->vme_next) == vm_map_to_entry(map)) {
                        vm_map_unlock_read(map);
                        return(KERN_INVALID_ADDRESS);
                }
        } else {
                entry = tmp_entry;
        }
        *size = entry->vme_end - entry->vme_start;
        start = entry->vme_start;
        base_addr = start;
        baddr_start_delta = *address - start;
        base_next = entry->vme_end;
        base_entry = entry;

        while(entry->is_sub_map && recurse_count) {
                recurse_count--;
                vm_map_lock_read(entry->object.sub_map);


                if(entry == base_entry) {
                        start = entry->offset;
                start += *address - entry->vme_start;
                }

                submap = entry->object.sub_map;
                vm_map_unlock_read(map);
                map = submap;

                if (!vm_map_lookup_entry(map, start, &tmp_entry)) {
                        if ((entry = tmp_entry->vme_next) 
                                                == vm_map_to_entry(map)) {
                                vm_map_unlock_read(map);
                                map = base_map;
                                start = base_next;
                                recurse_count = 0;
                                *nesting_depth = 0;
                                goto LOOKUP_NEXT_BASE_ENTRY;
                        }
                } else {
                        entry = tmp_entry;

                }
                if(start <= entry->vme_start) {
                        vm_offset_t     old_start = start;
                        if(baddr_start_delta) {
                                base_addr += (baddr_start_delta);
                                *size -= baddr_start_delta;
                                baddr_start_delta = 0;
                        }
                        if(base_next <= 
                                (base_addr += (entry->vme_start - start))) {
                                vm_map_unlock_read(map);
                                map = base_map;
                                start = base_next;
                                recurse_count = 0;
                                *nesting_depth = 0;
                                goto LOOKUP_NEXT_BASE_ENTRY;
                        }
                        *size -= entry->vme_start - start;
                        if (*size > (entry->vme_end - entry->vme_start)) {
                                *size = entry->vme_end - entry->vme_start;
                        }
                        start = 0;
                } else {
                        if(baddr_start_delta) {
                                if((start - entry->vme_start) 
                                                < baddr_start_delta) {
                                        base_addr += start - entry->vme_start;
                                        *size -= start - entry->vme_start;
                                } else {
                                        base_addr += baddr_start_delta;
                                        *size += baddr_start_delta;
                                }
                                baddr_start_delta = 0;
                        }
                        base_addr += entry->vme_start;
                        if(base_addr >= base_next) {
                                vm_map_unlock_read(map);
                                map = base_map;
                                start = base_next;
                                recurse_count = 0;
                                *nesting_depth = 0;
                                goto LOOKUP_NEXT_BASE_ENTRY;
                        }
                        if (*size > (entry->vme_end - start))
                                *size = entry->vme_end - start;

                        start = entry->vme_start - start;
                }

                start += entry->offset;

        }
        *nesting_depth -= recurse_count;
        if(entry != base_entry) {
                start = entry->vme_start + (start - entry->offset);
        }


        submap_info->user_tag = entry->alias;
        submap_info->offset = entry->offset;
        submap_info->protection = entry->protection;
        submap_info->inheritance = entry->inheritance;
        submap_info->max_protection = entry->max_protection;
        submap_info->behavior = entry->behavior;
        submap_info->user_wired_count = entry->user_wired_count;
        submap_info->is_submap = entry->is_sub_map;
        submap_info->object_id = (vm_offset_t)entry->object.vm_object;
        *address = base_addr;


        extended.pages_resident = 0;
        extended.pages_swapped_out = 0;
        extended.pages_shared_now_private = 0;
        extended.pages_referenced = 0;
        extended.external_pager = 0;
        extended.shadow_depth = 0;

        if(!entry->is_sub_map) {
                vm_region_walk(entry, &extended, entry->offset, 
                                entry->vme_end - start, map, start);
                submap_info->share_mode = extended.share_mode;
                if (extended.external_pager && extended.ref_count == 2 
                                        && extended.share_mode == SM_SHARED)
                        submap_info->share_mode = SM_PRIVATE;
                submap_info->ref_count = extended.ref_count;
        } else {
                if(entry->use_pmap) 
                        submap_info->share_mode =  SM_TRUESHARED;
                else
                        submap_info->share_mode = SM_PRIVATE;
                submap_info->ref_count = entry->object.sub_map->ref_count;
        }

        submap_info->pages_resident = extended.pages_resident;
        submap_info->pages_swapped_out = extended.pages_swapped_out;
        submap_info->pages_shared_now_private = 
                                extended.pages_shared_now_private;
        submap_info->pages_referenced = extended.pages_referenced;
        submap_info->external_pager = extended.external_pager;
        submap_info->shadow_depth = extended.shadow_depth;

        vm_map_unlock_read(map);
        return(KERN_SUCCESS);
}

/*
 *      TEMPORARYTEMPORARYTEMPORARYTEMPORARYTEMPORARYTEMPORARY
 *      Goes away after regular vm_region_recurse function migrates to
 *      64 bits
 *      vm_region_recurse: A form of vm_region which follows the
 *      submaps in a target map
 *
 */

kern_return_t
vm_region_recurse_64(
        vm_map_t                 map,
        vm_offset_t             *address,               /* IN/OUT */
        vm_size_t               *size,                  /* OUT */
        natural_t               *nesting_depth,         /* IN/OUT */
        vm_region_recurse_info_t info,                  /* IN/OUT */
        mach_msg_type_number_t  *count)                 /* IN/OUT */
{
        vm_map_entry_t          tmp_entry;
        register
        vm_map_entry_t          entry;
        register
        vm_offset_t             start;

        unsigned int                    recurse_count;
        vm_map_t                        submap;
        vm_map_t                        base_map;
        vm_map_entry_t                  base_entry;
        vm_offset_t                     base_next;
        vm_offset_t                     base_addr;
        vm_offset_t                     baddr_start_delta;
        vm_region_submap_info_64_t      submap_info;
        vm_region_extended_info_data_t  extended;

        if (map == VM_MAP_NULL) 
                return(KERN_INVALID_ARGUMENT);

        submap_info = (vm_region_submap_info_64_t) info;
        *count = VM_REGION_SUBMAP_INFO_COUNT;

        if (*count < VM_REGION_SUBMAP_INFO_COUNT)
                return(KERN_INVALID_ARGUMENT);

        start = *address;
        base_map = map;
        recurse_count = *nesting_depth;

LOOKUP_NEXT_BASE_ENTRY:
        vm_map_lock_read(map);
        if (!vm_map_lookup_entry(map, start, &tmp_entry)) {
                if ((entry = tmp_entry->vme_next) == vm_map_to_entry(map)) {
                        vm_map_unlock_read(map);
                        return(KERN_INVALID_ADDRESS);
                }
        } else {
                entry = tmp_entry;
        }
        *size = entry->vme_end - entry->vme_start;
        start = entry->vme_start;
        base_addr = start;
        baddr_start_delta = *address - start;
        base_next = entry->vme_end;
        base_entry = entry;

        while(entry->is_sub_map && recurse_count) {
                recurse_count--;
                vm_map_lock_read(entry->object.sub_map);


                if(entry == base_entry) {
                        start = entry->offset;
                start += *address - entry->vme_start;
                }

                submap = entry->object.sub_map;
                vm_map_unlock_read(map);
                map = submap;

                if (!vm_map_lookup_entry(map, start, &tmp_entry)) {
                        if ((entry = tmp_entry->vme_next) 
                                                == vm_map_to_entry(map)) {
                                vm_map_unlock_read(map);
                                map = base_map;
                                start = base_next;
                                recurse_count = 0;
                                *nesting_depth = 0;
                                goto LOOKUP_NEXT_BASE_ENTRY;
                        }
                } else {
                        entry = tmp_entry;

                }
                if(start <= entry->vme_start) {
                        vm_offset_t     old_start = start;
                        if(baddr_start_delta) {
                                base_addr += (baddr_start_delta);
                                *size -= baddr_start_delta;
                                baddr_start_delta = 0;
                        }
                        if(base_next <= 
                                (base_addr += (entry->vme_start - start))) {
                                vm_map_unlock_read(map);
                                map = base_map;
                                start = base_next;
                                recurse_count = 0;
                                *nesting_depth = 0;
                                goto LOOKUP_NEXT_BASE_ENTRY;
                        }
                        *size -= entry->vme_start - start;
                        if (*size > (entry->vme_end - entry->vme_start)) {
                                *size = entry->vme_end - entry->vme_start;
                        }
                        start = 0;
                } else {
                        if(baddr_start_delta) {
                                if((start - entry->vme_start) 
                                                < baddr_start_delta) {
                                        base_addr += start - entry->vme_start;
                                        *size -= start - entry->vme_start;
                                } else {
                                        base_addr += baddr_start_delta;
                                        *size += baddr_start_delta;
                                }
                                baddr_start_delta = 0;
                        }
                        base_addr += entry->vme_start;
                        if(base_addr >= base_next) {
                                vm_map_unlock_read(map);
                                map = base_map;
                                start = base_next;
                                recurse_count = 0;
                                *nesting_depth = 0;
                                goto LOOKUP_NEXT_BASE_ENTRY;
                        }
                        if (*size > (entry->vme_end - start))
                                *size = entry->vme_end - start;

                        start = entry->vme_start - start;
                }

                start += entry->offset;

        }
        *nesting_depth -= recurse_count;
        if(entry != base_entry) {
                start = entry->vme_start + (start - entry->offset);
        }


        submap_info->user_tag = entry->alias;
        submap_info->offset = entry->offset;
        submap_info->protection = entry->protection;
        submap_info->inheritance = entry->inheritance;
        submap_info->max_protection = entry->max_protection;
        submap_info->behavior = entry->behavior;
        submap_info->user_wired_count = entry->user_wired_count;
        submap_info->is_submap = entry->is_sub_map;
        submap_info->object_id = (vm_offset_t)entry->object.vm_object;
        *address = base_addr;


        extended.pages_resident = 0;
        extended.pages_swapped_out = 0;
        extended.pages_shared_now_private = 0;
        extended.pages_referenced = 0;
        extended.external_pager = 0;
        extended.shadow_depth = 0;

        if(!entry->is_sub_map) {
                vm_region_walk(entry, &extended, entry->offset, 
                                entry->vme_end - start, map, start);
                submap_info->share_mode = extended.share_mode;
                if (extended.external_pager && extended.ref_count == 2 
                                        && extended.share_mode == SM_SHARED)
                        submap_info->share_mode = SM_PRIVATE;
                submap_info->ref_count = extended.ref_count;
        } else {
                if(entry->use_pmap) 
                        submap_info->share_mode =  SM_TRUESHARED;
                else
                        submap_info->share_mode = SM_PRIVATE;
                submap_info->ref_count = entry->object.sub_map->ref_count;
        }

        submap_info->pages_resident = extended.pages_resident;
        submap_info->pages_swapped_out = extended.pages_swapped_out;
        submap_info->pages_shared_now_private = 
                                extended.pages_shared_now_private;
        submap_info->pages_referenced = extended.pages_referenced;
        submap_info->external_pager = extended.external_pager;
        submap_info->shadow_depth = extended.shadow_depth;

        vm_map_unlock_read(map);
        return(KERN_SUCCESS);
}


/*
 *      TEMPORARYTEMPORARYTEMPORARYTEMPORARYTEMPORARYTEMPORARY
 *      Goes away after regular vm_region function migrates to
 *      64 bits
 */


kern_return_t
vm_region_64(
        vm_map_t                 map,
        vm_offset_t             *address,               /* IN/OUT */
        vm_size_t               *size,                  /* OUT */
        vm_region_flavor_t       flavor,                /* IN */
        vm_region_info_t         info,                  /* OUT */
        mach_msg_type_number_t  *count,                 /* IN/OUT */
        ipc_port_t              *object_name)           /* OUT */
{
        vm_map_entry_t          tmp_entry;
        register
        vm_map_entry_t          entry;
        register
        vm_offset_t             start;
        vm_region_basic_info_64_t       basic;
        vm_region_extended_info_t       extended;
        vm_region_top_info_t    top;

        if (map == VM_MAP_NULL) 
                return(KERN_INVALID_ARGUMENT);

        switch (flavor) {
        
        case VM_REGION_BASIC_INFO:
        {
            if (*count < VM_REGION_BASIC_INFO_COUNT)
                return(KERN_INVALID_ARGUMENT);

            basic = (vm_region_basic_info_64_t) info;
            *count = VM_REGION_BASIC_INFO_COUNT;

            vm_map_lock_read(map);

            start = *address;
            if (!vm_map_lookup_entry(map, start, &tmp_entry)) {
                if ((entry = tmp_entry->vme_next) == vm_map_to_entry(map)) {
                        vm_map_unlock_read(map);
                        return(KERN_INVALID_ADDRESS);
                }
            } else {
                entry = tmp_entry;
            }

            start = entry->vme_start;

            basic->offset = entry->offset;
            basic->protection = entry->protection;
            basic->inheritance = entry->inheritance;
            basic->max_protection = entry->max_protection;
            basic->behavior = entry->behavior;
            basic->user_wired_count = entry->user_wired_count;
            basic->reserved = entry->is_sub_map;
            *address = start;
            *size = (entry->vme_end - start);

            if (object_name) *object_name = IP_NULL;
            if (entry->is_sub_map) {
                basic->shared = FALSE;
            } else {
                basic->shared = entry->is_shared;
            }

            vm_map_unlock_read(map);
            return(KERN_SUCCESS);
        }
        case VM_REGION_EXTENDED_INFO:
        {

            if (*count < VM_REGION_EXTENDED_INFO_COUNT)
                return(KERN_INVALID_ARGUMENT);

            extended = (vm_region_extended_info_t) info;
            *count = VM_REGION_EXTENDED_INFO_COUNT;

            vm_map_lock_read(map);

            start = *address;
            if (!vm_map_lookup_entry(map, start, &tmp_entry)) {
                if ((entry = tmp_entry->vme_next) == vm_map_to_entry(map)) {
                        vm_map_unlock_read(map);
                        return(KERN_INVALID_ADDRESS);
                }
            } else {
                entry = tmp_entry;
            }
            start = entry->vme_start;

            extended->protection = entry->protection;
            extended->user_tag = entry->alias;
            extended->pages_resident = 0;
            extended->pages_swapped_out = 0;
            extended->pages_shared_now_private = 0;
            extended->pages_referenced = 0;
            extended->external_pager = 0;
            extended->shadow_depth = 0;

            vm_region_walk(entry, extended, entry->offset, entry->vme_end - start, map, start);

            if (extended->external_pager && extended->ref_count == 2 && extended->share_mode == SM_SHARED)
                    extended->share_mode = SM_PRIVATE;

            if (object_name)
                *object_name = IP_NULL;
            *address = start;
            *size = (entry->vme_end - start);

            vm_map_unlock_read(map);
            return(KERN_SUCCESS);
        }
        case VM_REGION_TOP_INFO:
        {   

            if (*count < VM_REGION_TOP_INFO_COUNT)
                return(KERN_INVALID_ARGUMENT);

            top = (vm_region_top_info_t) info;
            *count = VM_REGION_TOP_INFO_COUNT;

            vm_map_lock_read(map);

            start = *address;
            if (!vm_map_lookup_entry(map, start, &tmp_entry)) {
                if ((entry = tmp_entry->vme_next) == vm_map_to_entry(map)) {
                        vm_map_unlock_read(map);
                        return(KERN_INVALID_ADDRESS);
                }
            } else {
                entry = tmp_entry;

            }
            start = entry->vme_start;

            top->private_pages_resident = 0;
            top->shared_pages_resident = 0;

            vm_region_top_walk(entry, top);

            if (object_name)
                *object_name = IP_NULL;
            *address = start;
            *size = (entry->vme_end - start);

            vm_map_unlock_read(map);
            return(KERN_SUCCESS);
        }
        default:
            return(KERN_INVALID_ARGUMENT);
        }
}

void
vm_region_top_walk(
        vm_map_entry_t             entry,
        vm_region_top_info_t       top)
{
        register struct vm_object *obj, *tmp_obj;

        if (entry->object.vm_object == 0) {
            top->share_mode = SM_EMPTY;
            top->ref_count = 0;
            top->obj_id = 0;
            return;
        }
        if (entry->is_sub_map)
            vm_region_top_walk((vm_map_entry_t)entry->object.sub_map, top);
        else {
            obj = entry->object.vm_object;

            vm_object_lock(obj);

            if (obj->shadow) {
                if (obj->ref_count == 1)
                    top->private_pages_resident = obj->resident_page_count;
                else
                    top->shared_pages_resident = obj->resident_page_count;
                top->ref_count  = obj->ref_count;
                top->share_mode = SM_COW;
            
                while (tmp_obj = obj->shadow) {
                    vm_object_lock(tmp_obj);
                    vm_object_unlock(obj);
                    obj = tmp_obj;

                    top->shared_pages_resident += obj->resident_page_count;
                    top->ref_count += obj->ref_count - 1;
                }
            } else {
                if (entry->needs_copy) {
                    top->share_mode = SM_COW;
                    top->shared_pages_resident = obj->resident_page_count;
                } else {
                    if (obj->ref_count == 1 ||
                       (obj->ref_count == 2 && !(obj->pager_trusted) && !(obj->internal))) {
                        top->share_mode = SM_PRIVATE;
                        top->private_pages_resident = obj->resident_page_count;
                    } else {
                        top->share_mode = SM_SHARED;
                        top->shared_pages_resident = obj->resident_page_count;
                    }
                }
                top->ref_count = obj->ref_count;
            }
            top->obj_id = (int)obj;

            vm_object_unlock(obj);
        }
}

void
vm_region_walk(
        vm_map_entry_t             entry,
        vm_region_extended_info_t  extended,
        vm_object_offset_t         offset,
        vm_offset_t                range,
        vm_map_t                   map,
        vm_offset_t                va)
{
        register struct vm_object *obj, *tmp_obj;
        register vm_offset_t       last_offset;
        register int               i;
        void vm_region_look_for_page();

        if (entry->object.vm_object == 0) {
            extended->share_mode = SM_EMPTY;
            extended->ref_count = 0;
            return;
        }
        if (entry->is_sub_map)
            vm_region_walk((vm_map_entry_t)entry->object.sub_map, extended, offset + entry->offset,
                           range, map, va);
        else {
            obj = entry->object.vm_object;

            vm_object_lock(obj);

            for (last_offset = offset + range; offset < last_offset; offset += PAGE_SIZE_64, va += PAGE_SIZE)
                vm_region_look_for_page(obj, extended, offset, obj->ref_count, 0, map, va);

            if (extended->shadow_depth || entry->needs_copy)
                extended->share_mode = SM_COW;
            else {
                if (obj->ref_count == 1)
                    extended->share_mode = SM_PRIVATE;
                else {
                    if (obj->true_share)
                        extended->share_mode = SM_TRUESHARED;
                    else
                        extended->share_mode = SM_SHARED;
                }
            }
            extended->ref_count = obj->ref_count - extended->shadow_depth;
            
            for (i = 0; i < extended->shadow_depth; i++) {
                if ((tmp_obj = obj->shadow) == 0)
                    break;
                vm_object_lock(tmp_obj);
                vm_object_unlock(obj);
                extended->ref_count += tmp_obj->ref_count;
                obj = tmp_obj;
            }
            vm_object_unlock(obj);

            if (extended->share_mode == SM_SHARED) {
                register vm_map_entry_t      cur;
                register vm_map_entry_t      last;
                int      my_refs;

                obj = entry->object.vm_object;
                last = vm_map_to_entry(map);
                my_refs = 0;

                for (cur = vm_map_first_entry(map); cur != last; cur = cur->vme_next)
                    my_refs += vm_region_count_obj_refs(cur, obj);

                if (my_refs == obj->ref_count)
                    extended->share_mode = SM_PRIVATE_ALIASED;
                else if (my_refs > 1)
                    extended->share_mode = SM_SHARED_ALIASED;
            }
        }
}



void
vm_region_look_for_page(
        vm_object_t                object,
        vm_region_extended_info_t  extended,
        vm_object_offset_t         offset,
        int                        max_refcnt,
        int                        depth,
        vm_map_t                   map,
        vm_offset_t                va)
{
        register vm_page_t         p;
        register vm_object_t       shadow;
        
        shadow = object->shadow;

        if ( !(object->pager_trusted) && !(object->internal))
            extended->external_pager = 1;

        if ((p = vm_page_lookup(object, offset)) != VM_PAGE_NULL) {
                if (shadow && (max_refcnt == 1))
                    extended->pages_shared_now_private++;

                if (pmap_extract(vm_map_pmap(map), va))
                    extended->pages_referenced++;
                extended->pages_resident++;

                return;
        }
        if (object->existence_map) {
            if (vm_external_state_get(object->existence_map, offset) == VM_EXTERNAL_STATE_EXISTS) {
                if (shadow && (max_refcnt == 1))
                    extended->pages_shared_now_private++;
                extended->pages_swapped_out++;

                return;
            }
        }
        if (shadow) {
            vm_object_lock(shadow);

            if (++depth > extended->shadow_depth)
                extended->shadow_depth = depth;

            if (shadow->ref_count > max_refcnt)
                max_refcnt = shadow->ref_count;
        
            vm_region_look_for_page(shadow, extended, offset + object->shadow_offset,
                                    max_refcnt, depth, map, va);
            vm_object_unlock(shadow);

            return;
        }
}


vm_region_count_obj_refs(
        vm_map_entry_t    entry,
        vm_object_t       object)
{
        register int ref_count;
        register vm_object_t chk_obj;
        register vm_object_t tmp_obj;

        if (entry->object.vm_object == 0)
            return(0);

        if (entry->is_sub_map)
            ref_count = vm_region_count_obj_refs((vm_map_entry_t)entry->object.sub_map, object);
        else {
            ref_count = 0;

            chk_obj = entry->object.vm_object;
            vm_object_lock(chk_obj);

            while (chk_obj) {
                if (chk_obj == object)
                    ref_count++;
                if (tmp_obj = chk_obj->shadow)
                    vm_object_lock(tmp_obj);
                vm_object_unlock(chk_obj);
                
                chk_obj = tmp_obj;
            }
        }
        return(ref_count);
}


/*
 *      Routine:        vm_map_simplify
 *
 *      Description:
 *              Attempt to simplify the map representation in
 *              the vicinity of the given starting address.
 *      Note:
 *              This routine is intended primarily to keep the
 *              kernel maps more compact -- they generally don't
 *              benefit from the "expand a map entry" technology
 *              at allocation time because the adjacent entry
 *              is often wired down.
 */
void
vm_map_simplify(
        vm_map_t        map,
        vm_offset_t     start)
{
        vm_map_entry_t  this_entry;
        vm_map_entry_t  prev_entry;
        vm_map_entry_t  next_entry;

        vm_map_lock(map);
        if (
                (vm_map_lookup_entry(map, start, &this_entry)) &&
                ((prev_entry = this_entry->vme_prev) != vm_map_to_entry(map)) &&

                (prev_entry->vme_end == this_entry->vme_start) &&

                (prev_entry->is_shared == FALSE) &&
                (prev_entry->is_sub_map == FALSE) &&

                (this_entry->is_shared == FALSE) &&
                (this_entry->is_sub_map == FALSE) &&

                (prev_entry->inheritance == this_entry->inheritance) &&
                (prev_entry->protection == this_entry->protection) &&
                (prev_entry->max_protection == this_entry->max_protection) &&
                (prev_entry->behavior == this_entry->behavior) &&
                (prev_entry->wired_count == this_entry->wired_count) &&
                (prev_entry->user_wired_count == this_entry->user_wired_count)&&
                (prev_entry->in_transition == FALSE) &&
                (this_entry->in_transition == FALSE) &&

                (prev_entry->needs_copy == this_entry->needs_copy) &&

                (prev_entry->object.vm_object == this_entry->object.vm_object)&&
                ((prev_entry->offset +
                 (prev_entry->vme_end - prev_entry->vme_start))
                     == this_entry->offset)
        ) {
                SAVE_HINT(map, prev_entry);
                vm_map_entry_unlink(map, this_entry);
                prev_entry->vme_end = this_entry->vme_end;
                UPDATE_FIRST_FREE(map, map->first_free);
                vm_object_deallocate(this_entry->object.vm_object);
                vm_map_entry_dispose(map, this_entry);
                counter(c_vm_map_simplified_lower++);
        }
        if (
                (vm_map_lookup_entry(map, start, &this_entry)) &&
                ((next_entry = this_entry->vme_next) != vm_map_to_entry(map)) &&

                (next_entry->vme_start == this_entry->vme_end) &&

                (next_entry->is_shared == FALSE) &&
                (next_entry->is_sub_map == FALSE) &&

                (next_entry->is_shared == FALSE) &&
                (next_entry->is_sub_map == FALSE) &&

                (next_entry->inheritance == this_entry->inheritance) &&
                (next_entry->protection == this_entry->protection) &&
                (next_entry->max_protection == this_entry->max_protection) &&
                (next_entry->behavior == this_entry->behavior) &&
                (next_entry->wired_count == this_entry->wired_count) &&
                (next_entry->user_wired_count == this_entry->user_wired_count)&&
                (this_entry->in_transition == FALSE) &&
                (next_entry->in_transition == FALSE) &&

                (next_entry->needs_copy == this_entry->needs_copy) &&

                (next_entry->object.vm_object == this_entry->object.vm_object)&&
                ((this_entry->offset +
                 (this_entry->vme_end - this_entry->vme_start))
                     == next_entry->offset)
        ) {
                vm_map_entry_unlink(map, next_entry);
                this_entry->vme_end = next_entry->vme_end;
                UPDATE_FIRST_FREE(map, map->first_free);
                vm_object_deallocate(next_entry->object.vm_object);
                vm_map_entry_dispose(map, next_entry);
                counter(c_vm_map_simplified_upper++);
        }
        counter(c_vm_map_simplify_called++);
        vm_map_unlock(map);
}


/*
 *      Routine:        vm_map_machine_attribute
 *      Purpose:
 *              Provide machine-specific attributes to mappings,
 *              such as cachability etc. for machines that provide
 *              them.  NUMA architectures and machines with big/strange
 *              caches will use this.
 *      Note:
 *              Responsibilities for locking and checking are handled here,
 *              everything else in the pmap module. If any non-volatile
 *              information must be kept, the pmap module should handle
 *              it itself. [This assumes that attributes do not
 *              need to be inherited, which seems ok to me]
 */
kern_return_t
vm_map_machine_attribute(
        vm_map_t        map,
        vm_offset_t     address,
        vm_size_t       size,
        vm_machine_attribute_t  attribute,
        vm_machine_attribute_val_t* value)              /* IN/OUT */
{
        kern_return_t   ret;

        if (address < vm_map_min(map) ||
            (address + size) > vm_map_max(map))
                return KERN_INVALID_ADDRESS;

        vm_map_lock(map);

        ret = pmap_attribute(map->pmap, address, size, attribute, value);

        vm_map_unlock(map);

        return ret;
}

/*
 *      vm_map_behavior_set:
 *
 *      Sets the paging reference behavior of the specified address
 *      range in the target map.  Paging reference behavior affects
 *      how pagein operations resulting from faults on the map will be 
 *      clustered.
 */
kern_return_t 
vm_map_behavior_set(
        vm_map_t        map,
        vm_offset_t     start,
        vm_offset_t     end,
        vm_behavior_t   new_behavior)
{
        register vm_map_entry_t entry;
        vm_map_entry_t  temp_entry;

        XPR(XPR_VM_MAP,
                "vm_map_behavior_set, 0x%X start 0x%X end 0x%X behavior %d",
                (integer_t)map, start, end, new_behavior, 0);

        switch (new_behavior) {
        case VM_BEHAVIOR_DEFAULT:
        case VM_BEHAVIOR_RANDOM:
        case VM_BEHAVIOR_SEQUENTIAL:
        case VM_BEHAVIOR_RSEQNTL:
                break;
        default:
                return(KERN_INVALID_ARGUMENT);
        }

        vm_map_lock(map);

        /*
         *      The entire address range must be valid for the map.
         *      Note that vm_map_range_check() does a 
         *      vm_map_lookup_entry() internally and returns the
         *      entry containing the start of the address range if
         *      the entire range is valid.
         */
        if (vm_map_range_check(map, start, end, &temp_entry)) {
                entry = temp_entry;
                vm_map_clip_start(map, entry, start);
        }
        else {
                vm_map_unlock(map);
                return(KERN_INVALID_ADDRESS);
        }

        while ((entry != vm_map_to_entry(map)) && (entry->vme_start < end)) {
                vm_map_clip_end(map, entry, end);

                entry->behavior = new_behavior;

                entry = entry->vme_next;
        }

        vm_map_unlock(map);
        return(KERN_SUCCESS);
}


int
vm_map_copy_cont_is_valid(
        vm_map_copy_t   copy)
{
        vm_map_copy_cont_t      cont;

        assert(copy->type == VM_MAP_COPY_PAGE_LIST);
        cont = copy->cpy_cont;
        if (
            cont != vm_map_copy_discard_cont &&
            cont != vm_map_copyin_page_list_cont ) {
                printf("vm_map_copy_cont_is_valid:  bogus cont 0x%x\n", cont);
                assert((integer_t) cont == 0xdeadbeef);
        }
        return 1;
}

#include <mach_kdb.h>
#if     MACH_KDB
#include <ddb/db_output.h>
#include <vm/vm_print.h>

#define printf  db_printf

/*
 * Forward declarations for internal functions.
 */
extern void vm_map_links_print(
                struct vm_map_links     *links);

extern void vm_map_header_print(
                struct vm_map_header    *header);

extern void vm_map_entry_print(
                vm_map_entry_t          entry);

extern void vm_follow_entry(
                vm_map_entry_t          entry);

extern void vm_follow_map(
                vm_map_t                map);

/*
 *      vm_map_links_print:     [ debug ]
 */
void
vm_map_links_print(
        struct vm_map_links     *links)
{
        iprintf("prev=0x%x, next=0x%x, start=0x%x, end=0x%x\n",
                links->prev,
                links->next,
                links->start,
                links->end);
}

/*
 *      vm_map_header_print:    [ debug ]
 */
void
vm_map_header_print(
        struct vm_map_header    *header)
{
        vm_map_links_print(&header->links);
        iprintf("nentries=0x%x, %sentries_pageable\n",
                header->nentries,
                (header->entries_pageable ? "" : "!"));
}

/*
 *      vm_follow_entry:        [ debug ]
 */
void
vm_follow_entry(
        vm_map_entry_t entry)
{
        extern int db_indent;
        int shadows;

        iprintf("map entry 0x%x:\n", entry);

        db_indent += 2;

        shadows = vm_follow_object(entry->object.vm_object);
        iprintf("Total objects : %d\n",shadows);

        db_indent -= 2;
}

/*
 *      vm_map_entry_print:     [ debug ]
 */
void
vm_map_entry_print(
        register vm_map_entry_t entry)
{
        extern int db_indent;
        static char *inheritance_name[4] = { "share", "copy", "none", "?"};
        static char *behavior_name[4] = { "dflt", "rand", "seqtl", "rseqntl" };
        
        iprintf("map entry 0x%x:\n", entry);

        db_indent += 2;

        vm_map_links_print(&entry->links);

        iprintf("start=0x%x, end=0x%x, prot=%x/%x/%s\n",
                entry->vme_start,
                entry->vme_end,
                entry->protection,
                entry->max_protection,
                inheritance_name[(entry->inheritance & 0x3)]);

        iprintf("behavior=%s, wired_count=%d, user_wired_count=%d\n",
                behavior_name[(entry->behavior & 0x3)],
                entry->wired_count,
                entry->user_wired_count);
        iprintf("%sin_transition, %sneeds_wakeup\n",
                (entry->in_transition ? "" : "!"),
                (entry->needs_wakeup ? "" : "!"));

        if (entry->is_sub_map) {
                iprintf("submap=0x%x, offset=0x%x\n",
                       entry->object.sub_map,
                       entry->offset);
        } else {
                iprintf("object=0x%x, offset=0x%x, ",
                        entry->object.vm_object,
                        entry->offset);
                printf("%sis_shared, %sneeds_copy\n",
                       (entry->is_shared ? "" : "!"),
                       (entry->needs_copy ? "" : "!"));
        }

        db_indent -= 2;
}

/*
 *      vm_follow_map:  [ debug ]
 */
void
vm_follow_map(
        vm_map_t map)
{
        register vm_map_entry_t entry;
        extern int db_indent;

        iprintf("task map 0x%x:\n", map);

        db_indent += 2;

        for (entry = vm_map_first_entry(map);
             entry && entry != vm_map_to_entry(map);
             entry = entry->vme_next) {
            vm_follow_entry(entry);
        }

        db_indent -= 2;
}

/*
 *      vm_map_print:   [ debug ]
 */
void
vm_map_print(
        register vm_map_t       map)
{
        register vm_map_entry_t entry;
        extern int db_indent;
        char *swstate;

        iprintf("task map 0x%x:\n", map);

        db_indent += 2;

        vm_map_header_print(&map->hdr);

        iprintf("pmap=0x%x, size=%d, ref=%d, hint=0x%x, first_free=0x%x\n",
                map->pmap,
                map->size,
                map->ref_count,
                map->hint,
                map->first_free);

        iprintf("%swait_for_space, %swiring_required, timestamp=%d\n",
                (map->wait_for_space ? "" : "!"),
                (map->wiring_required ? "" : "!"),
                map->timestamp);

#if     TASK_SWAPPER
        switch (map->sw_state) {
            case MAP_SW_IN:
                swstate = "SW_IN";
                break;
            case MAP_SW_OUT:
                swstate = "SW_OUT";
                break;
            default:
                swstate = "????";
                break;
        }
        iprintf("res=%d, sw_state=%s\n", map->res_count, swstate);
#endif  /* TASK_SWAPPER */

        for (entry = vm_map_first_entry(map);
             entry && entry != vm_map_to_entry(map);
             entry = entry->vme_next) {
                vm_map_entry_print(entry);
        }

        db_indent -= 2;
}

/*
 *      Routine:        vm_map_copy_print
 *      Purpose:
 *              Pretty-print a copy object for ddb.
 */

void
vm_map_copy_print(
        vm_map_copy_t   copy)
{
        extern int db_indent;
        int i, npages;
        vm_map_entry_t entry;

        printf("copy object 0x%x\n", copy);

        db_indent += 2;

        iprintf("type=%d", copy->type);
        switch (copy->type) {
                case VM_MAP_COPY_ENTRY_LIST:
                printf("[entry_list]");
                break;
                
                case VM_MAP_COPY_OBJECT:
                printf("[object]");
                break;
                
                case VM_MAP_COPY_PAGE_LIST:
                printf("[page_list]");
                break;
                
                case VM_MAP_COPY_KERNEL_BUFFER:
                printf("[kernel_buffer]");
                break;

                default:
                printf("[bad type]");
                break;
        }
        printf(", offset=0x%x", copy->offset);
        printf(", size=0x%x\n", copy->size);

        switch (copy->type) {
                case VM_MAP_COPY_ENTRY_LIST:
                vm_map_header_print(&copy->cpy_hdr);
                for (entry = vm_map_copy_first_entry(copy);
                     entry && entry != vm_map_copy_to_entry(copy);
                     entry = entry->vme_next) {
                        vm_map_entry_print(entry);
                }
                break;

                case VM_MAP_COPY_OBJECT:
                iprintf("object=0x%x\n", copy->cpy_object);
                break;

                case VM_MAP_COPY_KERNEL_BUFFER:
                iprintf("kernel buffer=0x%x", copy->cpy_kdata);
                printf(", kalloc_size=0x%x\n", copy->cpy_kalloc_size);
                break;

                case VM_MAP_COPY_PAGE_LIST:
                iprintf("npages=%d", copy->cpy_npages);
                printf(", cont=%x", copy->cpy_cont);
                printf(", cont_args=%x\n", copy->cpy_cont_args);
                if (copy->cpy_npages < 0) {
                        npages = 0;
                } else if (copy->cpy_npages > VM_MAP_COPY_PAGE_LIST_MAX) {
                        npages = VM_MAP_COPY_PAGE_LIST_MAX;
                } else {
                        npages = copy->cpy_npages;
                }
                iprintf("copy->cpy_page_list[0..%d] = {", npages);
                for (i = 0; i < npages - 1; i++) {
                        printf("0x%x, ", copy->cpy_page_list[i]);
                }
                if (npages > 0) {
                        printf("0x%x", copy->cpy_page_list[npages - 1]);
                }
                printf("}\n");
                break;
        }

        db_indent -=2;
}

/*
 *      db_vm_map_total_size(map)       [ debug ]
 *
 *      return the total virtual size (in bytes) of the map
 */
vm_size_t
db_vm_map_total_size(
        vm_map_t        map)
{
        vm_map_entry_t  entry;
        vm_size_t       total;

        total = 0;
        for (entry = vm_map_first_entry(map);
             entry != vm_map_to_entry(map);
             entry = entry->vme_next) {
                total += entry->vme_end - entry->vme_start;
        }

        return total;
}

#endif  /* MACH_KDB */

/*
 *      Routine:        vm_map_entry_insert
 *
 *      Descritpion:    This routine inserts a new vm_entry in a locked map.
 */
vm_map_entry_t
vm_map_entry_insert(
        vm_map_t                map,
        vm_map_entry_t          insp_entry,
        vm_offset_t             start,
        vm_offset_t             end,
        vm_object_t             object,
        vm_object_offset_t      offset,
        boolean_t               needs_copy,
        boolean_t               is_shared,
        boolean_t               in_transition,
        vm_prot_t               cur_protection,
        vm_prot_t               max_protection,
        vm_behavior_t           behavior,
        vm_inherit_t            inheritance,
        unsigned                wired_count)
{
        vm_map_entry_t  new_entry;

        assert(insp_entry != (vm_map_entry_t)0);

        new_entry = vm_map_entry_create(map);

        new_entry->vme_start = start;
        new_entry->vme_end = end;
        assert(page_aligned(new_entry->vme_start));
        assert(page_aligned(new_entry->vme_end));

        new_entry->object.vm_object = object;
        new_entry->offset = offset;
        new_entry->is_shared = is_shared;
        new_entry->is_sub_map = FALSE;
        new_entry->needs_copy = needs_copy;
        new_entry->in_transition = in_transition;
        new_entry->needs_wakeup = FALSE;
        new_entry->inheritance = inheritance;
        new_entry->protection = cur_protection;
        new_entry->max_protection = max_protection;
        new_entry->behavior = behavior;
        new_entry->wired_count = wired_count;
        new_entry->user_wired_count = 0;
        new_entry->use_pmap = FALSE;

        /*
         *      Insert the new entry into the list.
         */

        vm_map_entry_link(map, insp_entry, new_entry);
        map->size += end - start;

        /*
         *      Update the free space hint and the lookup hint.
         */

        SAVE_HINT(map, new_entry);
        return new_entry;
}

/*
 *      Routine:        vm_remap_extract
 *
 *      Descritpion:    This routine returns a vm_entry list from a map.
 */
kern_return_t
vm_remap_extract(
        vm_map_t                map,
        vm_offset_t             addr,
        vm_size_t               size,
        boolean_t               copy,
        struct vm_map_header    *map_header,
        vm_prot_t               *cur_protection,
        vm_prot_t               *max_protection,
        /* What, no behavior? */
        vm_inherit_t            inheritance,
        boolean_t               pageable)
{
        kern_return_t           result;
        vm_size_t               mapped_size;
        vm_size_t               tmp_size;
        vm_map_entry_t          src_entry;     /* result of last map lookup */
        vm_map_entry_t          new_entry;
        vm_object_offset_t      offset;
        vm_offset_t             map_address;
        vm_offset_t             src_start;     /* start of entry to map */
        vm_offset_t             src_end;       /* end of region to be mapped */
        vm_object_t             object;    
        vm_map_version_t        version;
        boolean_t               src_needs_copy;
        boolean_t               new_entry_needs_copy;

        assert(map != VM_MAP_NULL);
        assert(size != 0 && size == round_page(size));
        assert(inheritance == VM_INHERIT_NONE ||
               inheritance == VM_INHERIT_COPY ||
               inheritance == VM_INHERIT_SHARE);

        /*
         *      Compute start and end of region.
         */
        src_start = trunc_page(addr);
        src_end = round_page(src_start + size);

        /*
         *      Initialize map_header.
         */
        map_header->links.next = (struct vm_map_entry *)&map_header->links;
        map_header->links.prev = (struct vm_map_entry *)&map_header->links;
        map_header->nentries = 0;
        map_header->entries_pageable = pageable;

        *cur_protection = VM_PROT_ALL;
        *max_protection = VM_PROT_ALL;

        map_address = 0;
        mapped_size = 0;
        result = KERN_SUCCESS;

        /*  
         *      The specified source virtual space might correspond to
         *      multiple map entries, need to loop on them.
         */
        vm_map_lock(map);
        while (mapped_size != size) {
                vm_size_t       entry_size;

                /*
                 *      Find the beginning of the region.
                 */ 
                if (! vm_map_lookup_entry(map, src_start, &src_entry)) {
                        result = KERN_INVALID_ADDRESS;
                        break;
                }

                if (src_start < src_entry->vme_start ||
                    (mapped_size && src_start != src_entry->vme_start)) {
                        result = KERN_INVALID_ADDRESS;
                        break;
                }

                if(src_entry->is_sub_map) {
                        result = KERN_INVALID_ADDRESS;
                        break;
                }

                tmp_size = size - mapped_size;
                if (src_end > src_entry->vme_end)
                        tmp_size -= (src_end - src_entry->vme_end);

                entry_size = (vm_size_t)(src_entry->vme_end -
                                         src_entry->vme_start);

                if(src_entry->is_sub_map) {
                        vm_map_reference(src_entry->object.sub_map);
                } else {
                        object = src_entry->object.vm_object;

                        if (object == VM_OBJECT_NULL) {
                                object = vm_object_allocate(entry_size);
                                src_entry->offset = 0;
                                src_entry->object.vm_object = object;
                        } else if (object->copy_strategy !=
                                   MEMORY_OBJECT_COPY_SYMMETRIC) {
                                /*
                                 *      We are already using an asymmetric
                                 *      copy, and therefore we already have
                                 *      the right object.
                                 */
                                assert(!src_entry->needs_copy);
                        } else if (src_entry->needs_copy || object->shadowed ||
                                   (object->internal && !object->true_share &&
                                   !src_entry->is_shared &&
                                    object->size > entry_size)) {

                                vm_object_shadow(&src_entry->object.vm_object,
                                                 &src_entry->offset,
                                                 entry_size);

                                if (!src_entry->needs_copy &&
                                    (src_entry->protection & VM_PROT_WRITE)) {
                                        pmap_protect(vm_map_pmap(map),
                                             src_entry->vme_start,
                                             src_entry->vme_end,
                                             src_entry->protection &
                                                     ~VM_PROT_WRITE);
                                }

                                object = src_entry->object.vm_object;
                                src_entry->needs_copy = FALSE;
                        }


                        vm_object_lock(object);
                        object->ref_count++;    /* object ref. for new entry */
                        VM_OBJ_RES_INCR(object);
                        if (object->copy_strategy == 
                                        MEMORY_OBJECT_COPY_SYMMETRIC) {
                                object->copy_strategy = 
                                        MEMORY_OBJECT_COPY_DELAY;
                        }
                        vm_object_unlock(object);
                }

                offset = src_entry->offset + (src_start - src_entry->vme_start);

                new_entry = _vm_map_entry_create(map_header);
                vm_map_entry_copy(new_entry, src_entry);
                new_entry->use_pmap = FALSE; /* clr address space specifics */

                new_entry->vme_start = map_address;
                new_entry->vme_end = map_address + tmp_size;
                new_entry->inheritance = inheritance;
                new_entry->offset = offset;

                /*
                 * The new region has to be copied now if required.
                 */
        RestartCopy:
                if (!copy) {
                        src_entry->is_shared = TRUE;
                        new_entry->is_shared = TRUE;
                        if (!(new_entry->is_sub_map)) 
                                new_entry->needs_copy = FALSE;

                } else if (src_entry->is_sub_map) {
                        /* make this a COW sub_map if not already */
                        new_entry->needs_copy = TRUE;
                } else if (src_entry->wired_count == 0 &&
                         vm_object_copy_quickly(&new_entry->object.vm_object,
                                                new_entry->offset,
                                                (new_entry->vme_end -
                                                    new_entry->vme_start),
                                                &src_needs_copy,
                                                &new_entry_needs_copy)) {

                        new_entry->needs_copy = new_entry_needs_copy;
                        new_entry->is_shared = FALSE;

                        /*
                         * Handle copy_on_write semantics.
                         */
                        if (src_needs_copy && !src_entry->needs_copy) {
                                vm_object_pmap_protect(object,
                                                       offset,
                                                       entry_size,
                                                       (src_entry->is_shared ?
                                                        PMAP_NULL : map->pmap),
                                                       src_entry->vme_start,
                                                       src_entry->protection &
                                                       ~VM_PROT_WRITE);

                                src_entry->needs_copy = TRUE;
                        }
                        /*
                         * Throw away the old object reference of the new entry.
                         */
                        vm_object_deallocate(object);

                } else {
                        new_entry->is_shared = FALSE;

                        /*
                         * The map can be safely unlocked since we
                         * already hold a reference on the object.
                         *
                         * Record the timestamp of the map for later
                         * verification, and unlock the map.
                         */
                        version.main_timestamp = map->timestamp;
                        vm_map_unlock(map);

                        /*
                         * Perform the copy.
                         */
                        if (src_entry->wired_count > 0) {
                                vm_object_lock(object);
                                result = vm_object_copy_slowly(
                                                object,
                                                offset,
                                                entry_size,
                                                THREAD_UNINT,
                                                &new_entry->object.vm_object);

                                new_entry->offset = 0;
                                new_entry->needs_copy = FALSE;
                        } else {
                                result = vm_object_copy_strategically(
                                                object,
                                                offset,
                                                entry_size,
                                                &new_entry->object.vm_object,
                                                &new_entry->offset,
                                                &new_entry_needs_copy);

                                new_entry->needs_copy = new_entry_needs_copy;
                        }

                        /*
                         * Throw away the old object reference of the new entry.
                         */
                        vm_object_deallocate(object);

                        if (result != KERN_SUCCESS &&
                            result != KERN_MEMORY_RESTART_COPY) {
                                _vm_map_entry_dispose(map_header, new_entry);
                                break;
                        }

                        /*
                         * Verify that the map has not substantially
                         * changed while the copy was being made.
                         */

                        vm_map_lock(map);       /* Increments timestamp once! */
                        if (version.main_timestamp + 1 != map->timestamp) {
                                /*
                                 * Simple version comparison failed.
                                 *
                                 * Retry the lookup and verify that the
                                 * same object/offset are still present.
                                 */
                                vm_object_deallocate(new_entry->
                                                     object.vm_object);
                                _vm_map_entry_dispose(map_header, new_entry);
                                if (result == KERN_MEMORY_RESTART_COPY)
                                        result = KERN_SUCCESS;
                                continue;
                        }

                        if (result == KERN_MEMORY_RESTART_COPY) {
                                vm_object_reference(object);
                                goto RestartCopy;
                        }
                }

                _vm_map_entry_link(map_header,
                                   map_header->links.prev, new_entry);

                *cur_protection &= src_entry->protection;
                *max_protection &= src_entry->max_protection;

                map_address += tmp_size;
                mapped_size += tmp_size;
                src_start += tmp_size;

        } /* end while */

        vm_map_unlock(map);
        if (result != KERN_SUCCESS) {
                /*
                 * Free all allocated elements.
                 */
                for (src_entry = map_header->links.next;
                     src_entry != (struct vm_map_entry *)&map_header->links;
                     src_entry = new_entry) {
                        new_entry = src_entry->vme_next;
                        _vm_map_entry_unlink(map_header, src_entry);
                        vm_object_deallocate(src_entry->object.vm_object);
                        _vm_map_entry_dispose(map_header, src_entry);
                }
        }
        return result;
}

/*
 *      Routine:        vm_remap
 *
 *                      Map portion of a task's address space.
 *                      Mapped region must not overlap more than
 *                      one vm memory object. Protections and
 *                      inheritance attributes remain the same
 *                      as in the original task and are out parameters.
 *                      Source and Target task can be identical
 *                      Other attributes are identical as for vm_map()
 */
kern_return_t
vm_remap(
        vm_map_t                target_map,
        vm_offset_t             *address,
        vm_size_t               size,
        vm_offset_t             mask,
        boolean_t               anywhere,
        vm_map_t                src_map,
        vm_offset_t             memory_address,
        boolean_t               copy,
        vm_prot_t               *cur_protection,
        vm_prot_t               *max_protection,
        vm_inherit_t            inheritance)
{
        kern_return_t           result;
        vm_map_entry_t          entry;
        vm_map_entry_t          insp_entry;
        vm_map_entry_t          new_entry;
        struct vm_map_header    map_header;

        if (target_map == VM_MAP_NULL)
                return KERN_INVALID_ARGUMENT;

        switch (inheritance) {
            case VM_INHERIT_NONE:
            case VM_INHERIT_COPY:
            case VM_INHERIT_SHARE:
                if (size != 0 && src_map != VM_MAP_NULL)
                        break;
                /*FALL THRU*/
            default:
                return KERN_INVALID_ARGUMENT;
        }

        size = round_page(size);

        result = vm_remap_extract(src_map, memory_address,
                                  size, copy, &map_header,
                                  cur_protection,
                                  max_protection,
                                  inheritance,
                                  target_map->hdr.
                                  entries_pageable);
        vm_map_deallocate(src_map);

        if (result != KERN_SUCCESS) {
                return result;
        }

        /*
         * Allocate/check a range of free virtual address
         * space for the target
         */
        *address = trunc_page(*address);
        vm_map_lock(target_map);
        result = vm_remap_range_allocate(target_map, address, size,
                                         mask, anywhere, &insp_entry);

        for (entry = map_header.links.next;
             entry != (struct vm_map_entry *)&map_header.links;
             entry = new_entry) {
                new_entry = entry->vme_next;
                _vm_map_entry_unlink(&map_header, entry);
                if (result == KERN_SUCCESS) {
                        entry->vme_start += *address;
                        entry->vme_end += *address;
                        vm_map_entry_link(target_map, insp_entry, entry);
                        insp_entry = entry;
                } else {
                        if (!entry->is_sub_map) {
                                vm_object_deallocate(entry->object.vm_object);
                        } else {
                                vm_map_deallocate(entry->object.sub_map);
                           }
                        _vm_map_entry_dispose(&map_header, entry);
                }
        }

        if (result == KERN_SUCCESS) {
                target_map->size += size;
                SAVE_HINT(target_map, insp_entry);
        }
        vm_map_unlock(target_map);

        if (result == KERN_SUCCESS && target_map->wiring_required)
                result = vm_map_wire(target_map, *address,
                                     *address + size, *cur_protection, TRUE);
        return result;
}

/*
 *      Routine:        vm_remap_range_allocate
 *
 *      Description:
 *              Allocate a range in the specified virtual address map.
 *              returns the address and the map entry just before the allocated
 *              range
 *
 *      Map must be locked.
 */

kern_return_t
vm_remap_range_allocate(
        vm_map_t        map,
        vm_offset_t     *address,       /* IN/OUT */
        vm_size_t       size,
        vm_offset_t     mask,
        boolean_t       anywhere,
        vm_map_entry_t  *map_entry)     /* OUT */
{
        register vm_map_entry_t entry;
        register vm_offset_t    start;
        register vm_offset_t    end;
        kern_return_t           result = KERN_SUCCESS;

 StartAgain: ;

    start = *address;

    if (anywhere)
    {
        /*
         *      Calculate the first possible address.
         */

        if (start < map->min_offset)
            start = map->min_offset;
        if (start > map->max_offset)
            return(KERN_NO_SPACE);
                
        /*
         *      Look for the first possible address;
         *      if there's already something at this
         *      address, we have to start after it.
         */

        assert(first_free_is_valid(map));
        if (start == map->min_offset) {
            if ((entry = map->first_free) != vm_map_to_entry(map))
                start = entry->vme_end;
        } else {
            vm_map_entry_t      tmp_entry;
            if (vm_map_lookup_entry(map, start, &tmp_entry))
                start = tmp_entry->vme_end;
            entry = tmp_entry;
        }
                
        /*
         *      In any case, the "entry" always precedes
         *      the proposed new region throughout the
         *      loop:
         */

        while (TRUE) {
            register vm_map_entry_t     next;

            /*
             *  Find the end of the proposed new region.
             *  Be sure we didn't go beyond the end, or
             *  wrap around the address.
             */

            end = ((start + mask) & ~mask);
            if (end < start)
                    return(KERN_NO_SPACE);
            start = end;
            end += size;

            if ((end > map->max_offset) || (end < start)) {
                if (map->wait_for_space) {
                    if (size <= (map->max_offset -
                                 map->min_offset)) {
                        assert_wait((event_t) map, THREAD_INTERRUPTIBLE);
                        vm_map_unlock(map);
                        thread_block((void (*)(void))0);
                        vm_map_lock(map);
                        goto StartAgain;
                    }
                }
                
                return(KERN_NO_SPACE);
            }

            /*
             *  If there are no more entries, we must win.
             */

            next = entry->vme_next;
            if (next == vm_map_to_entry(map))
                break;

            /*
             *  If there is another entry, it must be
             *  after the end of the potential new region.
             */

            if (next->vme_start >= end)
                break;

            /*
             *  Didn't fit -- move to the next entry.
             */

            entry = next;
            start = entry->vme_end;
        }
        *address = start;
    } else {
        vm_map_entry_t          temp_entry;
        
        /*
         *      Verify that:
         *              the address doesn't itself violate
         *              the mask requirement.
         */

        if ((start & mask) != 0)
            return(KERN_NO_SPACE);


        /*
         *      ...     the address is within bounds
         */

        end = start + size;

        if ((start < map->min_offset) ||
            (end > map->max_offset) ||
            (start >= end)) {
            return(KERN_INVALID_ADDRESS);
        }

        /*
         *      ...     the starting address isn't allocated
         */

        if (vm_map_lookup_entry(map, start, &temp_entry))
            return(KERN_NO_SPACE);

        entry = temp_entry;

        /*
         *      ...     the next region doesn't overlap the
         *              end point.
         */

        if ((entry->vme_next != vm_map_to_entry(map)) &&
            (entry->vme_next->vme_start < end))
            return(KERN_NO_SPACE);
    }
    *map_entry = entry;
    return(KERN_SUCCESS);
}

/*
 *      vm_map_switch:
 *
 *      Set the address map for the current thr_act to the specified map
 */

vm_map_t
vm_map_switch(
        vm_map_t        map)
{
        int             mycpu;
        thread_act_t    thr_act = current_act();
        vm_map_t        oldmap = thr_act->map;

        mp_disable_preemption();
        mycpu = cpu_number();

        /*
         *      Deactivate the current map and activate the requested map
         */
        PMAP_SWITCH_USER(thr_act, map, mycpu);

        mp_enable_preemption();
        return(oldmap);
}


/*
 *      Routine:        vm_map_write_user
 *
 *      Description:
 *              Copy out data from a kernel space into space in the
 *              destination map. The space must already exist in the
 *              destination map.
 *              NOTE:  This routine should only be called by threads
 *              which can block on a page fault. i.e. kernel mode user
 *              threads.
 *
 */
kern_return_t
vm_map_write_user(
        vm_map_t        map,
        vm_offset_t     src_addr,
        vm_offset_t     dst_addr,
        vm_size_t       size)
{
        thread_act_t    thr_act = current_act();
        kern_return_t   kr = KERN_SUCCESS;

        if(thr_act->map == map) {
                if (copyout((char *)src_addr, (char *)dst_addr, size)) {
                        kr = KERN_INVALID_ADDRESS;
                }
        } else {
                vm_map_t        oldmap;

                /* take on the identity of the target map while doing */
                /* the transfer */

                vm_map_reference(map);
                oldmap = vm_map_switch(map);
                if (copyout((char *)src_addr, (char *)dst_addr, size)) {
                        kr = KERN_INVALID_ADDRESS;
                }
                vm_map_switch(oldmap);
                vm_map_deallocate(map);
        }
        return kr;
}

/*
 *      Routine:        vm_map_read_user
 *
 *      Description:
 *              Copy in data from a user space source map into the
 *              kernel map. The space must already exist in the
 *              kernel map.
 *              NOTE:  This routine should only be called by threads
 *              which can block on a page fault. i.e. kernel mode user
 *              threads.
 *
 */
kern_return_t
vm_map_read_user(
        vm_map_t        map,
        vm_offset_t     src_addr,
        vm_offset_t     dst_addr,
        vm_size_t       size)
{
        thread_act_t    thr_act = current_act();
        kern_return_t   kr = KERN_SUCCESS;

        if(thr_act->map == map) {
                if (copyin((char *)src_addr, (char *)dst_addr, size)) {
                        kr = KERN_INVALID_ADDRESS;
                }
        } else {
                vm_map_t        oldmap;

                /* take on the identity of the target map while doing */
                /* the transfer */

                vm_map_reference(map);
                oldmap = vm_map_switch(map);
                if (copyin((char *)src_addr, (char *)dst_addr, size)) {
                        kr = KERN_INVALID_ADDRESS;
                }
                vm_map_switch(oldmap);
                vm_map_deallocate(map);
        }
        return kr;
}

/* Takes existing source and destination sub-maps and clones the contents of */
/* the source map */

kern_return_t
vm_region_clone(
        ipc_port_t      src_region, 
        ipc_port_t      dst_region)
{
        vm_named_entry_t        src_object;
        vm_named_entry_t        dst_object;
        vm_map_t                src_map;
        vm_map_t                dst_map;
        vm_offset_t             addr;
        vm_offset_t             max_off;
        vm_map_entry_t          entry;
        vm_map_entry_t          new_entry;
        vm_map_entry_t          insert_point;

        src_object = (vm_named_entry_t)src_region->ip_kobject;
        dst_object = (vm_named_entry_t)dst_region->ip_kobject;
        if((!src_object->is_sub_map) || (!dst_object->is_sub_map)) {
                return KERN_INVALID_ARGUMENT;
        }
        src_map = (vm_map_t)src_object->backing.map;
        dst_map = (vm_map_t)dst_object->backing.map;
        /* destination map is assumed to be unavailable to any other */
        /* activity.  i.e. it is new */
        vm_map_lock(src_map);
        if((src_map->min_offset != dst_map->min_offset) 
                        || (src_map->max_offset != dst_map->max_offset)) {
                vm_map_unlock(src_map);
                return KERN_INVALID_ARGUMENT;
        }
        addr = src_map->min_offset;
        vm_map_lookup_entry(dst_map, addr, &entry);
        if(entry == vm_map_to_entry(dst_map)) {
                entry = entry->vme_next;
        }
        if(entry == vm_map_to_entry(dst_map)) {
                max_off = src_map->max_offset;
        } else {
                max_off =  entry->vme_start;
        }
        vm_map_lookup_entry(src_map, addr, &entry);
        if(entry == vm_map_to_entry(src_map)) {
                entry = entry->vme_next;
        }
        vm_map_lookup_entry(dst_map, addr, &insert_point);
        while((entry != vm_map_to_entry(src_map)) && 
                                        (entry->vme_end <= max_off)) {
                addr = entry->vme_start;
                new_entry = vm_map_entry_create(dst_map);
                vm_map_entry_copy(new_entry, entry);
                vm_map_entry_link(dst_map, insert_point, new_entry);
                insert_point = new_entry;
                if (entry->object.vm_object != VM_OBJECT_NULL) {
                        if (new_entry->is_sub_map) {
                                vm_map_reference(new_entry->object.sub_map);
                        } else {
                                vm_object_reference(
                                        new_entry->object.vm_object);
                        }
                }
                dst_map->size += new_entry->vme_end - new_entry->vme_start;
                entry = entry->vme_next;
        }
        vm_map_unlock(src_map);
        return KERN_SUCCESS;
}

/*
 * Export routines to other components for the things we access locally through
 * macros.
 */
#undef current_map
vm_map_t
current_map(void)
{
        return (current_map_fast());
}

/*
 *      vm_map_check_protection:
 *
 *      Assert that the target map allows the specified
 *      privilege on the entire address region given.
 *      The entire region must be allocated.
 */
boolean_t vm_map_check_protection(map, start, end, protection)
        register vm_map_t       map;
        register vm_offset_t    start;
        register vm_offset_t    end;
        register vm_prot_t      protection;
{
        register vm_map_entry_t entry;
        vm_map_entry_t          tmp_entry;

        vm_map_lock(map);

    if (start < vm_map_min(map) || end > vm_map_max(map) || start > end)
        {
                        vm_map_unlock(map);
                        return (FALSE);
        }

        if (!vm_map_lookup_entry(map, start, &tmp_entry)) {
                vm_map_unlock(map);
                return(FALSE);
        }

        entry = tmp_entry;

        while (start < end) {
                if (entry == vm_map_to_entry(map)) {
                        vm_map_unlock(map);
                        return(FALSE);
                }

                /*
                 *      No holes allowed!
                 */

                if (start < entry->vme_start) {
                        vm_map_unlock(map);
                        return(FALSE);
                }

                /*
                 * Check protection associated with entry.
                 */

                if ((entry->protection & protection) != protection) {
                        vm_map_unlock(map);
                        return(FALSE);
                }

                /* go to next entry */

                start = entry->vme_end;
                entry = entry->vme_next;
        }
        vm_map_unlock(map);
        return(TRUE);
}